@delali/sirannon-db 0.1.7 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +117 -969
- package/dist/backup-scheduler/index.d.ts +1 -1
- package/dist/backup-scheduler/index.mjs +2 -2
- package/dist/baseline-Br77Fnhb.d.ts +6 -0
- package/dist/{change-tracker-CbmaMO-N.d.ts → change-tracker-DKRVUC3l.d.ts} +5 -8
- package/dist/{chunk-DJLX6CAE.mjs → chunk-4ISB7XMA.mjs} +2 -1
- package/dist/chunk-5NOIGN5Y.mjs +18 -0
- package/dist/chunk-67M7KAH6.mjs +90 -0
- package/dist/{chunk-GEZUUIKV.mjs → chunk-D7LAYTKN.mjs} +5 -2
- package/dist/{chunk-4IGMIJQK.mjs → chunk-EBJXPQQO.mjs} +59 -20
- package/dist/{chunk-BNUTBHHH.mjs → chunk-FHWTZFI4.mjs} +1 -1
- package/dist/chunk-GVCNMPOS.mjs +130 -0
- package/dist/chunk-H237TXZW.mjs +1407 -0
- package/dist/{chunk-HHRMRFFR.mjs → chunk-H6PIVVDN.mjs} +353 -142
- package/dist/chunk-HR5CWTLC.mjs +661 -0
- package/dist/chunk-JU64Y7HM.mjs +299 -0
- package/dist/{chunk-CW6S3WL5.mjs → chunk-JZGINXTN.mjs} +7 -5
- package/dist/{chunk-CJLYFDP5.mjs → chunk-LFZ37BSX.mjs} +1 -1
- package/dist/chunk-LNY2VVHE.mjs +18 -0
- package/dist/chunk-NVQS53NT.mjs +665 -0
- package/dist/{chunk-H5AB6NIR.mjs → chunk-O7SLN3GI.mjs} +2 -15
- package/dist/chunk-OQVZBEBY.mjs +283 -0
- package/dist/{chunk-VLTICJOD.mjs → chunk-PIKHN33N.mjs} +1 -1
- package/dist/chunk-TJF5GZSV.mjs +187 -0
- package/dist/chunk-TUD5CJ76.mjs +126 -0
- package/dist/{chunk-YPYVQJ4C.mjs → chunk-UC3SCMIN.mjs} +9 -1
- package/dist/client/index.d.ts +216 -321
- package/dist/client/index.mjs +1192 -1342
- package/dist/client/topology.d.ts +61 -0
- package/dist/client/topology.mjs +682 -0
- package/dist/client-base-CLWmH5Ln.d.ts +234 -0
- package/dist/codegen/cli.d.ts +2 -0
- package/dist/codegen/cli.mjs +11 -0
- package/dist/codegen/index.d.ts +30 -0
- package/dist/codegen/index.mjs +5 -0
- package/dist/core/index.d.ts +37 -16
- package/dist/core/index.mjs +2322 -298
- package/dist/core/writer-worker.mjs +28 -6
- package/dist/{database-DuGp0Rtr.d.ts → database-BY0L5Q2n.d.ts} +59 -14
- package/dist/driver/better-sqlite3.d.ts +1 -1
- package/dist/driver/better-sqlite3.mjs +6 -6
- package/dist/driver/node.d.ts +1 -1
- package/dist/driver/node.mjs +6 -6
- package/dist/driver/wa-sqlite.d.ts +1 -136
- package/dist/{errors-5Nf5ZAEC.d.ts → errors-Bw5MdNCu.d.ts} +5 -1
- package/dist/file-migrations/index.d.ts +7 -5
- package/dist/file-migrations/index.mjs +42 -37
- package/dist/operation-registry-9DcvxcE5.d.ts +45 -0
- package/dist/primary-wins-DPAm2AKG.d.ts +61 -0
- package/dist/protocol-6KrSq2Hy.d.ts +66 -0
- package/dist/react/index.d.ts +20 -0
- package/dist/react/index.mjs +137 -0
- package/dist/replication/coordinator/etcd.d.ts +5 -3
- package/dist/replication/coordinator/etcd.mjs +554 -462
- package/dist/replication/index.d.ts +51 -105
- package/dist/replication/index.mjs +1415 -2061
- package/dist/server/index.d.ts +111 -87
- package/dist/server/index.mjs +1917 -489
- package/dist/server-options-1JHu8pid.d.ts +191 -0
- package/dist/{sirannon-4SspRvP5.d.ts → sirannon-DaQSyhbJ.d.ts} +7 -2
- package/dist/transport/grpc.d.ts +7 -4
- package/dist/transport/grpc.mjs +191 -97
- package/dist/transport/memory.d.ts +5 -134
- package/dist/transport/memory.mjs +3 -1
- package/dist/{types-BsjobKbl.d.ts → types-B7gmEsZW.d.ts} +8 -61
- package/dist/{types-BEu1I_9_.d.ts → types-BsVabqSI.d.ts} +1 -1
- package/dist/types-CL6piSnD.d.ts +43 -0
- package/dist/types-C_D8IhpO.d.ts +60 -0
- package/dist/{types-D_hQW1hr.d.ts → types-zhnRXrsb.d.ts} +55 -165
- package/package.json +44 -6
- package/dist/chunk-TKGHYWQ6.mjs +0 -35
- package/dist/types-D4p4UyDK.d.ts +0 -26
|
@@ -1,141 +1,20 @@
|
|
|
1
1
|
import { compatibilityAllowsPromotion } from '../chunk-ER7ODTDA.mjs';
|
|
2
|
-
import {
|
|
3
|
-
export { AuthorityError,
|
|
4
|
-
import { Transaction } from '../chunk-
|
|
5
|
-
|
|
6
|
-
import
|
|
7
|
-
import '../chunk-
|
|
8
|
-
import {
|
|
2
|
+
import { WriteConcernError, AuthorityError, SyncError, TopologyError, CoordinatorError, StalePrimaryError, NodeDrainingError, NodeNotInSyncError, ReadConcernError, ProtocolVersionMismatchError } from '../chunk-O7SLN3GI.mjs';
|
|
3
|
+
export { AuthorityError, ConflictError, CoordinatorError, FailoverError, NoSafePrimaryError, NodeDrainingError, NodeNotInSyncError, ProtocolVersionMismatchError, ReadConcernError, StalePrimaryError, SyncError, TopologyError, TransportError, UnsafeRecoveryRequiredError, WriteConcernError } from '../chunk-O7SLN3GI.mjs';
|
|
4
|
+
import { LWWResolver, HLC, BatchReader, BatchApplier, StampOps, isWellFormedHlc, Transaction, loadPersistedHlc } from '../chunk-HR5CWTLC.mjs';
|
|
5
|
+
export { HLC, LWWResolver } from '../chunk-HR5CWTLC.mjs';
|
|
6
|
+
import '../chunk-JZGINXTN.mjs';
|
|
7
|
+
import { DumpOps, dumpSchema, tablesInFkOrder } from '../chunk-OQVZBEBY.mjs';
|
|
8
|
+
import { canonicaliseForChecksum, PkResolver, setForeignKeysEnabled, selectMaxAppliedSourceSeqByNode, selectMaxChangeHlc, selectMaxColumnVersionHlc, selectMaxAppliedSourceSeq, upsertPeerAckedSeq, selectPeerAckedSeq, selectMaxChangeSeq, selectMinPeerAckedSeq, upsertSyncTableStatus, upsertSyncMeta, selectSyncMetaRow, selectCompletedSyncTableNames, ensureChangesTable, ensureReplicationStateTables, ensureMetaTable, IDENTIFIER_RE, deleteSyncTableStates, extractDroppedTable, insertDdlChange, prepareInsertAppliedChange } from '../chunk-NVQS53NT.mjs';
|
|
9
|
+
import { ReplicationError, BatchValidationError } from '../chunk-5NOIGN5Y.mjs';
|
|
10
|
+
export { BatchValidationError, ReplicationError } from '../chunk-5NOIGN5Y.mjs';
|
|
11
|
+
import { CHANGES_TABLE, INTERNAL_TABLE_PREFIX } from '../chunk-D7LAYTKN.mjs';
|
|
12
|
+
import '../chunk-GVCNMPOS.mjs';
|
|
13
|
+
import '../chunk-UC3SCMIN.mjs';
|
|
9
14
|
import { EventEmitter } from 'events';
|
|
15
|
+
import { randomBytes, randomUUID, createHash } from 'crypto';
|
|
10
16
|
|
|
11
|
-
// src/
|
|
12
|
-
var MAX_LOGICAL = 65535;
|
|
13
|
-
var HLC = class _HLC {
|
|
14
|
-
wallMs;
|
|
15
|
-
logical;
|
|
16
|
-
nodeId;
|
|
17
|
-
constructor(nodeId) {
|
|
18
|
-
this.nodeId = nodeId;
|
|
19
|
-
this.wallMs = 0;
|
|
20
|
-
this.logical = 0;
|
|
21
|
-
}
|
|
22
|
-
/** Generate a new HLC timestamp, advancing the clock. */
|
|
23
|
-
now() {
|
|
24
|
-
const physicalMs = Date.now();
|
|
25
|
-
if (physicalMs > this.wallMs) {
|
|
26
|
-
this.wallMs = physicalMs;
|
|
27
|
-
this.logical = 0;
|
|
28
|
-
} else {
|
|
29
|
-
this.logical += 1;
|
|
30
|
-
if (this.logical > MAX_LOGICAL) {
|
|
31
|
-
throw new ReplicationError("HLC logical counter overflow");
|
|
32
|
-
}
|
|
33
|
-
}
|
|
34
|
-
return this.encode(this.wallMs, this.logical, this.nodeId);
|
|
35
|
-
}
|
|
36
|
-
/** Merge a remote HLC timestamp into the local clock and return the updated value. */
|
|
37
|
-
receive(remote) {
|
|
38
|
-
const r = _HLC.decode(remote);
|
|
39
|
-
const physicalMs = Date.now();
|
|
40
|
-
if (physicalMs > this.wallMs && physicalMs > r.wallMs) {
|
|
41
|
-
this.wallMs = physicalMs;
|
|
42
|
-
this.logical = 0;
|
|
43
|
-
} else if (r.wallMs > this.wallMs) {
|
|
44
|
-
this.wallMs = r.wallMs;
|
|
45
|
-
this.logical = r.logical + 1;
|
|
46
|
-
} else if (this.wallMs > r.wallMs) {
|
|
47
|
-
this.logical += 1;
|
|
48
|
-
} else {
|
|
49
|
-
this.logical = Math.max(this.logical, r.logical) + 1;
|
|
50
|
-
}
|
|
51
|
-
if (this.logical > MAX_LOGICAL) {
|
|
52
|
-
throw new ReplicationError("HLC logical counter overflow");
|
|
53
|
-
}
|
|
54
|
-
return this.encode(this.wallMs, this.logical, this.nodeId);
|
|
55
|
-
}
|
|
56
|
-
/** Lexicographic comparison of two encoded HLC strings. Returns -1, 0, or 1. */
|
|
57
|
-
static compare(a, b) {
|
|
58
|
-
if (a < b) return -1;
|
|
59
|
-
if (a > b) return 1;
|
|
60
|
-
return 0;
|
|
61
|
-
}
|
|
62
|
-
/** Parse an encoded HLC string into its wall-clock, logical, and nodeId components. */
|
|
63
|
-
static decode(hlc) {
|
|
64
|
-
const parts = hlc.split("-");
|
|
65
|
-
if (parts.length < 3) {
|
|
66
|
-
throw new ReplicationError(`Invalid HLC format: ${hlc}`);
|
|
67
|
-
}
|
|
68
|
-
return {
|
|
69
|
-
wallMs: Number.parseInt(parts[0], 16),
|
|
70
|
-
logical: Number.parseInt(parts[1], 16),
|
|
71
|
-
nodeId: parts.slice(2).join("-")
|
|
72
|
-
};
|
|
73
|
-
}
|
|
74
|
-
encode(wallMs, logical, nodeId) {
|
|
75
|
-
const wallHex = wallMs.toString(16).padStart(12, "0");
|
|
76
|
-
const logicalHex = logical.toString(16).padStart(4, "0");
|
|
77
|
-
return `${wallHex}-${logicalHex}-${nodeId}`;
|
|
78
|
-
}
|
|
79
|
-
};
|
|
80
|
-
|
|
81
|
-
// src/replication/log/canonicalise.ts
|
|
82
|
-
function canonicaliseForChecksum(value) {
|
|
83
|
-
return canonicaliseValue(value);
|
|
84
|
-
}
|
|
85
|
-
function canonicaliseValue(v) {
|
|
86
|
-
if (v === null || v === void 0) return "null";
|
|
87
|
-
if (typeof v === "boolean") return v ? "true" : "false";
|
|
88
|
-
if (typeof v === "string") return JSON.stringify(v);
|
|
89
|
-
if (typeof v === "bigint") return `{"__sint":"${v.toString()}"}`;
|
|
90
|
-
if (typeof v === "number") {
|
|
91
|
-
if (Number.isInteger(v)) return `{"__sint":"${v.toString()}"}`;
|
|
92
|
-
if (Number.isFinite(v)) return JSON.stringify(v);
|
|
93
|
-
return "null";
|
|
94
|
-
}
|
|
95
|
-
if (typeof v === "object") {
|
|
96
|
-
if (v instanceof Uint8Array) {
|
|
97
|
-
const buf = Buffer.isBuffer(v) ? v : Buffer.from(v);
|
|
98
|
-
return `{"__blob":"${buf.toString("base64")}"}`;
|
|
99
|
-
}
|
|
100
|
-
if (typeof Buffer !== "undefined" && Buffer.isBuffer(v)) {
|
|
101
|
-
return `{"__blob":"${v.toString("base64")}"}`;
|
|
102
|
-
}
|
|
103
|
-
if (Array.isArray(v)) {
|
|
104
|
-
return `[${v.map(canonicaliseValue).join(",")}]`;
|
|
105
|
-
}
|
|
106
|
-
const obj = v;
|
|
107
|
-
const parts = [];
|
|
108
|
-
for (const key of Object.keys(obj).sort()) {
|
|
109
|
-
const val = obj[key];
|
|
110
|
-
if (val === void 0) continue;
|
|
111
|
-
parts.push(`${JSON.stringify(key)}:${canonicaliseValue(val)}`);
|
|
112
|
-
}
|
|
113
|
-
return `{${parts.join(",")}}`;
|
|
114
|
-
}
|
|
115
|
-
return "null";
|
|
116
|
-
}
|
|
117
|
-
|
|
118
|
-
// src/replication/conflict/lww.ts
|
|
119
|
-
var LWWResolver = class {
|
|
120
|
-
resolve(ctx) {
|
|
121
|
-
if (ctx.localHlc === null) {
|
|
122
|
-
return { action: "accept_remote" };
|
|
123
|
-
}
|
|
124
|
-
const cmp = HLC.compare(ctx.remoteHlc, ctx.localHlc);
|
|
125
|
-
if (cmp > 0) {
|
|
126
|
-
return { action: "accept_remote" };
|
|
127
|
-
}
|
|
128
|
-
if (cmp < 0) {
|
|
129
|
-
return { action: "keep_local" };
|
|
130
|
-
}
|
|
131
|
-
if (ctx.remoteChange.nodeId > (ctx.localChange?.nodeId ?? "")) {
|
|
132
|
-
return { action: "accept_remote" };
|
|
133
|
-
}
|
|
134
|
-
return { action: "keep_local" };
|
|
135
|
-
}
|
|
136
|
-
};
|
|
137
|
-
|
|
138
|
-
// src/replication/conflict/field-merge.ts
|
|
17
|
+
// src/core/sync/conflict/field-merge.ts
|
|
139
18
|
var FieldMergeResolver = class {
|
|
140
19
|
getColumnVersions;
|
|
141
20
|
lww = new LWWResolver();
|
|
@@ -209,7 +88,7 @@ var FieldMergeResolver = class {
|
|
|
209
88
|
}
|
|
210
89
|
};
|
|
211
90
|
|
|
212
|
-
// src/
|
|
91
|
+
// src/core/sync/conflict/primary-wins.ts
|
|
213
92
|
var PrimaryWinsResolver = class {
|
|
214
93
|
primaryNodeId;
|
|
215
94
|
lww = new LWWResolver();
|
|
@@ -227,690 +106,6 @@ var PrimaryWinsResolver = class {
|
|
|
227
106
|
}
|
|
228
107
|
};
|
|
229
108
|
|
|
230
|
-
// src/replication/engine/constants.ts
|
|
231
|
-
var DEFAULT_BATCH_SIZE = 100;
|
|
232
|
-
var DEFAULT_BATCH_INTERVAL_MS = 100;
|
|
233
|
-
var DEFAULT_MAX_CLOCK_DRIFT_MS = 6e4;
|
|
234
|
-
var DEFAULT_MAX_PENDING_BATCHES = 10;
|
|
235
|
-
var DEFAULT_MAX_BATCH_CHANGES = 1e3;
|
|
236
|
-
var DEFAULT_ACK_TIMEOUT_MS = 5e3;
|
|
237
|
-
var DEFAULT_SYNC_BATCH_SIZE = 1e4;
|
|
238
|
-
var DEFAULT_MAX_CONCURRENT_SYNCS = 2;
|
|
239
|
-
var DEFAULT_MAX_SYNC_DURATION_MS = 18e5;
|
|
240
|
-
var DEFAULT_MAX_SYNC_LAG_BEFORE_READY = 100;
|
|
241
|
-
var DEFAULT_SYNC_ACK_TIMEOUT_MS = 3e4;
|
|
242
|
-
var DEFAULT_CATCH_UP_DEADLINE_MS = 6e5;
|
|
243
|
-
var DDL_PREFIX_RE = /^\s*(CREATE\s+TABLE|ALTER\s+TABLE\s+\S+\s+ADD\s+COLUMN|DROP\s+TABLE|CREATE\s+INDEX|DROP\s+INDEX)\b/i;
|
|
244
|
-
var SAFE_SQL_PREFIX_RE = /^\s*(INSERT|UPDATE|DELETE|SELECT|CREATE\s+TABLE|ALTER\s+TABLE|DROP\s+TABLE|CREATE\s+INDEX|DROP\s+INDEX)\b/i;
|
|
245
|
-
var IDENTIFIER_RE = /^[a-zA-Z_][a-zA-Z0-9_]*$/;
|
|
246
|
-
var DDL_DENY_RE = /\b(load_extension|ATTACH|randomblob|zeroblob|writefile|readfile|fts3_tokenizer)\b/i;
|
|
247
|
-
var DROP_TABLE_RE = /^\s*DROP\s+TABLE\s+(?:IF\s+EXISTS\s+)?"?([A-Za-z_][A-Za-z0-9_]*)"?\s*;?\s*$/i;
|
|
248
|
-
function extractDroppedTable(sql) {
|
|
249
|
-
const m = DROP_TABLE_RE.exec(sql);
|
|
250
|
-
return m?.[1] ?? null;
|
|
251
|
-
}
|
|
252
|
-
var SYNC_SAFE_DDL_PREFIX_RE = /^\s*(CREATE\s+TABLE|ALTER\s+TABLE\s+\S+\s+ADD\s+COLUMN|DROP\s+TABLE|CREATE\s+INDEX|DROP\s+INDEX)\b/i;
|
|
253
|
-
function isSyncSafeDdl(sql) {
|
|
254
|
-
if (!SYNC_SAFE_DDL_PREFIX_RE.test(sql)) return false;
|
|
255
|
-
if (sql.includes(";")) return false;
|
|
256
|
-
if (/\bAS\s+SELECT\b/i.test(sql)) return false;
|
|
257
|
-
if (DDL_DENY_RE.test(sql)) return false;
|
|
258
|
-
const body = sql.replace(SYNC_SAFE_DDL_PREFIX_RE, "");
|
|
259
|
-
if (/\bSELECT\b/i.test(body)) return false;
|
|
260
|
-
return true;
|
|
261
|
-
}
|
|
262
|
-
var BatchReader = class {
|
|
263
|
-
constructor(conn, localNodeId, hlc, changesTable, pkResolver) {
|
|
264
|
-
this.conn = conn;
|
|
265
|
-
this.localNodeId = localNodeId;
|
|
266
|
-
this.hlc = hlc;
|
|
267
|
-
this.changesTable = changesTable;
|
|
268
|
-
this.pkResolver = pkResolver;
|
|
269
|
-
}
|
|
270
|
-
async readBatch(afterSeq, batchSize) {
|
|
271
|
-
const stmt = await this.conn.prepare(
|
|
272
|
-
`SELECT seq, table_name, operation, row_id, changed_at, old_data, new_data, node_id, tx_id, hlc
|
|
273
|
-
FROM "${this.changesTable}"
|
|
274
|
-
WHERE seq > ? AND node_id = ?
|
|
275
|
-
ORDER BY seq ASC
|
|
276
|
-
LIMIT ?`
|
|
277
|
-
);
|
|
278
|
-
const rows = await stmt.all(afterSeq.toString(), this.localNodeId, batchSize);
|
|
279
|
-
if (rows.length === 0) {
|
|
280
|
-
return null;
|
|
281
|
-
}
|
|
282
|
-
const changes = [];
|
|
283
|
-
let minHlc = rows[0].hlc;
|
|
284
|
-
let maxHlc = rows[0].hlc;
|
|
285
|
-
for (const row of rows) {
|
|
286
|
-
const operation = row.operation.toLowerCase();
|
|
287
|
-
const isDdl = operation === "ddl";
|
|
288
|
-
const rawNewData = row.new_data ? decodeTaggedValues(JSON.parse(row.new_data)) : null;
|
|
289
|
-
const rawOldData = row.old_data ? decodeTaggedValues(JSON.parse(row.old_data)) : null;
|
|
290
|
-
let ddlStatement;
|
|
291
|
-
let newData = rawNewData;
|
|
292
|
-
let oldData = rawOldData;
|
|
293
|
-
if (isDdl) {
|
|
294
|
-
const candidate = rawNewData?.ddlStatement;
|
|
295
|
-
if (typeof candidate === "string") {
|
|
296
|
-
ddlStatement = candidate;
|
|
297
|
-
}
|
|
298
|
-
newData = null;
|
|
299
|
-
oldData = null;
|
|
300
|
-
}
|
|
301
|
-
const pkColumns = isDdl ? [] : await this.pkResolver.forTable(row.table_name);
|
|
302
|
-
const primaryKey = {};
|
|
303
|
-
if (!isDdl) {
|
|
304
|
-
const sourceData = rawNewData ?? rawOldData ?? {};
|
|
305
|
-
for (const col of pkColumns) {
|
|
306
|
-
if (col in sourceData) {
|
|
307
|
-
primaryKey[col] = sourceData[col];
|
|
308
|
-
}
|
|
309
|
-
}
|
|
310
|
-
}
|
|
311
|
-
const change = {
|
|
312
|
-
table: row.table_name,
|
|
313
|
-
operation,
|
|
314
|
-
rowId: String(row.row_id),
|
|
315
|
-
primaryKey,
|
|
316
|
-
hlc: row.hlc,
|
|
317
|
-
txId: row.tx_id,
|
|
318
|
-
nodeId: row.node_id,
|
|
319
|
-
newData,
|
|
320
|
-
oldData
|
|
321
|
-
};
|
|
322
|
-
if (ddlStatement !== void 0) {
|
|
323
|
-
change.ddlStatement = ddlStatement;
|
|
324
|
-
}
|
|
325
|
-
changes.push(change);
|
|
326
|
-
if (HLC.compare(row.hlc, minHlc) < 0) {
|
|
327
|
-
minHlc = row.hlc;
|
|
328
|
-
}
|
|
329
|
-
if (HLC.compare(row.hlc, maxHlc) > 0) {
|
|
330
|
-
maxHlc = row.hlc;
|
|
331
|
-
}
|
|
332
|
-
}
|
|
333
|
-
const fromSeq = BigInt(rows[0].seq);
|
|
334
|
-
const toSeq = BigInt(rows[rows.length - 1].seq);
|
|
335
|
-
const checksum = computeChecksum(changes);
|
|
336
|
-
return {
|
|
337
|
-
sourceNodeId: this.localNodeId,
|
|
338
|
-
batchId: `${this.localNodeId}-${fromSeq}-${toSeq}`,
|
|
339
|
-
fromSeq,
|
|
340
|
-
toSeq,
|
|
341
|
-
hlcRange: { min: minHlc, max: maxHlc },
|
|
342
|
-
changes,
|
|
343
|
-
checksum
|
|
344
|
-
};
|
|
345
|
-
}
|
|
346
|
-
async stampChanges(tx, afterSeq, txId) {
|
|
347
|
-
const hlcValue = this.hlc.now();
|
|
348
|
-
const stmt = await tx.prepare(
|
|
349
|
-
`UPDATE "${this.changesTable}" SET node_id = ?, tx_id = ?, hlc = ? WHERE seq > ? AND node_id = ''`
|
|
350
|
-
);
|
|
351
|
-
await stmt.run(this.localNodeId, txId, hlcValue, afterSeq.toString());
|
|
352
|
-
}
|
|
353
|
-
async updateColumnVersions(tx, afterSeq) {
|
|
354
|
-
const selectStmt = await tx.prepare(
|
|
355
|
-
`SELECT seq, table_name, operation, row_id, old_data, new_data, hlc, node_id
|
|
356
|
-
FROM "${this.changesTable}"
|
|
357
|
-
WHERE seq > ? AND node_id = ?
|
|
358
|
-
ORDER BY seq ASC`
|
|
359
|
-
);
|
|
360
|
-
const rows = await selectStmt.all(afterSeq.toString(), this.localNodeId);
|
|
361
|
-
for (const row of rows) {
|
|
362
|
-
if (row.operation === "DELETE") {
|
|
363
|
-
const delStmt = await tx.prepare(`DELETE FROM ${COLUMN_VERSIONS_TABLE} WHERE table_name = ? AND row_id = ?`);
|
|
364
|
-
await delStmt.run(row.table_name, String(row.row_id));
|
|
365
|
-
continue;
|
|
366
|
-
}
|
|
367
|
-
const oldData = row.old_data ? decodeTaggedValues(JSON.parse(row.old_data)) : {};
|
|
368
|
-
const newData = row.new_data ? decodeTaggedValues(JSON.parse(row.new_data)) : {};
|
|
369
|
-
const changedCols = [];
|
|
370
|
-
if (row.operation === "INSERT") {
|
|
371
|
-
for (const key of Object.keys(newData)) {
|
|
372
|
-
changedCols.push(key);
|
|
373
|
-
}
|
|
374
|
-
} else {
|
|
375
|
-
for (const key of Object.keys(newData)) {
|
|
376
|
-
if (canonicaliseForChecksum(newData[key]) !== canonicaliseForChecksum(oldData[key])) {
|
|
377
|
-
changedCols.push(key);
|
|
378
|
-
}
|
|
379
|
-
}
|
|
380
|
-
}
|
|
381
|
-
const upsertStmt = await tx.prepare(
|
|
382
|
-
`INSERT INTO ${COLUMN_VERSIONS_TABLE} (table_name, row_id, column_name, hlc, node_id)
|
|
383
|
-
VALUES (?, ?, ?, ?, ?)
|
|
384
|
-
ON CONFLICT(table_name, row_id, column_name)
|
|
385
|
-
DO UPDATE SET hlc = excluded.hlc, node_id = excluded.node_id`
|
|
386
|
-
);
|
|
387
|
-
for (const col of changedCols) {
|
|
388
|
-
await upsertStmt.run(row.table_name, String(row.row_id), col, row.hlc, row.node_id);
|
|
389
|
-
}
|
|
390
|
-
}
|
|
391
|
-
}
|
|
392
|
-
};
|
|
393
|
-
function computeChecksum(changes) {
|
|
394
|
-
const hash = createHash("sha256");
|
|
395
|
-
hash.update(canonicaliseForChecksum(changes));
|
|
396
|
-
return hash.digest("hex");
|
|
397
|
-
}
|
|
398
|
-
|
|
399
|
-
// src/replication/log/validators.ts
|
|
400
|
-
var IDENTIFIER_RE2 = /^[a-zA-Z_][a-zA-Z0-9_]*$/;
|
|
401
|
-
var SAFE_DDL_RE = /^\s*(CREATE\s+TABLE|ALTER\s+TABLE\s+\S+\s+ADD\s+COLUMN|DROP\s+TABLE|CREATE\s+INDEX|DROP\s+INDEX)\b/i;
|
|
402
|
-
var DDL_DENY_RE2 = /\b(load_extension|ATTACH|randomblob|zeroblob|writefile|readfile|fts3_tokenizer)\b/i;
|
|
403
|
-
function validateIdentifier(name) {
|
|
404
|
-
return IDENTIFIER_RE2.test(name);
|
|
405
|
-
}
|
|
406
|
-
function validateDdlSafety(sql) {
|
|
407
|
-
if (!SAFE_DDL_RE.test(sql)) return false;
|
|
408
|
-
if (sql.includes(";")) return false;
|
|
409
|
-
if (/\bAS\s+SELECT\b/i.test(sql)) return false;
|
|
410
|
-
if (DDL_DENY_RE2.test(sql)) return false;
|
|
411
|
-
const body = sql.replace(SAFE_DDL_RE, "");
|
|
412
|
-
if (/\bSELECT\b/i.test(body)) return false;
|
|
413
|
-
return true;
|
|
414
|
-
}
|
|
415
|
-
|
|
416
|
-
// src/replication/log/batch-applier.ts
|
|
417
|
-
var BatchApplier = class {
|
|
418
|
-
constructor(conn, localNodeId, hlc, pkResolver, getLastAppliedSeq, tracker) {
|
|
419
|
-
this.conn = conn;
|
|
420
|
-
this.localNodeId = localNodeId;
|
|
421
|
-
this.hlc = hlc;
|
|
422
|
-
this.pkResolver = pkResolver;
|
|
423
|
-
this.getLastAppliedSeq = getLastAppliedSeq;
|
|
424
|
-
this.tracker = tracker;
|
|
425
|
-
}
|
|
426
|
-
async applyBatch(batch, resolver) {
|
|
427
|
-
const expectedChecksum = computeChecksum(batch.changes);
|
|
428
|
-
if (batch.checksum !== expectedChecksum) {
|
|
429
|
-
throw new BatchValidationError(`Checksum mismatch: expected ${expectedChecksum}, got ${batch.checksum}`);
|
|
430
|
-
}
|
|
431
|
-
for (const change of batch.changes) {
|
|
432
|
-
if (change.operation !== "ddl") {
|
|
433
|
-
if (!IDENTIFIER_RE2.test(change.table)) {
|
|
434
|
-
throw new BatchValidationError(`Invalid table name: ${change.table}`);
|
|
435
|
-
}
|
|
436
|
-
}
|
|
437
|
-
}
|
|
438
|
-
const lastApplied = await this.getLastAppliedSeq(batch.sourceNodeId);
|
|
439
|
-
if (batch.toSeq <= lastApplied) {
|
|
440
|
-
return { applied: 0, skipped: batch.changes.length, conflicts: 0, droppedTables: [] };
|
|
441
|
-
}
|
|
442
|
-
const needsPartialDedup = batch.fromSeq <= lastApplied;
|
|
443
|
-
let appliedSeqSet = null;
|
|
444
|
-
if (needsPartialDedup) {
|
|
445
|
-
appliedSeqSet = /* @__PURE__ */ new Set();
|
|
446
|
-
const checkStmt = await this.conn.prepare(
|
|
447
|
-
`SELECT source_seq FROM ${APPLIED_CHANGES_TABLE} WHERE source_node_id = ? AND source_seq >= ? AND source_seq <= ?`
|
|
448
|
-
);
|
|
449
|
-
const applied2 = await checkStmt.all(
|
|
450
|
-
batch.sourceNodeId,
|
|
451
|
-
batch.fromSeq.toString(),
|
|
452
|
-
batch.toSeq.toString()
|
|
453
|
-
);
|
|
454
|
-
for (const row of applied2) {
|
|
455
|
-
appliedSeqSet.add(String(row.source_seq));
|
|
456
|
-
}
|
|
457
|
-
}
|
|
458
|
-
let applied = 0;
|
|
459
|
-
let skipped = 0;
|
|
460
|
-
let conflicts = 0;
|
|
461
|
-
const droppedTables = [];
|
|
462
|
-
const changesByTx = /* @__PURE__ */ new Map();
|
|
463
|
-
for (const change of batch.changes) {
|
|
464
|
-
const txGroup = changesByTx.get(change.txId);
|
|
465
|
-
if (txGroup) {
|
|
466
|
-
txGroup.push(change);
|
|
467
|
-
} else {
|
|
468
|
-
changesByTx.set(change.txId, [change]);
|
|
469
|
-
}
|
|
470
|
-
}
|
|
471
|
-
for (const [_txId, txChanges] of changesByTx) {
|
|
472
|
-
const txDroppedTables = [];
|
|
473
|
-
const result = await this.conn.transaction(async (tx) => {
|
|
474
|
-
let txApplied = 0;
|
|
475
|
-
let txSkipped = 0;
|
|
476
|
-
let txConflicts = 0;
|
|
477
|
-
const ddlChanges = txChanges.filter((c) => c.operation === "ddl");
|
|
478
|
-
const dataChanges = txChanges.filter((c) => c.operation !== "ddl");
|
|
479
|
-
for (const ddl of ddlChanges) {
|
|
480
|
-
const ddlSql = ddl.ddlStatement;
|
|
481
|
-
if (!ddlSql || !validateDdlSafety(ddlSql)) {
|
|
482
|
-
throw new BatchValidationError(`Unsafe or missing DDL statement: ${ddlSql ?? "none"}`);
|
|
483
|
-
}
|
|
484
|
-
await tx.exec(ddlSql);
|
|
485
|
-
const droppedTable = extractDroppedTable(ddlSql);
|
|
486
|
-
if (droppedTable !== null) {
|
|
487
|
-
txDroppedTables.push(droppedTable);
|
|
488
|
-
}
|
|
489
|
-
if (this.tracker) {
|
|
490
|
-
await this.tracker.refreshAllTriggersUsingConnection(tx);
|
|
491
|
-
}
|
|
492
|
-
txApplied += 1;
|
|
493
|
-
}
|
|
494
|
-
for (const change of dataChanges) {
|
|
495
|
-
const existingRow = await this.findExistingRow(tx, change);
|
|
496
|
-
if (existingRow === void 0) {
|
|
497
|
-
if (change.operation === "insert" && change.newData) {
|
|
498
|
-
await this.insertRow(tx, change);
|
|
499
|
-
await this.recordColumnVersions(tx, change, change.newData);
|
|
500
|
-
txApplied += 1;
|
|
501
|
-
} else if (change.operation === "delete") {
|
|
502
|
-
txApplied += 1;
|
|
503
|
-
} else {
|
|
504
|
-
txSkipped += 1;
|
|
505
|
-
}
|
|
506
|
-
} else {
|
|
507
|
-
txConflicts += 1;
|
|
508
|
-
const localHlc = await this.getLocalHlcForRow(tx, change.table, change.rowId);
|
|
509
|
-
const localChange = {
|
|
510
|
-
table: change.table,
|
|
511
|
-
operation: "update",
|
|
512
|
-
rowId: change.rowId,
|
|
513
|
-
primaryKey: change.primaryKey,
|
|
514
|
-
hlc: localHlc ?? "",
|
|
515
|
-
txId: "",
|
|
516
|
-
nodeId: this.localNodeId,
|
|
517
|
-
newData: existingRow,
|
|
518
|
-
oldData: null
|
|
519
|
-
};
|
|
520
|
-
const changeResolver = typeof resolver === "function" ? resolver(change.table) : resolver;
|
|
521
|
-
const resolution = await changeResolver.resolve({
|
|
522
|
-
table: change.table,
|
|
523
|
-
rowId: change.rowId,
|
|
524
|
-
localChange,
|
|
525
|
-
remoteChange: change,
|
|
526
|
-
localHlc,
|
|
527
|
-
remoteHlc: change.hlc
|
|
528
|
-
});
|
|
529
|
-
if (resolution.action === "accept_remote") {
|
|
530
|
-
await this.applyRemoteChange(tx, change);
|
|
531
|
-
await this.recordColumnVersions(tx, change, change.newData);
|
|
532
|
-
txApplied += 1;
|
|
533
|
-
} else if (resolution.action === "merge" && resolution.mergedData) {
|
|
534
|
-
await this.applyMergedData(tx, change, resolution.mergedData);
|
|
535
|
-
await this.recordColumnVersions(tx, change, resolution.mergedData);
|
|
536
|
-
txApplied += 1;
|
|
537
|
-
} else {
|
|
538
|
-
txSkipped += 1;
|
|
539
|
-
}
|
|
540
|
-
}
|
|
541
|
-
}
|
|
542
|
-
return { txApplied, txSkipped, txConflicts };
|
|
543
|
-
});
|
|
544
|
-
applied += result.txApplied;
|
|
545
|
-
skipped += result.txSkipped;
|
|
546
|
-
conflicts += result.txConflicts;
|
|
547
|
-
if (txDroppedTables.length > 0) {
|
|
548
|
-
droppedTables.push(...txDroppedTables);
|
|
549
|
-
}
|
|
550
|
-
}
|
|
551
|
-
const recordStmt = await this.conn.prepare(
|
|
552
|
-
`INSERT OR IGNORE INTO ${APPLIED_CHANGES_TABLE} (source_node_id, source_seq, applied_at) VALUES (?, ?, ?)`
|
|
553
|
-
);
|
|
554
|
-
const nowSec = Date.now() / 1e3;
|
|
555
|
-
for (let seq = batch.fromSeq; seq <= batch.toSeq; seq += 1n) {
|
|
556
|
-
if (appliedSeqSet?.has(seq.toString())) continue;
|
|
557
|
-
await recordStmt.run(batch.sourceNodeId, seq.toString(), nowSec);
|
|
558
|
-
}
|
|
559
|
-
try {
|
|
560
|
-
this.hlc.receive(batch.hlcRange.max);
|
|
561
|
-
} catch {
|
|
562
|
-
}
|
|
563
|
-
return { applied, skipped, conflicts, droppedTables };
|
|
564
|
-
}
|
|
565
|
-
async findExistingRow(tx, change) {
|
|
566
|
-
if (!IDENTIFIER_RE2.test(change.table)) return void 0;
|
|
567
|
-
const pkColumns = await this.pkResolver.forTable(change.table);
|
|
568
|
-
const result = await findRowByPk(tx, change.table, pkColumns, change.newData ?? change.oldData ?? {});
|
|
569
|
-
if (result) return result;
|
|
570
|
-
if (change.operation === "update" && change.oldData) {
|
|
571
|
-
return findRowByPk(tx, change.table, pkColumns, change.oldData);
|
|
572
|
-
}
|
|
573
|
-
return void 0;
|
|
574
|
-
}
|
|
575
|
-
async getLocalHlcForRow(tx, table, rowId) {
|
|
576
|
-
const stmt = await tx.prepare(
|
|
577
|
-
`SELECT MAX(hlc) as max_hlc FROM ${COLUMN_VERSIONS_TABLE} WHERE table_name = ? AND row_id = ?`
|
|
578
|
-
);
|
|
579
|
-
const row = await stmt.get(table, rowId);
|
|
580
|
-
return row?.max_hlc ?? null;
|
|
581
|
-
}
|
|
582
|
-
async insertRow(tx, change) {
|
|
583
|
-
if (!change.newData) return;
|
|
584
|
-
const columns = Object.keys(change.newData).filter(validateIdentifier);
|
|
585
|
-
if (columns.length === 0) return;
|
|
586
|
-
const placeholders = columns.map(() => "?").join(", ");
|
|
587
|
-
const colNames = columns.map((c) => `"${c}"`).join(", ");
|
|
588
|
-
const values = columns.map((c) => change.newData?.[c]);
|
|
589
|
-
const stmt = await tx.prepare(`INSERT INTO "${change.table}" (${colNames}) VALUES (${placeholders})`);
|
|
590
|
-
await stmt.run(...values);
|
|
591
|
-
}
|
|
592
|
-
async applyRemoteChange(tx, change) {
|
|
593
|
-
if (change.operation === "delete") {
|
|
594
|
-
await this.deleteRow(tx, change);
|
|
595
|
-
return;
|
|
596
|
-
}
|
|
597
|
-
if (!change.newData) return;
|
|
598
|
-
const pkColumns = await this.pkResolver.forTable(change.table);
|
|
599
|
-
const sourceData = change.newData;
|
|
600
|
-
const wherePkSource = change.oldData ?? sourceData;
|
|
601
|
-
const setClauses = [];
|
|
602
|
-
const setValues = [];
|
|
603
|
-
const whereConditions = [];
|
|
604
|
-
const whereValues = [];
|
|
605
|
-
const pkSet = new Set(pkColumns);
|
|
606
|
-
for (const [col, val] of Object.entries(sourceData)) {
|
|
607
|
-
if (!validateIdentifier(col)) continue;
|
|
608
|
-
if (pkSet.has(col)) continue;
|
|
609
|
-
setClauses.push(`"${col}" = ?`);
|
|
610
|
-
setValues.push(val);
|
|
611
|
-
}
|
|
612
|
-
for (const col of pkColumns) {
|
|
613
|
-
if (!validateIdentifier(col)) continue;
|
|
614
|
-
whereConditions.push(`"${col}" = ?`);
|
|
615
|
-
whereValues.push(wherePkSource[col]);
|
|
616
|
-
}
|
|
617
|
-
if (setClauses.length === 0 || whereConditions.length === 0) return;
|
|
618
|
-
const stmt = await tx.prepare(
|
|
619
|
-
`UPDATE "${change.table}" SET ${setClauses.join(", ")} WHERE ${whereConditions.join(" AND ")}`
|
|
620
|
-
);
|
|
621
|
-
await stmt.run(...setValues, ...whereValues);
|
|
622
|
-
}
|
|
623
|
-
async applyMergedData(tx, change, mergedData) {
|
|
624
|
-
const pkColumns = await this.pkResolver.forTable(change.table);
|
|
625
|
-
const sourceData = change.newData ?? change.oldData ?? {};
|
|
626
|
-
const setClauses = [];
|
|
627
|
-
const setValues = [];
|
|
628
|
-
const whereConditions = [];
|
|
629
|
-
const whereValues = [];
|
|
630
|
-
for (const [col, val] of Object.entries(mergedData)) {
|
|
631
|
-
if (!validateIdentifier(col)) continue;
|
|
632
|
-
if (!pkColumns.includes(col)) {
|
|
633
|
-
setClauses.push(`"${col}" = ?`);
|
|
634
|
-
setValues.push(val);
|
|
635
|
-
}
|
|
636
|
-
}
|
|
637
|
-
for (const col of pkColumns) {
|
|
638
|
-
if (!validateIdentifier(col)) continue;
|
|
639
|
-
whereConditions.push(`"${col}" = ?`);
|
|
640
|
-
whereValues.push(sourceData[col]);
|
|
641
|
-
}
|
|
642
|
-
if (setClauses.length === 0 || whereConditions.length === 0) return;
|
|
643
|
-
const stmt = await tx.prepare(
|
|
644
|
-
`UPDATE "${change.table}" SET ${setClauses.join(", ")} WHERE ${whereConditions.join(" AND ")}`
|
|
645
|
-
);
|
|
646
|
-
await stmt.run(...setValues, ...whereValues);
|
|
647
|
-
}
|
|
648
|
-
async deleteRow(tx, change) {
|
|
649
|
-
const pkColumns = await this.pkResolver.forTable(change.table);
|
|
650
|
-
const sourceData = change.oldData ?? change.newData ?? {};
|
|
651
|
-
const conditions = [];
|
|
652
|
-
const values = [];
|
|
653
|
-
for (const col of pkColumns) {
|
|
654
|
-
if (!validateIdentifier(col)) continue;
|
|
655
|
-
conditions.push(`"${col}" = ?`);
|
|
656
|
-
values.push(sourceData[col]);
|
|
657
|
-
}
|
|
658
|
-
if (conditions.length === 0) return;
|
|
659
|
-
const stmt = await tx.prepare(`DELETE FROM "${change.table}" WHERE ${conditions.join(" AND ")}`);
|
|
660
|
-
await stmt.run(...values);
|
|
661
|
-
}
|
|
662
|
-
async recordColumnVersions(tx, change, data) {
|
|
663
|
-
if (change.operation === "delete") {
|
|
664
|
-
const delStmt = await tx.prepare(`DELETE FROM ${COLUMN_VERSIONS_TABLE} WHERE table_name = ? AND row_id = ?`);
|
|
665
|
-
await delStmt.run(change.table, change.rowId);
|
|
666
|
-
return;
|
|
667
|
-
}
|
|
668
|
-
if (!data) return;
|
|
669
|
-
const upsertStmt = await tx.prepare(
|
|
670
|
-
`INSERT INTO ${COLUMN_VERSIONS_TABLE} (table_name, row_id, column_name, hlc, node_id)
|
|
671
|
-
VALUES (?, ?, ?, ?, ?)
|
|
672
|
-
ON CONFLICT(table_name, row_id, column_name)
|
|
673
|
-
DO UPDATE SET hlc = excluded.hlc, node_id = excluded.node_id`
|
|
674
|
-
);
|
|
675
|
-
for (const col of Object.keys(data)) {
|
|
676
|
-
if (!validateIdentifier(col)) continue;
|
|
677
|
-
await upsertStmt.run(change.table, change.rowId, col, change.hlc, change.nodeId);
|
|
678
|
-
}
|
|
679
|
-
}
|
|
680
|
-
};
|
|
681
|
-
async function findRowByPk(tx, table, pkColumns, sourceData) {
|
|
682
|
-
const conditions = [];
|
|
683
|
-
const values = [];
|
|
684
|
-
for (const col of pkColumns) {
|
|
685
|
-
if (!validateIdentifier(col)) return void 0;
|
|
686
|
-
if (!(col in sourceData)) return void 0;
|
|
687
|
-
conditions.push(`"${col}" = ?`);
|
|
688
|
-
values.push(sourceData[col]);
|
|
689
|
-
}
|
|
690
|
-
if (conditions.length === 0) return void 0;
|
|
691
|
-
const stmt = await tx.prepare(`SELECT * FROM "${table}" WHERE ${conditions.join(" AND ")} LIMIT 1`);
|
|
692
|
-
return stmt.get(...values);
|
|
693
|
-
}
|
|
694
|
-
var DumpOps = class {
|
|
695
|
-
constructor(conn, localNodeId, hlc, pkResolver) {
|
|
696
|
-
this.conn = conn;
|
|
697
|
-
this.localNodeId = localNodeId;
|
|
698
|
-
this.hlc = hlc;
|
|
699
|
-
this.pkResolver = pkResolver;
|
|
700
|
-
}
|
|
701
|
-
async *dumpTable(table, batchSize) {
|
|
702
|
-
if (!IDENTIFIER_RE2.test(table)) {
|
|
703
|
-
throw new ReplicationError(`Invalid table name: ${table}`);
|
|
704
|
-
}
|
|
705
|
-
const countStmt = await this.conn.prepare(`SELECT COUNT(*) as cnt FROM "${table}"`);
|
|
706
|
-
const countRow = await countStmt.get();
|
|
707
|
-
const total = countRow?.cnt ?? 0;
|
|
708
|
-
if (total === 0) return;
|
|
709
|
-
const pkColumns = await this.pkResolver.forTable(table);
|
|
710
|
-
let offset = 0;
|
|
711
|
-
let batchNum = 0;
|
|
712
|
-
const needsRowid = pkColumns.length === 1 && pkColumns[0] === "rowid";
|
|
713
|
-
while (offset < total) {
|
|
714
|
-
const selectSql = needsRowid ? `SELECT rowid, * FROM "${table}" LIMIT ? OFFSET ?` : `SELECT * FROM "${table}" LIMIT ? OFFSET ?`;
|
|
715
|
-
const selectStmt = await this.conn.prepare(selectSql);
|
|
716
|
-
const rows = await selectStmt.all(batchSize, offset);
|
|
717
|
-
if (rows.length === 0) break;
|
|
718
|
-
const changes = [];
|
|
719
|
-
const hlcValue = this.hlc.now();
|
|
720
|
-
for (const row of rows) {
|
|
721
|
-
const primaryKey = {};
|
|
722
|
-
for (const col of pkColumns) {
|
|
723
|
-
if (col in row) {
|
|
724
|
-
primaryKey[col] = row[col];
|
|
725
|
-
}
|
|
726
|
-
}
|
|
727
|
-
const rowId = pkColumns.map((col) => String(row[col] ?? "")).join("-");
|
|
728
|
-
changes.push({
|
|
729
|
-
table,
|
|
730
|
-
operation: "insert",
|
|
731
|
-
rowId,
|
|
732
|
-
primaryKey,
|
|
733
|
-
hlc: hlcValue,
|
|
734
|
-
txId: `dump-${table}-${batchNum}`,
|
|
735
|
-
nodeId: this.localNodeId,
|
|
736
|
-
newData: row,
|
|
737
|
-
oldData: null
|
|
738
|
-
});
|
|
739
|
-
}
|
|
740
|
-
const checksum = computeChecksum(changes);
|
|
741
|
-
const dumpEpoch = BigInt(Date.now()) * 1000000n;
|
|
742
|
-
const fromSeq = dumpEpoch + BigInt(offset);
|
|
743
|
-
const toSeq = dumpEpoch + BigInt(offset + rows.length - 1);
|
|
744
|
-
const dumpNodeId = `dump-${this.localNodeId}`;
|
|
745
|
-
yield {
|
|
746
|
-
sourceNodeId: dumpNodeId,
|
|
747
|
-
batchId: `dump-${table}-${batchNum}`,
|
|
748
|
-
fromSeq,
|
|
749
|
-
toSeq,
|
|
750
|
-
hlcRange: { min: hlcValue, max: hlcValue },
|
|
751
|
-
changes,
|
|
752
|
-
checksum
|
|
753
|
-
};
|
|
754
|
-
offset += rows.length;
|
|
755
|
-
batchNum += 1;
|
|
756
|
-
}
|
|
757
|
-
}
|
|
758
|
-
async *dumpTableOnConnection(conn, table, batchSize) {
|
|
759
|
-
if (!IDENTIFIER_RE2.test(table)) {
|
|
760
|
-
throw new ReplicationError(`Invalid table name: ${table}`);
|
|
761
|
-
}
|
|
762
|
-
const pkColumns = await this.pkResolver.forTableOnConnection(conn, table);
|
|
763
|
-
const needsRowid = pkColumns.length === 1 && pkColumns[0] === "rowid";
|
|
764
|
-
let lastPkValues = null;
|
|
765
|
-
let done = false;
|
|
766
|
-
while (!done) {
|
|
767
|
-
let selectSql;
|
|
768
|
-
const params = [];
|
|
769
|
-
if (lastPkValues === null) {
|
|
770
|
-
selectSql = needsRowid ? `SELECT rowid, * FROM "${table}" ORDER BY rowid LIMIT ?` : `SELECT * FROM "${table}" ORDER BY ${pkColumns.map((c) => `"${c}"`).join(", ")} LIMIT ?`;
|
|
771
|
-
params.push(batchSize);
|
|
772
|
-
} else if (needsRowid) {
|
|
773
|
-
selectSql = `SELECT rowid, * FROM "${table}" WHERE rowid > ? ORDER BY rowid LIMIT ?`;
|
|
774
|
-
params.push(lastPkValues[0], batchSize);
|
|
775
|
-
} else if (pkColumns.length === 1) {
|
|
776
|
-
selectSql = `SELECT * FROM "${table}" WHERE "${pkColumns[0]}" > ? ORDER BY "${pkColumns[0]}" LIMIT ?`;
|
|
777
|
-
params.push(lastPkValues[0], batchSize);
|
|
778
|
-
} else {
|
|
779
|
-
const colList = pkColumns.map((c) => `"${c}"`).join(", ");
|
|
780
|
-
const placeholders = pkColumns.map(() => "?").join(", ");
|
|
781
|
-
selectSql = `SELECT * FROM "${table}" WHERE (${colList}) > (${placeholders}) ORDER BY ${colList} LIMIT ?`;
|
|
782
|
-
params.push(...lastPkValues, batchSize);
|
|
783
|
-
}
|
|
784
|
-
const stmt = await conn.prepare(selectSql);
|
|
785
|
-
const rows = await stmt.all(...params);
|
|
786
|
-
if (rows.length === 0) {
|
|
787
|
-
break;
|
|
788
|
-
}
|
|
789
|
-
const lastRow = rows[rows.length - 1];
|
|
790
|
-
lastPkValues = pkColumns.map((col) => lastRow[col]);
|
|
791
|
-
let isLast = rows.length < batchSize;
|
|
792
|
-
if (!isLast) {
|
|
793
|
-
let peekSql;
|
|
794
|
-
const peekParams = [];
|
|
795
|
-
if (needsRowid) {
|
|
796
|
-
peekSql = `SELECT 1 FROM "${table}" WHERE rowid > ? LIMIT 1`;
|
|
797
|
-
peekParams.push(lastPkValues[0]);
|
|
798
|
-
} else if (pkColumns.length === 1) {
|
|
799
|
-
peekSql = `SELECT 1 FROM "${table}" WHERE "${pkColumns[0]}" > ? LIMIT 1`;
|
|
800
|
-
peekParams.push(lastPkValues[0]);
|
|
801
|
-
} else {
|
|
802
|
-
const colList = pkColumns.map((c) => `"${c}"`).join(", ");
|
|
803
|
-
const placeholders = pkColumns.map(() => "?").join(", ");
|
|
804
|
-
peekSql = `SELECT 1 FROM "${table}" WHERE (${colList}) > (${placeholders}) LIMIT 1`;
|
|
805
|
-
peekParams.push(...lastPkValues);
|
|
806
|
-
}
|
|
807
|
-
const peekStmt = await conn.prepare(peekSql);
|
|
808
|
-
const peekRow = await peekStmt.get(...peekParams);
|
|
809
|
-
if (!peekRow) {
|
|
810
|
-
isLast = true;
|
|
811
|
-
}
|
|
812
|
-
}
|
|
813
|
-
done = isLast;
|
|
814
|
-
const checksum = createHash("sha256").update(canonicaliseForChecksum(rows)).digest("hex");
|
|
815
|
-
yield { rows, checksum, isLast };
|
|
816
|
-
}
|
|
817
|
-
}
|
|
818
|
-
async generateManifest(conn, table) {
|
|
819
|
-
if (!IDENTIFIER_RE2.test(table)) {
|
|
820
|
-
throw new ReplicationError(`Invalid table name: ${table}`);
|
|
821
|
-
}
|
|
822
|
-
const countStmt = await conn.prepare(`SELECT COUNT(*) as cnt FROM "${table}"`);
|
|
823
|
-
const countRow = await countStmt.get();
|
|
824
|
-
const rowCount = countRow?.cnt ?? 0;
|
|
825
|
-
const hash = createHash("sha256");
|
|
826
|
-
const pkColumns = await this.pkResolver.forTableOnConnection(conn, table);
|
|
827
|
-
const needsRowid = pkColumns.length === 1 && pkColumns[0] === "rowid";
|
|
828
|
-
let lastPkValues = null;
|
|
829
|
-
const PK_BATCH_SIZE = 1e4;
|
|
830
|
-
let hasRows = true;
|
|
831
|
-
while (hasRows) {
|
|
832
|
-
let selectSql;
|
|
833
|
-
const params = [];
|
|
834
|
-
if (needsRowid) {
|
|
835
|
-
if (lastPkValues === null) {
|
|
836
|
-
selectSql = `SELECT rowid FROM "${table}" ORDER BY rowid LIMIT ?`;
|
|
837
|
-
} else {
|
|
838
|
-
selectSql = `SELECT rowid FROM "${table}" WHERE rowid > ? ORDER BY rowid LIMIT ?`;
|
|
839
|
-
params.push(lastPkValues[0]);
|
|
840
|
-
}
|
|
841
|
-
} else if (pkColumns.length === 1) {
|
|
842
|
-
const col = pkColumns[0];
|
|
843
|
-
if (lastPkValues === null) {
|
|
844
|
-
selectSql = `SELECT "${col}" FROM "${table}" ORDER BY "${col}" LIMIT ?`;
|
|
845
|
-
} else {
|
|
846
|
-
selectSql = `SELECT "${col}" FROM "${table}" WHERE "${col}" > ? ORDER BY "${col}" LIMIT ?`;
|
|
847
|
-
params.push(lastPkValues[0]);
|
|
848
|
-
}
|
|
849
|
-
} else {
|
|
850
|
-
const colList = pkColumns.map((c) => `"${c}"`).join(", ");
|
|
851
|
-
if (lastPkValues === null) {
|
|
852
|
-
selectSql = `SELECT ${colList} FROM "${table}" ORDER BY ${colList} LIMIT ?`;
|
|
853
|
-
} else {
|
|
854
|
-
const placeholders = pkColumns.map(() => "?").join(", ");
|
|
855
|
-
selectSql = `SELECT ${colList} FROM "${table}" WHERE (${colList}) > (${placeholders}) ORDER BY ${colList} LIMIT ?`;
|
|
856
|
-
params.push(...lastPkValues);
|
|
857
|
-
}
|
|
858
|
-
}
|
|
859
|
-
params.push(PK_BATCH_SIZE);
|
|
860
|
-
const stmt = await conn.prepare(selectSql);
|
|
861
|
-
const rows = await stmt.all(...params);
|
|
862
|
-
if (rows.length === 0) {
|
|
863
|
-
hasRows = false;
|
|
864
|
-
break;
|
|
865
|
-
}
|
|
866
|
-
for (const row of rows) {
|
|
867
|
-
const pkVal = pkColumns.map((col) => String(row[col] ?? "")).join("-");
|
|
868
|
-
hash.update(pkVal);
|
|
869
|
-
}
|
|
870
|
-
if (rows.length < PK_BATCH_SIZE) {
|
|
871
|
-
hasRows = false;
|
|
872
|
-
} else {
|
|
873
|
-
const lastRow = rows[rows.length - 1];
|
|
874
|
-
lastPkValues = pkColumns.map((col) => lastRow[col]);
|
|
875
|
-
}
|
|
876
|
-
}
|
|
877
|
-
return { table, rowCount, pkHash: hash.digest("hex") };
|
|
878
|
-
}
|
|
879
|
-
async verifyManifest(manifest) {
|
|
880
|
-
const local = await this.generateManifest(this.conn, manifest.table);
|
|
881
|
-
return local.rowCount === manifest.rowCount && local.pkHash === manifest.pkHash;
|
|
882
|
-
}
|
|
883
|
-
};
|
|
884
|
-
|
|
885
|
-
// src/replication/log/pk.ts
|
|
886
|
-
var PkResolver = class {
|
|
887
|
-
constructor(conn) {
|
|
888
|
-
this.conn = conn;
|
|
889
|
-
}
|
|
890
|
-
cache = /* @__PURE__ */ new Map();
|
|
891
|
-
async forTable(table) {
|
|
892
|
-
const cached = this.cache.get(table);
|
|
893
|
-
if (cached) return cached;
|
|
894
|
-
const stmt = await this.conn.prepare(`PRAGMA table_info("${table}")`);
|
|
895
|
-
const info = await stmt.all();
|
|
896
|
-
const pkCols = info.filter((col) => col.pk > 0).sort((a, b) => a.pk - b.pk).map((col) => col.name);
|
|
897
|
-
if (pkCols.length === 0) {
|
|
898
|
-
pkCols.push("rowid");
|
|
899
|
-
}
|
|
900
|
-
this.cache.set(table, pkCols);
|
|
901
|
-
return pkCols;
|
|
902
|
-
}
|
|
903
|
-
async forTableOnConnection(conn, table) {
|
|
904
|
-
const stmt = await conn.prepare(`PRAGMA table_info("${table}")`);
|
|
905
|
-
const info = await stmt.all();
|
|
906
|
-
const pkCols = info.filter((col) => col.pk > 0).sort((a, b) => a.pk - b.pk).map((col) => col.name);
|
|
907
|
-
if (pkCols.length === 0) {
|
|
908
|
-
pkCols.push("rowid");
|
|
909
|
-
}
|
|
910
|
-
return pkCols;
|
|
911
|
-
}
|
|
912
|
-
};
|
|
913
|
-
|
|
914
109
|
// src/replication/log/schema.ts
|
|
915
110
|
var SchemaOps = class {
|
|
916
111
|
constructor(conn, changesTable) {
|
|
@@ -918,150 +113,23 @@ var SchemaOps = class {
|
|
|
918
113
|
this.changesTable = changesTable;
|
|
919
114
|
}
|
|
920
115
|
async ensureReplicationTables() {
|
|
921
|
-
await this.conn.
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
table_name TEXT NOT NULL,
|
|
925
|
-
operation TEXT NOT NULL,
|
|
926
|
-
row_id TEXT NOT NULL,
|
|
927
|
-
changed_at REAL NOT NULL DEFAULT (unixepoch('subsec')),
|
|
928
|
-
old_data TEXT,
|
|
929
|
-
new_data TEXT,
|
|
930
|
-
node_id TEXT NOT NULL DEFAULT '',
|
|
931
|
-
tx_id TEXT NOT NULL DEFAULT '',
|
|
932
|
-
hlc TEXT NOT NULL DEFAULT ''
|
|
933
|
-
)`);
|
|
934
|
-
await this.conn.exec(`
|
|
935
|
-
CREATE TABLE IF NOT EXISTS ${PEER_STATE_TABLE} (
|
|
936
|
-
peer_node_id TEXT PRIMARY KEY,
|
|
937
|
-
last_acked_seq INTEGER NOT NULL DEFAULT 0,
|
|
938
|
-
last_received_hlc TEXT NOT NULL DEFAULT '',
|
|
939
|
-
updated_at REAL NOT NULL
|
|
940
|
-
)`);
|
|
941
|
-
await this.conn.exec(`
|
|
942
|
-
CREATE TABLE IF NOT EXISTS ${APPLIED_CHANGES_TABLE} (
|
|
943
|
-
source_node_id TEXT NOT NULL,
|
|
944
|
-
source_seq INTEGER NOT NULL,
|
|
945
|
-
applied_at REAL NOT NULL,
|
|
946
|
-
PRIMARY KEY (source_node_id, source_seq)
|
|
947
|
-
)`);
|
|
948
|
-
await this.conn.exec(`
|
|
949
|
-
CREATE TABLE IF NOT EXISTS ${COLUMN_VERSIONS_TABLE} (
|
|
950
|
-
table_name TEXT NOT NULL,
|
|
951
|
-
row_id TEXT NOT NULL,
|
|
952
|
-
column_name TEXT NOT NULL,
|
|
953
|
-
hlc TEXT NOT NULL,
|
|
954
|
-
node_id TEXT NOT NULL,
|
|
955
|
-
PRIMARY KEY (table_name, row_id, column_name)
|
|
956
|
-
)`);
|
|
957
|
-
await this.conn.exec(`
|
|
958
|
-
CREATE TABLE IF NOT EXISTS ${SYNC_STATE_TABLE} (
|
|
959
|
-
table_name TEXT PRIMARY KEY,
|
|
960
|
-
status TEXT NOT NULL DEFAULT 'pending',
|
|
961
|
-
row_count INTEGER NOT NULL DEFAULT 0,
|
|
962
|
-
pk_hash TEXT NOT NULL DEFAULT '',
|
|
963
|
-
snapshot_seq INTEGER,
|
|
964
|
-
source_peer_id TEXT,
|
|
965
|
-
started_at REAL,
|
|
966
|
-
completed_at REAL,
|
|
967
|
-
request_id TEXT
|
|
968
|
-
)`);
|
|
116
|
+
await ensureChangesTable(this.conn, this.changesTable);
|
|
117
|
+
await ensureReplicationStateTables(this.conn);
|
|
118
|
+
await ensureMetaTable(this.conn);
|
|
969
119
|
}
|
|
970
120
|
async dumpSchema(conn, excludePrefix = INTERNAL_TABLE_PREFIX) {
|
|
971
|
-
|
|
972
|
-
`SELECT type, name, sql FROM sqlite_master
|
|
973
|
-
WHERE type IN ('table', 'index') AND name NOT LIKE ? AND name NOT LIKE 'sqlite_%' AND sql IS NOT NULL`
|
|
974
|
-
);
|
|
975
|
-
const rows = await stmt.all(`${excludePrefix}%`);
|
|
976
|
-
const filtered = rows.filter((row) => {
|
|
977
|
-
if (row.name.startsWith(excludePrefix)) return false;
|
|
978
|
-
if (row.name.startsWith("sqlite_")) return false;
|
|
979
|
-
return validateDdlSafety(row.sql);
|
|
980
|
-
});
|
|
981
|
-
const tables = [];
|
|
982
|
-
const indexes = [];
|
|
983
|
-
for (const row of filtered) {
|
|
984
|
-
if (row.type === "table") {
|
|
985
|
-
tables.push({ name: row.name, sql: row.sql });
|
|
986
|
-
} else {
|
|
987
|
-
indexes.push({ sql: row.sql });
|
|
988
|
-
}
|
|
989
|
-
}
|
|
990
|
-
const tableOrder = await this.getTablesInFkOrder(conn);
|
|
991
|
-
const orderMap = /* @__PURE__ */ new Map();
|
|
992
|
-
for (let i = 0; i < tableOrder.length; i++) {
|
|
993
|
-
orderMap.set(tableOrder[i], i);
|
|
994
|
-
}
|
|
995
|
-
tables.sort((a, b) => (orderMap.get(a.name) ?? 999) - (orderMap.get(b.name) ?? 999));
|
|
996
|
-
const result = [];
|
|
997
|
-
for (const t of tables) {
|
|
998
|
-
result.push(t.sql);
|
|
999
|
-
}
|
|
1000
|
-
for (const idx of indexes) {
|
|
1001
|
-
result.push(idx.sql);
|
|
1002
|
-
}
|
|
1003
|
-
return result;
|
|
121
|
+
return dumpSchema(conn, excludePrefix);
|
|
1004
122
|
}
|
|
1005
123
|
async getTablesInFkOrder(conn) {
|
|
1006
|
-
|
|
1007
|
-
`SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE '${INTERNAL_TABLE_PREFIX}%' AND name NOT LIKE 'sqlite_%'`
|
|
1008
|
-
);
|
|
1009
|
-
const tableRows = await stmt.all();
|
|
1010
|
-
const tableNames = tableRows.map((r) => r.name);
|
|
1011
|
-
const adjacency = /* @__PURE__ */ new Map();
|
|
1012
|
-
const inDegree = /* @__PURE__ */ new Map();
|
|
1013
|
-
for (const name of tableNames) {
|
|
1014
|
-
adjacency.set(name, /* @__PURE__ */ new Set());
|
|
1015
|
-
inDegree.set(name, 0);
|
|
1016
|
-
}
|
|
1017
|
-
for (const name of tableNames) {
|
|
1018
|
-
const fkStmt = await conn.prepare(`PRAGMA foreign_key_list("${name}")`);
|
|
1019
|
-
const fks = await fkStmt.all();
|
|
1020
|
-
for (const fk of fks) {
|
|
1021
|
-
if (tableNames.includes(fk.table) && fk.table !== name) {
|
|
1022
|
-
const deps = adjacency.get(fk.table);
|
|
1023
|
-
if (deps && !deps.has(name)) {
|
|
1024
|
-
deps.add(name);
|
|
1025
|
-
inDegree.set(name, (inDegree.get(name) ?? 0) + 1);
|
|
1026
|
-
}
|
|
1027
|
-
}
|
|
1028
|
-
}
|
|
1029
|
-
}
|
|
1030
|
-
const queue = [];
|
|
1031
|
-
for (const name of tableNames) {
|
|
1032
|
-
if ((inDegree.get(name) ?? 0) === 0) {
|
|
1033
|
-
queue.push(name);
|
|
1034
|
-
}
|
|
1035
|
-
}
|
|
1036
|
-
const sorted = [];
|
|
1037
|
-
while (queue.length > 0) {
|
|
1038
|
-
const current = queue.shift();
|
|
1039
|
-
if (current === void 0) break;
|
|
1040
|
-
sorted.push(current);
|
|
1041
|
-
const deps = adjacency.get(current);
|
|
1042
|
-
if (deps) {
|
|
1043
|
-
for (const dep of deps) {
|
|
1044
|
-
const newDeg = (inDegree.get(dep) ?? 1) - 1;
|
|
1045
|
-
inDegree.set(dep, newDeg);
|
|
1046
|
-
if (newDeg === 0) {
|
|
1047
|
-
queue.push(dep);
|
|
1048
|
-
}
|
|
1049
|
-
}
|
|
1050
|
-
}
|
|
1051
|
-
}
|
|
1052
|
-
if (sorted.length < tableNames.length) {
|
|
1053
|
-
const remaining = tableNames.filter((n) => !sorted.includes(n)).sort();
|
|
1054
|
-
sorted.push(...remaining);
|
|
1055
|
-
}
|
|
1056
|
-
return sorted;
|
|
124
|
+
return tablesInFkOrder(conn);
|
|
1057
125
|
}
|
|
1058
126
|
async wipeTables(conn, tables, tracker) {
|
|
1059
127
|
for (const table of tables) {
|
|
1060
|
-
if (!
|
|
128
|
+
if (!IDENTIFIER_RE.test(table)) {
|
|
1061
129
|
throw new ReplicationError(`Invalid table name: ${table}`);
|
|
1062
130
|
}
|
|
1063
131
|
}
|
|
1064
|
-
await conn
|
|
132
|
+
await setForeignKeysEnabled(conn, false);
|
|
1065
133
|
try {
|
|
1066
134
|
await conn.transaction(async (tx) => {
|
|
1067
135
|
for (const table of tables) {
|
|
@@ -1071,10 +139,10 @@ CREATE TABLE IF NOT EXISTS ${SYNC_STATE_TABLE} (
|
|
|
1071
139
|
for (const table of reversed) {
|
|
1072
140
|
await tx.exec(`DELETE FROM "${table}"`);
|
|
1073
141
|
}
|
|
1074
|
-
await tx
|
|
142
|
+
await deleteSyncTableStates(tx);
|
|
1075
143
|
});
|
|
1076
144
|
} finally {
|
|
1077
|
-
await conn
|
|
145
|
+
await setForeignKeysEnabled(conn, true);
|
|
1078
146
|
}
|
|
1079
147
|
}
|
|
1080
148
|
};
|
|
@@ -1085,98 +153,26 @@ var StateOps = class {
|
|
|
1085
153
|
this.conn = conn;
|
|
1086
154
|
}
|
|
1087
155
|
activeSyncSnapshotSeqs = /* @__PURE__ */ new Set();
|
|
1088
|
-
/**
|
|
1089
|
-
* Returns the highest valid HLC observed in this database, or `null` when
|
|
1090
|
-
* no replication evidence exists yet.
|
|
1091
|
-
*
|
|
1092
|
-
* Reads two durable sources and takes the lexicographic maximum:
|
|
1093
|
-
* - `MAX(hlc) FROM <changes table>` covers HLCs stamped onto local writes.
|
|
1094
|
-
* - `MAX(hlc) FROM _sirannon_column_versions` covers HLCs applied from
|
|
1095
|
-
* remote batches and persisted alongside per-column versions.
|
|
1096
|
-
*
|
|
1097
|
-
* Combining both is necessary: a primary that has never received remote
|
|
1098
|
-
* batches has no rows in `_sirannon_column_versions` for inbound HLCs, and
|
|
1099
|
-
* a replica that has never written locally has no rows in the changes
|
|
1100
|
-
* table. Either source can independently hold the highest value depending
|
|
1101
|
-
* on traffic patterns.
|
|
1102
|
-
*
|
|
1103
|
-
* The changes table is subject to retention pruning by `ChangeTracker`,
|
|
1104
|
-
* which means its `MAX(hlc)` can lag behind the highest HLC ever emitted.
|
|
1105
|
-
* Cross-checking against `_sirannon_column_versions` (which is upserted
|
|
1106
|
-
* per `(table, row, column)` and not subject to time-based retention)
|
|
1107
|
-
* keeps the recovered value above any HLC currently still observable by
|
|
1108
|
-
* any peer's conflict-resolution decisions.
|
|
1109
|
-
*
|
|
1110
|
-
* Filters out the empty-string sentinel that the CDC triggers write before
|
|
1111
|
-
* the local-stamp pass fills in a real HLC, and rejects any decoded
|
|
1112
|
-
* timestamp whose components are not finite, non-negative integers. The
|
|
1113
|
-
* SQL filter narrows the result set; the JavaScript-level guard catches a
|
|
1114
|
-
* future producer that writes a malformed HLC.
|
|
1115
|
-
*/
|
|
1116
156
|
async recoverMaxObservedHlc(changesTable) {
|
|
1117
157
|
let best = null;
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
best = mergeCandidate(best, changesRow?.max_hlc ?? null);
|
|
1121
|
-
const versionsStmt = await this.conn.prepare(
|
|
1122
|
-
`SELECT MAX(hlc) AS max_hlc FROM ${COLUMN_VERSIONS_TABLE} WHERE hlc != ''`
|
|
1123
|
-
);
|
|
1124
|
-
const versionsRow = await versionsStmt.get();
|
|
1125
|
-
best = mergeCandidate(best, versionsRow?.max_hlc ?? null);
|
|
158
|
+
best = mergeCandidate(best, await selectMaxChangeHlc(this.conn, changesTable));
|
|
159
|
+
best = mergeCandidate(best, await selectMaxColumnVersionHlc(this.conn));
|
|
1126
160
|
return best;
|
|
1127
161
|
}
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
`SELECT MAX(source_seq) as max_seq FROM ${APPLIED_CHANGES_TABLE} WHERE source_node_id = ?`
|
|
1131
|
-
);
|
|
1132
|
-
const row = await stmt.get(fromNodeId);
|
|
1133
|
-
if (!row || row.max_seq === null) {
|
|
1134
|
-
return 0n;
|
|
1135
|
-
}
|
|
1136
|
-
return BigInt(row.max_seq);
|
|
1137
|
-
}
|
|
1138
|
-
async setLastAppliedSeq(fromNodeId, seq) {
|
|
1139
|
-
const stmt = await this.conn.prepare(
|
|
1140
|
-
`INSERT INTO ${PEER_STATE_TABLE} (peer_node_id, last_acked_seq, updated_at)
|
|
1141
|
-
VALUES (?, ?, ?)
|
|
1142
|
-
ON CONFLICT(peer_node_id)
|
|
1143
|
-
DO UPDATE SET
|
|
1144
|
-
last_acked_seq = max(${PEER_STATE_TABLE}.last_acked_seq, excluded.last_acked_seq),
|
|
1145
|
-
updated_at = CASE
|
|
1146
|
-
WHEN excluded.last_acked_seq >= ${PEER_STATE_TABLE}.last_acked_seq THEN excluded.updated_at
|
|
1147
|
-
ELSE ${PEER_STATE_TABLE}.updated_at
|
|
1148
|
-
END`
|
|
1149
|
-
);
|
|
1150
|
-
await stmt.run(fromNodeId, seq.toString(), Date.now() / 1e3);
|
|
162
|
+
getLastAppliedSeq(fromNodeId) {
|
|
163
|
+
return selectMaxAppliedSourceSeq(this.conn, fromNodeId);
|
|
1151
164
|
}
|
|
1152
|
-
|
|
1153
|
-
|
|
1154
|
-
|
|
1155
|
-
|
|
1156
|
-
|
|
1157
|
-
}
|
|
1158
|
-
return BigInt(row.last_acked_seq);
|
|
165
|
+
setLastAppliedSeq(fromNodeId, seq) {
|
|
166
|
+
return upsertPeerAckedSeq(this.conn, fromNodeId, seq, Date.now() / 1e3);
|
|
167
|
+
}
|
|
168
|
+
getPeerAckedSeq(peerNodeId) {
|
|
169
|
+
return selectPeerAckedSeq(this.conn, peerNodeId);
|
|
1159
170
|
}
|
|
1160
171
|
async getLocalSeq(changesTable) {
|
|
1161
|
-
|
|
1162
|
-
const row = await stmt.get();
|
|
1163
|
-
if (!row || row.max_seq === null) {
|
|
1164
|
-
return 0n;
|
|
1165
|
-
}
|
|
1166
|
-
return BigInt(row.max_seq);
|
|
172
|
+
return selectMaxChangeSeq(this.conn, changesTable);
|
|
1167
173
|
}
|
|
1168
174
|
async getMinAckedSeq() {
|
|
1169
|
-
|
|
1170
|
-
`SELECT MIN(last_acked_seq) as min_seq, COUNT(*) as cnt FROM ${PEER_STATE_TABLE}`
|
|
1171
|
-
);
|
|
1172
|
-
const row = await stmt.get();
|
|
1173
|
-
const hasPeers = row !== void 0 && row.cnt > 0;
|
|
1174
|
-
let result = null;
|
|
1175
|
-
if (hasPeers && row.min_seq !== null) {
|
|
1176
|
-
result = BigInt(row.min_seq);
|
|
1177
|
-
} else if (hasPeers) {
|
|
1178
|
-
result = 0n;
|
|
1179
|
-
}
|
|
175
|
+
let result = await selectMinPeerAckedSeq(this.conn);
|
|
1180
176
|
for (const syncSeq of this.activeSyncSnapshotSeqs) {
|
|
1181
177
|
if (result === null || syncSeq < result) {
|
|
1182
178
|
result = syncSeq;
|
|
@@ -1190,75 +186,41 @@ var StateOps = class {
|
|
|
1190
186
|
unregisterActiveSyncSeq(seq) {
|
|
1191
187
|
this.activeSyncSnapshotSeqs.delete(seq);
|
|
1192
188
|
}
|
|
1193
|
-
|
|
1194
|
-
|
|
1195
|
-
|
|
1196
|
-
|
|
1197
|
-
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
|
|
1203
|
-
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
-
|
|
1207
|
-
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
status = excluded.status,
|
|
1211
|
-
snapshot_seq = COALESCE(excluded.snapshot_seq, snapshot_seq),
|
|
1212
|
-
source_peer_id = COALESCE(excluded.source_peer_id, source_peer_id),
|
|
1213
|
-
started_at = COALESCE(excluded.started_at, started_at),
|
|
1214
|
-
request_id = COALESCE(excluded.request_id, request_id)`
|
|
1215
|
-
);
|
|
1216
|
-
await stmt.run(
|
|
1217
|
-
phase,
|
|
1218
|
-
snapshotSeq !== void 0 ? snapshotSeq.toString() : null,
|
|
1219
|
-
sourcePeerId ?? null,
|
|
1220
|
-
phase === "syncing" ? Date.now() / 1e3 : null,
|
|
1221
|
-
requestId ?? null
|
|
1222
|
-
);
|
|
189
|
+
setSyncTableStatus(table, status, rowCount, pkHash) {
|
|
190
|
+
return upsertSyncTableStatus(this.conn, {
|
|
191
|
+
tableName: table,
|
|
192
|
+
status,
|
|
193
|
+
rowCount: rowCount ?? 0,
|
|
194
|
+
pkHash: pkHash ?? "",
|
|
195
|
+
completedAt: status === "completed" ? Date.now() / 1e3 : null
|
|
196
|
+
});
|
|
197
|
+
}
|
|
198
|
+
setSyncMeta(phase, snapshotSeq, sourcePeerId, requestId) {
|
|
199
|
+
return upsertSyncMeta(this.conn, {
|
|
200
|
+
status: phase,
|
|
201
|
+
snapshotSeq,
|
|
202
|
+
sourcePeerId,
|
|
203
|
+
startedAt: phase === "syncing" ? Date.now() / 1e3 : null,
|
|
204
|
+
requestId
|
|
205
|
+
});
|
|
1223
206
|
}
|
|
1224
207
|
async getSyncState() {
|
|
1225
|
-
const
|
|
1226
|
-
`SELECT status, snapshot_seq, source_peer_id FROM ${SYNC_STATE_TABLE} WHERE table_name = '__sync_meta__'`
|
|
1227
|
-
);
|
|
1228
|
-
const meta = await metaStmt.get();
|
|
208
|
+
const meta = await selectSyncMetaRow(this.conn);
|
|
1229
209
|
if (!meta) {
|
|
1230
210
|
return { phase: "ready", completedTables: [], snapshotSeq: null, sourcePeerId: null };
|
|
1231
211
|
}
|
|
1232
|
-
const completedStmt = await this.conn.prepare(
|
|
1233
|
-
`SELECT table_name FROM ${SYNC_STATE_TABLE} WHERE table_name != '__sync_meta__' AND status = 'completed'`
|
|
1234
|
-
);
|
|
1235
|
-
const completedRows = await completedStmt.all();
|
|
1236
212
|
return {
|
|
1237
213
|
phase: meta.status,
|
|
1238
|
-
completedTables:
|
|
1239
|
-
snapshotSeq: meta.
|
|
1240
|
-
sourcePeerId: meta.
|
|
214
|
+
completedTables: await selectCompletedSyncTableNames(this.conn),
|
|
215
|
+
snapshotSeq: meta.snapshotSeq,
|
|
216
|
+
sourcePeerId: meta.sourcePeerId
|
|
1241
217
|
};
|
|
1242
218
|
}
|
|
1243
219
|
async isSyncCompleted() {
|
|
1244
|
-
const
|
|
1245
|
-
|
|
1246
|
-
return row?.status === "ready";
|
|
220
|
+
const meta = await selectSyncMetaRow(this.conn);
|
|
221
|
+
return meta?.status === "ready";
|
|
1247
222
|
}
|
|
1248
223
|
};
|
|
1249
|
-
function isWellFormedHlc(candidate) {
|
|
1250
|
-
if (candidate.length === 0) return false;
|
|
1251
|
-
let decoded;
|
|
1252
|
-
try {
|
|
1253
|
-
decoded = HLC.decode(candidate);
|
|
1254
|
-
} catch {
|
|
1255
|
-
return false;
|
|
1256
|
-
}
|
|
1257
|
-
if (!Number.isInteger(decoded.wallMs) || decoded.wallMs < 0) return false;
|
|
1258
|
-
if (!Number.isInteger(decoded.logical) || decoded.logical < 0) return false;
|
|
1259
|
-
if (decoded.nodeId.length === 0) return false;
|
|
1260
|
-
return true;
|
|
1261
|
-
}
|
|
1262
224
|
function mergeCandidate(current, candidate) {
|
|
1263
225
|
if (candidate === null || candidate.length === 0) return current;
|
|
1264
226
|
if (!isWellFormedHlc(candidate)) return current;
|
|
@@ -1273,15 +235,17 @@ var ReplicationLog = class {
|
|
|
1273
235
|
this.pkResolver = new PkResolver(conn);
|
|
1274
236
|
this.state = new StateOps(conn);
|
|
1275
237
|
this.schema = new SchemaOps(conn, changesTable);
|
|
1276
|
-
this.batchReader = new BatchReader(conn, localNodeId,
|
|
238
|
+
this.batchReader = new BatchReader(conn, localNodeId, changesTable, this.pkResolver);
|
|
1277
239
|
this.batchApplier = new BatchApplier(
|
|
1278
240
|
conn,
|
|
1279
241
|
localNodeId,
|
|
1280
242
|
hlc,
|
|
1281
243
|
this.pkResolver,
|
|
1282
244
|
(fromNodeId) => this.state.getLastAppliedSeq(fromNodeId),
|
|
1283
|
-
tracker
|
|
245
|
+
tracker,
|
|
246
|
+
changesTable
|
|
1284
247
|
);
|
|
248
|
+
this.stampOps = new StampOps(localNodeId, hlc, changesTable);
|
|
1285
249
|
this.dump = new DumpOps(conn, localNodeId, hlc, this.pkResolver);
|
|
1286
250
|
}
|
|
1287
251
|
pkResolver;
|
|
@@ -1289,6 +253,7 @@ var ReplicationLog = class {
|
|
|
1289
253
|
schema;
|
|
1290
254
|
batchReader;
|
|
1291
255
|
batchApplier;
|
|
256
|
+
stampOps;
|
|
1292
257
|
dump;
|
|
1293
258
|
ensureReplicationTables() {
|
|
1294
259
|
return this.schema.ensureReplicationTables();
|
|
@@ -1297,10 +262,10 @@ var ReplicationLog = class {
|
|
|
1297
262
|
return this.batchReader.readBatch(afterSeq, batchSize);
|
|
1298
263
|
}
|
|
1299
264
|
stampChanges(tx, afterSeq, txId) {
|
|
1300
|
-
return this.
|
|
265
|
+
return this.stampOps.stampChanges(tx, afterSeq, txId);
|
|
1301
266
|
}
|
|
1302
267
|
updateColumnVersions(tx, afterSeq) {
|
|
1303
|
-
return this.
|
|
268
|
+
return this.stampOps.updateColumnVersions(tx, afterSeq);
|
|
1304
269
|
}
|
|
1305
270
|
applyBatch(batch, resolver) {
|
|
1306
271
|
return this.batchApplier.applyBatch(batch, resolver);
|
|
@@ -1465,153 +430,984 @@ var PeerTracker = class {
|
|
|
1465
430
|
this.waiters.add(waiter);
|
|
1466
431
|
});
|
|
1467
432
|
}
|
|
1468
|
-
waitForAll(seq, timeoutMs) {
|
|
1469
|
-
const connected = this.connectedPeerCount();
|
|
1470
|
-
if (connected === 0 || this.countAcked(seq) >= connected) {
|
|
1471
|
-
return Promise.resolve();
|
|
433
|
+
waitForAll(seq, timeoutMs) {
|
|
434
|
+
const connected = this.connectedPeerCount();
|
|
435
|
+
if (connected === 0 || this.countAcked(seq) >= connected) {
|
|
436
|
+
return Promise.resolve();
|
|
437
|
+
}
|
|
438
|
+
return new Promise((resolve, reject) => {
|
|
439
|
+
const timer = setTimeout(() => {
|
|
440
|
+
this.waiters.delete(waiter);
|
|
441
|
+
reject(new WriteConcernError(`Timed out waiting for all peers to ACK seq ${seq}`));
|
|
442
|
+
}, timeoutMs);
|
|
443
|
+
timer.unref?.();
|
|
444
|
+
const waiter = { seq, kind: "all", resolve, reject, timer };
|
|
445
|
+
this.waiters.add(waiter);
|
|
446
|
+
});
|
|
447
|
+
}
|
|
448
|
+
waitForConfiguredMajority(seq, localNodeId, votingNodeIds, timeoutMs) {
|
|
449
|
+
const needed = Math.floor(votingNodeIds.length / 2) + 1;
|
|
450
|
+
if (this.countAckedConfigured(seq, localNodeId, votingNodeIds) >= needed) {
|
|
451
|
+
return Promise.resolve();
|
|
452
|
+
}
|
|
453
|
+
return new Promise((resolve, reject) => {
|
|
454
|
+
const timer = setTimeout(() => {
|
|
455
|
+
this.waiters.delete(waiter);
|
|
456
|
+
reject(new WriteConcernError(`Timed out waiting for configured majority ACK of seq ${seq}`));
|
|
457
|
+
}, timeoutMs);
|
|
458
|
+
timer.unref?.();
|
|
459
|
+
const waiter = {
|
|
460
|
+
seq,
|
|
461
|
+
kind: "configured-majority",
|
|
462
|
+
localNodeId,
|
|
463
|
+
votingNodeIds: [...votingNodeIds],
|
|
464
|
+
resolve,
|
|
465
|
+
reject,
|
|
466
|
+
timer
|
|
467
|
+
};
|
|
468
|
+
this.waiters.add(waiter);
|
|
469
|
+
});
|
|
470
|
+
}
|
|
471
|
+
waitForConfiguredAll(seq, localNodeId, votingNodeIds, timeoutMs) {
|
|
472
|
+
if (this.countAckedConfigured(seq, localNodeId, votingNodeIds) >= votingNodeIds.length) {
|
|
473
|
+
return Promise.resolve();
|
|
474
|
+
}
|
|
475
|
+
return new Promise((resolve, reject) => {
|
|
476
|
+
const timer = setTimeout(() => {
|
|
477
|
+
this.waiters.delete(waiter);
|
|
478
|
+
reject(new WriteConcernError(`Timed out waiting for all configured voters to ACK seq ${seq}`));
|
|
479
|
+
}, timeoutMs);
|
|
480
|
+
timer.unref?.();
|
|
481
|
+
const waiter = {
|
|
482
|
+
seq,
|
|
483
|
+
kind: "configured-all",
|
|
484
|
+
localNodeId,
|
|
485
|
+
votingNodeIds: [...votingNodeIds],
|
|
486
|
+
resolve,
|
|
487
|
+
reject,
|
|
488
|
+
timer
|
|
489
|
+
};
|
|
490
|
+
this.waiters.add(waiter);
|
|
491
|
+
});
|
|
492
|
+
}
|
|
493
|
+
ackedConfiguredNodeIds(seq, localNodeId, votingNodeIds) {
|
|
494
|
+
const acked = /* @__PURE__ */ new Set();
|
|
495
|
+
if (votingNodeIds.includes(localNodeId)) {
|
|
496
|
+
acked.add(localNodeId);
|
|
497
|
+
}
|
|
498
|
+
for (const nodeId of votingNodeIds) {
|
|
499
|
+
const peer = this.peers.get(nodeId);
|
|
500
|
+
if (peer && peer.lastAckedSeq >= seq) {
|
|
501
|
+
acked.add(nodeId);
|
|
502
|
+
}
|
|
503
|
+
}
|
|
504
|
+
return votingNodeIds.filter((nodeId) => acked.has(nodeId));
|
|
505
|
+
}
|
|
506
|
+
allPeerStates() {
|
|
507
|
+
return Array.from(this.peers.values());
|
|
508
|
+
}
|
|
509
|
+
countAcked(seq) {
|
|
510
|
+
let count = 0;
|
|
511
|
+
for (const peer of this.peers.values()) {
|
|
512
|
+
if (peer.connected && peer.lastAckedSeq >= seq) {
|
|
513
|
+
count += 1;
|
|
514
|
+
}
|
|
515
|
+
}
|
|
516
|
+
return count;
|
|
517
|
+
}
|
|
518
|
+
countAckedConfigured(seq, localNodeId, votingNodeIds) {
|
|
519
|
+
return this.ackedConfiguredNodeIds(seq, localNodeId, votingNodeIds).length;
|
|
520
|
+
}
|
|
521
|
+
checkWaiters() {
|
|
522
|
+
const connected = this.connectedPeerCount();
|
|
523
|
+
for (const waiter of this.waiters) {
|
|
524
|
+
let needed;
|
|
525
|
+
if (waiter.kind === "majority") {
|
|
526
|
+
needed = Math.floor(connected / 2) + 1;
|
|
527
|
+
} else if (waiter.kind === "configured-majority") {
|
|
528
|
+
const votingNodeIds = waiter.votingNodeIds ?? [];
|
|
529
|
+
const localNodeId = waiter.localNodeId ?? "";
|
|
530
|
+
needed = Math.floor(votingNodeIds.length / 2) + 1;
|
|
531
|
+
if (this.countAckedConfigured(waiter.seq, localNodeId, votingNodeIds) >= needed) {
|
|
532
|
+
clearTimeout(waiter.timer);
|
|
533
|
+
this.waiters.delete(waiter);
|
|
534
|
+
waiter.resolve();
|
|
535
|
+
}
|
|
536
|
+
continue;
|
|
537
|
+
} else if (waiter.kind === "configured-all") {
|
|
538
|
+
const votingNodeIds = waiter.votingNodeIds ?? [];
|
|
539
|
+
const localNodeId = waiter.localNodeId ?? "";
|
|
540
|
+
needed = votingNodeIds.length;
|
|
541
|
+
if (this.countAckedConfigured(waiter.seq, localNodeId, votingNodeIds) >= needed) {
|
|
542
|
+
clearTimeout(waiter.timer);
|
|
543
|
+
this.waiters.delete(waiter);
|
|
544
|
+
waiter.resolve();
|
|
545
|
+
}
|
|
546
|
+
continue;
|
|
547
|
+
} else {
|
|
548
|
+
needed = connected;
|
|
549
|
+
}
|
|
550
|
+
if (connected === 0 || this.countAcked(waiter.seq) >= needed) {
|
|
551
|
+
clearTimeout(waiter.timer);
|
|
552
|
+
this.waiters.delete(waiter);
|
|
553
|
+
waiter.resolve();
|
|
554
|
+
continue;
|
|
555
|
+
}
|
|
556
|
+
if (waiter.kind === "all" && connected < this.peers.size) {
|
|
557
|
+
clearTimeout(waiter.timer);
|
|
558
|
+
this.waiters.delete(waiter);
|
|
559
|
+
waiter.reject(
|
|
560
|
+
new WriteConcernError(
|
|
561
|
+
`Cannot satisfy 'all' write concern: only ${connected}/${this.peers.size} peers connected`
|
|
562
|
+
)
|
|
563
|
+
);
|
|
564
|
+
continue;
|
|
565
|
+
}
|
|
566
|
+
const totalNodes = this.peers.size + 1;
|
|
567
|
+
const majorityNeeded = Math.floor(totalNodes / 2) + 1;
|
|
568
|
+
if (waiter.kind === "majority" && connected + 1 < majorityNeeded) {
|
|
569
|
+
clearTimeout(waiter.timer);
|
|
570
|
+
this.waiters.delete(waiter);
|
|
571
|
+
waiter.reject(
|
|
572
|
+
new WriteConcernError(
|
|
573
|
+
`Cannot satisfy 'majority' write concern: only ${connected + 1}/${totalNodes} nodes reachable`
|
|
574
|
+
)
|
|
575
|
+
);
|
|
576
|
+
}
|
|
577
|
+
}
|
|
578
|
+
}
|
|
579
|
+
};
|
|
580
|
+
|
|
581
|
+
// src/replication/engine/constants.ts
|
|
582
|
+
var DEFAULT_BATCH_SIZE = 100;
|
|
583
|
+
var DEFAULT_BATCH_INTERVAL_MS = 100;
|
|
584
|
+
var DEFAULT_MAX_CLOCK_DRIFT_MS = 6e4;
|
|
585
|
+
var DEFAULT_MAX_PENDING_BATCHES = 10;
|
|
586
|
+
var DEFAULT_MAX_BATCH_CHANGES = 1e3;
|
|
587
|
+
var DEFAULT_ACK_TIMEOUT_MS = 5e3;
|
|
588
|
+
var DEFAULT_SYNC_BATCH_SIZE = 1e4;
|
|
589
|
+
var DEFAULT_MAX_CONCURRENT_SYNCS = 2;
|
|
590
|
+
var DEFAULT_MAX_SYNC_DURATION_MS = 18e5;
|
|
591
|
+
var DEFAULT_MAX_SYNC_LAG_BEFORE_READY = 100;
|
|
592
|
+
var DEFAULT_SYNC_ACK_TIMEOUT_MS = 3e4;
|
|
593
|
+
var DEFAULT_CATCH_UP_DEADLINE_MS = 6e5;
|
|
594
|
+
var DEFAULT_COORDINATOR_SESSION_TTL_MS = 1e4;
|
|
595
|
+
var DDL_PREFIX_RE = /^\s*(CREATE\s+TABLE|ALTER\s+TABLE\s+\S+\s+ADD\s+COLUMN|DROP\s+TABLE|CREATE\s+INDEX|DROP\s+INDEX)\b/i;
|
|
596
|
+
var SAFE_SQL_PREFIX_RE = /^\s*(INSERT|UPDATE|DELETE|SELECT|CREATE\s+TABLE|ALTER\s+TABLE|DROP\s+TABLE|CREATE\s+INDEX|DROP\s+INDEX)\b/i;
|
|
597
|
+
var IDENTIFIER_RE2 = /^[a-zA-Z_][a-zA-Z0-9_]*$/;
|
|
598
|
+
var DDL_DENY_RE = /\b(load_extension|ATTACH|randomblob|zeroblob|writefile|readfile|fts3_tokenizer)\b/i;
|
|
599
|
+
var SYNC_SAFE_DDL_PREFIX_RE = /^\s*(CREATE\s+TABLE|ALTER\s+TABLE\s+\S+\s+ADD\s+COLUMN|DROP\s+TABLE|CREATE\s+INDEX|DROP\s+INDEX)\b/i;
|
|
600
|
+
function isSyncSafeDdl(sql) {
|
|
601
|
+
if (!SYNC_SAFE_DDL_PREFIX_RE.test(sql)) return false;
|
|
602
|
+
if (sql.includes(";")) return false;
|
|
603
|
+
if (/\bAS\s+SELECT\b/i.test(sql)) return false;
|
|
604
|
+
if (DDL_DENY_RE.test(sql)) return false;
|
|
605
|
+
const body = sql.replace(SYNC_SAFE_DDL_PREFIX_RE, "");
|
|
606
|
+
if (/\bSELECT\b/i.test(body)) return false;
|
|
607
|
+
return true;
|
|
608
|
+
}
|
|
609
|
+
|
|
610
|
+
// src/replication/engine/coordinator-authority.ts
|
|
611
|
+
function arraysEqual(left, right) {
|
|
612
|
+
if (left.length !== right.length) return false;
|
|
613
|
+
for (let i = 0; i < left.length; i++) {
|
|
614
|
+
if (left[i] !== right[i]) return false;
|
|
615
|
+
}
|
|
616
|
+
return true;
|
|
617
|
+
}
|
|
618
|
+
function errorDetails(engine, state) {
|
|
619
|
+
return {
|
|
620
|
+
currentPrimary: state?.currentPrimary ?? null,
|
|
621
|
+
primaryTerm: state?.primaryTerm.toString(),
|
|
622
|
+
replicationGroupId: engine.config.coordinator?.groupId
|
|
623
|
+
};
|
|
624
|
+
}
|
|
625
|
+
function hasCurrentPrimaryAuthorityFor(engine, state) {
|
|
626
|
+
return state.currentPrimary?.nodeId === engine.nodeId;
|
|
627
|
+
}
|
|
628
|
+
function noteCoordinatorContact(engine) {
|
|
629
|
+
engine.coordinatorLastContactMs = Date.now();
|
|
630
|
+
}
|
|
631
|
+
function isCoordinatorConnected(engine) {
|
|
632
|
+
const config = engine.config.coordinator;
|
|
633
|
+
if (!config) return false;
|
|
634
|
+
if (engine.coordinatorLastContactMs === 0) return false;
|
|
635
|
+
const sessionTtlMs = config.sessionTtlMs ?? DEFAULT_COORDINATOR_SESSION_TTL_MS;
|
|
636
|
+
return Date.now() - engine.coordinatorLastContactMs < sessionTtlMs;
|
|
637
|
+
}
|
|
638
|
+
function effectiveTopologyRole(engine) {
|
|
639
|
+
const state = engine.coordinatorState;
|
|
640
|
+
if (!engine.isCoordinatorMode() || !state) return engine.config.topology.role;
|
|
641
|
+
return state.currentPrimary?.nodeId === engine.nodeId ? "primary" : "replica";
|
|
642
|
+
}
|
|
643
|
+
async function refreshCoordinatorState(engine) {
|
|
644
|
+
const config = engine.config.coordinator;
|
|
645
|
+
if (!config) return null;
|
|
646
|
+
try {
|
|
647
|
+
const state = await config.coordinator.getReplicationGroupState(config.clusterId, config.groupId);
|
|
648
|
+
noteCoordinatorContact(engine);
|
|
649
|
+
engine.coordinatorState = state;
|
|
650
|
+
engine.coordinatorAuthority = state ? hasCurrentPrimaryAuthorityFor(engine, state) : false;
|
|
651
|
+
return state;
|
|
652
|
+
} catch {
|
|
653
|
+
engine.coordinatorAuthority = false;
|
|
654
|
+
throw new CoordinatorError(
|
|
655
|
+
"Coordinator is unavailable while refreshing replication-group authority",
|
|
656
|
+
errorDetails(engine, engine.coordinatorState)
|
|
657
|
+
);
|
|
658
|
+
}
|
|
659
|
+
}
|
|
660
|
+
function getCoordinatorMessageFields(engine) {
|
|
661
|
+
const config = engine.config.coordinator;
|
|
662
|
+
if (!config) return {};
|
|
663
|
+
return {
|
|
664
|
+
groupId: config.groupId,
|
|
665
|
+
primaryTerm: engine.coordinatorState?.primaryTerm ?? engine.config.transportConfig?.primaryTerm
|
|
666
|
+
};
|
|
667
|
+
}
|
|
668
|
+
function getForwardingPrimaryPeerId(engine) {
|
|
669
|
+
if (engine.isCoordinatorMode()) {
|
|
670
|
+
const primaryId = engine.coordinatorState?.currentPrimary?.nodeId;
|
|
671
|
+
if (primaryId && engine.config.transport.peers().has(primaryId)) {
|
|
672
|
+
return primaryId;
|
|
673
|
+
}
|
|
674
|
+
return null;
|
|
675
|
+
}
|
|
676
|
+
for (const [peerId, info] of engine.config.transport.peers()) {
|
|
677
|
+
if (info.role === "primary") {
|
|
678
|
+
return peerId;
|
|
679
|
+
}
|
|
680
|
+
}
|
|
681
|
+
return null;
|
|
682
|
+
}
|
|
683
|
+
function assertLocalCompatibility(engine, state) {
|
|
684
|
+
if (compatibilityAllowsPromotion(state.compatibility, engine.config.coordinator?.compatibility)) return;
|
|
685
|
+
throw new ProtocolVersionMismatchError("Node compatibility metadata is incompatible with the replication group", {
|
|
686
|
+
...errorDetails(engine, state),
|
|
687
|
+
groupCompatibility: state.compatibility,
|
|
688
|
+
localCompatibility: engine.config.coordinator?.compatibility
|
|
689
|
+
});
|
|
690
|
+
}
|
|
691
|
+
async function canAcceptLocalWrite(engine) {
|
|
692
|
+
if (!engine.isCoordinatorMode()) {
|
|
693
|
+
return engine.config.topology.canWrite();
|
|
694
|
+
}
|
|
695
|
+
const state = await refreshCoordinatorState(engine);
|
|
696
|
+
if (!state) {
|
|
697
|
+
throw new CoordinatorError("Cannot prove write authority without replication-group state");
|
|
698
|
+
}
|
|
699
|
+
if (!hasCurrentPrimaryAuthorityFor(engine, state)) {
|
|
700
|
+
return false;
|
|
701
|
+
}
|
|
702
|
+
assertLocalCompatibility(engine, state);
|
|
703
|
+
if (state.drainingNodeIds.includes(engine.nodeId)) {
|
|
704
|
+
throw new NodeDrainingError("Node is draining and cannot accept writes", errorDetails(engine, state));
|
|
705
|
+
}
|
|
706
|
+
if (state.repairingNodeIds.includes(engine.nodeId) || state.faultedNodeIds.includes(engine.nodeId)) {
|
|
707
|
+
throw new AuthorityError(
|
|
708
|
+
"Node is not eligible to accept writes while repair or fault state is active",
|
|
709
|
+
"AUTHORITY_LOST",
|
|
710
|
+
errorDetails(engine, state)
|
|
711
|
+
);
|
|
712
|
+
}
|
|
713
|
+
return true;
|
|
714
|
+
}
|
|
715
|
+
function throwNotCurrentPrimary(engine) {
|
|
716
|
+
if (engine.isCoordinatorMode()) {
|
|
717
|
+
throw new StalePrimaryError("This node is not the current primary", errorDetails(engine, engine.coordinatorState));
|
|
718
|
+
}
|
|
719
|
+
throw new TopologyError("This node cannot accept writes");
|
|
720
|
+
}
|
|
721
|
+
async function assertReadConcern(engine, level) {
|
|
722
|
+
if (!engine.isCoordinatorMode()) return;
|
|
723
|
+
const readConcern = level ?? "majority";
|
|
724
|
+
if (readConcern === "local") return;
|
|
725
|
+
const state = await refreshCoordinatorState(engine);
|
|
726
|
+
if (!state) {
|
|
727
|
+
throw new CoordinatorError("Cannot satisfy read concern without replication-group state");
|
|
728
|
+
}
|
|
729
|
+
if (readConcern === "linearizable") {
|
|
730
|
+
await verifyPrimaryAuthority(engine);
|
|
731
|
+
return;
|
|
732
|
+
}
|
|
733
|
+
if (readConcern === "majority") {
|
|
734
|
+
if (!state.inSyncNodeIds.includes(engine.nodeId)) {
|
|
735
|
+
throw new NodeNotInSyncError("Node is not in the in-sync set for majority reads", errorDetails(engine, state));
|
|
736
|
+
}
|
|
737
|
+
if (state.drainingNodeIds.includes(engine.nodeId) || state.repairingNodeIds.includes(engine.nodeId)) {
|
|
738
|
+
throw new ReadConcernError(
|
|
739
|
+
"Node cannot serve majority reads while draining or repairing",
|
|
740
|
+
errorDetails(engine, state)
|
|
741
|
+
);
|
|
742
|
+
}
|
|
743
|
+
return;
|
|
744
|
+
}
|
|
745
|
+
throw new ReadConcernError(`Unsupported read concern '${readConcern}'`, errorDetails(engine, state));
|
|
746
|
+
}
|
|
747
|
+
async function verifyPrimaryAuthority(engine) {
|
|
748
|
+
const state = await refreshCoordinatorState(engine);
|
|
749
|
+
if (!state) {
|
|
750
|
+
throw new CoordinatorError("Cannot prove primary authority without replication-group state");
|
|
751
|
+
}
|
|
752
|
+
if (!hasCurrentPrimaryAuthorityFor(engine, state)) {
|
|
753
|
+
throw new StalePrimaryError(
|
|
754
|
+
"Node is not the current primary for this replication group",
|
|
755
|
+
errorDetails(engine, state)
|
|
756
|
+
);
|
|
757
|
+
}
|
|
758
|
+
assertLocalCompatibility(engine, state);
|
|
759
|
+
if (state.drainingNodeIds.includes(engine.nodeId)) {
|
|
760
|
+
throw new NodeDrainingError("Node is draining and cannot accept writes", errorDetails(engine, state));
|
|
761
|
+
}
|
|
762
|
+
if (state.repairingNodeIds.includes(engine.nodeId) || state.faultedNodeIds.includes(engine.nodeId)) {
|
|
763
|
+
throw new AuthorityError(
|
|
764
|
+
"Node is not eligible to accept writes while repair or fault state is active",
|
|
765
|
+
"AUTHORITY_LOST",
|
|
766
|
+
errorDetails(engine, state)
|
|
767
|
+
);
|
|
768
|
+
}
|
|
769
|
+
return state;
|
|
770
|
+
}
|
|
771
|
+
async function assertInboundCoordinatorMessage(engine, message, fromPeerId, direction) {
|
|
772
|
+
const config = engine.config.coordinator;
|
|
773
|
+
if (!config) return;
|
|
774
|
+
const state = engine.coordinatorState ?? await refreshCoordinatorState(engine);
|
|
775
|
+
if (!state) {
|
|
776
|
+
throw new CoordinatorError("Cannot validate coordinator message without replication-group state");
|
|
777
|
+
}
|
|
778
|
+
if (message.groupId !== config.groupId || message.primaryTerm !== state.primaryTerm) {
|
|
779
|
+
throw new StalePrimaryError(
|
|
780
|
+
`Rejected ${direction} message for stale or wrong replication group term`,
|
|
781
|
+
errorDetails(engine, state)
|
|
782
|
+
);
|
|
783
|
+
}
|
|
784
|
+
if ((direction === "batch" || direction === "sync-data") && fromPeerId !== state.currentPrimary?.nodeId) {
|
|
785
|
+
throw new StalePrimaryError(
|
|
786
|
+
`Rejected ${direction} message from non-current primary '${fromPeerId}'`,
|
|
787
|
+
errorDetails(engine, state)
|
|
788
|
+
);
|
|
789
|
+
}
|
|
790
|
+
if ((direction === "ack" || direction === "forward" || direction === "sync-request") && !hasCurrentPrimaryAuthorityFor(engine, state)) {
|
|
791
|
+
throw new StalePrimaryError(
|
|
792
|
+
`Rejected ${direction} message because this node is not current primary`,
|
|
793
|
+
errorDetails(engine, state)
|
|
794
|
+
);
|
|
795
|
+
}
|
|
796
|
+
}
|
|
797
|
+
async function updateCoordinatorProgressAfterWrite(engine, seq, state) {
|
|
798
|
+
const config = engine.config.coordinator;
|
|
799
|
+
if (!config) return;
|
|
800
|
+
const ackedNodeIds = engine.peerTracker.ackedConfiguredNodeIds(seq, engine.nodeId, state.votingDataBearingNodeIds);
|
|
801
|
+
const retainedInSync = state.inSyncNodeIds.filter((nodeId) => ackedNodeIds.includes(nodeId));
|
|
802
|
+
let nextState = state;
|
|
803
|
+
if (!arraysEqual(retainedInSync, state.inSyncNodeIds) || state.durabilityPointSeq < seq) {
|
|
804
|
+
const updated = await config.coordinator.updateInSyncSet({
|
|
805
|
+
clusterId: config.clusterId,
|
|
806
|
+
groupId: config.groupId,
|
|
807
|
+
inSyncNodeIds: retainedInSync,
|
|
808
|
+
durabilityPointSeq: seq
|
|
809
|
+
});
|
|
810
|
+
if (!updated) {
|
|
811
|
+
throw new CoordinatorError("Failed to update in-sync set before acknowledging majority write");
|
|
812
|
+
}
|
|
813
|
+
nextState = updated;
|
|
814
|
+
}
|
|
815
|
+
for (const nodeId of ackedNodeIds) {
|
|
816
|
+
if (nextState.inSyncNodeIds.includes(nodeId)) continue;
|
|
817
|
+
if (nodeId === engine.nodeId) continue;
|
|
818
|
+
const admitted = await config.coordinator.admitNodeToInSyncSet({
|
|
819
|
+
clusterId: config.clusterId,
|
|
820
|
+
groupId: config.groupId,
|
|
821
|
+
nodeId,
|
|
822
|
+
sourceNodeId: engine.nodeId,
|
|
823
|
+
appliedSeq: seq
|
|
824
|
+
});
|
|
825
|
+
if (!admitted) {
|
|
826
|
+
throw new CoordinatorError("Failed to admit ACKing node to in-sync set before acknowledging majority write");
|
|
827
|
+
}
|
|
828
|
+
nextState = admitted;
|
|
829
|
+
}
|
|
830
|
+
engine.coordinatorState = nextState;
|
|
831
|
+
}
|
|
832
|
+
async function waitForWriteConcern(engine, seq, wc) {
|
|
833
|
+
const timeout = wc.timeoutMs ?? 5e3;
|
|
834
|
+
if (engine.isCoordinatorMode()) {
|
|
835
|
+
const state = engine.coordinatorState ?? await refreshCoordinatorState(engine);
|
|
836
|
+
if (!state) {
|
|
837
|
+
throw new CoordinatorError("Cannot satisfy write concern without replication-group state");
|
|
838
|
+
}
|
|
839
|
+
if (wc.level === "majority") {
|
|
840
|
+
await engine.peerTracker.waitForConfiguredMajority(seq, engine.nodeId, state.votingDataBearingNodeIds, timeout);
|
|
841
|
+
await updateCoordinatorProgressAfterWrite(engine, seq, state);
|
|
842
|
+
} else if (wc.level === "all") {
|
|
843
|
+
const eligibleVoters = state.votingDataBearingNodeIds.filter((nodeId) => !state.drainingNodeIds.includes(nodeId));
|
|
844
|
+
await engine.peerTracker.waitForConfiguredAll(seq, engine.nodeId, eligibleVoters, timeout);
|
|
845
|
+
await updateCoordinatorProgressAfterWrite(engine, seq, state);
|
|
846
|
+
}
|
|
847
|
+
return;
|
|
848
|
+
}
|
|
849
|
+
if (wc.level === "majority") {
|
|
850
|
+
await engine.peerTracker.waitForMajority(seq, timeout);
|
|
851
|
+
} else if (wc.level === "all") {
|
|
852
|
+
await engine.peerTracker.waitForAll(seq, timeout);
|
|
853
|
+
}
|
|
854
|
+
}
|
|
855
|
+
function getCoordinatorRuntimeStatus(engine) {
|
|
856
|
+
const config = engine.config.coordinator;
|
|
857
|
+
const state = engine.coordinatorState;
|
|
858
|
+
if (!config || !state) return void 0;
|
|
859
|
+
return {
|
|
860
|
+
clusterId: config.clusterId,
|
|
861
|
+
groupId: config.groupId,
|
|
862
|
+
currentPrimary: state.currentPrimary ? { ...state.currentPrimary } : null,
|
|
863
|
+
primaryTerm: state.primaryTerm,
|
|
864
|
+
inSyncNodeIds: [...state.inSyncNodeIds],
|
|
865
|
+
drainingNodeIds: [...state.drainingNodeIds],
|
|
866
|
+
repairingNodeIds: [...state.repairingNodeIds],
|
|
867
|
+
faultedNodeIds: [...state.faultedNodeIds],
|
|
868
|
+
votingDataBearingNodeIds: [...state.votingDataBearingNodeIds],
|
|
869
|
+
authority: engine.coordinatorAuthority,
|
|
870
|
+
connected: isCoordinatorConnected(engine),
|
|
871
|
+
controllerState: engine.controllerState
|
|
872
|
+
};
|
|
873
|
+
}
|
|
874
|
+
|
|
875
|
+
// src/replication/engine/coordinator-membership.ts
|
|
876
|
+
function requiresCoordinatorRejoin(engine, state) {
|
|
877
|
+
if (state.currentPrimary?.nodeId === engine.nodeId) return false;
|
|
878
|
+
return state.repairingNodeIds.includes(engine.nodeId) || state.faultedNodeIds.includes(engine.nodeId);
|
|
879
|
+
}
|
|
880
|
+
function requiresCoordinatorRejoinSync(engine, state) {
|
|
881
|
+
if (!state || state.currentPrimary?.nodeId === engine.nodeId) return false;
|
|
882
|
+
if (state.faultedNodeIds.includes(engine.nodeId)) return false;
|
|
883
|
+
return state.repairingNodeIds.includes(engine.nodeId);
|
|
884
|
+
}
|
|
885
|
+
async function prepareCoordinatorRejoinIfNeeded(engine) {
|
|
886
|
+
const state = engine.coordinatorState ?? await refreshCoordinatorState(engine);
|
|
887
|
+
if (!state || !requiresCoordinatorRejoin(engine, state)) return;
|
|
888
|
+
await evaluateFormerPrimaryHistory(engine, state);
|
|
889
|
+
}
|
|
890
|
+
async function markCoordinatorSyncReady(engine) {
|
|
891
|
+
const config = engine.config.coordinator;
|
|
892
|
+
const state = engine.coordinatorState ?? await refreshCoordinatorState(engine);
|
|
893
|
+
if (!config || !state || state.currentPrimary?.nodeId === engine.nodeId) return;
|
|
894
|
+
if (!state.votingDataBearingNodeIds.includes(engine.nodeId)) return;
|
|
895
|
+
if (state.faultedNodeIds.includes(engine.nodeId)) return;
|
|
896
|
+
if (state.drainingNodeIds.includes(engine.nodeId)) {
|
|
897
|
+
await removeLocalNodeFromCoordinatorInSyncSet(engine, state);
|
|
898
|
+
return;
|
|
899
|
+
}
|
|
900
|
+
const needsAdmission = !state.inSyncNodeIds.includes(engine.nodeId) || state.repairingNodeIds.includes(engine.nodeId);
|
|
901
|
+
if (!needsAdmission) {
|
|
902
|
+
engine.coordinatorState = state;
|
|
903
|
+
engine.coordinatorAuthority = hasCurrentPrimaryAuthorityFor(engine, state);
|
|
904
|
+
return;
|
|
905
|
+
}
|
|
906
|
+
const sourceNodeId = state.currentPrimary?.nodeId;
|
|
907
|
+
if (!sourceNodeId || engine.syncState.sourcePeerId !== sourceNodeId) {
|
|
908
|
+
engine.coordinatorState = state;
|
|
909
|
+
engine.coordinatorAuthority = hasCurrentPrimaryAuthorityFor(engine, state);
|
|
910
|
+
return;
|
|
911
|
+
}
|
|
912
|
+
const appliedSeq = await engine.log.getLastAppliedSeq(sourceNodeId);
|
|
913
|
+
if (appliedSeq < state.durabilityPointSeq) {
|
|
914
|
+
engine.coordinatorState = state;
|
|
915
|
+
engine.coordinatorAuthority = hasCurrentPrimaryAuthorityFor(engine, state);
|
|
916
|
+
return;
|
|
917
|
+
}
|
|
918
|
+
const admitted = await config.coordinator.admitNodeToInSyncSet({
|
|
919
|
+
clusterId: config.clusterId,
|
|
920
|
+
groupId: config.groupId,
|
|
921
|
+
nodeId: engine.nodeId,
|
|
922
|
+
sourceNodeId,
|
|
923
|
+
appliedSeq
|
|
924
|
+
});
|
|
925
|
+
engine.coordinatorState = admitted ?? state;
|
|
926
|
+
engine.coordinatorAuthority = hasCurrentPrimaryAuthorityFor(engine, engine.coordinatorState);
|
|
927
|
+
}
|
|
928
|
+
async function handleCoordinatorAckProgress(engine, nodeId, ackedSeq) {
|
|
929
|
+
const config = engine.config.coordinator;
|
|
930
|
+
const state = engine.coordinatorState;
|
|
931
|
+
if (!config || !state) return;
|
|
932
|
+
if (nodeId === engine.nodeId) return;
|
|
933
|
+
if (!hasCurrentPrimaryAuthorityFor(engine, state)) return;
|
|
934
|
+
if (state.inSyncNodeIds.includes(nodeId)) return;
|
|
935
|
+
if (!state.votingDataBearingNodeIds.includes(nodeId)) return;
|
|
936
|
+
if (ackedSeq < state.durabilityPointSeq) return;
|
|
937
|
+
const admitted = await config.coordinator.admitNodeToInSyncSet({
|
|
938
|
+
clusterId: config.clusterId,
|
|
939
|
+
groupId: config.groupId,
|
|
940
|
+
nodeId,
|
|
941
|
+
sourceNodeId: engine.nodeId,
|
|
942
|
+
appliedSeq: ackedSeq
|
|
943
|
+
});
|
|
944
|
+
if (!admitted) return;
|
|
945
|
+
engine.coordinatorState = admitted;
|
|
946
|
+
engine.coordinatorAuthority = hasCurrentPrimaryAuthorityFor(engine, admitted);
|
|
947
|
+
}
|
|
948
|
+
async function reconcileInSyncSet(engine) {
|
|
949
|
+
if (!engine.running || engine.inSyncReconciling) return;
|
|
950
|
+
const state = engine.coordinatorState;
|
|
951
|
+
if (!state || !hasCurrentPrimaryAuthorityFor(engine, state)) return;
|
|
952
|
+
const caughtUpNodeIds = engine.peerTracker.ackedConfiguredNodeIds(
|
|
953
|
+
state.durabilityPointSeq,
|
|
954
|
+
engine.nodeId,
|
|
955
|
+
state.votingDataBearingNodeIds
|
|
956
|
+
);
|
|
957
|
+
const pending = caughtUpNodeIds.filter((nodeId) => nodeId !== engine.nodeId && !state.inSyncNodeIds.includes(nodeId));
|
|
958
|
+
if (pending.length === 0) return;
|
|
959
|
+
engine.inSyncReconciling = true;
|
|
960
|
+
try {
|
|
961
|
+
for (const nodeId of pending) {
|
|
962
|
+
await handleCoordinatorAckProgress(engine, nodeId, state.durabilityPointSeq);
|
|
963
|
+
}
|
|
964
|
+
} finally {
|
|
965
|
+
engine.inSyncReconciling = false;
|
|
966
|
+
}
|
|
967
|
+
}
|
|
968
|
+
async function removeLocalNodeFromCoordinatorInSyncSet(engine, state) {
|
|
969
|
+
const config = engine.config.coordinator;
|
|
970
|
+
if (!config) return;
|
|
971
|
+
const inSyncNodeIds = state.inSyncNodeIds.filter((nodeId) => nodeId !== engine.nodeId);
|
|
972
|
+
let nextState = state;
|
|
973
|
+
if (!arraysEqual(inSyncNodeIds, state.inSyncNodeIds)) {
|
|
974
|
+
const updated = await config.coordinator.updateInSyncSet({
|
|
975
|
+
clusterId: config.clusterId,
|
|
976
|
+
groupId: config.groupId,
|
|
977
|
+
inSyncNodeIds
|
|
978
|
+
});
|
|
979
|
+
if (!updated) {
|
|
980
|
+
throw new CoordinatorError("Failed to mark repaired node as in sync");
|
|
981
|
+
}
|
|
982
|
+
nextState = updated;
|
|
983
|
+
}
|
|
984
|
+
engine.coordinatorState = nextState;
|
|
985
|
+
engine.coordinatorAuthority = hasCurrentPrimaryAuthorityFor(engine, nextState);
|
|
986
|
+
}
|
|
987
|
+
function startCoordinatorRejoinSyncIfReady(engine, state) {
|
|
988
|
+
if (!engine.running) return;
|
|
989
|
+
if (engine.coordinatorRejoinSyncStarting) return;
|
|
990
|
+
if (!requiresCoordinatorRejoinSync(engine, state)) return;
|
|
991
|
+
if (engine.syncState.phase !== "pending") return;
|
|
992
|
+
const sourceNodeId = state.currentPrimary?.nodeId;
|
|
993
|
+
if (!sourceNodeId || !engine.config.transport.peers().has(sourceNodeId)) return;
|
|
994
|
+
engine.coordinatorRejoinSyncStarting = true;
|
|
995
|
+
engine.syncJoiner.initiateSync().catch((err) => {
|
|
996
|
+
const wrappedErr = err instanceof Error ? err : new Error(String(err));
|
|
997
|
+
engine.emitError({ error: wrappedErr, operation: "coordinator-rejoin-sync", recoverable: true });
|
|
998
|
+
}).finally(() => {
|
|
999
|
+
engine.coordinatorRejoinSyncStarting = false;
|
|
1000
|
+
});
|
|
1001
|
+
}
|
|
1002
|
+
async function handleFormerPrimaryDemotion(engine, state) {
|
|
1003
|
+
const repairingState = await markLocalNodeRepairing(engine, state);
|
|
1004
|
+
const evaluatedState = await evaluateFormerPrimaryHistory(engine, repairingState);
|
|
1005
|
+
if (!requiresCoordinatorRejoinSync(engine, evaluatedState)) return;
|
|
1006
|
+
if (!engine.running || engine.syncState.phase === "syncing" || engine.syncState.phase === "catching-up") return;
|
|
1007
|
+
engine.syncState = {
|
|
1008
|
+
phase: "pending",
|
|
1009
|
+
sourcePeerId: null,
|
|
1010
|
+
snapshotSeq: null,
|
|
1011
|
+
completedTables: [],
|
|
1012
|
+
totalTables: 0,
|
|
1013
|
+
startedAt: null,
|
|
1014
|
+
error: null
|
|
1015
|
+
};
|
|
1016
|
+
await engine.log.setSyncMeta("pending");
|
|
1017
|
+
await engine.syncJoiner.initiateSync();
|
|
1018
|
+
}
|
|
1019
|
+
async function markLocalNodeRepairing(engine, state) {
|
|
1020
|
+
const config = engine.config.coordinator;
|
|
1021
|
+
if (!config || state.repairingNodeIds.includes(engine.nodeId) || state.faultedNodeIds.includes(engine.nodeId)) {
|
|
1022
|
+
return state;
|
|
1023
|
+
}
|
|
1024
|
+
const updated = await config.coordinator.updateNodeMaintenance({
|
|
1025
|
+
clusterId: config.clusterId,
|
|
1026
|
+
groupId: config.groupId,
|
|
1027
|
+
nodeId: engine.nodeId,
|
|
1028
|
+
repairing: true
|
|
1029
|
+
});
|
|
1030
|
+
if (!updated) {
|
|
1031
|
+
throw new CoordinatorError("Failed to mark former primary for repair");
|
|
1032
|
+
}
|
|
1033
|
+
engine.coordinatorState = updated;
|
|
1034
|
+
engine.coordinatorAuthority = hasCurrentPrimaryAuthorityFor(engine, updated);
|
|
1035
|
+
return updated;
|
|
1036
|
+
}
|
|
1037
|
+
async function evaluateFormerPrimaryHistory(engine, state) {
|
|
1038
|
+
const config = engine.config.coordinator;
|
|
1039
|
+
const currentPrimary = state.currentPrimary;
|
|
1040
|
+
if (!config || !currentPrimary || state.faultedNodeIds.includes(engine.nodeId)) return state;
|
|
1041
|
+
const localSeq = await engine.log.getLocalSeq();
|
|
1042
|
+
const currentPrimaryAckedSeq = await engine.log.getPeerAckedSeq(currentPrimary.nodeId);
|
|
1043
|
+
if (localSeq <= currentPrimaryAckedSeq) {
|
|
1044
|
+
return state;
|
|
1045
|
+
}
|
|
1046
|
+
const inSyncNodeIds = state.inSyncNodeIds.filter((nodeId) => nodeId !== engine.nodeId);
|
|
1047
|
+
let nextState = state;
|
|
1048
|
+
if (inSyncNodeIds.length !== state.inSyncNodeIds.length) {
|
|
1049
|
+
const updated = await config.coordinator.updateInSyncSet({
|
|
1050
|
+
clusterId: config.clusterId,
|
|
1051
|
+
groupId: config.groupId,
|
|
1052
|
+
inSyncNodeIds
|
|
1053
|
+
});
|
|
1054
|
+
if (!updated) {
|
|
1055
|
+
throw new CoordinatorError("Failed to remove divergent former primary from in-sync set");
|
|
1056
|
+
}
|
|
1057
|
+
nextState = updated;
|
|
1058
|
+
}
|
|
1059
|
+
const faulted = await config.coordinator.updateNodeMaintenance({
|
|
1060
|
+
clusterId: config.clusterId,
|
|
1061
|
+
groupId: config.groupId,
|
|
1062
|
+
nodeId: engine.nodeId,
|
|
1063
|
+
repairing: false,
|
|
1064
|
+
faulted: true
|
|
1065
|
+
});
|
|
1066
|
+
nextState = faulted ?? nextState;
|
|
1067
|
+
engine.coordinatorState = nextState;
|
|
1068
|
+
engine.coordinatorAuthority = false;
|
|
1069
|
+
engine.emitError({
|
|
1070
|
+
error: new AuthorityError("Former primary returned with local-only writes and was quarantined", "AUTHORITY_LOST", {
|
|
1071
|
+
...errorDetails(engine, nextState),
|
|
1072
|
+
localSeq: localSeq.toString(),
|
|
1073
|
+
currentPrimaryAckedSeq: currentPrimaryAckedSeq.toString()
|
|
1074
|
+
}),
|
|
1075
|
+
operation: "coordinator-former-primary-divergence",
|
|
1076
|
+
recoverable: false
|
|
1077
|
+
});
|
|
1078
|
+
return nextState;
|
|
1079
|
+
}
|
|
1080
|
+
|
|
1081
|
+
// src/replication/engine/coordinator-lifecycle.ts
|
|
1082
|
+
var DEFAULT_CONTROLLER_LEASE_TTL_MS = 1e4;
|
|
1083
|
+
var DEFAULT_CONTROLLER_TICK_INTERVAL_MS = 1e3;
|
|
1084
|
+
var IN_SYNC_RECONCILE_INTERVAL_MS = 1e3;
|
|
1085
|
+
function unrefTimer(timer) {
|
|
1086
|
+
const unref = timer.unref;
|
|
1087
|
+
unref?.call(timer);
|
|
1088
|
+
}
|
|
1089
|
+
function localCoordinatorPrimary(engine) {
|
|
1090
|
+
const endpoint = engine.config.coordinator?.endpoint;
|
|
1091
|
+
return endpoint ? { nodeId: engine.nodeId, endpoint } : { nodeId: engine.nodeId };
|
|
1092
|
+
}
|
|
1093
|
+
async function startCoordinatorMode(engine) {
|
|
1094
|
+
const config = engine.config.coordinator;
|
|
1095
|
+
if (!config) return;
|
|
1096
|
+
const coordinator = config.coordinator;
|
|
1097
|
+
let state = await coordinator.getReplicationGroupState(config.clusterId, config.groupId);
|
|
1098
|
+
if (!state && config.votingDataBearingNodeIds) {
|
|
1099
|
+
state = await coordinator.setReplicationGroupState({
|
|
1100
|
+
clusterId: config.clusterId,
|
|
1101
|
+
groupId: config.groupId,
|
|
1102
|
+
votingDataBearingNodeIds: config.votingDataBearingNodeIds,
|
|
1103
|
+
currentPrimary: engine.config.topology.role === "primary" ? localCoordinatorPrimary(engine) : null,
|
|
1104
|
+
primaryTerm: 1n,
|
|
1105
|
+
inSyncNodeIds: [engine.nodeId],
|
|
1106
|
+
compatibility: config.compatibility
|
|
1107
|
+
});
|
|
1108
|
+
}
|
|
1109
|
+
if (!state) {
|
|
1110
|
+
throw new CoordinatorError(`Replication group '${config.groupId}' is not registered`);
|
|
1111
|
+
}
|
|
1112
|
+
noteCoordinatorContact(engine);
|
|
1113
|
+
engine.coordinatorState = state;
|
|
1114
|
+
engine.coordinatorAuthority = hasCurrentPrimaryAuthorityFor(engine, state);
|
|
1115
|
+
const session = await coordinator.registerNodeSession({
|
|
1116
|
+
clusterId: config.clusterId,
|
|
1117
|
+
nodeId: engine.nodeId,
|
|
1118
|
+
ttlMs: config.sessionTtlMs ?? DEFAULT_COORDINATOR_SESSION_TTL_MS,
|
|
1119
|
+
endpoint: config.endpoint,
|
|
1120
|
+
groupIds: [config.groupId],
|
|
1121
|
+
dataBearing: true,
|
|
1122
|
+
voting: state.votingDataBearingNodeIds.includes(engine.nodeId),
|
|
1123
|
+
compatibility: config.compatibility
|
|
1124
|
+
});
|
|
1125
|
+
engine.nodeSessionLeaseId = session.lease.id;
|
|
1126
|
+
noteCoordinatorContact(engine);
|
|
1127
|
+
engine.coordinatorWatchDisposer = await coordinator.watchReplicationGroup(config.clusterId, config.groupId, (next) => {
|
|
1128
|
+
handleCoordinatorStateUpdate(engine, next);
|
|
1129
|
+
});
|
|
1130
|
+
startCoordinatorLeaseRenewal(engine);
|
|
1131
|
+
startInSyncReconcileLoop(engine);
|
|
1132
|
+
startControllerLoop(engine);
|
|
1133
|
+
}
|
|
1134
|
+
function startInSyncReconcileLoop(engine) {
|
|
1135
|
+
const timer = setInterval(() => {
|
|
1136
|
+
reconcileInSyncSet(engine).catch((err) => {
|
|
1137
|
+
const wrappedErr = err instanceof Error ? err : new Error(String(err));
|
|
1138
|
+
engine.emitError({ error: wrappedErr, operation: "coordinator-in-sync-reconcile", recoverable: true });
|
|
1139
|
+
});
|
|
1140
|
+
}, IN_SYNC_RECONCILE_INTERVAL_MS);
|
|
1141
|
+
unrefTimer(timer);
|
|
1142
|
+
engine.inSyncReconcileTimer = timer;
|
|
1143
|
+
}
|
|
1144
|
+
function handleCoordinatorStateUpdate(engine, next) {
|
|
1145
|
+
const previous = engine.coordinatorState;
|
|
1146
|
+
const wasPrimary = previous ? hasCurrentPrimaryAuthorityFor(engine, previous) : engine.coordinatorAuthority;
|
|
1147
|
+
noteCoordinatorContact(engine);
|
|
1148
|
+
engine.coordinatorState = next;
|
|
1149
|
+
engine.coordinatorAuthority = hasCurrentPrimaryAuthorityFor(engine, next);
|
|
1150
|
+
if (wasPrimary && !engine.coordinatorAuthority && next.primaryTerm > (previous?.primaryTerm ?? 0n)) {
|
|
1151
|
+
handleFormerPrimaryDemotion(engine, next).catch((err) => {
|
|
1152
|
+
const wrappedErr = err instanceof Error ? err : new Error(String(err));
|
|
1153
|
+
engine.emitError({ error: wrappedErr, operation: "coordinator-former-primary-demotion", recoverable: false });
|
|
1154
|
+
});
|
|
1155
|
+
return;
|
|
1156
|
+
}
|
|
1157
|
+
startCoordinatorRejoinSyncIfReady(engine, next);
|
|
1158
|
+
}
|
|
1159
|
+
function startCoordinatorLeaseRenewal(engine) {
|
|
1160
|
+
const config = engine.config.coordinator;
|
|
1161
|
+
if (!config || !engine.nodeSessionLeaseId) return;
|
|
1162
|
+
const ttlMs = config.sessionTtlMs ?? DEFAULT_COORDINATOR_SESSION_TTL_MS;
|
|
1163
|
+
const timer = setInterval(
|
|
1164
|
+
() => {
|
|
1165
|
+
void keepCoordinatorSessionAlive(engine, ttlMs);
|
|
1166
|
+
},
|
|
1167
|
+
Math.max(1e3, Math.floor(ttlMs / 3))
|
|
1168
|
+
);
|
|
1169
|
+
unrefTimer(timer);
|
|
1170
|
+
engine.coordinatorLeaseTimer = timer;
|
|
1171
|
+
}
|
|
1172
|
+
function applyAuthorityFromKnownState(engine) {
|
|
1173
|
+
const state = engine.coordinatorState;
|
|
1174
|
+
engine.coordinatorAuthority = state ? hasCurrentPrimaryAuthorityFor(engine, state) : false;
|
|
1175
|
+
}
|
|
1176
|
+
async function keepCoordinatorSessionAlive(engine, ttlMs) {
|
|
1177
|
+
const config = engine.config.coordinator;
|
|
1178
|
+
if (!config || !engine.running || engine.coordinatorSessionRestoring) return;
|
|
1179
|
+
const leaseId = engine.nodeSessionLeaseId;
|
|
1180
|
+
if (leaseId) {
|
|
1181
|
+
try {
|
|
1182
|
+
if (await config.coordinator.renewLease(leaseId, ttlMs)) {
|
|
1183
|
+
noteCoordinatorContact(engine);
|
|
1184
|
+
applyAuthorityFromKnownState(engine);
|
|
1185
|
+
return;
|
|
1186
|
+
}
|
|
1187
|
+
} catch (err) {
|
|
1188
|
+
engine.coordinatorAuthority = false;
|
|
1189
|
+
const wrappedErr = err instanceof Error ? err : new Error(String(err));
|
|
1190
|
+
engine.emitError({ error: wrappedErr, operation: "coordinator-session-renew", recoverable: true });
|
|
1191
|
+
return;
|
|
1472
1192
|
}
|
|
1473
|
-
return new Promise((resolve, reject) => {
|
|
1474
|
-
const timer = setTimeout(() => {
|
|
1475
|
-
this.waiters.delete(waiter);
|
|
1476
|
-
reject(new WriteConcernError(`Timed out waiting for all peers to ACK seq ${seq}`));
|
|
1477
|
-
}, timeoutMs);
|
|
1478
|
-
timer.unref?.();
|
|
1479
|
-
const waiter = { seq, kind: "all", resolve, reject, timer };
|
|
1480
|
-
this.waiters.add(waiter);
|
|
1481
|
-
});
|
|
1482
1193
|
}
|
|
1483
|
-
|
|
1484
|
-
|
|
1485
|
-
|
|
1486
|
-
|
|
1194
|
+
engine.coordinatorAuthority = false;
|
|
1195
|
+
engine.nodeSessionLeaseId = null;
|
|
1196
|
+
await restoreCoordinatorSession(engine, ttlMs);
|
|
1197
|
+
}
|
|
1198
|
+
async function restoreCoordinatorSession(engine, ttlMs) {
|
|
1199
|
+
const config = engine.config.coordinator;
|
|
1200
|
+
if (!config) return;
|
|
1201
|
+
engine.coordinatorSessionRestoring = true;
|
|
1202
|
+
try {
|
|
1203
|
+
const state = await config.coordinator.getReplicationGroupState(config.clusterId, config.groupId);
|
|
1204
|
+
if (!state) {
|
|
1205
|
+
throw new CoordinatorError(`Replication group '${config.groupId}' is not registered`);
|
|
1487
1206
|
}
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
|
|
1491
|
-
|
|
1492
|
-
|
|
1493
|
-
|
|
1494
|
-
|
|
1495
|
-
|
|
1496
|
-
|
|
1497
|
-
localNodeId,
|
|
1498
|
-
votingNodeIds: [...votingNodeIds],
|
|
1499
|
-
resolve,
|
|
1500
|
-
reject,
|
|
1501
|
-
timer
|
|
1502
|
-
};
|
|
1503
|
-
this.waiters.add(waiter);
|
|
1207
|
+
const session = await config.coordinator.registerNodeSession({
|
|
1208
|
+
clusterId: config.clusterId,
|
|
1209
|
+
nodeId: engine.nodeId,
|
|
1210
|
+
ttlMs,
|
|
1211
|
+
endpoint: config.endpoint,
|
|
1212
|
+
groupIds: [config.groupId],
|
|
1213
|
+
dataBearing: true,
|
|
1214
|
+
voting: state.votingDataBearingNodeIds.includes(engine.nodeId),
|
|
1215
|
+
compatibility: config.compatibility
|
|
1504
1216
|
});
|
|
1217
|
+
if (!engine.running) {
|
|
1218
|
+
await config.coordinator.releaseLease(session.lease.id).catch(() => void 0);
|
|
1219
|
+
return;
|
|
1220
|
+
}
|
|
1221
|
+
engine.nodeSessionLeaseId = session.lease.id;
|
|
1222
|
+
engine.coordinatorState = state;
|
|
1223
|
+
noteCoordinatorContact(engine);
|
|
1224
|
+
applyAuthorityFromKnownState(engine);
|
|
1225
|
+
} catch (err) {
|
|
1226
|
+
const wrappedErr = err instanceof Error ? err : new Error(String(err));
|
|
1227
|
+
engine.emitError({ error: wrappedErr, operation: "coordinator-session-restore", recoverable: true });
|
|
1228
|
+
} finally {
|
|
1229
|
+
engine.coordinatorSessionRestoring = false;
|
|
1505
1230
|
}
|
|
1506
|
-
|
|
1507
|
-
|
|
1508
|
-
|
|
1231
|
+
}
|
|
1232
|
+
function startControllerLoop(engine) {
|
|
1233
|
+
const config = engine.config.coordinator;
|
|
1234
|
+
if (!config) return;
|
|
1235
|
+
const controllerConfig = typeof config.controller === "object" ? config.controller : {};
|
|
1236
|
+
const enabled = typeof config.controller === "boolean" ? config.controller : controllerConfig.enabled ?? true;
|
|
1237
|
+
if (!enabled) {
|
|
1238
|
+
engine.controllerState = "disabled";
|
|
1239
|
+
return;
|
|
1240
|
+
}
|
|
1241
|
+
engine.controllerState = "standby";
|
|
1242
|
+
const ttlMs = controllerConfig.leaseTtlMs ?? DEFAULT_CONTROLLER_LEASE_TTL_MS;
|
|
1243
|
+
const holderId = controllerConfig.holderId ?? engine.nodeId;
|
|
1244
|
+
const tickMs = controllerConfig.tickIntervalMs ?? DEFAULT_CONTROLLER_TICK_INTERVAL_MS;
|
|
1245
|
+
const timer = setInterval(() => {
|
|
1246
|
+
controllerTick(engine, holderId, ttlMs).catch((err) => {
|
|
1247
|
+
const wrappedErr = err instanceof Error ? err : new Error(String(err));
|
|
1248
|
+
engine.controllerState = "lost";
|
|
1249
|
+
engine.emitError({ error: wrappedErr, operation: "coordinator-controller", recoverable: true });
|
|
1250
|
+
});
|
|
1251
|
+
}, tickMs);
|
|
1252
|
+
unrefTimer(timer);
|
|
1253
|
+
engine.controllerTimer = timer;
|
|
1254
|
+
}
|
|
1255
|
+
async function controllerTick(engine, holderId, ttlMs) {
|
|
1256
|
+
const config = engine.config.coordinator;
|
|
1257
|
+
if (!config) return;
|
|
1258
|
+
if (engine.controllerLeaseId) {
|
|
1259
|
+
const renewed = await config.coordinator.renewLease(engine.controllerLeaseId, ttlMs);
|
|
1260
|
+
if (!renewed) {
|
|
1261
|
+
engine.controllerLeaseId = null;
|
|
1262
|
+
engine.controllerState = "lost";
|
|
1263
|
+
return;
|
|
1509
1264
|
}
|
|
1510
|
-
|
|
1511
|
-
|
|
1512
|
-
|
|
1513
|
-
|
|
1514
|
-
|
|
1515
|
-
|
|
1516
|
-
|
|
1517
|
-
|
|
1518
|
-
|
|
1519
|
-
|
|
1520
|
-
|
|
1521
|
-
|
|
1522
|
-
|
|
1523
|
-
|
|
1524
|
-
|
|
1525
|
-
|
|
1265
|
+
noteCoordinatorContact(engine);
|
|
1266
|
+
engine.controllerState = "active";
|
|
1267
|
+
await runControllerPromotionCheck(engine);
|
|
1268
|
+
return;
|
|
1269
|
+
}
|
|
1270
|
+
const acquired = await config.coordinator.tryAcquireControllerLease({
|
|
1271
|
+
clusterId: config.clusterId,
|
|
1272
|
+
holderId,
|
|
1273
|
+
ttlMs
|
|
1274
|
+
});
|
|
1275
|
+
noteCoordinatorContact(engine);
|
|
1276
|
+
if (acquired.acquired) {
|
|
1277
|
+
engine.controllerLeaseId = acquired.lease.id;
|
|
1278
|
+
engine.controllerState = "active";
|
|
1279
|
+
await runControllerPromotionCheck(engine);
|
|
1280
|
+
} else {
|
|
1281
|
+
engine.controllerState = "standby";
|
|
1282
|
+
}
|
|
1283
|
+
}
|
|
1284
|
+
async function runControllerPromotionCheck(engine) {
|
|
1285
|
+
const config = engine.config.coordinator;
|
|
1286
|
+
if (!config) return;
|
|
1287
|
+
const state = await refreshCoordinatorState(engine);
|
|
1288
|
+
if (!state) return;
|
|
1289
|
+
const primaryNodeId = state.currentPrimary?.nodeId;
|
|
1290
|
+
const primaryLive = primaryNodeId ? await config.coordinator.getLiveNodeSession(config.clusterId, primaryNodeId) : null;
|
|
1291
|
+
const primaryCanKeepDuty = primaryNodeId && primaryLive && compatibilityAllowsPromotion(state.compatibility, primaryLive.compatibility) && !state.drainingNodeIds.includes(primaryNodeId) && !state.repairingNodeIds.includes(primaryNodeId) && !state.faultedNodeIds.includes(primaryNodeId);
|
|
1292
|
+
if (primaryCanKeepDuty) {
|
|
1293
|
+
return;
|
|
1294
|
+
}
|
|
1295
|
+
try {
|
|
1296
|
+
const promoted = await config.coordinator.promoteEligibleReplica({
|
|
1297
|
+
clusterId: config.clusterId,
|
|
1298
|
+
groupId: config.groupId,
|
|
1299
|
+
excludeNodeIds: primaryNodeId ? [primaryNodeId] : []
|
|
1526
1300
|
});
|
|
1301
|
+
engine.coordinatorState = promoted;
|
|
1302
|
+
engine.coordinatorAuthority = hasCurrentPrimaryAuthorityFor(engine, promoted);
|
|
1303
|
+
} catch (err) {
|
|
1304
|
+
const wrappedErr = err instanceof Error ? err : new Error(String(err));
|
|
1305
|
+
engine.emitError({ error: wrappedErr, operation: "coordinator-promotion", recoverable: true });
|
|
1527
1306
|
}
|
|
1528
|
-
|
|
1529
|
-
|
|
1530
|
-
|
|
1531
|
-
|
|
1532
|
-
|
|
1533
|
-
for (const nodeId of votingNodeIds) {
|
|
1534
|
-
const peer = this.peers.get(nodeId);
|
|
1535
|
-
if (peer && peer.lastAckedSeq >= seq) {
|
|
1536
|
-
acked.add(nodeId);
|
|
1537
|
-
}
|
|
1538
|
-
}
|
|
1539
|
-
return votingNodeIds.filter((nodeId) => acked.has(nodeId));
|
|
1307
|
+
}
|
|
1308
|
+
function stopCoordinatorTimers(engine) {
|
|
1309
|
+
if (engine.coordinatorLeaseTimer) {
|
|
1310
|
+
clearInterval(engine.coordinatorLeaseTimer);
|
|
1311
|
+
engine.coordinatorLeaseTimer = null;
|
|
1540
1312
|
}
|
|
1541
|
-
|
|
1542
|
-
|
|
1313
|
+
if (engine.inSyncReconcileTimer) {
|
|
1314
|
+
clearInterval(engine.inSyncReconcileTimer);
|
|
1315
|
+
engine.inSyncReconcileTimer = null;
|
|
1543
1316
|
}
|
|
1544
|
-
|
|
1545
|
-
|
|
1546
|
-
|
|
1547
|
-
if (peer.connected && peer.lastAckedSeq >= seq) {
|
|
1548
|
-
count += 1;
|
|
1549
|
-
}
|
|
1550
|
-
}
|
|
1551
|
-
return count;
|
|
1317
|
+
if (engine.controllerTimer) {
|
|
1318
|
+
clearInterval(engine.controllerTimer);
|
|
1319
|
+
engine.controllerTimer = null;
|
|
1552
1320
|
}
|
|
1553
|
-
|
|
1554
|
-
|
|
1321
|
+
}
|
|
1322
|
+
async function stopCoordinatorMode(engine) {
|
|
1323
|
+
const config = engine.config.coordinator;
|
|
1324
|
+
if (!config) return;
|
|
1325
|
+
if (engine.coordinatorWatchDisposer) {
|
|
1326
|
+
await engine.coordinatorWatchDisposer();
|
|
1327
|
+
engine.coordinatorWatchDisposer = null;
|
|
1328
|
+
}
|
|
1329
|
+
if (engine.controllerLeaseId) {
|
|
1330
|
+
const leaseId = engine.controllerLeaseId;
|
|
1331
|
+
engine.controllerLeaseId = null;
|
|
1332
|
+
await config.coordinator.releaseLease(leaseId).catch((err) => {
|
|
1333
|
+
const wrappedErr = err instanceof Error ? err : new Error(String(err));
|
|
1334
|
+
engine.emitError({ error: wrappedErr, operation: "coordinator-controller-release", recoverable: true });
|
|
1335
|
+
});
|
|
1555
1336
|
}
|
|
1556
|
-
|
|
1557
|
-
const
|
|
1558
|
-
|
|
1559
|
-
|
|
1560
|
-
|
|
1561
|
-
|
|
1562
|
-
|
|
1563
|
-
|
|
1564
|
-
|
|
1565
|
-
|
|
1566
|
-
|
|
1567
|
-
|
|
1568
|
-
|
|
1569
|
-
|
|
1570
|
-
|
|
1571
|
-
|
|
1572
|
-
|
|
1573
|
-
|
|
1574
|
-
|
|
1575
|
-
|
|
1576
|
-
|
|
1577
|
-
|
|
1578
|
-
|
|
1579
|
-
|
|
1580
|
-
|
|
1581
|
-
|
|
1582
|
-
|
|
1583
|
-
|
|
1584
|
-
}
|
|
1585
|
-
if (connected === 0 || this.countAcked(waiter.seq) >= needed) {
|
|
1586
|
-
clearTimeout(waiter.timer);
|
|
1587
|
-
this.waiters.delete(waiter);
|
|
1588
|
-
waiter.resolve();
|
|
1589
|
-
continue;
|
|
1590
|
-
}
|
|
1591
|
-
if (waiter.kind === "all" && connected < this.peers.size) {
|
|
1592
|
-
clearTimeout(waiter.timer);
|
|
1593
|
-
this.waiters.delete(waiter);
|
|
1594
|
-
waiter.reject(
|
|
1595
|
-
new WriteConcernError(
|
|
1596
|
-
`Cannot satisfy 'all' write concern: only ${connected}/${this.peers.size} peers connected`
|
|
1597
|
-
)
|
|
1598
|
-
);
|
|
1599
|
-
continue;
|
|
1600
|
-
}
|
|
1601
|
-
const totalNodes = this.peers.size + 1;
|
|
1602
|
-
const majorityNeeded = Math.floor(totalNodes / 2) + 1;
|
|
1603
|
-
if (waiter.kind === "majority" && connected + 1 < majorityNeeded) {
|
|
1604
|
-
clearTimeout(waiter.timer);
|
|
1605
|
-
this.waiters.delete(waiter);
|
|
1606
|
-
waiter.reject(
|
|
1607
|
-
new WriteConcernError(
|
|
1608
|
-
`Cannot satisfy 'majority' write concern: only ${connected + 1}/${totalNodes} nodes reachable`
|
|
1609
|
-
)
|
|
1610
|
-
);
|
|
1611
|
-
}
|
|
1337
|
+
await config.coordinator.deregisterNodeSession(config.clusterId, engine.nodeId).catch((err) => {
|
|
1338
|
+
const wrappedErr = err instanceof Error ? err : new Error(String(err));
|
|
1339
|
+
engine.emitError({ error: wrappedErr, operation: "coordinator-session-deregister", recoverable: true });
|
|
1340
|
+
});
|
|
1341
|
+
}
|
|
1342
|
+
function assertReadyPhase(engine, activity) {
|
|
1343
|
+
if (engine.syncState.phase !== "ready") {
|
|
1344
|
+
throw new SyncError(`Node is in '${engine.syncState.phase}' phase and cannot ${activity}`);
|
|
1345
|
+
}
|
|
1346
|
+
}
|
|
1347
|
+
async function query(engine, sql, params, options) {
|
|
1348
|
+
assertReadyPhase(engine, "serve reads");
|
|
1349
|
+
await assertReadConcern(engine, options?.readConcern?.level);
|
|
1350
|
+
return engine.database.query(sql, params, options);
|
|
1351
|
+
}
|
|
1352
|
+
async function execute(engine, sql, params, options) {
|
|
1353
|
+
assertReadyPhase(engine, "accept operations");
|
|
1354
|
+
if (!await canAcceptLocalWrite(engine)) {
|
|
1355
|
+
if (engine.config.writeForwarding) {
|
|
1356
|
+
const result = await forwardStatements(engine, [{ sql, params }]);
|
|
1357
|
+
const first = result.results[0];
|
|
1358
|
+
if (!first) {
|
|
1359
|
+
return { changes: 0, lastInsertRowId: 0 };
|
|
1360
|
+
}
|
|
1361
|
+
return {
|
|
1362
|
+
changes: first.changes,
|
|
1363
|
+
lastInsertRowId: typeof first.lastInsertRowId === "string" ? BigInt(first.lastInsertRowId) : first.lastInsertRowId
|
|
1364
|
+
};
|
|
1612
1365
|
}
|
|
1366
|
+
throwNotCurrentPrimary(engine);
|
|
1613
1367
|
}
|
|
1614
|
-
|
|
1368
|
+
return engine.localExecutor.executeLocally(sql, params, options);
|
|
1369
|
+
}
|
|
1370
|
+
async function executeBatch(engine, sql, paramsBatch, options) {
|
|
1371
|
+
assertReadyPhase(engine, "accept operations");
|
|
1372
|
+
if (!await canAcceptLocalWrite(engine)) {
|
|
1373
|
+
if (engine.config.writeForwarding) {
|
|
1374
|
+
const statements = paramsBatch.map((p) => ({ sql, params: p }));
|
|
1375
|
+
const result = await forwardStatements(engine, statements);
|
|
1376
|
+
return result.results.map((r) => ({
|
|
1377
|
+
changes: r.changes,
|
|
1378
|
+
lastInsertRowId: typeof r.lastInsertRowId === "string" ? BigInt(r.lastInsertRowId) : Number(r.lastInsertRowId)
|
|
1379
|
+
}));
|
|
1380
|
+
}
|
|
1381
|
+
throwNotCurrentPrimary(engine);
|
|
1382
|
+
}
|
|
1383
|
+
const results = [];
|
|
1384
|
+
for (const params of paramsBatch) {
|
|
1385
|
+
const r = await engine.localExecutor.executeLocally(sql, params, options);
|
|
1386
|
+
results.push(r);
|
|
1387
|
+
}
|
|
1388
|
+
return results;
|
|
1389
|
+
}
|
|
1390
|
+
async function transaction(engine, fn, options) {
|
|
1391
|
+
assertReadyPhase(engine, "accept operations");
|
|
1392
|
+
if (!await canAcceptLocalWrite(engine)) {
|
|
1393
|
+
throw new TopologyError("This node cannot accept writes in transaction mode");
|
|
1394
|
+
}
|
|
1395
|
+
return engine.localExecutor.executeTransactionLocally(fn, options);
|
|
1396
|
+
}
|
|
1397
|
+
async function forwardStatements(engine, statements, _options) {
|
|
1398
|
+
if (await canAcceptLocalWrite(engine)) {
|
|
1399
|
+
return engine.localExecutor.executeForwardedLocally(statements);
|
|
1400
|
+
}
|
|
1401
|
+
const primaryPeerId = getForwardingPrimaryPeerId(engine);
|
|
1402
|
+
if (primaryPeerId === null) {
|
|
1403
|
+
throw new TopologyError("No primary node available for write forwarding");
|
|
1404
|
+
}
|
|
1405
|
+
return engine.config.transport.forward(primaryPeerId, {
|
|
1406
|
+
statements,
|
|
1407
|
+
requestId: randomUUID(),
|
|
1408
|
+
...getCoordinatorMessageFields(engine)
|
|
1409
|
+
});
|
|
1410
|
+
}
|
|
1615
1411
|
|
|
1616
1412
|
// src/replication/engine/replication-transaction.ts
|
|
1617
1413
|
var ReplicationTransaction = class extends Transaction {
|
|
@@ -1672,12 +1468,12 @@ var LocalExecutor = class {
|
|
|
1672
1468
|
const stmt = await tx.prepare(sql);
|
|
1673
1469
|
const r = await stmt.run(...bindValues);
|
|
1674
1470
|
if (isDdl) {
|
|
1675
|
-
|
|
1676
|
-
|
|
1677
|
-
|
|
1678
|
-
|
|
1679
|
-
|
|
1680
|
-
|
|
1471
|
+
await insertDdlChange(tx, CHANGES_TABLE, {
|
|
1472
|
+
ddlStatement: sql,
|
|
1473
|
+
nodeId: engine.nodeId,
|
|
1474
|
+
txId,
|
|
1475
|
+
hlc: engine.hlc.now()
|
|
1476
|
+
});
|
|
1681
1477
|
} else {
|
|
1682
1478
|
await engine.log.stampChanges(tx, seqBefore, txId);
|
|
1683
1479
|
await engine.log.updateColumnVersions(tx, seqBefore);
|
|
@@ -1738,12 +1534,12 @@ var LocalExecutor = class {
|
|
|
1738
1534
|
if (droppedTable !== null) {
|
|
1739
1535
|
droppedTables.push(droppedTable);
|
|
1740
1536
|
}
|
|
1741
|
-
|
|
1742
|
-
|
|
1743
|
-
|
|
1744
|
-
|
|
1745
|
-
|
|
1746
|
-
|
|
1537
|
+
await insertDdlChange(tx, CHANGES_TABLE, {
|
|
1538
|
+
ddlStatement: sql,
|
|
1539
|
+
nodeId: engine.nodeId,
|
|
1540
|
+
txId,
|
|
1541
|
+
hlc: engine.hlc.now()
|
|
1542
|
+
});
|
|
1747
1543
|
if (engine.tracker) {
|
|
1748
1544
|
await engine.tracker.refreshAllTriggersUsingConnection(tx);
|
|
1749
1545
|
}
|
|
@@ -1789,12 +1585,12 @@ var LocalExecutor = class {
|
|
|
1789
1585
|
if (droppedTable !== null) {
|
|
1790
1586
|
hooks.droppedTables.push(droppedTable);
|
|
1791
1587
|
}
|
|
1792
|
-
|
|
1793
|
-
|
|
1794
|
-
|
|
1795
|
-
|
|
1796
|
-
|
|
1797
|
-
|
|
1588
|
+
await insertDdlChange(tx, CHANGES_TABLE, {
|
|
1589
|
+
ddlStatement: sql,
|
|
1590
|
+
nodeId: engine.nodeId,
|
|
1591
|
+
txId,
|
|
1592
|
+
hlc: engine.hlc.now()
|
|
1593
|
+
});
|
|
1798
1594
|
if (engine.tracker) {
|
|
1799
1595
|
await engine.tracker.refreshAllTriggersUsingConnection(tx);
|
|
1800
1596
|
}
|
|
@@ -1823,6 +1619,54 @@ var LocalExecutor = class {
|
|
|
1823
1619
|
}
|
|
1824
1620
|
};
|
|
1825
1621
|
|
|
1622
|
+
// src/replication/engine/node-health.ts
|
|
1623
|
+
function isSidelined(state, nodeId) {
|
|
1624
|
+
return state.drainingNodeIds.includes(nodeId) || state.repairingNodeIds.includes(nodeId) || state.faultedNodeIds.includes(nodeId);
|
|
1625
|
+
}
|
|
1626
|
+
function excludingReason(state, nodeId) {
|
|
1627
|
+
if (state.drainingNodeIds.includes(nodeId)) return "draining";
|
|
1628
|
+
if (state.faultedNodeIds.includes(nodeId)) return "faulted";
|
|
1629
|
+
return "sync-pending";
|
|
1630
|
+
}
|
|
1631
|
+
function staticHealth(engine, phase) {
|
|
1632
|
+
if (phase !== "ready") {
|
|
1633
|
+
return { state: "unavailable", reason: "sync-pending", canRead: false, canWrite: false };
|
|
1634
|
+
}
|
|
1635
|
+
return { state: "healthy", reason: "in-sync", canRead: true, canWrite: engine.config.topology.canWrite() };
|
|
1636
|
+
}
|
|
1637
|
+
function computeNodeHealth(engine) {
|
|
1638
|
+
const phase = engine.syncState.phase;
|
|
1639
|
+
if (phase === "syncing" || phase === "catching-up") {
|
|
1640
|
+
return { state: "syncing", reason: "sync-pending", canRead: false, canWrite: false };
|
|
1641
|
+
}
|
|
1642
|
+
if (!engine.isCoordinatorMode()) {
|
|
1643
|
+
return staticHealth(engine, phase);
|
|
1644
|
+
}
|
|
1645
|
+
const state = engine.coordinatorState;
|
|
1646
|
+
if (!state) {
|
|
1647
|
+
return { state: "unavailable", reason: "no-group-state", canRead: false, canWrite: false };
|
|
1648
|
+
}
|
|
1649
|
+
const nodeId = engine.nodeId;
|
|
1650
|
+
const canRead = phase === "ready" && !isSidelined(state, nodeId);
|
|
1651
|
+
const canWrite = canRead && engine.coordinatorAuthority;
|
|
1652
|
+
if (state.repairingNodeIds.includes(nodeId)) {
|
|
1653
|
+
return { state: "repairing", reason: "repairing", canRead, canWrite };
|
|
1654
|
+
}
|
|
1655
|
+
if (engine.coordinatorAuthority && !canWrite) {
|
|
1656
|
+
return { state: "failing_over", reason: excludingReason(state, nodeId), canRead, canWrite };
|
|
1657
|
+
}
|
|
1658
|
+
if (!canRead) {
|
|
1659
|
+
return { state: "unavailable", reason: excludingReason(state, nodeId), canRead, canWrite };
|
|
1660
|
+
}
|
|
1661
|
+
if (!isCoordinatorConnected(engine)) {
|
|
1662
|
+
return { state: "degraded", reason: "coordinator-unreachable", canRead, canWrite };
|
|
1663
|
+
}
|
|
1664
|
+
if (!state.inSyncNodeIds.includes(nodeId)) {
|
|
1665
|
+
return { state: "degraded", reason: "lagging", canRead, canWrite };
|
|
1666
|
+
}
|
|
1667
|
+
return { state: "healthy", reason: "in-sync", canRead, canWrite };
|
|
1668
|
+
}
|
|
1669
|
+
|
|
1826
1670
|
// src/replication/engine/sender-loop.ts
|
|
1827
1671
|
var SenderLoop = class {
|
|
1828
1672
|
constructor(engine) {
|
|
@@ -1952,6 +1796,24 @@ async function delayAckIfConfigured(engine) {
|
|
|
1952
1796
|
}
|
|
1953
1797
|
|
|
1954
1798
|
// src/replication/engine/transport-wiring.ts
|
|
1799
|
+
function replicatesFrom(engine, peer) {
|
|
1800
|
+
if (engine.isCoordinatorMode()) {
|
|
1801
|
+
return engine.coordinatorState?.currentPrimary?.nodeId === peer.id;
|
|
1802
|
+
}
|
|
1803
|
+
return engine.config.topology.shouldAcceptFrom(peer.id, peer.role);
|
|
1804
|
+
}
|
|
1805
|
+
function servesItsData(phase) {
|
|
1806
|
+
return phase === "ready" || phase === "catching-up";
|
|
1807
|
+
}
|
|
1808
|
+
async function reportAppliedPosition(engine, peer, appliedSeq) {
|
|
1809
|
+
if (appliedSeq === 0n || !replicatesFrom(engine, peer)) return;
|
|
1810
|
+
if (!servesItsData(engine.syncState.phase)) return;
|
|
1811
|
+
if (!servesItsData((await engine.log.getSyncState()).phase)) return;
|
|
1812
|
+
await engine.config.transport.sendAck(
|
|
1813
|
+
peer.id,
|
|
1814
|
+
engine.decorateAck({ batchId: "", ackedSeq: appliedSeq, nodeId: engine.nodeId })
|
|
1815
|
+
);
|
|
1816
|
+
}
|
|
1955
1817
|
function persistAckProgress(engine, nodeId, ackedSeq) {
|
|
1956
1818
|
engine.log.setLastAppliedSeq(nodeId, ackedSeq).then(() => engine.handleCoordinatorAckProgress(nodeId, ackedSeq)).catch((err) => {
|
|
1957
1819
|
const wrappedErr = err instanceof Error ? err : new Error(String(err));
|
|
@@ -2073,6 +1935,7 @@ function wireTransportHandlers(engine) {
|
|
|
2073
1935
|
engine.peerTracker.onAckReceived(peer.id, ackedSeq);
|
|
2074
1936
|
await engine.log.setLastAppliedSeq(peer.id, ackedSeq);
|
|
2075
1937
|
await engine.handleCoordinatorAckProgress(peer.id, ackedSeq);
|
|
1938
|
+
await reportAppliedPosition(engine, peer, ackedSeq);
|
|
2076
1939
|
}).catch((err) => {
|
|
2077
1940
|
const wrappedErr = err instanceof Error ? err : new Error(String(err));
|
|
2078
1941
|
engine.emitError({ error: wrappedErr, operation: "peer-state-initialise", peerId: peer.id, recoverable: true });
|
|
@@ -2090,11 +1953,12 @@ function wireTransportHandlers(engine) {
|
|
|
2090
1953
|
if (engine.syncState.phase === "syncing" && engine.syncState.sourcePeerId === peerId) {
|
|
2091
1954
|
engine.syncState.phase = "pending";
|
|
2092
1955
|
engine.syncState.sourcePeerId = null;
|
|
2093
|
-
engine.writerConn
|
|
1956
|
+
setForeignKeysEnabled(engine.writerConn, true).catch((err) => {
|
|
2094
1957
|
const wrappedErr = err instanceof Error ? err : new Error(String(err));
|
|
2095
1958
|
engine.emitError({ error: wrappedErr, operation: "peer-disconnect-pragma-restore", peerId, recoverable: false });
|
|
2096
1959
|
});
|
|
2097
1960
|
engine.expectedBatchIndex.clear();
|
|
1961
|
+
engine.syncTableDigests.clear();
|
|
2098
1962
|
engine.log.setSyncMeta("pending").catch((err) => {
|
|
2099
1963
|
const wrappedErr = err instanceof Error ? err : new Error(String(err));
|
|
2100
1964
|
engine.emitError({ error: wrappedErr, operation: "sync-meta-write", peerId, recoverable: false });
|
|
@@ -2118,6 +1982,10 @@ function wireTransportHandlers(engine) {
|
|
|
2118
1982
|
});
|
|
2119
1983
|
engine.config.transport.onSyncAckReceived((ack, fromPeerId) => {
|
|
2120
1984
|
if (!engine.running) return;
|
|
1985
|
+
if (engine.syncJoiner.isSourceRejection(ack, fromPeerId)) {
|
|
1986
|
+
engine.syncJoiner.handleSourceRejection(ack, fromPeerId);
|
|
1987
|
+
return;
|
|
1988
|
+
}
|
|
2121
1989
|
if (!engine.isCoordinatorMode()) {
|
|
2122
1990
|
if (ack.joinerNodeId !== fromPeerId) {
|
|
2123
1991
|
engine.emitError({
|
|
@@ -2145,6 +2013,10 @@ function wireTransportHandlers(engine) {
|
|
|
2145
2013
|
|
|
2146
2014
|
// src/replication/engine/startup.ts
|
|
2147
2015
|
async function recoverHlcFromDurableState(engine) {
|
|
2016
|
+
const persisted = await loadPersistedHlc(engine.writerConn);
|
|
2017
|
+
if (persisted !== null) {
|
|
2018
|
+
engine.hlc.receive(persisted);
|
|
2019
|
+
}
|
|
2148
2020
|
const maxObserved = await engine.log.recoverMaxObservedHlc();
|
|
2149
2021
|
if (maxObserved === null) return;
|
|
2150
2022
|
engine.hlc.receive(maxObserved);
|
|
@@ -2170,8 +2042,8 @@ async function startEngine(engine) {
|
|
|
2170
2042
|
await engine.config.transport.connect(engine.nodeId, transportConfig);
|
|
2171
2043
|
const isPrimary = engine.isCoordinatorMode() ? engine.hasCoordinatorWriteAuthority() : engine.config.topology.role === "primary";
|
|
2172
2044
|
const syncCompleted = await engine.log.isSyncCompleted();
|
|
2173
|
-
const
|
|
2174
|
-
if (engine.initialSync && !isPrimary && (!syncCompleted ||
|
|
2045
|
+
const requiresCoordinatorRejoinSync2 = engine.requiresCoordinatorRejoinSync();
|
|
2046
|
+
if (engine.initialSync && !isPrimary && (!syncCompleted || requiresCoordinatorRejoinSync2)) {
|
|
2175
2047
|
const savedState = await engine.log.getSyncState();
|
|
2176
2048
|
if (savedState.phase === "syncing") {
|
|
2177
2049
|
if (!engine.tracker) {
|
|
@@ -2229,11 +2101,135 @@ async function startEngine(engine) {
|
|
|
2229
2101
|
}
|
|
2230
2102
|
engine.startSenderLoop();
|
|
2231
2103
|
}
|
|
2104
|
+
|
|
2105
|
+
// src/replication/engine/sync-recovery.ts
|
|
2106
|
+
var SYNC_RETRY_MIN_DELAY_MS = 1e3;
|
|
2107
|
+
var SYNC_RETRY_MAX_DELAY_MS = 3e4;
|
|
2108
|
+
function unreffed(timer) {
|
|
2109
|
+
const unreffable = timer;
|
|
2110
|
+
unreffable.unref?.();
|
|
2111
|
+
return unreffable;
|
|
2112
|
+
}
|
|
2113
|
+
var SyncRecovery = class {
|
|
2114
|
+
constructor(engine, retryAttempt) {
|
|
2115
|
+
this.engine = engine;
|
|
2116
|
+
this.retryAttempt = retryAttempt;
|
|
2117
|
+
}
|
|
2118
|
+
activeRequestId = null;
|
|
2119
|
+
responseTimer = null;
|
|
2120
|
+
retryTimer = null;
|
|
2121
|
+
retryDelayMs = SYNC_RETRY_MIN_DELAY_MS;
|
|
2122
|
+
markRequestSent(requestId, sourcePeerId) {
|
|
2123
|
+
this.activeRequestId = requestId;
|
|
2124
|
+
this.clearResponseTimer();
|
|
2125
|
+
this.responseTimer = unreffed(
|
|
2126
|
+
setTimeout(() => {
|
|
2127
|
+
this.responseTimer = null;
|
|
2128
|
+
if (!this.engine.running || this.activeRequestId !== requestId) return;
|
|
2129
|
+
const reason = `The sync source sent no data within ${this.engine.syncAckTimeoutMs}ms`;
|
|
2130
|
+
this.report(reason, requestId, "sync-response-timeout", sourcePeerId);
|
|
2131
|
+
void this.abandonAndRetry(reason);
|
|
2132
|
+
}, this.engine.syncAckTimeoutMs)
|
|
2133
|
+
);
|
|
2134
|
+
}
|
|
2135
|
+
markRequestFailed() {
|
|
2136
|
+
this.activeRequestId = null;
|
|
2137
|
+
this.clearResponseTimer();
|
|
2138
|
+
this.scheduleRetry();
|
|
2139
|
+
}
|
|
2140
|
+
noteSourceResponded(requestId) {
|
|
2141
|
+
if (requestId !== this.activeRequestId) return;
|
|
2142
|
+
this.clearResponseTimer();
|
|
2143
|
+
}
|
|
2144
|
+
markSyncReady() {
|
|
2145
|
+
this.activeRequestId = null;
|
|
2146
|
+
this.clearResponseTimer();
|
|
2147
|
+
this.cancelRetry();
|
|
2148
|
+
this.retryDelayMs = SYNC_RETRY_MIN_DELAY_MS;
|
|
2149
|
+
}
|
|
2150
|
+
isSourceRejection(ack, fromPeerId) {
|
|
2151
|
+
const engine = this.engine;
|
|
2152
|
+
return ack.success === false && this.activeRequestId !== null && ack.requestId === this.activeRequestId && ack.joinerNodeId === engine.nodeId && engine.syncState.phase === "syncing" && engine.syncState.sourcePeerId === fromPeerId;
|
|
2153
|
+
}
|
|
2154
|
+
handleSourceRejection(ack, fromPeerId) {
|
|
2155
|
+
const reason = ack.error ?? "The sync source refused the request";
|
|
2156
|
+
this.report(reason, ack.requestId, "sync-refused", fromPeerId);
|
|
2157
|
+
void this.abandonAndRetry(reason);
|
|
2158
|
+
}
|
|
2159
|
+
stop() {
|
|
2160
|
+
this.clearResponseTimer();
|
|
2161
|
+
this.cancelRetry();
|
|
2162
|
+
}
|
|
2163
|
+
report(reason, requestId, operation, peerId) {
|
|
2164
|
+
this.engine.emitError({ error: new SyncError(reason, requestId), operation, peerId, recoverable: true });
|
|
2165
|
+
}
|
|
2166
|
+
async abandonAndRetry(reason) {
|
|
2167
|
+
this.activeRequestId = null;
|
|
2168
|
+
this.clearResponseTimer();
|
|
2169
|
+
await abandonActiveSync(this.engine, reason);
|
|
2170
|
+
if (!this.engine.running) return;
|
|
2171
|
+
this.scheduleRetry();
|
|
2172
|
+
}
|
|
2173
|
+
scheduleRetry() {
|
|
2174
|
+
this.cancelRetry();
|
|
2175
|
+
this.retryTimer = unreffed(
|
|
2176
|
+
setTimeout(() => {
|
|
2177
|
+
this.retryTimer = null;
|
|
2178
|
+
this.retryAttempt();
|
|
2179
|
+
}, this.retryDelayMs)
|
|
2180
|
+
);
|
|
2181
|
+
this.retryDelayMs = Math.min(this.retryDelayMs * 2, SYNC_RETRY_MAX_DELAY_MS);
|
|
2182
|
+
}
|
|
2183
|
+
cancelRetry() {
|
|
2184
|
+
if (this.retryTimer === null) return;
|
|
2185
|
+
clearTimeout(this.retryTimer);
|
|
2186
|
+
this.retryTimer = null;
|
|
2187
|
+
}
|
|
2188
|
+
clearResponseTimer() {
|
|
2189
|
+
if (this.responseTimer === null) return;
|
|
2190
|
+
clearTimeout(this.responseTimer);
|
|
2191
|
+
this.responseTimer = null;
|
|
2192
|
+
}
|
|
2193
|
+
};
|
|
2194
|
+
async function abandonActiveSync(engine, reason) {
|
|
2195
|
+
engine.syncState.phase = "pending";
|
|
2196
|
+
engine.syncState.sourcePeerId = null;
|
|
2197
|
+
engine.syncState.startedAt = null;
|
|
2198
|
+
engine.syncState.error = reason;
|
|
2199
|
+
engine.expectedBatchIndex.clear();
|
|
2200
|
+
engine.syncTableDigests.clear();
|
|
2201
|
+
try {
|
|
2202
|
+
await setForeignKeysEnabled(engine.writerConn, true);
|
|
2203
|
+
} catch (err) {
|
|
2204
|
+
const wrappedErr = err instanceof Error ? err : new Error(String(err));
|
|
2205
|
+
engine.emitError({ error: wrappedErr, operation: "sync-abandon-pragma-restore", recoverable: false });
|
|
2206
|
+
}
|
|
2207
|
+
try {
|
|
2208
|
+
await engine.log.setSyncMeta("pending");
|
|
2209
|
+
} catch (err) {
|
|
2210
|
+
const wrappedErr = err instanceof Error ? err : new Error(String(err));
|
|
2211
|
+
engine.emitError({ error: wrappedErr, operation: "sync-meta-write", recoverable: false });
|
|
2212
|
+
}
|
|
2213
|
+
}
|
|
2214
|
+
function advanceStreamDigest(previous, batchChecksum, rowsInBatch) {
|
|
2215
|
+
const digest = createHash("sha256").update(previous?.digest ?? "").update(batchChecksum).digest("hex");
|
|
2216
|
+
return { rowCount: (previous?.rowCount ?? 0) + rowsInBatch, digest };
|
|
2217
|
+
}
|
|
2218
|
+
function matchesStreamDigest(manifest, local) {
|
|
2219
|
+
if (manifest.batchDigest === void 0 || local === void 0) return false;
|
|
2220
|
+
return local.rowCount === manifest.rowCount && local.digest === manifest.batchDigest;
|
|
2221
|
+
}
|
|
2222
|
+
|
|
2223
|
+
// src/replication/engine/sync-joiner.ts
|
|
2232
2224
|
var SyncJoiner = class {
|
|
2233
2225
|
constructor(engine) {
|
|
2234
2226
|
this.engine = engine;
|
|
2227
|
+
this.recovery = new SyncRecovery(engine, () => {
|
|
2228
|
+
this.retryInitiateSync();
|
|
2229
|
+
});
|
|
2235
2230
|
}
|
|
2236
2231
|
catchUpCheckTimer = null;
|
|
2232
|
+
recovery;
|
|
2237
2233
|
async initiateSync() {
|
|
2238
2234
|
const engine = this.engine;
|
|
2239
2235
|
if (engine.syncState.phase !== "pending") return;
|
|
@@ -2270,17 +2266,21 @@ var SyncJoiner = class {
|
|
|
2270
2266
|
const savedState = await engine.log.getSyncState();
|
|
2271
2267
|
const completedTables = savedState.phase === "pending" ? [] : savedState.completedTables;
|
|
2272
2268
|
const requestId = randomUUID();
|
|
2269
|
+
engine.syncTableDigests.clear();
|
|
2273
2270
|
await engine.log.setSyncMeta("syncing", void 0, sourcePeerId, requestId);
|
|
2274
2271
|
await engine.config.transport.requestSync(
|
|
2275
2272
|
sourcePeerId,
|
|
2276
2273
|
engine.decorateSyncRequest({
|
|
2277
2274
|
requestId,
|
|
2278
2275
|
joinerNodeId: engine.nodeId,
|
|
2279
|
-
completedTables
|
|
2276
|
+
completedTables,
|
|
2277
|
+
supportsStreamVerification: true
|
|
2280
2278
|
})
|
|
2281
2279
|
);
|
|
2280
|
+
this.recovery.markRequestSent(requestId, sourcePeerId);
|
|
2282
2281
|
} catch (err) {
|
|
2283
2282
|
const wrappedErr = err instanceof Error ? err : new Error(String(err));
|
|
2283
|
+
this.recovery.markRequestFailed();
|
|
2284
2284
|
if (engine.syncState.phase === "syncing" && engine.syncState.sourcePeerId === sourcePeerId) {
|
|
2285
2285
|
engine.syncState.phase = "pending";
|
|
2286
2286
|
engine.syncState.sourcePeerId = null;
|
|
@@ -2305,6 +2305,7 @@ var SyncJoiner = class {
|
|
|
2305
2305
|
const engine = this.engine;
|
|
2306
2306
|
if (engine.syncState.phase !== "syncing") return;
|
|
2307
2307
|
if (fromPeerId !== engine.syncState.sourcePeerId) return;
|
|
2308
|
+
this.recovery.noteSourceResponded(batch.requestId);
|
|
2308
2309
|
if (!engine.tracker) {
|
|
2309
2310
|
throw new SyncError("Initial sync requires a ChangeTracker in ReplicationConfig");
|
|
2310
2311
|
}
|
|
@@ -2332,7 +2333,7 @@ var SyncJoiner = class {
|
|
|
2332
2333
|
throw new SyncError(`Unsafe DDL in schema batch: ${ddl}`);
|
|
2333
2334
|
}
|
|
2334
2335
|
}
|
|
2335
|
-
await engine.writerConn
|
|
2336
|
+
await setForeignKeysEnabled(engine.writerConn, false);
|
|
2336
2337
|
for (const ddl of batch.schema) {
|
|
2337
2338
|
let saferDdl = ddl.replace(/^\s*CREATE\s+TABLE\b/i, "CREATE TABLE IF NOT EXISTS");
|
|
2338
2339
|
saferDdl = saferDdl.replace(/^\s*CREATE\s+INDEX\b/i, "CREATE INDEX IF NOT EXISTS");
|
|
@@ -2348,7 +2349,7 @@ var SyncJoiner = class {
|
|
|
2348
2349
|
});
|
|
2349
2350
|
return;
|
|
2350
2351
|
}
|
|
2351
|
-
if (!
|
|
2352
|
+
if (!IDENTIFIER_RE2.test(batch.table)) {
|
|
2352
2353
|
throw new SyncError(`Invalid table name in sync batch: ${batch.table}`);
|
|
2353
2354
|
}
|
|
2354
2355
|
const expectedChecksum = createHash("sha256").update(canonicaliseForChecksum(batch.rows)).digest("hex");
|
|
@@ -2356,24 +2357,42 @@ var SyncJoiner = class {
|
|
|
2356
2357
|
throw new SyncError(`Checksum mismatch for ${batch.table} batch ${batch.batchIndex}`);
|
|
2357
2358
|
}
|
|
2358
2359
|
if (batch.batchIndex === 0 && !engine.syncState.completedTables.includes(batch.table)) {
|
|
2360
|
+
engine.syncTableDigests.delete(batch.table);
|
|
2359
2361
|
await engine.tracker.unwatch(engine.writerConn, batch.table);
|
|
2360
2362
|
await engine.writerConn.exec(`DELETE FROM "${batch.table}"`);
|
|
2361
2363
|
}
|
|
2362
2364
|
if (batch.rows.length > 0) {
|
|
2363
|
-
const columns = Object.keys(batch.rows[0])
|
|
2364
|
-
if (columns.length
|
|
2365
|
-
|
|
2366
|
-
|
|
2367
|
-
|
|
2368
|
-
|
|
2369
|
-
|
|
2370
|
-
|
|
2371
|
-
const values = columns.map((c) => row[c]);
|
|
2372
|
-
await stmt.run(...values);
|
|
2373
|
-
}
|
|
2374
|
-
});
|
|
2365
|
+
const columns = Object.keys(batch.rows[0]);
|
|
2366
|
+
if (columns.length === 0) {
|
|
2367
|
+
throw new SyncError(`No columns in sync batch for ${batch.table}`);
|
|
2368
|
+
}
|
|
2369
|
+
for (const col of columns) {
|
|
2370
|
+
if (!IDENTIFIER_RE2.test(col)) {
|
|
2371
|
+
throw new SyncError(`Invalid column name '${col}' in sync batch for ${batch.table}`);
|
|
2372
|
+
}
|
|
2375
2373
|
}
|
|
2374
|
+
const columnSet = new Set(columns);
|
|
2375
|
+
for (const row of batch.rows) {
|
|
2376
|
+
const keys = Object.keys(row);
|
|
2377
|
+
if (keys.length !== columns.length || keys.some((k) => !columnSet.has(k))) {
|
|
2378
|
+
throw new SyncError(`Row columns do not match batch columns for ${batch.table}`);
|
|
2379
|
+
}
|
|
2380
|
+
}
|
|
2381
|
+
const placeholders = columns.map(() => "?").join(", ");
|
|
2382
|
+
const colNames = columns.map((c) => `"${c}"`).join(", ");
|
|
2383
|
+
const insertSql = `INSERT INTO "${batch.table}" (${colNames}) VALUES (${placeholders})`;
|
|
2384
|
+
await engine.writerConn.transaction(async (tx) => {
|
|
2385
|
+
const stmt = await tx.prepare(insertSql);
|
|
2386
|
+
for (const row of batch.rows) {
|
|
2387
|
+
const values = columns.map((c) => row[c]);
|
|
2388
|
+
await stmt.run(...values);
|
|
2389
|
+
}
|
|
2390
|
+
});
|
|
2376
2391
|
}
|
|
2392
|
+
engine.syncTableDigests.set(
|
|
2393
|
+
batch.table,
|
|
2394
|
+
advanceStreamDigest(engine.syncTableDigests.get(batch.table), batch.checksum, batch.rows.length)
|
|
2395
|
+
);
|
|
2377
2396
|
if (batch.isLastBatchForTable) {
|
|
2378
2397
|
await engine.log.setSyncTableStatus(batch.table, "completed");
|
|
2379
2398
|
engine.syncState.completedTables.push(batch.table);
|
|
@@ -2411,31 +2430,31 @@ var SyncJoiner = class {
|
|
|
2411
2430
|
}
|
|
2412
2431
|
try {
|
|
2413
2432
|
for (const manifest of complete.manifests) {
|
|
2414
|
-
if (
|
|
2433
|
+
if (IDENTIFIER_RE2.test(manifest.table)) {
|
|
2415
2434
|
await engine.tracker.watch(engine.writerConn, manifest.table);
|
|
2416
2435
|
}
|
|
2417
2436
|
}
|
|
2418
|
-
await engine.writerConn
|
|
2437
|
+
await setForeignKeysEnabled(engine.writerConn, true);
|
|
2419
2438
|
for (const manifest of complete.manifests) {
|
|
2420
|
-
const valid = await engine.log.verifyManifest(manifest);
|
|
2439
|
+
const valid = manifest.batchDigest !== void 0 ? matchesStreamDigest(manifest, engine.syncTableDigests.get(manifest.table)) : await engine.log.verifyManifest(manifest);
|
|
2421
2440
|
if (!valid) {
|
|
2422
2441
|
await engine.log.wipeTables(
|
|
2423
2442
|
engine.writerConn,
|
|
2424
|
-
complete.manifests.map((m) => m.table).filter((t) =>
|
|
2443
|
+
complete.manifests.map((m) => m.table).filter((t) => IDENTIFIER_RE2.test(t)),
|
|
2425
2444
|
engine.tracker
|
|
2426
2445
|
);
|
|
2427
2446
|
engine.syncState.phase = "pending";
|
|
2428
2447
|
engine.syncState.completedTables = [];
|
|
2429
2448
|
engine.expectedBatchIndex.clear();
|
|
2449
|
+
engine.syncTableDigests.clear();
|
|
2430
2450
|
await engine.log.setSyncMeta("pending");
|
|
2431
2451
|
await this.initiateSync();
|
|
2432
2452
|
return;
|
|
2433
2453
|
}
|
|
2434
2454
|
}
|
|
2455
|
+
engine.syncTableDigests.clear();
|
|
2435
2456
|
await engine.log.setLastAppliedSeq(fromPeerId, complete.snapshotSeq);
|
|
2436
|
-
const recordStmt = await engine.writerConn
|
|
2437
|
-
`INSERT OR IGNORE INTO ${APPLIED_CHANGES_TABLE} (source_node_id, source_seq, applied_at) VALUES (?, ?, ?)`
|
|
2438
|
-
);
|
|
2457
|
+
const recordStmt = await prepareInsertAppliedChange(engine.writerConn);
|
|
2439
2458
|
await recordStmt.run(fromPeerId, complete.snapshotSeq.toString(), Date.now() / 1e3);
|
|
2440
2459
|
const previousApplied = engine.appliedSeqByPeer.get(fromPeerId) ?? 0n;
|
|
2441
2460
|
if (complete.snapshotSeq > previousApplied) {
|
|
@@ -2448,7 +2467,7 @@ var SyncJoiner = class {
|
|
|
2448
2467
|
this.startCatchUpCheck();
|
|
2449
2468
|
} catch {
|
|
2450
2469
|
try {
|
|
2451
|
-
await engine.writerConn
|
|
2470
|
+
await setForeignKeysEnabled(engine.writerConn, true);
|
|
2452
2471
|
} catch (pragmaErr) {
|
|
2453
2472
|
const wrappedErr = pragmaErr instanceof Error ? pragmaErr : new Error(String(pragmaErr));
|
|
2454
2473
|
engine.emitError({ error: wrappedErr, operation: "sync-complete-pragma-restore", recoverable: false });
|
|
@@ -2501,6 +2520,23 @@ var SyncJoiner = class {
|
|
|
2501
2520
|
this.catchUpCheckTimer = null;
|
|
2502
2521
|
}
|
|
2503
2522
|
}
|
|
2523
|
+
stopTimers() {
|
|
2524
|
+
this.stopCatchUpCheck();
|
|
2525
|
+
this.recovery.stop();
|
|
2526
|
+
}
|
|
2527
|
+
isSourceRejection(ack, fromPeerId) {
|
|
2528
|
+
return this.recovery.isSourceRejection(ack, fromPeerId);
|
|
2529
|
+
}
|
|
2530
|
+
handleSourceRejection(ack, fromPeerId) {
|
|
2531
|
+
this.recovery.handleSourceRejection(ack, fromPeerId);
|
|
2532
|
+
}
|
|
2533
|
+
retryInitiateSync() {
|
|
2534
|
+
if (!this.engine.running) return;
|
|
2535
|
+
this.initiateSync().catch((err) => {
|
|
2536
|
+
const wrappedErr = err instanceof Error ? err : new Error(String(err));
|
|
2537
|
+
this.engine.emitError({ error: wrappedErr, operation: "sync-retry", recoverable: true });
|
|
2538
|
+
});
|
|
2539
|
+
}
|
|
2504
2540
|
async sendSyncAck(peerId, ack) {
|
|
2505
2541
|
await delayAckIfConfigured(this.engine);
|
|
2506
2542
|
await this.engine.config.transport.sendSyncAck(peerId, this.engine.decorateSyncAck(ack));
|
|
@@ -2510,6 +2546,7 @@ var SyncJoiner = class {
|
|
|
2510
2546
|
await engine.log.setSyncMeta("ready");
|
|
2511
2547
|
await engine.markCoordinatorSyncReady();
|
|
2512
2548
|
engine.syncState.phase = "ready";
|
|
2549
|
+
this.recovery.markSyncReady();
|
|
2513
2550
|
this.stopCatchUpCheck();
|
|
2514
2551
|
}
|
|
2515
2552
|
};
|
|
@@ -2519,6 +2556,19 @@ var SyncServer = class {
|
|
|
2519
2556
|
}
|
|
2520
2557
|
activeSyncs = /* @__PURE__ */ new Map();
|
|
2521
2558
|
syncAckWaiters = /* @__PURE__ */ new Map();
|
|
2559
|
+
async rejectSyncRequest(fromPeerId, request, error) {
|
|
2560
|
+
await this.engine.config.transport.sendSyncAck(
|
|
2561
|
+
fromPeerId,
|
|
2562
|
+
this.engine.decorateSyncAck({
|
|
2563
|
+
requestId: request.requestId,
|
|
2564
|
+
joinerNodeId: request.joinerNodeId,
|
|
2565
|
+
table: "__schema__",
|
|
2566
|
+
batchIndex: -1,
|
|
2567
|
+
success: false,
|
|
2568
|
+
error
|
|
2569
|
+
})
|
|
2570
|
+
);
|
|
2571
|
+
}
|
|
2522
2572
|
async handleSyncRequest(request, fromPeerId) {
|
|
2523
2573
|
const engine = this.engine;
|
|
2524
2574
|
const knownPeers = engine.config.transport.peers();
|
|
@@ -2528,88 +2578,28 @@ var SyncServer = class {
|
|
|
2528
2578
|
await engine.verifyPrimaryAuthority();
|
|
2529
2579
|
} catch (err) {
|
|
2530
2580
|
const wrappedErr = err instanceof Error ? err : new Error(String(err));
|
|
2531
|
-
await
|
|
2532
|
-
fromPeerId,
|
|
2533
|
-
engine.decorateSyncAck({
|
|
2534
|
-
requestId: request.requestId,
|
|
2535
|
-
joinerNodeId: request.joinerNodeId,
|
|
2536
|
-
table: "__schema__",
|
|
2537
|
-
batchIndex: -1,
|
|
2538
|
-
success: false,
|
|
2539
|
-
error: wrappedErr.message
|
|
2540
|
-
})
|
|
2541
|
-
);
|
|
2581
|
+
await this.rejectSyncRequest(fromPeerId, request, wrappedErr.message);
|
|
2542
2582
|
return;
|
|
2543
2583
|
}
|
|
2544
2584
|
} else if (!engine.config.topology.canWrite()) {
|
|
2545
|
-
await
|
|
2546
|
-
fromPeerId,
|
|
2547
|
-
engine.decorateSyncAck({
|
|
2548
|
-
requestId: request.requestId,
|
|
2549
|
-
joinerNodeId: request.joinerNodeId,
|
|
2550
|
-
table: "__schema__",
|
|
2551
|
-
batchIndex: -1,
|
|
2552
|
-
success: false,
|
|
2553
|
-
error: "This node cannot serve syncs"
|
|
2554
|
-
})
|
|
2555
|
-
);
|
|
2585
|
+
await this.rejectSyncRequest(fromPeerId, request, "This node cannot serve syncs");
|
|
2556
2586
|
return;
|
|
2557
2587
|
}
|
|
2558
2588
|
if (this.activeSyncs.has(request.requestId)) {
|
|
2559
|
-
await
|
|
2560
|
-
fromPeerId,
|
|
2561
|
-
engine.decorateSyncAck({
|
|
2562
|
-
requestId: request.requestId,
|
|
2563
|
-
joinerNodeId: request.joinerNodeId,
|
|
2564
|
-
table: "__schema__",
|
|
2565
|
-
batchIndex: -1,
|
|
2566
|
-
success: false,
|
|
2567
|
-
error: "Duplicate requestId"
|
|
2568
|
-
})
|
|
2569
|
-
);
|
|
2589
|
+
await this.rejectSyncRequest(fromPeerId, request, "Duplicate requestId");
|
|
2570
2590
|
return;
|
|
2571
2591
|
}
|
|
2572
2592
|
if (this.activeSyncs.size >= engine.maxConcurrentSyncs) {
|
|
2573
|
-
await
|
|
2574
|
-
fromPeerId,
|
|
2575
|
-
engine.decorateSyncAck({
|
|
2576
|
-
requestId: request.requestId,
|
|
2577
|
-
joinerNodeId: request.joinerNodeId,
|
|
2578
|
-
table: "__schema__",
|
|
2579
|
-
batchIndex: -1,
|
|
2580
|
-
success: false,
|
|
2581
|
-
error: "Sync capacity reached, retry later"
|
|
2582
|
-
})
|
|
2583
|
-
);
|
|
2593
|
+
await this.rejectSyncRequest(fromPeerId, request, "Sync capacity reached, retry later");
|
|
2584
2594
|
return;
|
|
2585
2595
|
}
|
|
2586
2596
|
if (!engine.snapshotConnectionFactory) {
|
|
2587
|
-
await
|
|
2588
|
-
fromPeerId,
|
|
2589
|
-
engine.decorateSyncAck({
|
|
2590
|
-
requestId: request.requestId,
|
|
2591
|
-
joinerNodeId: request.joinerNodeId,
|
|
2592
|
-
table: "__schema__",
|
|
2593
|
-
batchIndex: -1,
|
|
2594
|
-
success: false,
|
|
2595
|
-
error: "No snapshot connection factory configured"
|
|
2596
|
-
})
|
|
2597
|
-
);
|
|
2597
|
+
await this.rejectSyncRequest(fromPeerId, request, "No snapshot connection factory configured");
|
|
2598
2598
|
return;
|
|
2599
2599
|
}
|
|
2600
2600
|
for (const table of request.completedTables) {
|
|
2601
|
-
if (!
|
|
2602
|
-
await
|
|
2603
|
-
fromPeerId,
|
|
2604
|
-
engine.decorateSyncAck({
|
|
2605
|
-
requestId: request.requestId,
|
|
2606
|
-
joinerNodeId: request.joinerNodeId,
|
|
2607
|
-
table: "__schema__",
|
|
2608
|
-
batchIndex: -1,
|
|
2609
|
-
success: false,
|
|
2610
|
-
error: `Invalid table name in completedTables: ${table}`
|
|
2611
|
-
})
|
|
2612
|
-
);
|
|
2601
|
+
if (!IDENTIFIER_RE2.test(table)) {
|
|
2602
|
+
await this.rejectSyncRequest(fromPeerId, request, `Invalid table name in completedTables: ${table}`);
|
|
2613
2603
|
return;
|
|
2614
2604
|
}
|
|
2615
2605
|
}
|
|
@@ -2617,16 +2607,10 @@ var SyncServer = class {
|
|
|
2617
2607
|
try {
|
|
2618
2608
|
readConn = await engine.snapshotConnectionFactory();
|
|
2619
2609
|
} catch (err) {
|
|
2620
|
-
await
|
|
2610
|
+
await this.rejectSyncRequest(
|
|
2621
2611
|
fromPeerId,
|
|
2622
|
-
|
|
2623
|
-
|
|
2624
|
-
joinerNodeId: request.joinerNodeId,
|
|
2625
|
-
table: "__schema__",
|
|
2626
|
-
batchIndex: -1,
|
|
2627
|
-
success: false,
|
|
2628
|
-
error: `Failed to open snapshot connection: ${err instanceof Error ? err.message : String(err)}`
|
|
2629
|
-
})
|
|
2612
|
+
request,
|
|
2613
|
+
`Failed to open snapshot connection: ${err instanceof Error ? err.message : String(err)}`
|
|
2630
2614
|
);
|
|
2631
2615
|
return;
|
|
2632
2616
|
}
|
|
@@ -2653,7 +2637,9 @@ var SyncServer = class {
|
|
|
2653
2637
|
completedTables: new Set(request.completedTables),
|
|
2654
2638
|
startedAt: Date.now(),
|
|
2655
2639
|
timeoutTimer,
|
|
2656
|
-
aborted: false
|
|
2640
|
+
aborted: false,
|
|
2641
|
+
streamVerification: request.supportsStreamVerification === true,
|
|
2642
|
+
tableDigests: /* @__PURE__ */ new Map()
|
|
2657
2643
|
};
|
|
2658
2644
|
this.activeSyncs.set(request.requestId, session);
|
|
2659
2645
|
this.serveSyncSession(session).catch((err) => {
|
|
@@ -2764,6 +2750,15 @@ var SyncServer = class {
|
|
|
2764
2750
|
}
|
|
2765
2751
|
return ackPromise;
|
|
2766
2752
|
}
|
|
2753
|
+
refreshSessionDeadline(session) {
|
|
2754
|
+
clearTimeout(session.timeoutTimer);
|
|
2755
|
+
const timer = setTimeout(
|
|
2756
|
+
() => this.abortSyncSession(session.requestId),
|
|
2757
|
+
this.engine.maxSyncDurationMs
|
|
2758
|
+
);
|
|
2759
|
+
timer.unref?.();
|
|
2760
|
+
session.timeoutTimer = timer;
|
|
2761
|
+
}
|
|
2767
2762
|
async serveSyncSession(session) {
|
|
2768
2763
|
const engine = this.engine;
|
|
2769
2764
|
const schemaDdl = await engine.log.dumpSchema(session.readConn);
|
|
@@ -2782,6 +2777,7 @@ var SyncServer = class {
|
|
|
2782
2777
|
this.abortSyncSession(session.requestId);
|
|
2783
2778
|
return;
|
|
2784
2779
|
}
|
|
2780
|
+
this.refreshSessionDeadline(session);
|
|
2785
2781
|
for (const table of session.tables) {
|
|
2786
2782
|
if (session.aborted) return;
|
|
2787
2783
|
let batchIndex = 0;
|
|
@@ -2803,28 +2799,41 @@ var SyncServer = class {
|
|
|
2803
2799
|
this.abortSyncSession(session.requestId);
|
|
2804
2800
|
return;
|
|
2805
2801
|
}
|
|
2802
|
+
this.refreshSessionDeadline(session);
|
|
2803
|
+
session.tableDigests.set(table, advanceStreamDigest(session.tableDigests.get(table), checksum, rows.length));
|
|
2806
2804
|
batchIndex += 1;
|
|
2807
2805
|
}
|
|
2808
2806
|
if (batchIndex === 0) {
|
|
2807
|
+
const emptyChecksum = createHash("sha256").update(canonicaliseForChecksum([])).digest("hex");
|
|
2809
2808
|
const emptyAck = await this.sendSyncBatchAndWaitForAck(session.joinerNodeId, {
|
|
2810
2809
|
requestId: session.requestId,
|
|
2811
2810
|
table,
|
|
2812
2811
|
batchIndex: 0,
|
|
2813
2812
|
rows: [],
|
|
2814
|
-
checksum:
|
|
2813
|
+
checksum: emptyChecksum,
|
|
2815
2814
|
isLastBatchForTable: true
|
|
2816
2815
|
});
|
|
2817
2816
|
if (!emptyAck.success) {
|
|
2818
2817
|
this.abortSyncSession(session.requestId);
|
|
2819
2818
|
return;
|
|
2820
2819
|
}
|
|
2820
|
+
this.refreshSessionDeadline(session);
|
|
2821
|
+
session.tableDigests.set(table, advanceStreamDigest(void 0, emptyChecksum, 0));
|
|
2821
2822
|
}
|
|
2822
2823
|
session.completedTables.add(table);
|
|
2823
2824
|
}
|
|
2824
2825
|
if (session.aborted) return;
|
|
2825
2826
|
const manifests = [];
|
|
2826
2827
|
for (const table of session.tables) {
|
|
2827
|
-
|
|
2828
|
+
if (session.streamVerification) {
|
|
2829
|
+
const streamDigest = session.tableDigests.get(table);
|
|
2830
|
+
if (!streamDigest) {
|
|
2831
|
+
throw new SyncError(`Missing stream digest for table ${table}`, session.requestId);
|
|
2832
|
+
}
|
|
2833
|
+
manifests.push({ table, rowCount: streamDigest.rowCount, batchDigest: streamDigest.digest });
|
|
2834
|
+
} else {
|
|
2835
|
+
manifests.push(await engine.log.generateManifest(session.readConn, table));
|
|
2836
|
+
}
|
|
2828
2837
|
}
|
|
2829
2838
|
await engine.config.transport.sendSyncComplete(
|
|
2830
2839
|
session.joinerNodeId,
|
|
@@ -2863,9 +2872,6 @@ var SyncServer = class {
|
|
|
2863
2872
|
};
|
|
2864
2873
|
|
|
2865
2874
|
// src/replication/engine/engine.ts
|
|
2866
|
-
var DEFAULT_COORDINATOR_SESSION_TTL_MS = 1e4;
|
|
2867
|
-
var DEFAULT_CONTROLLER_LEASE_TTL_MS = 1e4;
|
|
2868
|
-
var DEFAULT_CONTROLLER_TICK_INTERVAL_MS = 1e3;
|
|
2869
2875
|
var ReplicationEngine = class extends EventEmitter {
|
|
2870
2876
|
database;
|
|
2871
2877
|
writerConn;
|
|
@@ -2901,11 +2907,16 @@ var ReplicationEngine = class extends EventEmitter {
|
|
|
2901
2907
|
coordinatorLeaseTimer = null;
|
|
2902
2908
|
controllerTimer = null;
|
|
2903
2909
|
coordinatorRejoinSyncStarting = false;
|
|
2910
|
+
coordinatorSessionRestoring = false;
|
|
2911
|
+
coordinatorLastContactMs = 0;
|
|
2912
|
+
inSyncReconcileTimer = null;
|
|
2913
|
+
inSyncReconciling = false;
|
|
2904
2914
|
lastSentSeq = 0n;
|
|
2905
2915
|
lastLocalSeq = 0n;
|
|
2906
2916
|
highestSourceSeqSeen = 0n;
|
|
2907
2917
|
appliedSeqByPeer = /* @__PURE__ */ new Map();
|
|
2908
2918
|
expectedBatchIndex = /* @__PURE__ */ new Map();
|
|
2919
|
+
syncTableDigests = /* @__PURE__ */ new Map();
|
|
2909
2920
|
syncState = {
|
|
2910
2921
|
phase: "ready",
|
|
2911
2922
|
sourcePeerId: null,
|
|
@@ -2959,12 +2970,12 @@ var ReplicationEngine = class extends EventEmitter {
|
|
|
2959
2970
|
async stop() {
|
|
2960
2971
|
if (!this.running) return;
|
|
2961
2972
|
this.running = false;
|
|
2962
|
-
|
|
2963
|
-
this.syncJoiner.
|
|
2973
|
+
stopCoordinatorTimers(this);
|
|
2974
|
+
this.syncJoiner.stopTimers();
|
|
2964
2975
|
this.syncServer.abortAll();
|
|
2965
2976
|
if (this.syncState.phase === "syncing") {
|
|
2966
2977
|
try {
|
|
2967
|
-
await this.writerConn
|
|
2978
|
+
await setForeignKeysEnabled(this.writerConn, true);
|
|
2968
2979
|
} catch (err) {
|
|
2969
2980
|
const wrappedErr = err instanceof Error ? err : new Error(String(err));
|
|
2970
2981
|
this.emitError({ error: wrappedErr, operation: "engine-stop-pragma-restore", recoverable: false });
|
|
@@ -2974,18 +2985,19 @@ var ReplicationEngine = class extends EventEmitter {
|
|
|
2974
2985
|
if (this.tracker) {
|
|
2975
2986
|
this.tracker.clearPruneBoundary();
|
|
2976
2987
|
}
|
|
2977
|
-
await
|
|
2988
|
+
await stopCoordinatorMode(this);
|
|
2978
2989
|
await this.config.transport.disconnect();
|
|
2979
2990
|
}
|
|
2980
2991
|
status() {
|
|
2981
2992
|
return {
|
|
2982
2993
|
nodeId: this.nodeId,
|
|
2983
|
-
role: this
|
|
2994
|
+
role: effectiveTopologyRole(this),
|
|
2984
2995
|
peers: this.peerTracker.allPeerStates(),
|
|
2985
2996
|
localSeq: this.lastSentSeq,
|
|
2986
2997
|
replicating: this.running,
|
|
2998
|
+
health: computeNodeHealth(this),
|
|
2987
2999
|
syncState: { ...this.syncState },
|
|
2988
|
-
coordinator:
|
|
3000
|
+
coordinator: getCoordinatorRuntimeStatus(this)
|
|
2989
3001
|
};
|
|
2990
3002
|
}
|
|
2991
3003
|
getCurrentSeq() {
|
|
@@ -2994,77 +3006,20 @@ var ReplicationEngine = class extends EventEmitter {
|
|
|
2994
3006
|
getAppliedSeq(peerId) {
|
|
2995
3007
|
return this.appliedSeqByPeer.get(peerId) ?? 0n;
|
|
2996
3008
|
}
|
|
2997
|
-
|
|
2998
|
-
|
|
2999
|
-
throw new SyncError(`Node is in '${this.syncState.phase}' phase and cannot serve reads`);
|
|
3000
|
-
}
|
|
3001
|
-
await this.assertReadConcern(options?.readConcern?.level);
|
|
3002
|
-
return this.database.query(sql, params, options);
|
|
3009
|
+
query(sql, params, options) {
|
|
3010
|
+
return query(this, sql, params, options);
|
|
3003
3011
|
}
|
|
3004
|
-
|
|
3005
|
-
|
|
3006
|
-
throw new SyncError(`Node is in '${this.syncState.phase}' phase and cannot accept operations`);
|
|
3007
|
-
}
|
|
3008
|
-
if (!await this.canAcceptLocalWrite()) {
|
|
3009
|
-
if (this.config.writeForwarding) {
|
|
3010
|
-
const result = await this.forwardStatements([{ sql, params }], options);
|
|
3011
|
-
const first = result.results[0];
|
|
3012
|
-
if (!first) {
|
|
3013
|
-
return { changes: 0, lastInsertRowId: 0 };
|
|
3014
|
-
}
|
|
3015
|
-
return {
|
|
3016
|
-
changes: first.changes,
|
|
3017
|
-
lastInsertRowId: typeof first.lastInsertRowId === "string" ? BigInt(first.lastInsertRowId) : first.lastInsertRowId
|
|
3018
|
-
};
|
|
3019
|
-
}
|
|
3020
|
-
this.throwNotCurrentPrimary();
|
|
3021
|
-
}
|
|
3022
|
-
return this.localExecutor.executeLocally(sql, params, options);
|
|
3012
|
+
execute(sql, params, options) {
|
|
3013
|
+
return execute(this, sql, params, options);
|
|
3023
3014
|
}
|
|
3024
|
-
|
|
3025
|
-
|
|
3026
|
-
throw new SyncError(`Node is in '${this.syncState.phase}' phase and cannot accept operations`);
|
|
3027
|
-
}
|
|
3028
|
-
if (!await this.canAcceptLocalWrite()) {
|
|
3029
|
-
if (this.config.writeForwarding) {
|
|
3030
|
-
const statements = paramsBatch.map((p) => ({ sql, params: p }));
|
|
3031
|
-
const result = await this.forwardStatements(statements, options);
|
|
3032
|
-
return result.results.map((r) => ({
|
|
3033
|
-
changes: r.changes,
|
|
3034
|
-
lastInsertRowId: typeof r.lastInsertRowId === "string" ? BigInt(r.lastInsertRowId) : Number(r.lastInsertRowId)
|
|
3035
|
-
}));
|
|
3036
|
-
}
|
|
3037
|
-
this.throwNotCurrentPrimary();
|
|
3038
|
-
}
|
|
3039
|
-
const results = [];
|
|
3040
|
-
for (const params of paramsBatch) {
|
|
3041
|
-
const r = await this.localExecutor.executeLocally(sql, params, options);
|
|
3042
|
-
results.push(r);
|
|
3043
|
-
}
|
|
3044
|
-
return results;
|
|
3015
|
+
executeBatch(sql, paramsBatch, options) {
|
|
3016
|
+
return executeBatch(this, sql, paramsBatch, options);
|
|
3045
3017
|
}
|
|
3046
|
-
|
|
3047
|
-
|
|
3048
|
-
throw new SyncError(`Node is in '${this.syncState.phase}' phase and cannot accept operations`);
|
|
3049
|
-
}
|
|
3050
|
-
if (!await this.canAcceptLocalWrite()) {
|
|
3051
|
-
throw new TopologyError("This node cannot accept writes in transaction mode");
|
|
3052
|
-
}
|
|
3053
|
-
return this.localExecutor.executeTransactionLocally(fn, options);
|
|
3018
|
+
transaction(fn, options) {
|
|
3019
|
+
return transaction(this, fn, options);
|
|
3054
3020
|
}
|
|
3055
|
-
|
|
3056
|
-
|
|
3057
|
-
return this.localExecutor.executeForwardedLocally(statements);
|
|
3058
|
-
}
|
|
3059
|
-
const primaryPeerId = this.getForwardingPrimaryPeerId();
|
|
3060
|
-
if (primaryPeerId === null) {
|
|
3061
|
-
throw new TopologyError("No primary node available for write forwarding");
|
|
3062
|
-
}
|
|
3063
|
-
return this.config.transport.forward(primaryPeerId, {
|
|
3064
|
-
statements,
|
|
3065
|
-
requestId: randomUUID(),
|
|
3066
|
-
...this.getCoordinatorMessageFields()
|
|
3067
|
-
});
|
|
3021
|
+
forwardStatements(statements, options) {
|
|
3022
|
+
return forwardStatements(this, statements);
|
|
3068
3023
|
}
|
|
3069
3024
|
startSenderLoop() {
|
|
3070
3025
|
this.senderLoop.start();
|
|
@@ -3098,240 +3053,61 @@ var ReplicationEngine = class extends EventEmitter {
|
|
|
3098
3053
|
}
|
|
3099
3054
|
}
|
|
3100
3055
|
}
|
|
3101
|
-
|
|
3102
|
-
|
|
3103
|
-
if (this.isCoordinatorMode()) {
|
|
3104
|
-
const state = this.coordinatorState ?? await this.refreshCoordinatorState();
|
|
3105
|
-
if (!state) {
|
|
3106
|
-
throw new CoordinatorError("Cannot satisfy write concern without replication-group state");
|
|
3107
|
-
}
|
|
3108
|
-
if (wc.level === "majority") {
|
|
3109
|
-
await this.peerTracker.waitForConfiguredMajority(seq, this.nodeId, state.votingDataBearingNodeIds, timeout);
|
|
3110
|
-
await this.updateCoordinatorProgressAfterWrite(seq, state);
|
|
3111
|
-
} else if (wc.level === "all") {
|
|
3112
|
-
const eligibleVoters = state.votingDataBearingNodeIds.filter((nodeId) => !state.drainingNodeIds.includes(nodeId));
|
|
3113
|
-
await this.peerTracker.waitForConfiguredAll(seq, this.nodeId, eligibleVoters, timeout);
|
|
3114
|
-
await this.updateCoordinatorProgressAfterWrite(seq, state);
|
|
3115
|
-
}
|
|
3116
|
-
return;
|
|
3117
|
-
}
|
|
3118
|
-
if (wc.level === "majority") {
|
|
3119
|
-
await this.peerTracker.waitForMajority(seq, timeout);
|
|
3120
|
-
} else if (wc.level === "all") {
|
|
3121
|
-
await this.peerTracker.waitForAll(seq, timeout);
|
|
3122
|
-
}
|
|
3056
|
+
waitForWriteConcern(seq, wc) {
|
|
3057
|
+
return waitForWriteConcern(this, seq, wc);
|
|
3123
3058
|
}
|
|
3124
3059
|
async loadAppliedSeqs() {
|
|
3125
|
-
const
|
|
3126
|
-
|
|
3127
|
-
);
|
|
3128
|
-
const rows = await stmt.all();
|
|
3129
|
-
for (const row of rows) {
|
|
3130
|
-
if (row.max_seq === null) continue;
|
|
3131
|
-
this.appliedSeqByPeer.set(row.source_node_id, BigInt(row.max_seq));
|
|
3060
|
+
for (const [nodeId, seq] of await selectMaxAppliedSourceSeqByNode(this.writerConn)) {
|
|
3061
|
+
this.appliedSeqByPeer.set(nodeId, seq);
|
|
3132
3062
|
}
|
|
3133
3063
|
}
|
|
3134
3064
|
isCoordinatorMode() {
|
|
3135
3065
|
return this.config.coordinator !== void 0;
|
|
3136
3066
|
}
|
|
3137
|
-
|
|
3138
|
-
|
|
3139
|
-
if (!config) return;
|
|
3140
|
-
const coordinator = config.coordinator;
|
|
3141
|
-
let state = await coordinator.getReplicationGroupState(config.clusterId, config.groupId);
|
|
3142
|
-
if (!state && config.votingDataBearingNodeIds) {
|
|
3143
|
-
state = await coordinator.setReplicationGroupState({
|
|
3144
|
-
clusterId: config.clusterId,
|
|
3145
|
-
groupId: config.groupId,
|
|
3146
|
-
votingDataBearingNodeIds: config.votingDataBearingNodeIds,
|
|
3147
|
-
currentPrimary: this.config.topology.role === "primary" ? this.localCoordinatorPrimary() : null,
|
|
3148
|
-
primaryTerm: 1n,
|
|
3149
|
-
inSyncNodeIds: [this.nodeId],
|
|
3150
|
-
compatibility: config.compatibility
|
|
3151
|
-
});
|
|
3152
|
-
}
|
|
3153
|
-
if (!state) {
|
|
3154
|
-
throw new CoordinatorError(`Replication group '${config.groupId}' is not registered`);
|
|
3155
|
-
}
|
|
3156
|
-
this.coordinatorState = state;
|
|
3157
|
-
this.coordinatorAuthority = this.hasCurrentPrimaryAuthorityFor(state);
|
|
3158
|
-
const session = await coordinator.registerNodeSession({
|
|
3159
|
-
clusterId: config.clusterId,
|
|
3160
|
-
nodeId: this.nodeId,
|
|
3161
|
-
ttlMs: config.sessionTtlMs ?? DEFAULT_COORDINATOR_SESSION_TTL_MS,
|
|
3162
|
-
endpoint: config.endpoint,
|
|
3163
|
-
groupIds: [config.groupId],
|
|
3164
|
-
dataBearing: true,
|
|
3165
|
-
voting: state.votingDataBearingNodeIds.includes(this.nodeId),
|
|
3166
|
-
compatibility: config.compatibility
|
|
3167
|
-
});
|
|
3168
|
-
this.nodeSessionLeaseId = session.lease.id;
|
|
3169
|
-
this.coordinatorWatchDisposer = await coordinator.watchReplicationGroup(config.clusterId, config.groupId, (next) => {
|
|
3170
|
-
this.handleCoordinatorStateUpdate(next);
|
|
3171
|
-
});
|
|
3172
|
-
this.startCoordinatorLeaseRenewal();
|
|
3173
|
-
this.startControllerLoop();
|
|
3067
|
+
startCoordinatorMode() {
|
|
3068
|
+
return startCoordinatorMode(this);
|
|
3174
3069
|
}
|
|
3175
|
-
|
|
3176
|
-
|
|
3177
|
-
if (!state || !this.requiresCoordinatorRejoin(state)) return;
|
|
3178
|
-
await this.evaluateFormerPrimaryHistory(state);
|
|
3070
|
+
prepareCoordinatorRejoinIfNeeded() {
|
|
3071
|
+
return prepareCoordinatorRejoinIfNeeded(this);
|
|
3179
3072
|
}
|
|
3180
3073
|
hasCoordinatorWriteAuthority() {
|
|
3181
3074
|
return this.coordinatorAuthority;
|
|
3182
3075
|
}
|
|
3183
3076
|
requiresCoordinatorRejoinSync(state = this.coordinatorState) {
|
|
3184
|
-
|
|
3185
|
-
if (state.faultedNodeIds.includes(this.nodeId)) return false;
|
|
3186
|
-
return state.repairingNodeIds.includes(this.nodeId);
|
|
3187
|
-
}
|
|
3188
|
-
async markCoordinatorSyncReady() {
|
|
3189
|
-
const config = this.config.coordinator;
|
|
3190
|
-
const state = this.coordinatorState ?? await this.refreshCoordinatorState();
|
|
3191
|
-
if (!config || !state || state.currentPrimary?.nodeId === this.nodeId) return;
|
|
3192
|
-
if (!state.votingDataBearingNodeIds.includes(this.nodeId)) return;
|
|
3193
|
-
if (state.faultedNodeIds.includes(this.nodeId)) return;
|
|
3194
|
-
if (state.drainingNodeIds.includes(this.nodeId)) {
|
|
3195
|
-
await this.removeLocalNodeFromCoordinatorInSyncSet(state);
|
|
3196
|
-
return;
|
|
3197
|
-
}
|
|
3198
|
-
const needsAdmission = !state.inSyncNodeIds.includes(this.nodeId) || state.repairingNodeIds.includes(this.nodeId);
|
|
3199
|
-
if (!needsAdmission) {
|
|
3200
|
-
this.coordinatorState = state;
|
|
3201
|
-
this.coordinatorAuthority = this.hasCurrentPrimaryAuthorityFor(state);
|
|
3202
|
-
return;
|
|
3203
|
-
}
|
|
3204
|
-
const sourceNodeId = state.currentPrimary?.nodeId;
|
|
3205
|
-
if (!sourceNodeId || this.syncState.sourcePeerId !== sourceNodeId) {
|
|
3206
|
-
this.coordinatorState = state;
|
|
3207
|
-
this.coordinatorAuthority = this.hasCurrentPrimaryAuthorityFor(state);
|
|
3208
|
-
return;
|
|
3209
|
-
}
|
|
3210
|
-
const appliedSeq = await this.log.getLastAppliedSeq(sourceNodeId);
|
|
3211
|
-
if (appliedSeq < state.durabilityPointSeq) {
|
|
3212
|
-
this.coordinatorState = state;
|
|
3213
|
-
this.coordinatorAuthority = this.hasCurrentPrimaryAuthorityFor(state);
|
|
3214
|
-
return;
|
|
3215
|
-
}
|
|
3216
|
-
const admitted = await config.coordinator.admitNodeToInSyncSet({
|
|
3217
|
-
clusterId: config.clusterId,
|
|
3218
|
-
groupId: config.groupId,
|
|
3219
|
-
nodeId: this.nodeId,
|
|
3220
|
-
sourceNodeId,
|
|
3221
|
-
appliedSeq
|
|
3222
|
-
});
|
|
3223
|
-
this.coordinatorState = admitted ?? state;
|
|
3224
|
-
this.coordinatorAuthority = this.hasCurrentPrimaryAuthorityFor(this.coordinatorState);
|
|
3225
|
-
}
|
|
3226
|
-
async handleCoordinatorAckProgress(nodeId, ackedSeq) {
|
|
3227
|
-
const config = this.config.coordinator;
|
|
3228
|
-
const state = this.coordinatorState;
|
|
3229
|
-
if (!config || !state) return;
|
|
3230
|
-
if (nodeId === this.nodeId) return;
|
|
3231
|
-
if (!this.hasCurrentPrimaryAuthorityFor(state)) return;
|
|
3232
|
-
if (state.inSyncNodeIds.includes(nodeId)) return;
|
|
3233
|
-
if (!state.votingDataBearingNodeIds.includes(nodeId)) return;
|
|
3234
|
-
if (ackedSeq < state.durabilityPointSeq) return;
|
|
3235
|
-
const admitted = await config.coordinator.admitNodeToInSyncSet({
|
|
3236
|
-
clusterId: config.clusterId,
|
|
3237
|
-
groupId: config.groupId,
|
|
3238
|
-
nodeId,
|
|
3239
|
-
sourceNodeId: this.nodeId,
|
|
3240
|
-
appliedSeq: ackedSeq
|
|
3241
|
-
});
|
|
3242
|
-
if (!admitted) return;
|
|
3243
|
-
this.coordinatorState = admitted;
|
|
3244
|
-
this.coordinatorAuthority = this.hasCurrentPrimaryAuthorityFor(admitted);
|
|
3245
|
-
}
|
|
3246
|
-
async removeLocalNodeFromCoordinatorInSyncSet(state) {
|
|
3247
|
-
const config = this.config.coordinator;
|
|
3248
|
-
if (!config) return;
|
|
3249
|
-
const inSyncNodeIds = state.inSyncNodeIds.filter((nodeId) => nodeId !== this.nodeId);
|
|
3250
|
-
let nextState = state;
|
|
3251
|
-
if (!arraysEqual(inSyncNodeIds, state.inSyncNodeIds)) {
|
|
3252
|
-
const updated = await config.coordinator.updateInSyncSet({
|
|
3253
|
-
clusterId: config.clusterId,
|
|
3254
|
-
groupId: config.groupId,
|
|
3255
|
-
inSyncNodeIds
|
|
3256
|
-
});
|
|
3257
|
-
if (!updated) {
|
|
3258
|
-
throw new CoordinatorError("Failed to mark repaired node as in sync");
|
|
3259
|
-
}
|
|
3260
|
-
nextState = updated;
|
|
3261
|
-
}
|
|
3262
|
-
this.coordinatorState = nextState;
|
|
3263
|
-
this.coordinatorAuthority = this.hasCurrentPrimaryAuthorityFor(this.coordinatorState);
|
|
3077
|
+
return requiresCoordinatorRejoinSync(this, state);
|
|
3264
3078
|
}
|
|
3265
|
-
|
|
3266
|
-
|
|
3267
|
-
if (!state) {
|
|
3268
|
-
throw new CoordinatorError("Cannot prove primary authority without replication-group state");
|
|
3269
|
-
}
|
|
3270
|
-
if (!this.hasCurrentPrimaryAuthorityFor(state)) {
|
|
3271
|
-
throw new StalePrimaryError(
|
|
3272
|
-
"Node is not the current primary for this replication group",
|
|
3273
|
-
this.errorDetails(state)
|
|
3274
|
-
);
|
|
3275
|
-
}
|
|
3276
|
-
this.assertLocalCompatibility(state);
|
|
3277
|
-
if (state.drainingNodeIds.includes(this.nodeId)) {
|
|
3278
|
-
throw new NodeDrainingError("Node is draining and cannot accept writes", this.errorDetails(state));
|
|
3279
|
-
}
|
|
3280
|
-
if (state.repairingNodeIds.includes(this.nodeId) || state.faultedNodeIds.includes(this.nodeId)) {
|
|
3281
|
-
throw new AuthorityError(
|
|
3282
|
-
"Node is not eligible to accept writes while repair or fault state is active",
|
|
3283
|
-
"AUTHORITY_LOST",
|
|
3284
|
-
this.errorDetails(state)
|
|
3285
|
-
);
|
|
3286
|
-
}
|
|
3287
|
-
return state;
|
|
3079
|
+
markCoordinatorSyncReady() {
|
|
3080
|
+
return markCoordinatorSyncReady(this);
|
|
3288
3081
|
}
|
|
3289
|
-
|
|
3290
|
-
|
|
3291
|
-
|
|
3292
|
-
|
|
3293
|
-
|
|
3294
|
-
|
|
3295
|
-
|
|
3296
|
-
|
|
3297
|
-
throw new StalePrimaryError(
|
|
3298
|
-
`Rejected ${direction} message for stale or wrong replication group term`,
|
|
3299
|
-
this.errorDetails(state)
|
|
3300
|
-
);
|
|
3301
|
-
}
|
|
3302
|
-
if ((direction === "batch" || direction === "sync-data") && fromPeerId !== state.currentPrimary?.nodeId) {
|
|
3303
|
-
throw new StalePrimaryError(
|
|
3304
|
-
`Rejected ${direction} message from non-current primary '${fromPeerId}'`,
|
|
3305
|
-
this.errorDetails(state)
|
|
3306
|
-
);
|
|
3307
|
-
}
|
|
3308
|
-
if ((direction === "ack" || direction === "forward" || direction === "sync-request") && !this.hasCurrentPrimaryAuthorityFor(state)) {
|
|
3309
|
-
throw new StalePrimaryError(
|
|
3310
|
-
`Rejected ${direction} message because this node is not current primary`,
|
|
3311
|
-
this.errorDetails(state)
|
|
3312
|
-
);
|
|
3313
|
-
}
|
|
3082
|
+
handleCoordinatorAckProgress(nodeId, ackedSeq) {
|
|
3083
|
+
return handleCoordinatorAckProgress(this, nodeId, ackedSeq);
|
|
3084
|
+
}
|
|
3085
|
+
verifyPrimaryAuthority() {
|
|
3086
|
+
return verifyPrimaryAuthority(this);
|
|
3087
|
+
}
|
|
3088
|
+
assertInboundCoordinatorMessage(message, fromPeerId, direction) {
|
|
3089
|
+
return assertInboundCoordinatorMessage(this, message, fromPeerId, direction);
|
|
3314
3090
|
}
|
|
3315
3091
|
decorateBatch(batch) {
|
|
3316
|
-
return { ...batch, ...
|
|
3092
|
+
return { ...batch, ...getCoordinatorMessageFields(this) };
|
|
3317
3093
|
}
|
|
3318
3094
|
decorateAck(ack) {
|
|
3319
|
-
return { ...ack, ...
|
|
3095
|
+
return { ...ack, ...getCoordinatorMessageFields(this) };
|
|
3320
3096
|
}
|
|
3321
3097
|
decorateForwardResult(result) {
|
|
3322
|
-
return { ...result, ...
|
|
3098
|
+
return { ...result, ...getCoordinatorMessageFields(this) };
|
|
3323
3099
|
}
|
|
3324
3100
|
decorateSyncRequest(request) {
|
|
3325
|
-
return { ...request, ...
|
|
3101
|
+
return { ...request, ...getCoordinatorMessageFields(this) };
|
|
3326
3102
|
}
|
|
3327
3103
|
decorateSyncBatch(batch) {
|
|
3328
|
-
return { ...batch, ...
|
|
3104
|
+
return { ...batch, ...getCoordinatorMessageFields(this) };
|
|
3329
3105
|
}
|
|
3330
3106
|
decorateSyncComplete(complete) {
|
|
3331
|
-
return { ...complete, ...
|
|
3107
|
+
return { ...complete, ...getCoordinatorMessageFields(this) };
|
|
3332
3108
|
}
|
|
3333
3109
|
decorateSyncAck(ack) {
|
|
3334
|
-
return { ...ack, ...
|
|
3110
|
+
return { ...ack, ...getCoordinatorMessageFields(this) };
|
|
3335
3111
|
}
|
|
3336
3112
|
resolveWriteConcern(wc) {
|
|
3337
3113
|
if (wc) return wc;
|
|
@@ -3341,431 +3117,9 @@ var ReplicationEngine = class extends EventEmitter {
|
|
|
3341
3117
|
return void 0;
|
|
3342
3118
|
}
|
|
3343
3119
|
getCurrentPrimaryPeerId() {
|
|
3344
|
-
return
|
|
3345
|
-
}
|
|
3346
|
-
async canAcceptLocalWrite() {
|
|
3347
|
-
if (!this.isCoordinatorMode()) {
|
|
3348
|
-
return this.config.topology.canWrite();
|
|
3349
|
-
}
|
|
3350
|
-
const state = await this.refreshCoordinatorState();
|
|
3351
|
-
if (!state) {
|
|
3352
|
-
throw new CoordinatorError("Cannot prove write authority without replication-group state");
|
|
3353
|
-
}
|
|
3354
|
-
if (!this.hasCurrentPrimaryAuthorityFor(state)) {
|
|
3355
|
-
return false;
|
|
3356
|
-
}
|
|
3357
|
-
this.assertLocalCompatibility(state);
|
|
3358
|
-
if (state.drainingNodeIds.includes(this.nodeId)) {
|
|
3359
|
-
throw new NodeDrainingError("Node is draining and cannot accept writes", this.errorDetails(state));
|
|
3360
|
-
}
|
|
3361
|
-
if (state.repairingNodeIds.includes(this.nodeId) || state.faultedNodeIds.includes(this.nodeId)) {
|
|
3362
|
-
throw new AuthorityError(
|
|
3363
|
-
"Node is not eligible to accept writes while repair or fault state is active",
|
|
3364
|
-
"AUTHORITY_LOST",
|
|
3365
|
-
this.errorDetails(state)
|
|
3366
|
-
);
|
|
3367
|
-
}
|
|
3368
|
-
return true;
|
|
3369
|
-
}
|
|
3370
|
-
throwNotCurrentPrimary() {
|
|
3371
|
-
if (this.isCoordinatorMode()) {
|
|
3372
|
-
throw new StalePrimaryError("This node is not the current primary", this.errorDetails(this.coordinatorState));
|
|
3373
|
-
}
|
|
3374
|
-
throw new TopologyError("This node cannot accept writes");
|
|
3375
|
-
}
|
|
3376
|
-
async assertReadConcern(level) {
|
|
3377
|
-
if (!this.isCoordinatorMode()) return;
|
|
3378
|
-
const readConcern = level ?? "majority";
|
|
3379
|
-
if (readConcern === "local") return;
|
|
3380
|
-
const state = await this.refreshCoordinatorState();
|
|
3381
|
-
if (!state) {
|
|
3382
|
-
throw new CoordinatorError("Cannot satisfy read concern without replication-group state");
|
|
3383
|
-
}
|
|
3384
|
-
if (readConcern === "linearizable") {
|
|
3385
|
-
await this.verifyPrimaryAuthority();
|
|
3386
|
-
return;
|
|
3387
|
-
}
|
|
3388
|
-
if (readConcern === "majority") {
|
|
3389
|
-
if (!state.inSyncNodeIds.includes(this.nodeId)) {
|
|
3390
|
-
throw new NodeNotInSyncError("Node is not in the in-sync set for majority reads", this.errorDetails(state));
|
|
3391
|
-
}
|
|
3392
|
-
if (state.drainingNodeIds.includes(this.nodeId) || state.repairingNodeIds.includes(this.nodeId)) {
|
|
3393
|
-
throw new ReadConcernError(
|
|
3394
|
-
"Node cannot serve majority reads while draining or repairing",
|
|
3395
|
-
this.errorDetails(state)
|
|
3396
|
-
);
|
|
3397
|
-
}
|
|
3398
|
-
return;
|
|
3399
|
-
}
|
|
3400
|
-
throw new ReadConcernError(`Unsupported read concern '${readConcern}'`, this.errorDetails(state));
|
|
3401
|
-
}
|
|
3402
|
-
async refreshCoordinatorState() {
|
|
3403
|
-
const config = this.config.coordinator;
|
|
3404
|
-
if (!config) return null;
|
|
3405
|
-
try {
|
|
3406
|
-
const state = await config.coordinator.getReplicationGroupState(config.clusterId, config.groupId);
|
|
3407
|
-
this.coordinatorState = state;
|
|
3408
|
-
this.coordinatorAuthority = state ? this.hasCurrentPrimaryAuthorityFor(state) : false;
|
|
3409
|
-
return state;
|
|
3410
|
-
} catch {
|
|
3411
|
-
this.coordinatorAuthority = false;
|
|
3412
|
-
throw new CoordinatorError(
|
|
3413
|
-
"Coordinator is unavailable while refreshing replication-group authority",
|
|
3414
|
-
this.errorDetails(this.coordinatorState)
|
|
3415
|
-
);
|
|
3416
|
-
}
|
|
3417
|
-
}
|
|
3418
|
-
hasCurrentPrimaryAuthorityFor(state) {
|
|
3419
|
-
return state.currentPrimary?.nodeId === this.nodeId;
|
|
3420
|
-
}
|
|
3421
|
-
getCoordinatorMessageFields() {
|
|
3422
|
-
const config = this.config.coordinator;
|
|
3423
|
-
if (!config) return {};
|
|
3424
|
-
return {
|
|
3425
|
-
groupId: config.groupId,
|
|
3426
|
-
primaryTerm: this.coordinatorState?.primaryTerm ?? this.config.transportConfig?.primaryTerm
|
|
3427
|
-
};
|
|
3428
|
-
}
|
|
3429
|
-
getForwardingPrimaryPeerId() {
|
|
3430
|
-
if (this.isCoordinatorMode()) {
|
|
3431
|
-
const primaryId = this.coordinatorState?.currentPrimary?.nodeId;
|
|
3432
|
-
if (primaryId && this.config.transport.peers().has(primaryId)) {
|
|
3433
|
-
return primaryId;
|
|
3434
|
-
}
|
|
3435
|
-
return null;
|
|
3436
|
-
}
|
|
3437
|
-
for (const [peerId, info] of this.config.transport.peers()) {
|
|
3438
|
-
if (info.role === "primary") {
|
|
3439
|
-
return peerId;
|
|
3440
|
-
}
|
|
3441
|
-
}
|
|
3442
|
-
return null;
|
|
3443
|
-
}
|
|
3444
|
-
async updateCoordinatorProgressAfterWrite(seq, state) {
|
|
3445
|
-
const config = this.config.coordinator;
|
|
3446
|
-
if (!config) return;
|
|
3447
|
-
const ackedNodeIds = this.peerTracker.ackedConfiguredNodeIds(seq, this.nodeId, state.votingDataBearingNodeIds);
|
|
3448
|
-
const retainedInSync = state.inSyncNodeIds.filter((nodeId) => ackedNodeIds.includes(nodeId));
|
|
3449
|
-
let nextState = state;
|
|
3450
|
-
if (!arraysEqual(retainedInSync, state.inSyncNodeIds) || state.durabilityPointSeq < seq) {
|
|
3451
|
-
const updated = await config.coordinator.updateInSyncSet({
|
|
3452
|
-
clusterId: config.clusterId,
|
|
3453
|
-
groupId: config.groupId,
|
|
3454
|
-
inSyncNodeIds: retainedInSync,
|
|
3455
|
-
durabilityPointSeq: seq
|
|
3456
|
-
});
|
|
3457
|
-
if (!updated) {
|
|
3458
|
-
throw new CoordinatorError("Failed to update in-sync set before acknowledging majority write");
|
|
3459
|
-
}
|
|
3460
|
-
nextState = updated;
|
|
3461
|
-
}
|
|
3462
|
-
for (const nodeId of ackedNodeIds) {
|
|
3463
|
-
if (nextState.inSyncNodeIds.includes(nodeId)) continue;
|
|
3464
|
-
if (nodeId === this.nodeId) continue;
|
|
3465
|
-
const admitted = await config.coordinator.admitNodeToInSyncSet({
|
|
3466
|
-
clusterId: config.clusterId,
|
|
3467
|
-
groupId: config.groupId,
|
|
3468
|
-
nodeId,
|
|
3469
|
-
sourceNodeId: this.nodeId,
|
|
3470
|
-
appliedSeq: seq
|
|
3471
|
-
});
|
|
3472
|
-
if (!admitted) {
|
|
3473
|
-
throw new CoordinatorError("Failed to admit ACKing node to in-sync set before acknowledging majority write");
|
|
3474
|
-
}
|
|
3475
|
-
nextState = admitted;
|
|
3476
|
-
}
|
|
3477
|
-
this.coordinatorState = nextState;
|
|
3478
|
-
}
|
|
3479
|
-
startCoordinatorLeaseRenewal() {
|
|
3480
|
-
const config = this.config.coordinator;
|
|
3481
|
-
const leaseId = this.nodeSessionLeaseId;
|
|
3482
|
-
if (!config || !leaseId) return;
|
|
3483
|
-
const ttlMs = config.sessionTtlMs ?? DEFAULT_COORDINATOR_SESSION_TTL_MS;
|
|
3484
|
-
const timer = setInterval(
|
|
3485
|
-
() => {
|
|
3486
|
-
config.coordinator.renewLease(leaseId, ttlMs).then((renewed) => {
|
|
3487
|
-
if (!renewed) {
|
|
3488
|
-
this.coordinatorAuthority = false;
|
|
3489
|
-
}
|
|
3490
|
-
}).catch((err) => {
|
|
3491
|
-
this.coordinatorAuthority = false;
|
|
3492
|
-
const wrappedErr = err instanceof Error ? err : new Error(String(err));
|
|
3493
|
-
this.emitError({ error: wrappedErr, operation: "coordinator-session-renew", recoverable: false });
|
|
3494
|
-
});
|
|
3495
|
-
},
|
|
3496
|
-
Math.max(1e3, Math.floor(ttlMs / 3))
|
|
3497
|
-
);
|
|
3498
|
-
this.unrefTimer(timer);
|
|
3499
|
-
this.coordinatorLeaseTimer = timer;
|
|
3500
|
-
}
|
|
3501
|
-
startControllerLoop() {
|
|
3502
|
-
const config = this.config.coordinator;
|
|
3503
|
-
if (!config) return;
|
|
3504
|
-
const controllerConfig = typeof config.controller === "object" ? config.controller : {};
|
|
3505
|
-
const enabled = typeof config.controller === "boolean" ? config.controller : controllerConfig.enabled ?? true;
|
|
3506
|
-
if (!enabled) {
|
|
3507
|
-
this.controllerState = "disabled";
|
|
3508
|
-
return;
|
|
3509
|
-
}
|
|
3510
|
-
this.controllerState = "standby";
|
|
3511
|
-
const ttlMs = controllerConfig.leaseTtlMs ?? DEFAULT_CONTROLLER_LEASE_TTL_MS;
|
|
3512
|
-
const holderId = controllerConfig.holderId ?? this.nodeId;
|
|
3513
|
-
const tickMs = controllerConfig.tickIntervalMs ?? DEFAULT_CONTROLLER_TICK_INTERVAL_MS;
|
|
3514
|
-
const timer = setInterval(() => {
|
|
3515
|
-
this.controllerTick(holderId, ttlMs).catch((err) => {
|
|
3516
|
-
const wrappedErr = err instanceof Error ? err : new Error(String(err));
|
|
3517
|
-
this.controllerState = "lost";
|
|
3518
|
-
this.emitError({ error: wrappedErr, operation: "coordinator-controller", recoverable: true });
|
|
3519
|
-
});
|
|
3520
|
-
}, tickMs);
|
|
3521
|
-
this.unrefTimer(timer);
|
|
3522
|
-
this.controllerTimer = timer;
|
|
3523
|
-
}
|
|
3524
|
-
async controllerTick(holderId, ttlMs) {
|
|
3525
|
-
const config = this.config.coordinator;
|
|
3526
|
-
if (!config) return;
|
|
3527
|
-
if (this.controllerLeaseId) {
|
|
3528
|
-
const renewed = await config.coordinator.renewLease(this.controllerLeaseId, ttlMs);
|
|
3529
|
-
if (!renewed) {
|
|
3530
|
-
this.controllerLeaseId = null;
|
|
3531
|
-
this.controllerState = "lost";
|
|
3532
|
-
return;
|
|
3533
|
-
}
|
|
3534
|
-
this.controllerState = "active";
|
|
3535
|
-
await this.runControllerPromotionCheck();
|
|
3536
|
-
return;
|
|
3537
|
-
}
|
|
3538
|
-
const acquired = await config.coordinator.tryAcquireControllerLease({
|
|
3539
|
-
clusterId: config.clusterId,
|
|
3540
|
-
holderId,
|
|
3541
|
-
ttlMs
|
|
3542
|
-
});
|
|
3543
|
-
if (acquired.acquired) {
|
|
3544
|
-
this.controllerLeaseId = acquired.lease.id;
|
|
3545
|
-
this.controllerState = "active";
|
|
3546
|
-
await this.runControllerPromotionCheck();
|
|
3547
|
-
} else {
|
|
3548
|
-
this.controllerState = "standby";
|
|
3549
|
-
}
|
|
3550
|
-
}
|
|
3551
|
-
async runControllerPromotionCheck() {
|
|
3552
|
-
const config = this.config.coordinator;
|
|
3553
|
-
if (!config) return;
|
|
3554
|
-
const state = await this.refreshCoordinatorState();
|
|
3555
|
-
if (!state) return;
|
|
3556
|
-
const primaryNodeId = state.currentPrimary?.nodeId;
|
|
3557
|
-
const primaryLive = primaryNodeId ? await config.coordinator.getLiveNodeSession(config.clusterId, primaryNodeId) : null;
|
|
3558
|
-
const primaryCanKeepDuty = primaryNodeId && primaryLive && compatibilityAllowsPromotion(state.compatibility, primaryLive.compatibility) && !state.drainingNodeIds.includes(primaryNodeId) && !state.repairingNodeIds.includes(primaryNodeId) && !state.faultedNodeIds.includes(primaryNodeId);
|
|
3559
|
-
if (primaryCanKeepDuty) {
|
|
3560
|
-
return;
|
|
3561
|
-
}
|
|
3562
|
-
try {
|
|
3563
|
-
const promoted = await config.coordinator.promoteEligibleReplica({
|
|
3564
|
-
clusterId: config.clusterId,
|
|
3565
|
-
groupId: config.groupId,
|
|
3566
|
-
excludeNodeIds: primaryNodeId ? [primaryNodeId] : []
|
|
3567
|
-
});
|
|
3568
|
-
this.coordinatorState = promoted;
|
|
3569
|
-
this.coordinatorAuthority = this.hasCurrentPrimaryAuthorityFor(promoted);
|
|
3570
|
-
} catch (err) {
|
|
3571
|
-
const wrappedErr = err instanceof Error ? err : new Error(String(err));
|
|
3572
|
-
this.emitError({ error: wrappedErr, operation: "coordinator-promotion", recoverable: true });
|
|
3573
|
-
}
|
|
3574
|
-
}
|
|
3575
|
-
handleCoordinatorStateUpdate(next) {
|
|
3576
|
-
const previous = this.coordinatorState;
|
|
3577
|
-
const wasPrimary = previous ? this.hasCurrentPrimaryAuthorityFor(previous) : this.coordinatorAuthority;
|
|
3578
|
-
this.coordinatorState = next;
|
|
3579
|
-
this.coordinatorAuthority = this.hasCurrentPrimaryAuthorityFor(next);
|
|
3580
|
-
if (wasPrimary && !this.coordinatorAuthority && next.primaryTerm > (previous?.primaryTerm ?? 0n)) {
|
|
3581
|
-
this.handleFormerPrimaryDemotion(next).catch((err) => {
|
|
3582
|
-
const wrappedErr = err instanceof Error ? err : new Error(String(err));
|
|
3583
|
-
this.emitError({ error: wrappedErr, operation: "coordinator-former-primary-demotion", recoverable: false });
|
|
3584
|
-
});
|
|
3585
|
-
return;
|
|
3586
|
-
}
|
|
3587
|
-
this.startCoordinatorRejoinSyncIfReady(next);
|
|
3588
|
-
}
|
|
3589
|
-
startCoordinatorRejoinSyncIfReady(state) {
|
|
3590
|
-
if (!this.running) return;
|
|
3591
|
-
if (this.coordinatorRejoinSyncStarting) return;
|
|
3592
|
-
if (!this.requiresCoordinatorRejoinSync(state)) return;
|
|
3593
|
-
if (this.syncState.phase !== "pending") return;
|
|
3594
|
-
const sourceNodeId = state.currentPrimary?.nodeId;
|
|
3595
|
-
if (!sourceNodeId || !this.config.transport.peers().has(sourceNodeId)) return;
|
|
3596
|
-
this.coordinatorRejoinSyncStarting = true;
|
|
3597
|
-
this.syncJoiner.initiateSync().catch((err) => {
|
|
3598
|
-
const wrappedErr = err instanceof Error ? err : new Error(String(err));
|
|
3599
|
-
this.emitError({ error: wrappedErr, operation: "coordinator-rejoin-sync", recoverable: true });
|
|
3600
|
-
}).finally(() => {
|
|
3601
|
-
this.coordinatorRejoinSyncStarting = false;
|
|
3602
|
-
});
|
|
3603
|
-
}
|
|
3604
|
-
async handleFormerPrimaryDemotion(state) {
|
|
3605
|
-
const repairingState = await this.markLocalNodeRepairing(state);
|
|
3606
|
-
const evaluatedState = await this.evaluateFormerPrimaryHistory(repairingState);
|
|
3607
|
-
if (!this.requiresCoordinatorRejoinSync(evaluatedState)) return;
|
|
3608
|
-
if (!this.running || this.syncState.phase === "syncing" || this.syncState.phase === "catching-up") return;
|
|
3609
|
-
this.syncState = {
|
|
3610
|
-
phase: "pending",
|
|
3611
|
-
sourcePeerId: null,
|
|
3612
|
-
snapshotSeq: null,
|
|
3613
|
-
completedTables: [],
|
|
3614
|
-
totalTables: 0,
|
|
3615
|
-
startedAt: null,
|
|
3616
|
-
error: null
|
|
3617
|
-
};
|
|
3618
|
-
await this.log.setSyncMeta("pending");
|
|
3619
|
-
await this.syncJoiner.initiateSync();
|
|
3620
|
-
}
|
|
3621
|
-
async markLocalNodeRepairing(state) {
|
|
3622
|
-
const config = this.config.coordinator;
|
|
3623
|
-
if (!config || state.repairingNodeIds.includes(this.nodeId) || state.faultedNodeIds.includes(this.nodeId)) {
|
|
3624
|
-
return state;
|
|
3625
|
-
}
|
|
3626
|
-
const updated = await config.coordinator.updateNodeMaintenance({
|
|
3627
|
-
clusterId: config.clusterId,
|
|
3628
|
-
groupId: config.groupId,
|
|
3629
|
-
nodeId: this.nodeId,
|
|
3630
|
-
repairing: true
|
|
3631
|
-
});
|
|
3632
|
-
if (!updated) {
|
|
3633
|
-
throw new CoordinatorError("Failed to mark former primary for repair");
|
|
3634
|
-
}
|
|
3635
|
-
this.coordinatorState = updated;
|
|
3636
|
-
this.coordinatorAuthority = this.hasCurrentPrimaryAuthorityFor(updated);
|
|
3637
|
-
return updated;
|
|
3638
|
-
}
|
|
3639
|
-
requiresCoordinatorRejoin(state) {
|
|
3640
|
-
if (state.currentPrimary?.nodeId === this.nodeId) return false;
|
|
3641
|
-
return state.repairingNodeIds.includes(this.nodeId) || state.faultedNodeIds.includes(this.nodeId);
|
|
3642
|
-
}
|
|
3643
|
-
assertLocalCompatibility(state) {
|
|
3644
|
-
if (compatibilityAllowsPromotion(state.compatibility, this.config.coordinator?.compatibility)) return;
|
|
3645
|
-
throw new ProtocolVersionMismatchError("Node compatibility metadata is incompatible with the replication group", {
|
|
3646
|
-
...this.errorDetails(state),
|
|
3647
|
-
groupCompatibility: state.compatibility,
|
|
3648
|
-
localCompatibility: this.config.coordinator?.compatibility
|
|
3649
|
-
});
|
|
3650
|
-
}
|
|
3651
|
-
async evaluateFormerPrimaryHistory(state) {
|
|
3652
|
-
const config = this.config.coordinator;
|
|
3653
|
-
const currentPrimary = state.currentPrimary;
|
|
3654
|
-
if (!config || !currentPrimary || state.faultedNodeIds.includes(this.nodeId)) return state;
|
|
3655
|
-
const localSeq = await this.log.getLocalSeq();
|
|
3656
|
-
const currentPrimaryAckedSeq = await this.log.getPeerAckedSeq(currentPrimary.nodeId);
|
|
3657
|
-
if (localSeq <= currentPrimaryAckedSeq) {
|
|
3658
|
-
return state;
|
|
3659
|
-
}
|
|
3660
|
-
const inSyncNodeIds = state.inSyncNodeIds.filter((nodeId) => nodeId !== this.nodeId);
|
|
3661
|
-
let nextState = state;
|
|
3662
|
-
if (inSyncNodeIds.length !== state.inSyncNodeIds.length) {
|
|
3663
|
-
const updated = await config.coordinator.updateInSyncSet({
|
|
3664
|
-
clusterId: config.clusterId,
|
|
3665
|
-
groupId: config.groupId,
|
|
3666
|
-
inSyncNodeIds
|
|
3667
|
-
});
|
|
3668
|
-
if (!updated) {
|
|
3669
|
-
throw new CoordinatorError("Failed to remove divergent former primary from in-sync set");
|
|
3670
|
-
}
|
|
3671
|
-
nextState = updated;
|
|
3672
|
-
}
|
|
3673
|
-
const faulted = await config.coordinator.updateNodeMaintenance({
|
|
3674
|
-
clusterId: config.clusterId,
|
|
3675
|
-
groupId: config.groupId,
|
|
3676
|
-
nodeId: this.nodeId,
|
|
3677
|
-
repairing: false,
|
|
3678
|
-
faulted: true
|
|
3679
|
-
});
|
|
3680
|
-
nextState = faulted ?? nextState;
|
|
3681
|
-
this.coordinatorState = nextState;
|
|
3682
|
-
this.coordinatorAuthority = false;
|
|
3683
|
-
this.emitError({
|
|
3684
|
-
error: new AuthorityError(
|
|
3685
|
-
"Former primary returned with local-only writes and was quarantined",
|
|
3686
|
-
"AUTHORITY_LOST",
|
|
3687
|
-
{
|
|
3688
|
-
...this.errorDetails(nextState),
|
|
3689
|
-
localSeq: localSeq.toString(),
|
|
3690
|
-
currentPrimaryAckedSeq: currentPrimaryAckedSeq.toString()
|
|
3691
|
-
}
|
|
3692
|
-
),
|
|
3693
|
-
operation: "coordinator-former-primary-divergence",
|
|
3694
|
-
recoverable: false
|
|
3695
|
-
});
|
|
3696
|
-
return nextState;
|
|
3697
|
-
}
|
|
3698
|
-
stopCoordinatorTimers() {
|
|
3699
|
-
if (this.coordinatorLeaseTimer) {
|
|
3700
|
-
clearInterval(this.coordinatorLeaseTimer);
|
|
3701
|
-
this.coordinatorLeaseTimer = null;
|
|
3702
|
-
}
|
|
3703
|
-
if (this.controllerTimer) {
|
|
3704
|
-
clearInterval(this.controllerTimer);
|
|
3705
|
-
this.controllerTimer = null;
|
|
3706
|
-
}
|
|
3707
|
-
}
|
|
3708
|
-
async stopCoordinatorMode() {
|
|
3709
|
-
const config = this.config.coordinator;
|
|
3710
|
-
if (!config) return;
|
|
3711
|
-
if (this.coordinatorWatchDisposer) {
|
|
3712
|
-
await this.coordinatorWatchDisposer();
|
|
3713
|
-
this.coordinatorWatchDisposer = null;
|
|
3714
|
-
}
|
|
3715
|
-
if (this.controllerLeaseId) {
|
|
3716
|
-
const leaseId = this.controllerLeaseId;
|
|
3717
|
-
this.controllerLeaseId = null;
|
|
3718
|
-
await config.coordinator.releaseLease(leaseId).catch((err) => {
|
|
3719
|
-
const wrappedErr = err instanceof Error ? err : new Error(String(err));
|
|
3720
|
-
this.emitError({ error: wrappedErr, operation: "coordinator-controller-release", recoverable: true });
|
|
3721
|
-
});
|
|
3722
|
-
}
|
|
3723
|
-
await config.coordinator.deregisterNodeSession(config.clusterId, this.nodeId).catch((err) => {
|
|
3724
|
-
const wrappedErr = err instanceof Error ? err : new Error(String(err));
|
|
3725
|
-
this.emitError({ error: wrappedErr, operation: "coordinator-session-deregister", recoverable: true });
|
|
3726
|
-
});
|
|
3727
|
-
}
|
|
3728
|
-
localCoordinatorPrimary() {
|
|
3729
|
-
const endpoint = this.config.coordinator?.endpoint;
|
|
3730
|
-
return endpoint ? { nodeId: this.nodeId, endpoint } : { nodeId: this.nodeId };
|
|
3731
|
-
}
|
|
3732
|
-
unrefTimer(timer) {
|
|
3733
|
-
const unref = timer.unref;
|
|
3734
|
-
unref?.call(timer);
|
|
3735
|
-
}
|
|
3736
|
-
getCoordinatorRuntimeStatus() {
|
|
3737
|
-
const config = this.config.coordinator;
|
|
3738
|
-
const state = this.coordinatorState;
|
|
3739
|
-
if (!config || !state) return void 0;
|
|
3740
|
-
return {
|
|
3741
|
-
clusterId: config.clusterId,
|
|
3742
|
-
groupId: config.groupId,
|
|
3743
|
-
currentPrimary: state.currentPrimary ? { ...state.currentPrimary } : null,
|
|
3744
|
-
primaryTerm: state.primaryTerm,
|
|
3745
|
-
inSyncNodeIds: [...state.inSyncNodeIds],
|
|
3746
|
-
drainingNodeIds: [...state.drainingNodeIds],
|
|
3747
|
-
repairingNodeIds: [...state.repairingNodeIds],
|
|
3748
|
-
faultedNodeIds: [...state.faultedNodeIds],
|
|
3749
|
-
votingDataBearingNodeIds: [...state.votingDataBearingNodeIds],
|
|
3750
|
-
authority: this.coordinatorAuthority,
|
|
3751
|
-
controllerState: this.controllerState
|
|
3752
|
-
};
|
|
3753
|
-
}
|
|
3754
|
-
errorDetails(state) {
|
|
3755
|
-
return {
|
|
3756
|
-
currentPrimary: state?.currentPrimary ?? null,
|
|
3757
|
-
primaryTerm: state?.primaryTerm.toString(),
|
|
3758
|
-
replicationGroupId: this.config.coordinator?.groupId
|
|
3759
|
-
};
|
|
3120
|
+
return getForwardingPrimaryPeerId(this);
|
|
3760
3121
|
}
|
|
3761
3122
|
};
|
|
3762
|
-
function arraysEqual(left, right) {
|
|
3763
|
-
if (left.length !== right.length) return false;
|
|
3764
|
-
for (let i = 0; i < left.length; i++) {
|
|
3765
|
-
if (left[i] !== right[i]) return false;
|
|
3766
|
-
}
|
|
3767
|
-
return true;
|
|
3768
|
-
}
|
|
3769
3123
|
|
|
3770
3124
|
// src/replication/topology/primary-replica.ts
|
|
3771
3125
|
var PrimaryReplicaTopology = class {
|
|
@@ -3787,4 +3141,4 @@ var PrimaryReplicaTopology = class {
|
|
|
3787
3141
|
}
|
|
3788
3142
|
};
|
|
3789
3143
|
|
|
3790
|
-
export { FieldMergeResolver,
|
|
3144
|
+
export { FieldMergeResolver, PeerTracker, PrimaryReplicaTopology, PrimaryWinsResolver, ReplicationEngine, ReplicationLog, generateNodeId, validateNodeId };
|