@delali/sirannon-db 0.1.8 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +117 -989
- package/dist/backup-scheduler/index.d.ts +1 -1
- package/dist/backup-scheduler/index.mjs +2 -2
- package/dist/baseline-Br77Fnhb.d.ts +6 -0
- package/dist/{change-tracker-CbmaMO-N.d.ts → change-tracker-DKRVUC3l.d.ts} +5 -8
- package/dist/chunk-5NOIGN5Y.mjs +18 -0
- package/dist/chunk-67M7KAH6.mjs +90 -0
- package/dist/{chunk-GEZUUIKV.mjs → chunk-D7LAYTKN.mjs} +5 -2
- package/dist/{chunk-UXLAO6ZH.mjs → chunk-EBJXPQQO.mjs} +8 -3
- package/dist/{chunk-BNUTBHHH.mjs → chunk-FHWTZFI4.mjs} +1 -1
- package/dist/chunk-GVCNMPOS.mjs +130 -0
- package/dist/chunk-H237TXZW.mjs +1407 -0
- package/dist/{chunk-HHRMRFFR.mjs → chunk-H6PIVVDN.mjs} +353 -142
- package/dist/chunk-HR5CWTLC.mjs +661 -0
- package/dist/chunk-JU64Y7HM.mjs +299 -0
- package/dist/{chunk-CW6S3WL5.mjs → chunk-JZGINXTN.mjs} +7 -5
- package/dist/{chunk-CJLYFDP5.mjs → chunk-LFZ37BSX.mjs} +1 -1
- package/dist/chunk-LNY2VVHE.mjs +18 -0
- package/dist/chunk-NVQS53NT.mjs +665 -0
- package/dist/{chunk-H5AB6NIR.mjs → chunk-O7SLN3GI.mjs} +2 -15
- package/dist/chunk-OQVZBEBY.mjs +283 -0
- package/dist/{chunk-VLTICJOD.mjs → chunk-PIKHN33N.mjs} +1 -1
- package/dist/chunk-TJF5GZSV.mjs +187 -0
- package/dist/chunk-TUD5CJ76.mjs +126 -0
- package/dist/{chunk-YPYVQJ4C.mjs → chunk-UC3SCMIN.mjs} +9 -1
- package/dist/client/index.d.ts +216 -321
- package/dist/client/index.mjs +1192 -1342
- package/dist/client/topology.d.ts +61 -0
- package/dist/client/topology.mjs +682 -0
- package/dist/client-base-CLWmH5Ln.d.ts +234 -0
- package/dist/codegen/cli.d.ts +2 -0
- package/dist/codegen/cli.mjs +11 -0
- package/dist/codegen/index.d.ts +30 -0
- package/dist/codegen/index.mjs +5 -0
- package/dist/core/index.d.ts +37 -16
- package/dist/core/index.mjs +2322 -298
- package/dist/core/writer-worker.mjs +3 -3
- package/dist/{database-DuGp0Rtr.d.ts → database-BY0L5Q2n.d.ts} +59 -14
- package/dist/driver/better-sqlite3.d.ts +1 -1
- package/dist/driver/better-sqlite3.mjs +5 -5
- package/dist/driver/node.d.ts +1 -1
- package/dist/driver/node.mjs +5 -5
- package/dist/driver/wa-sqlite.d.ts +1 -136
- package/dist/{errors-5Nf5ZAEC.d.ts → errors-Bw5MdNCu.d.ts} +5 -1
- package/dist/file-migrations/index.d.ts +7 -5
- package/dist/file-migrations/index.mjs +42 -37
- package/dist/operation-registry-9DcvxcE5.d.ts +45 -0
- package/dist/primary-wins-DPAm2AKG.d.ts +61 -0
- package/dist/protocol-6KrSq2Hy.d.ts +66 -0
- package/dist/react/index.d.ts +20 -0
- package/dist/react/index.mjs +137 -0
- package/dist/replication/coordinator/etcd.d.ts +5 -3
- package/dist/replication/coordinator/etcd.mjs +554 -462
- package/dist/replication/index.d.ts +51 -105
- package/dist/replication/index.mjs +1415 -2061
- package/dist/server/index.d.ts +111 -87
- package/dist/server/index.mjs +1917 -489
- package/dist/server-options-1JHu8pid.d.ts +191 -0
- package/dist/{sirannon-4SspRvP5.d.ts → sirannon-DaQSyhbJ.d.ts} +7 -2
- package/dist/transport/grpc.d.ts +7 -4
- package/dist/transport/grpc.mjs +191 -97
- package/dist/transport/memory.d.ts +5 -134
- package/dist/transport/memory.mjs +3 -1
- package/dist/{types-BsjobKbl.d.ts → types-B7gmEsZW.d.ts} +8 -61
- package/dist/{types-BEu1I_9_.d.ts → types-BsVabqSI.d.ts} +1 -1
- package/dist/types-CL6piSnD.d.ts +43 -0
- package/dist/types-C_D8IhpO.d.ts +60 -0
- package/dist/{types-D_hQW1hr.d.ts → types-zhnRXrsb.d.ts} +55 -165
- package/package.json +44 -6
- package/dist/chunk-TKGHYWQ6.mjs +0 -35
- package/dist/types-D4p4UyDK.d.ts +0 -26
|
@@ -1,6 +1,7 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { CHANGES_TABLE, isReservedIdentifier,
|
|
3
|
-
import {
|
|
1
|
+
import { tableColumnNames, tablePkColumns, selectMinChangeSeqSql, selectMaxChangeSeq, deleteChangesBeforeUpToSeqSql, deleteChangesBeforeSql, selectTableExists, ensureChangesTable, ensureMetaTable, insertMetaValueIfAbsent, selectMetaValue, selectChangesAfterSeqSql, selectTableChangesInRangeSql, selectTablesChangesInRangeSql } from './chunk-NVQS53NT.mjs';
|
|
2
|
+
import { CHANGES_TABLE, isReservedIdentifier, CDC_TRIGGER_PREFIX, encodeTaggedValues, decodeTaggedValues, SAFE_INT_BOUND_TEXT } from './chunk-D7LAYTKN.mjs';
|
|
3
|
+
import { synchronousPragmaValue } from './chunk-LFZ37BSX.mjs';
|
|
4
|
+
import { SirannonError, ForbiddenSqlError, CDCError, MigrationError } from './chunk-UC3SCMIN.mjs';
|
|
4
5
|
|
|
5
6
|
// src/core/bulk-load.ts
|
|
6
7
|
var DEFAULT_LOAD_DURABILITY = "off";
|
|
@@ -57,19 +58,106 @@ function delay(ms) {
|
|
|
57
58
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
58
59
|
}
|
|
59
60
|
|
|
61
|
+
// src/core/cdc/change-log-reader.ts
|
|
62
|
+
var MAX_TRANSACTION_CARRY_ROWS = 5e4;
|
|
63
|
+
function rowToEvent(row) {
|
|
64
|
+
return {
|
|
65
|
+
type: row.operation.toLowerCase(),
|
|
66
|
+
table: row.table_name,
|
|
67
|
+
row: row.new_data ? decodeTaggedValues(JSON.parse(row.new_data)) : {},
|
|
68
|
+
oldRow: row.old_data ? decodeTaggedValues(JSON.parse(row.old_data)) : void 0,
|
|
69
|
+
seq: BigInt(row.seq),
|
|
70
|
+
timestamp: row.changed_at,
|
|
71
|
+
rowId: String(row.row_id),
|
|
72
|
+
...row.node_id ? { origin: row.node_id } : {},
|
|
73
|
+
...row.hlc ? { hlc: row.hlc } : {},
|
|
74
|
+
...row.tx_id ? { txId: row.tx_id } : {}
|
|
75
|
+
};
|
|
76
|
+
}
|
|
77
|
+
async function pollChanges(conn, cache, changesTable, afterSeq, batchSize) {
|
|
78
|
+
const stmt = await cache.get(conn, "poll", selectChangesAfterSeqSql(changesTable));
|
|
79
|
+
const rows = await stmt.all(afterSeq.toString(), batchSize + 1);
|
|
80
|
+
if (rows.length === 0) return null;
|
|
81
|
+
const emitted = rows.length > batchSize ? rows.slice(0, batchSize) : rows;
|
|
82
|
+
const lastEmitted = emitted[emitted.length - 1];
|
|
83
|
+
const peekedBeyondBatch = rows[batchSize];
|
|
84
|
+
const openTxId = lastEmitted.tx_id;
|
|
85
|
+
const events = emitted.map(rowToEvent);
|
|
86
|
+
let lastSeq = BigInt(lastEmitted.seq);
|
|
87
|
+
if (peekedBeyondBatch === void 0 || !openTxId || (peekedBeyondBatch.tx_id ?? "") !== openTxId) {
|
|
88
|
+
return { events, lastSeq, atTxBoundary: true };
|
|
89
|
+
}
|
|
90
|
+
let carried = 0;
|
|
91
|
+
while (carried < MAX_TRANSACTION_CARRY_ROWS) {
|
|
92
|
+
const tail = await stmt.all(lastSeq.toString(), batchSize);
|
|
93
|
+
if (tail.length === 0) {
|
|
94
|
+
return { events, lastSeq, atTxBoundary: true };
|
|
95
|
+
}
|
|
96
|
+
for (const row of tail) {
|
|
97
|
+
if ((row.tx_id ?? "") !== openTxId) {
|
|
98
|
+
return { events, lastSeq, atTxBoundary: true };
|
|
99
|
+
}
|
|
100
|
+
events.push(rowToEvent(row));
|
|
101
|
+
lastSeq = BigInt(row.seq);
|
|
102
|
+
carried += 1;
|
|
103
|
+
}
|
|
104
|
+
if (tail.length < batchSize) {
|
|
105
|
+
return { events, lastSeq, atTxBoundary: true };
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
return { events, lastSeq, atTxBoundary: false };
|
|
109
|
+
}
|
|
110
|
+
async function readSinceOneTable(conn, cache, changesTable, table, afterSeq, upToSeq, limit) {
|
|
111
|
+
const stmt = await cache.get(conn, "read_since", selectTableChangesInRangeSql(changesTable));
|
|
112
|
+
const rows = await stmt.all(table, afterSeq.toString(), upToSeq.toString(), limit);
|
|
113
|
+
return rows.map(rowToEvent);
|
|
114
|
+
}
|
|
115
|
+
async function readSinceTables(conn, cache, changesTable, tables, afterSeq, upToSeq, limit) {
|
|
116
|
+
if (tables.length === 0) return [];
|
|
117
|
+
const stmt = await cache.get(
|
|
118
|
+
conn,
|
|
119
|
+
`read_since_tables_${tables.length}`,
|
|
120
|
+
selectTablesChangesInRangeSql(changesTable, tables.length)
|
|
121
|
+
);
|
|
122
|
+
const rows = await stmt.all(...tables, afterSeq.toString(), upToSeq.toString(), limit);
|
|
123
|
+
return rows.map(rowToEvent);
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
// src/core/cdc/statement-cache.ts
|
|
127
|
+
var StatementCache = class {
|
|
128
|
+
cache = /* @__PURE__ */ new WeakMap();
|
|
129
|
+
async get(conn, key, sql) {
|
|
130
|
+
let stmts = this.cache.get(conn);
|
|
131
|
+
if (!stmts) {
|
|
132
|
+
stmts = /* @__PURE__ */ new Map();
|
|
133
|
+
this.cache.set(conn, stmts);
|
|
134
|
+
}
|
|
135
|
+
const existing = stmts.get(key);
|
|
136
|
+
if (existing) return existing;
|
|
137
|
+
const pending = conn.prepare(sql);
|
|
138
|
+
stmts.set(key, pending);
|
|
139
|
+
try {
|
|
140
|
+
return await pending;
|
|
141
|
+
} catch (err) {
|
|
142
|
+
stmts.delete(key);
|
|
143
|
+
throw err;
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
};
|
|
147
|
+
|
|
60
148
|
// src/core/cdc/trigger-sql.ts
|
|
61
149
|
async function dropCdcTriggers(conn, table) {
|
|
62
150
|
await conn.exec(`DROP TRIGGER IF EXISTS "${CDC_TRIGGER_PREFIX}${table}_insert"`);
|
|
63
151
|
await conn.exec(`DROP TRIGGER IF EXISTS "${CDC_TRIGGER_PREFIX}${table}_update"`);
|
|
64
152
|
await conn.exec(`DROP TRIGGER IF EXISTS "${CDC_TRIGGER_PREFIX}${table}_delete"`);
|
|
65
153
|
}
|
|
66
|
-
async function installCdcTriggers(conn, changesTable, table, columns, pkColumns
|
|
154
|
+
async function installCdcTriggers(conn, changesTable, table, columns, pkColumns) {
|
|
67
155
|
const newJson = buildJsonObject(columns, "NEW");
|
|
68
156
|
const oldJson = buildJsonObject(columns, "OLD");
|
|
69
157
|
const newPk = buildPkRef(pkColumns, "NEW");
|
|
70
158
|
const oldPk = buildPkRef(pkColumns, "OLD");
|
|
71
|
-
const replCols =
|
|
72
|
-
const replVals =
|
|
159
|
+
const replCols = ", node_id, tx_id, hlc";
|
|
160
|
+
const replVals = ", '', '', ''";
|
|
73
161
|
await conn.exec(`
|
|
74
162
|
CREATE TRIGGER IF NOT EXISTS "${CDC_TRIGGER_PREFIX}${table}_insert"
|
|
75
163
|
AFTER INSERT ON "${table}"
|
|
@@ -128,16 +216,16 @@ var ChangeTracker = class {
|
|
|
128
216
|
retentionMs;
|
|
129
217
|
changesTable;
|
|
130
218
|
pollBatchSize;
|
|
131
|
-
replication;
|
|
132
219
|
changesTableReady = false;
|
|
220
|
+
changesTableEnsured = false;
|
|
133
221
|
watchedTablesCache = null;
|
|
134
|
-
stmtCache =
|
|
222
|
+
stmtCache = new StatementCache();
|
|
135
223
|
pruneBoundary = null;
|
|
224
|
+
lastPollAtTxBoundary = true;
|
|
136
225
|
constructor(options) {
|
|
137
226
|
this.retentionMs = options?.retention ?? DEFAULT_RETENTION_MS;
|
|
138
227
|
this.changesTable = options?.changesTable ?? CHANGES_TABLE;
|
|
139
228
|
this.pollBatchSize = options?.pollBatchSize ?? DEFAULT_POLL_BATCH_SIZE;
|
|
140
|
-
this.replication = options?.replication ?? false;
|
|
141
229
|
this.assertIdentifier(this.changesTable, "changes table name");
|
|
142
230
|
}
|
|
143
231
|
async watch(conn, table) {
|
|
@@ -146,14 +234,14 @@ var ChangeTracker = class {
|
|
|
146
234
|
throw new ForbiddenSqlError(`Table '${table}' is reserved for Sirannon and cannot be watched`);
|
|
147
235
|
}
|
|
148
236
|
await this.ensureChangesTable(conn);
|
|
149
|
-
const columns = await
|
|
237
|
+
const columns = await tableColumnNames(conn, table);
|
|
150
238
|
if (columns.length === 0) {
|
|
151
239
|
throw new CDCError(`Table '${table}' does not exist or has no columns`);
|
|
152
240
|
}
|
|
153
241
|
for (const col of columns) {
|
|
154
242
|
this.assertIdentifier(col, `column name in table '${table}'`);
|
|
155
243
|
}
|
|
156
|
-
const pkColumns = await
|
|
244
|
+
const pkColumns = await tablePkColumns(conn, table);
|
|
157
245
|
const existing = this.watched.get(table);
|
|
158
246
|
if (existing) {
|
|
159
247
|
const same = existing.columns.length === columns.length && existing.columns.every((col, i) => col === columns[i]);
|
|
@@ -161,7 +249,7 @@ var ChangeTracker = class {
|
|
|
161
249
|
return;
|
|
162
250
|
}
|
|
163
251
|
await conn.transaction(async (txConn) => {
|
|
164
|
-
await
|
|
252
|
+
await dropCdcTriggers(txConn, table);
|
|
165
253
|
await this.installTriggers(txConn, table, columns, pkColumns);
|
|
166
254
|
});
|
|
167
255
|
} else {
|
|
@@ -174,7 +262,7 @@ var ChangeTracker = class {
|
|
|
174
262
|
if (!this.watched.has(table)) {
|
|
175
263
|
return;
|
|
176
264
|
}
|
|
177
|
-
await
|
|
265
|
+
await dropCdcTriggers(conn, table);
|
|
178
266
|
this.watched.delete(table);
|
|
179
267
|
this.watchedTablesCache = null;
|
|
180
268
|
}
|
|
@@ -209,7 +297,7 @@ var ChangeTracker = class {
|
|
|
209
297
|
for (const table of tables) {
|
|
210
298
|
const existing = this.watched.get(table);
|
|
211
299
|
if (!existing) continue;
|
|
212
|
-
const columns = await
|
|
300
|
+
const columns = await tableColumnNames(conn, table);
|
|
213
301
|
if (columns.length === 0) {
|
|
214
302
|
continue;
|
|
215
303
|
}
|
|
@@ -220,8 +308,8 @@ var ChangeTracker = class {
|
|
|
220
308
|
if (same) {
|
|
221
309
|
continue;
|
|
222
310
|
}
|
|
223
|
-
const pkColumns = await
|
|
224
|
-
await
|
|
311
|
+
const pkColumns = await tablePkColumns(conn, table);
|
|
312
|
+
await dropCdcTriggers(conn, table);
|
|
225
313
|
await this.installTriggers(conn, table, columns, pkColumns);
|
|
226
314
|
this.watched.set(table, { table, columns, pkColumns });
|
|
227
315
|
anyMutated = true;
|
|
@@ -256,7 +344,7 @@ var ChangeTracker = class {
|
|
|
256
344
|
if (!this.watched.has(table)) {
|
|
257
345
|
continue;
|
|
258
346
|
}
|
|
259
|
-
await
|
|
347
|
+
await dropCdcTriggers(conn, table);
|
|
260
348
|
this.watched.delete(table);
|
|
261
349
|
mutated = true;
|
|
262
350
|
}
|
|
@@ -271,22 +359,17 @@ var ChangeTracker = class {
|
|
|
271
359
|
return [];
|
|
272
360
|
}
|
|
273
361
|
}
|
|
274
|
-
const
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
`SELECT seq, table_name, operation, row_id, changed_at, old_data, new_data
|
|
278
|
-
FROM "${this.changesTable}"
|
|
279
|
-
WHERE seq > ?
|
|
280
|
-
ORDER BY seq ASC
|
|
281
|
-
LIMIT ?`
|
|
282
|
-
);
|
|
283
|
-
const rows = await stmt.all(this.lastSeq.toString(), this.pollBatchSize);
|
|
284
|
-
if (rows.length === 0) {
|
|
362
|
+
const result = await pollChanges(conn, this.stmtCache, this.changesTable, this.lastSeq, this.pollBatchSize);
|
|
363
|
+
if (result === null) {
|
|
364
|
+
this.lastPollAtTxBoundary = true;
|
|
285
365
|
return [];
|
|
286
366
|
}
|
|
287
|
-
|
|
288
|
-
this.lastSeq =
|
|
289
|
-
return events;
|
|
367
|
+
this.lastPollAtTxBoundary = result.atTxBoundary;
|
|
368
|
+
this.lastSeq = result.lastSeq;
|
|
369
|
+
return result.events;
|
|
370
|
+
}
|
|
371
|
+
get pollEndedAtTxBoundary() {
|
|
372
|
+
return this.lastPollAtTxBoundary;
|
|
290
373
|
}
|
|
291
374
|
/** The highest seq already polled; live subscribers receive events beyond it. */
|
|
292
375
|
get cursor() {
|
|
@@ -304,17 +387,17 @@ var ChangeTracker = class {
|
|
|
304
387
|
return [];
|
|
305
388
|
}
|
|
306
389
|
}
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
return
|
|
390
|
+
return readSinceOneTable(conn, this.stmtCache, this.changesTable, table, afterSeq, upToSeq, limit);
|
|
391
|
+
}
|
|
392
|
+
async readSinceTables(conn, tables, afterSeq, upToSeq, limit) {
|
|
393
|
+
if (tables.length === 0) return [];
|
|
394
|
+
if (!this.changesTableReady) {
|
|
395
|
+
await this.detectChangesTable(conn);
|
|
396
|
+
if (!this.changesTableReady) {
|
|
397
|
+
return [];
|
|
398
|
+
}
|
|
399
|
+
}
|
|
400
|
+
return readSinceTables(conn, this.stmtCache, this.changesTable, tables, afterSeq, upToSeq, limit);
|
|
318
401
|
}
|
|
319
402
|
/** The lowest retained seq, or `null` when the change log is empty. */
|
|
320
403
|
async getMinSeq(conn) {
|
|
@@ -324,7 +407,7 @@ var ChangeTracker = class {
|
|
|
324
407
|
return null;
|
|
325
408
|
}
|
|
326
409
|
}
|
|
327
|
-
const stmt = await this.
|
|
410
|
+
const stmt = await this.stmtCache.get(conn, "min_seq", selectMinChangeSeqSql(this.changesTable));
|
|
328
411
|
const row = await stmt.get();
|
|
329
412
|
const seq = row?.seq;
|
|
330
413
|
if (seq === void 0 || seq === null) {
|
|
@@ -332,16 +415,6 @@ var ChangeTracker = class {
|
|
|
332
415
|
}
|
|
333
416
|
return typeof seq === "bigint" ? seq : BigInt(String(seq));
|
|
334
417
|
}
|
|
335
|
-
rowToEvent(row) {
|
|
336
|
-
return {
|
|
337
|
-
type: row.operation.toLowerCase(),
|
|
338
|
-
table: row.table_name,
|
|
339
|
-
row: row.new_data ? decodeTaggedValues(JSON.parse(row.new_data)) : {},
|
|
340
|
-
oldRow: row.old_data ? decodeTaggedValues(JSON.parse(row.old_data)) : void 0,
|
|
341
|
-
seq: BigInt(row.seq),
|
|
342
|
-
timestamp: row.changed_at
|
|
343
|
-
};
|
|
344
|
-
}
|
|
345
418
|
async advanceToLatest(conn) {
|
|
346
419
|
if (!this.changesTableReady) {
|
|
347
420
|
await this.detectChangesTable(conn);
|
|
@@ -349,13 +422,7 @@ var ChangeTracker = class {
|
|
|
349
422
|
return;
|
|
350
423
|
}
|
|
351
424
|
}
|
|
352
|
-
const
|
|
353
|
-
const row = await stmt.get();
|
|
354
|
-
const seq = row?.seq;
|
|
355
|
-
if (seq === void 0 || seq === null) {
|
|
356
|
-
return;
|
|
357
|
-
}
|
|
358
|
-
const latestSeq = typeof seq === "bigint" ? seq : BigInt(String(seq));
|
|
425
|
+
const latestSeq = await selectMaxChangeSeq(conn, this.changesTable);
|
|
359
426
|
if (latestSeq > this.lastSeq) {
|
|
360
427
|
this.lastSeq = latestSeq;
|
|
361
428
|
}
|
|
@@ -370,15 +437,15 @@ var ChangeTracker = class {
|
|
|
370
437
|
const cutoff = Date.now() / 1e3 - this.retentionMs / 1e3;
|
|
371
438
|
const seqBound = this.computeSeqBound();
|
|
372
439
|
if (seqBound !== null) {
|
|
373
|
-
const stmt2 = await this.
|
|
440
|
+
const stmt2 = await this.stmtCache.get(
|
|
374
441
|
conn,
|
|
375
442
|
"cleanup_coordinated",
|
|
376
|
-
|
|
443
|
+
deleteChangesBeforeUpToSeqSql(this.changesTable)
|
|
377
444
|
);
|
|
378
445
|
const result2 = await stmt2.run(cutoff, seqBound.toString());
|
|
379
446
|
return result2.changes;
|
|
380
447
|
}
|
|
381
|
-
const stmt = await this.
|
|
448
|
+
const stmt = await this.stmtCache.get(conn, "cleanup", deleteChangesBeforeSql(this.changesTable));
|
|
382
449
|
const result = await stmt.run(cutoff);
|
|
383
450
|
return result.changes;
|
|
384
451
|
}
|
|
@@ -415,94 +482,85 @@ var ChangeTracker = class {
|
|
|
415
482
|
}
|
|
416
483
|
}
|
|
417
484
|
async detectChangesTable(conn) {
|
|
418
|
-
|
|
419
|
-
const row = await stmt.get(this.changesTable);
|
|
420
|
-
if (row) {
|
|
485
|
+
if (await selectTableExists(conn, this.changesTable)) {
|
|
421
486
|
this.changesTableReady = true;
|
|
422
487
|
}
|
|
423
488
|
}
|
|
424
489
|
async ensureChangesTable(conn) {
|
|
425
|
-
if (this.
|
|
490
|
+
if (this.changesTableEnsured) {
|
|
426
491
|
return;
|
|
427
492
|
}
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
CREATE TABLE IF NOT EXISTS "${this.changesTable}" (
|
|
431
|
-
seq INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
432
|
-
table_name TEXT NOT NULL,
|
|
433
|
-
operation TEXT NOT NULL,
|
|
434
|
-
row_id TEXT NOT NULL,
|
|
435
|
-
changed_at REAL NOT NULL DEFAULT (unixepoch('subsec')),
|
|
436
|
-
old_data TEXT,
|
|
437
|
-
new_data TEXT,
|
|
438
|
-
node_id TEXT NOT NULL DEFAULT '',
|
|
439
|
-
tx_id TEXT NOT NULL DEFAULT '',
|
|
440
|
-
hlc TEXT NOT NULL DEFAULT ''
|
|
441
|
-
)`);
|
|
442
|
-
await conn.exec(
|
|
443
|
-
`CREATE INDEX IF NOT EXISTS "idx_${this.changesTable}_changed_at" ON "${this.changesTable}" (changed_at)`
|
|
444
|
-
);
|
|
445
|
-
await conn.exec(
|
|
446
|
-
`CREATE INDEX IF NOT EXISTS "idx_${this.changesTable}_node_id" ON "${this.changesTable}" (node_id)`
|
|
447
|
-
);
|
|
448
|
-
await conn.exec(`CREATE INDEX IF NOT EXISTS "idx_${this.changesTable}_hlc" ON "${this.changesTable}" (hlc)`);
|
|
449
|
-
} else {
|
|
450
|
-
await conn.exec(`
|
|
451
|
-
CREATE TABLE IF NOT EXISTS "${this.changesTable}" (
|
|
452
|
-
seq INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
453
|
-
table_name TEXT NOT NULL,
|
|
454
|
-
operation TEXT NOT NULL,
|
|
455
|
-
row_id TEXT NOT NULL,
|
|
456
|
-
changed_at REAL NOT NULL DEFAULT (unixepoch('subsec')),
|
|
457
|
-
old_data TEXT,
|
|
458
|
-
new_data TEXT
|
|
459
|
-
)`);
|
|
460
|
-
await conn.exec(
|
|
461
|
-
`CREATE INDEX IF NOT EXISTS "idx_${this.changesTable}_changed_at" ON "${this.changesTable}" (changed_at)`
|
|
462
|
-
);
|
|
463
|
-
}
|
|
493
|
+
await ensureChangesTable(conn, this.changesTable);
|
|
494
|
+
this.changesTableEnsured = true;
|
|
464
495
|
this.changesTableReady = true;
|
|
465
496
|
}
|
|
466
|
-
async getColumns(conn, table) {
|
|
467
|
-
const stmt = await conn.prepare(`PRAGMA table_info("${table}")`);
|
|
468
|
-
const info = await stmt.all();
|
|
469
|
-
return info.map((col) => col.name);
|
|
470
|
-
}
|
|
471
|
-
async getPkColumns(conn, table) {
|
|
472
|
-
const stmt = await conn.prepare(`PRAGMA table_info("${table}")`);
|
|
473
|
-
const info = await stmt.all();
|
|
474
|
-
return info.filter((col) => col.pk > 0).sort((a, b) => a.pk - b.pk).map((col) => col.name);
|
|
475
|
-
}
|
|
476
|
-
async dropTriggers(conn, table) {
|
|
477
|
-
await dropCdcTriggers(conn, table);
|
|
478
|
-
}
|
|
479
|
-
async getStmt(conn, key, sql) {
|
|
480
|
-
let stmts = this.stmtCache.get(conn);
|
|
481
|
-
if (!stmts) {
|
|
482
|
-
stmts = /* @__PURE__ */ new Map();
|
|
483
|
-
this.stmtCache.set(conn, stmts);
|
|
484
|
-
}
|
|
485
|
-
const existing = stmts.get(key);
|
|
486
|
-
if (existing) return existing;
|
|
487
|
-
const pending = conn.prepare(sql);
|
|
488
|
-
stmts.set(key, pending);
|
|
489
|
-
try {
|
|
490
|
-
return await pending;
|
|
491
|
-
} catch (err) {
|
|
492
|
-
stmts.delete(key);
|
|
493
|
-
throw err;
|
|
494
|
-
}
|
|
495
|
-
}
|
|
496
497
|
async installTriggers(conn, table, columns, pkColumns) {
|
|
497
|
-
await installCdcTriggers(conn, this.changesTable, table, columns, pkColumns
|
|
498
|
+
await installCdcTriggers(conn, this.changesTable, table, columns, pkColumns);
|
|
498
499
|
}
|
|
499
500
|
};
|
|
500
501
|
|
|
502
|
+
// src/core/migrations/checksum.ts
|
|
503
|
+
var FNV_OFFSET_BASIS = 0xcbf29ce484222325n;
|
|
504
|
+
var FNV_PRIME = 0x100000001b3n;
|
|
505
|
+
var MASK_64 = 0xffffffffffffffffn;
|
|
506
|
+
function migrationChecksum(sql) {
|
|
507
|
+
const bytes = new TextEncoder().encode(sql.replace(/\r\n?/g, "\n").trim());
|
|
508
|
+
let hash = FNV_OFFSET_BASIS;
|
|
509
|
+
for (let i = 0; i < bytes.length; i++) {
|
|
510
|
+
hash = (hash ^ BigInt(bytes[i])) * FNV_PRIME & MASK_64;
|
|
511
|
+
}
|
|
512
|
+
return hash.toString(16).padStart(16, "0");
|
|
513
|
+
}
|
|
514
|
+
function migrationContentChecksum(migration) {
|
|
515
|
+
return typeof migration.up === "string" ? migrationChecksum(migration.up) : null;
|
|
516
|
+
}
|
|
517
|
+
function reconcileMigrationChecksums(migrations, applied) {
|
|
518
|
+
const backfills = [];
|
|
519
|
+
for (const migration of migrations) {
|
|
520
|
+
const row = applied.get(migration.version);
|
|
521
|
+
if (!row) continue;
|
|
522
|
+
const computed = migrationContentChecksum(migration);
|
|
523
|
+
if (computed === null) continue;
|
|
524
|
+
if (row.checksum === null) {
|
|
525
|
+
backfills.push({ version: migration.version, checksum: computed });
|
|
526
|
+
continue;
|
|
527
|
+
}
|
|
528
|
+
if (row.checksum !== computed) {
|
|
529
|
+
throw new MigrationError(
|
|
530
|
+
`Migration ${migration.version}_${migration.name} was modified after it was applied`,
|
|
531
|
+
migration.version,
|
|
532
|
+
"MIGRATION_CHECKSUM_MISMATCH"
|
|
533
|
+
);
|
|
534
|
+
}
|
|
535
|
+
}
|
|
536
|
+
return backfills;
|
|
537
|
+
}
|
|
538
|
+
|
|
539
|
+
// src/core/random-hex.ts
|
|
540
|
+
function randomHex(byteLength) {
|
|
541
|
+
const bytes = new Uint8Array(byteLength);
|
|
542
|
+
globalThis.crypto.getRandomValues(bytes);
|
|
543
|
+
return Array.from(bytes, (value) => value.toString(16).padStart(2, "0")).join("");
|
|
544
|
+
}
|
|
545
|
+
|
|
546
|
+
// src/core/cdc/epoch.ts
|
|
547
|
+
var EPOCH_KEY = "cdc_epoch";
|
|
548
|
+
async function ensureCdcEpoch(conn) {
|
|
549
|
+
await ensureMetaTable(conn);
|
|
550
|
+
await insertMetaValueIfAbsent(conn, EPOCH_KEY, randomHex(16));
|
|
551
|
+
const value = await selectMetaValue(conn, EPOCH_KEY);
|
|
552
|
+
if (value === null || value.length === 0) {
|
|
553
|
+
throw new CDCError("Failed to read the CDC epoch identifier");
|
|
554
|
+
}
|
|
555
|
+
return value;
|
|
556
|
+
}
|
|
557
|
+
|
|
501
558
|
// src/core/cdc/subscription.ts
|
|
502
559
|
var SubscriptionManager = class {
|
|
503
560
|
nextId = 1;
|
|
504
561
|
subscriptions = /* @__PURE__ */ new Map();
|
|
505
562
|
byTable = /* @__PURE__ */ new Map();
|
|
563
|
+
batchEndListeners = /* @__PURE__ */ new Set();
|
|
506
564
|
subscribe(table, filter, callback) {
|
|
507
565
|
const id = this.nextId++;
|
|
508
566
|
this.subscriptions.set(id, { id, table, filter, callback });
|
|
@@ -532,16 +590,29 @@ var SubscriptionManager = class {
|
|
|
532
590
|
for (const id of ids) {
|
|
533
591
|
const sub = this.subscriptions.get(id);
|
|
534
592
|
if (!sub) continue;
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
}
|
|
593
|
+
const delivered = sub.filter === void 0 ? event : filteredChange(event, sub.filter);
|
|
594
|
+
if (delivered === null) continue;
|
|
538
595
|
try {
|
|
539
|
-
sub.callback(
|
|
596
|
+
sub.callback(delivered);
|
|
540
597
|
} catch {
|
|
541
598
|
}
|
|
542
599
|
}
|
|
543
600
|
}
|
|
544
601
|
}
|
|
602
|
+
addBatchEndListener(listener) {
|
|
603
|
+
this.batchEndListeners.add(listener);
|
|
604
|
+
return () => {
|
|
605
|
+
this.batchEndListeners.delete(listener);
|
|
606
|
+
};
|
|
607
|
+
}
|
|
608
|
+
endBatch(atTxBoundary) {
|
|
609
|
+
for (const listener of this.batchEndListeners) {
|
|
610
|
+
try {
|
|
611
|
+
listener(atTxBoundary);
|
|
612
|
+
} catch {
|
|
613
|
+
}
|
|
614
|
+
}
|
|
615
|
+
}
|
|
545
616
|
get size() {
|
|
546
617
|
return this.subscriptions.size;
|
|
547
618
|
}
|
|
@@ -579,6 +650,7 @@ function startPolling(conn, tracker, manager, intervalMs, onError, runExclusive)
|
|
|
579
650
|
if (events.length > 0) {
|
|
580
651
|
manager.dispatch(events);
|
|
581
652
|
}
|
|
653
|
+
manager.endBatch(tracker.pollEndedAtTxBoundary);
|
|
582
654
|
consecutiveErrors = 0;
|
|
583
655
|
tickCount++;
|
|
584
656
|
if (tickCount >= CLEANUP_INTERVAL_TICKS) {
|
|
@@ -604,10 +676,23 @@ function startPolling(conn, tracker, manager, intervalMs, onError, runExclusive)
|
|
|
604
676
|
};
|
|
605
677
|
return stop;
|
|
606
678
|
}
|
|
607
|
-
function
|
|
608
|
-
const
|
|
679
|
+
function filteredChange(event, filter) {
|
|
680
|
+
const matchedBefore = event.type !== "insert" && event.oldRow !== void 0 && rowMatchesFilter(event.oldRow, filter);
|
|
681
|
+
const matchedAfter = event.type !== "delete" && rowMatchesFilter(event.row, filter);
|
|
682
|
+
if (!matchedBefore && !matchedAfter) return null;
|
|
683
|
+
if (matchedBefore && matchedAfter) return event;
|
|
684
|
+
if (matchedAfter) {
|
|
685
|
+
if (event.type === "insert") return event;
|
|
686
|
+
const entering = { ...event, type: "insert" };
|
|
687
|
+
entering.oldRow = void 0;
|
|
688
|
+
return entering;
|
|
689
|
+
}
|
|
690
|
+
if (event.type === "delete") return event;
|
|
691
|
+
return { ...event, type: "delete", row: {} };
|
|
692
|
+
}
|
|
693
|
+
function rowMatchesFilter(row, filter) {
|
|
609
694
|
for (const [key, value] of Object.entries(filter)) {
|
|
610
|
-
if (!filterValueMatches(
|
|
695
|
+
if (!filterValueMatches(row[key], value)) {
|
|
611
696
|
return false;
|
|
612
697
|
}
|
|
613
698
|
}
|
|
@@ -631,4 +716,130 @@ function filterValueMatches(rowValue, filterValue) {
|
|
|
631
716
|
return false;
|
|
632
717
|
}
|
|
633
718
|
|
|
634
|
-
|
|
719
|
+
// src/core/cdc/primed-subscription.ts
|
|
720
|
+
var REPLAY_BATCH_SIZE = 1e3;
|
|
721
|
+
var DEFAULT_PRIME_BUFFER_LIMIT = 1e4;
|
|
722
|
+
var DEFAULT_PRIME_BUFFER_BYTES = 16 * 1048576;
|
|
723
|
+
function approxEventBytes(event) {
|
|
724
|
+
let bytes = 0;
|
|
725
|
+
if (event.row) bytes += JSON.stringify(encodeTaggedValues(event.row)).length;
|
|
726
|
+
if (event.oldRow) bytes += JSON.stringify(encodeTaggedValues(event.oldRow)).length;
|
|
727
|
+
return bytes;
|
|
728
|
+
}
|
|
729
|
+
function needsResync(sinceSeq, minSeq, boundary) {
|
|
730
|
+
if (sinceSeq >= boundary) return false;
|
|
731
|
+
if (minSeq === null) return true;
|
|
732
|
+
return minSeq > sinceSeq + 1n;
|
|
733
|
+
}
|
|
734
|
+
var PrimedSubscription = class {
|
|
735
|
+
constructor(sinceSeq, deliver, onOverload, bufferLimit = DEFAULT_PRIME_BUFFER_LIMIT, bufferByteLimit = DEFAULT_PRIME_BUFFER_BYTES) {
|
|
736
|
+
this.sinceSeq = sinceSeq;
|
|
737
|
+
this.deliver = deliver;
|
|
738
|
+
this.onOverload = onOverload;
|
|
739
|
+
this.bufferLimit = bufferLimit;
|
|
740
|
+
this.bufferByteLimit = bufferByteLimit;
|
|
741
|
+
}
|
|
742
|
+
mode = "priming";
|
|
743
|
+
buffer = [];
|
|
744
|
+
bufferBytes = 0;
|
|
745
|
+
bufferLimit;
|
|
746
|
+
bufferByteLimit;
|
|
747
|
+
onLiveEvent(event) {
|
|
748
|
+
if (this.mode === "stopped") return;
|
|
749
|
+
if (this.mode === "live") {
|
|
750
|
+
this.emit(event);
|
|
751
|
+
return;
|
|
752
|
+
}
|
|
753
|
+
const nextBytes = this.bufferBytes + approxEventBytes(event);
|
|
754
|
+
if (this.buffer.length >= this.bufferLimit || nextBytes > this.bufferByteLimit) {
|
|
755
|
+
this.stop();
|
|
756
|
+
this.onOverload();
|
|
757
|
+
return;
|
|
758
|
+
}
|
|
759
|
+
this.buffer.push(event);
|
|
760
|
+
this.bufferBytes = nextBytes;
|
|
761
|
+
}
|
|
762
|
+
replay(tracker, conn, table, filter, boundary) {
|
|
763
|
+
return this.replayTables(tracker, conn, [table], filter, boundary);
|
|
764
|
+
}
|
|
765
|
+
async replayTables(tracker, conn, tables, filter, boundary) {
|
|
766
|
+
let cursor = this.sinceSeq;
|
|
767
|
+
while (this.mode === "priming" && cursor < boundary) {
|
|
768
|
+
const events = await tracker.readSinceTables(conn, tables, cursor, boundary, REPLAY_BATCH_SIZE);
|
|
769
|
+
if (events.length === 0) break;
|
|
770
|
+
for (const event of events) {
|
|
771
|
+
const delivered = filter === void 0 ? event : filteredChange(event, filter);
|
|
772
|
+
if (delivered === null) continue;
|
|
773
|
+
if (!this.emit(delivered)) return;
|
|
774
|
+
}
|
|
775
|
+
const lastEvent = events[events.length - 1];
|
|
776
|
+
if (lastEvent === void 0) break;
|
|
777
|
+
cursor = lastEvent.seq;
|
|
778
|
+
if (events.length < REPLAY_BATCH_SIZE) break;
|
|
779
|
+
}
|
|
780
|
+
}
|
|
781
|
+
goLive() {
|
|
782
|
+
if (this.mode !== "priming") return;
|
|
783
|
+
const pending = this.buffer;
|
|
784
|
+
this.buffer = [];
|
|
785
|
+
for (const event of pending) {
|
|
786
|
+
if (!this.emit(event)) return;
|
|
787
|
+
}
|
|
788
|
+
if (this.mode === "priming") {
|
|
789
|
+
this.mode = "live";
|
|
790
|
+
}
|
|
791
|
+
}
|
|
792
|
+
emit(event) {
|
|
793
|
+
if (this.mode === "stopped") return false;
|
|
794
|
+
if (event.seq <= this.sinceSeq) return true;
|
|
795
|
+
if (!this.deliver(event)) {
|
|
796
|
+
this.stop();
|
|
797
|
+
return false;
|
|
798
|
+
}
|
|
799
|
+
return true;
|
|
800
|
+
}
|
|
801
|
+
stop() {
|
|
802
|
+
this.mode = "stopped";
|
|
803
|
+
this.buffer = [];
|
|
804
|
+
this.bufferBytes = 0;
|
|
805
|
+
}
|
|
806
|
+
};
|
|
807
|
+
|
|
808
|
+
// src/core/cdc/transaction-grouper.ts
|
|
809
|
+
var TransactionGrouper = class {
|
|
810
|
+
constructor(deliver) {
|
|
811
|
+
this.deliver = deliver;
|
|
812
|
+
}
|
|
813
|
+
pending;
|
|
814
|
+
stopped = false;
|
|
815
|
+
receive(event) {
|
|
816
|
+
if (this.stopped) return false;
|
|
817
|
+
const held = this.pending;
|
|
818
|
+
this.pending = void 0;
|
|
819
|
+
if (held !== void 0 && !this.emit(held, held.txId !== event.txId)) {
|
|
820
|
+
return false;
|
|
821
|
+
}
|
|
822
|
+
if (event.txId === void 0) {
|
|
823
|
+
return this.emit(event, true);
|
|
824
|
+
}
|
|
825
|
+
this.pending = event;
|
|
826
|
+
return true;
|
|
827
|
+
}
|
|
828
|
+
flush(atTxBoundary) {
|
|
829
|
+
if (this.stopped) return false;
|
|
830
|
+
if (!atTxBoundary) return true;
|
|
831
|
+
const held = this.pending;
|
|
832
|
+
this.pending = void 0;
|
|
833
|
+
if (held === void 0) return true;
|
|
834
|
+
return this.emit(held, true);
|
|
835
|
+
}
|
|
836
|
+
emit(event, txEnd) {
|
|
837
|
+
const outgoing = txEnd && event.txEnd !== true ? { ...event, txEnd: true } : event;
|
|
838
|
+
if (this.deliver(outgoing)) return true;
|
|
839
|
+
this.stopped = true;
|
|
840
|
+
this.pending = void 0;
|
|
841
|
+
return false;
|
|
842
|
+
}
|
|
843
|
+
};
|
|
844
|
+
|
|
845
|
+
export { ChangeTracker, PrimedSubscription, SubscriptionBuilderImpl, SubscriptionManager, TransactionGrouper, ensureCdcEpoch, filteredChange, isBulkLoadDurability, migrationChecksum, migrationContentChecksum, needsResync, randomHex, reconcileMigrationChecksums, runBulkLoad, startPolling };
|