@delali/sirannon-db 0.1.4 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +415 -39
- package/dist/backup-scheduler/index.d.ts +2 -2
- package/dist/change-tracker-CFTQ9TSn.d.ts +89 -0
- package/dist/chunk-3MCMONVP.mjs +115 -0
- package/dist/chunk-ER7ODTDA.mjs +23 -0
- package/dist/chunk-GS7T5YMI.mjs +51 -0
- package/dist/{chunk-AX66KWBR.mjs → chunk-UTO3ZAFS.mjs} +226 -64
- package/dist/chunk-UVMVN3OT.mjs +111 -0
- package/dist/client/index.d.ts +99 -42
- package/dist/client/index.mjs +726 -26
- package/dist/core/index.d.ts +11 -108
- package/dist/core/index.mjs +134 -168
- package/dist/{sirannon-B1oTfebD.d.ts → database-BVY1GqE7.d.ts} +8 -33
- package/dist/errors-C00ed08Q.d.ts +101 -0
- package/dist/file-migrations/index.d.ts +2 -2
- package/dist/{index-hXiis3N-.d.ts → index-CLdNrcPz.d.ts} +1 -1
- package/dist/replication/coordinator/etcd.d.ts +44 -0
- package/dist/replication/coordinator/etcd.mjs +650 -0
- package/dist/replication/index.d.ts +491 -0
- package/dist/replication/index.mjs +3784 -0
- package/dist/server/index.d.ts +14 -3
- package/dist/server/index.mjs +262 -44
- package/dist/sirannon-Cd-lK6T0.d.ts +31 -0
- package/dist/transport/grpc.d.ts +316 -0
- package/dist/transport/grpc.mjs +3341 -0
- package/dist/transport/memory.d.ts +221 -0
- package/dist/transport/memory.mjs +337 -0
- package/dist/types-B2byqt0B.d.ts +273 -0
- package/dist/types-BEu1I_9_.d.ts +139 -0
- package/dist/types-BeozgNPr.d.ts +26 -0
- package/dist/{types-DtDutWRU.d.ts → types-D-74JiXb.d.ts} +78 -2
- package/package.json +54 -10
- package/dist/types-DRkJlqex.d.ts +0 -38
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
import { a as SQLiteConnection } from './types-BFSsG77t.js';
|
|
2
|
+
import { C as ChangeEvent } from './types-D-74JiXb.js';
|
|
3
|
+
|
|
4
|
+
interface ChangeTrackerOptions {
|
|
5
|
+
retention?: number;
|
|
6
|
+
changesTable?: string;
|
|
7
|
+
pollBatchSize?: number;
|
|
8
|
+
replication?: boolean;
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
declare class ChangeTracker {
|
|
12
|
+
private readonly watched;
|
|
13
|
+
private lastSeq;
|
|
14
|
+
private readonly retentionMs;
|
|
15
|
+
private readonly changesTable;
|
|
16
|
+
private readonly pollBatchSize;
|
|
17
|
+
private readonly replication;
|
|
18
|
+
private changesTableReady;
|
|
19
|
+
private watchedTablesCache;
|
|
20
|
+
private readonly stmtCache;
|
|
21
|
+
private pruneBoundary;
|
|
22
|
+
constructor(options?: ChangeTrackerOptions);
|
|
23
|
+
watch(conn: SQLiteConnection, table: string): Promise<void>;
|
|
24
|
+
unwatch(conn: SQLiteConnection, table: string): Promise<void>;
|
|
25
|
+
/**
|
|
26
|
+
* Rebuilds CDC triggers for every watched table directly on the supplied
|
|
27
|
+
* connection, without opening a nested transaction.
|
|
28
|
+
*
|
|
29
|
+
* Required when a caller has already issued `BEGIN` on `conn` and runs a
|
|
30
|
+
* DDL statement that changes a watched table's column list: the next DML
|
|
31
|
+
* inside the same transaction must see triggers compiled against the new
|
|
32
|
+
* columns, otherwise CDC `new_data` silently omits them. Callers that are
|
|
33
|
+
* not inside an active transaction may also use this method; CREATE
|
|
34
|
+
* TRIGGER and DROP TRIGGER statements are committed individually by the
|
|
35
|
+
* driver in that case.
|
|
36
|
+
*
|
|
37
|
+
* Failure semantics:
|
|
38
|
+
* - If a watched table no longer exists (e.g. it was just dropped by the
|
|
39
|
+
* DDL), it is skipped. The watched-map entry is left alone so a separate
|
|
40
|
+
* cleanup path can handle it; throwing here would roll back the user's
|
|
41
|
+
* transaction over a benign condition.
|
|
42
|
+
* - If reading column metadata succeeds but the column list is unchanged,
|
|
43
|
+
* no triggers are touched.
|
|
44
|
+
* - If reading column metadata succeeds and the column list differs from
|
|
45
|
+
* the cached one, the existing triggers are dropped and reinstalled on
|
|
46
|
+
* `conn`. The cached column list is updated on success.
|
|
47
|
+
* - Any other error (driver failure, identifier validation failure) is
|
|
48
|
+
* rethrown so the caller's transaction can roll back deterministically.
|
|
49
|
+
*/
|
|
50
|
+
refreshAllTriggersUsingConnection(conn: SQLiteConnection): Promise<void>;
|
|
51
|
+
/**
|
|
52
|
+
* Removes the supplied tables from the watched map and drops any leftover
|
|
53
|
+
* CDC triggers carrying their identifier on the supplied connection.
|
|
54
|
+
*
|
|
55
|
+
* Intended to be called after a DDL transaction that dropped one or more
|
|
56
|
+
* watched tables has committed. On rollback the caller must not invoke
|
|
57
|
+
* this method; the rollback semantics rely on the caller discarding its
|
|
58
|
+
* captured drop list before reaching this call.
|
|
59
|
+
*
|
|
60
|
+
* Idempotent: tables not currently in the watched map are silently
|
|
61
|
+
* skipped. `dropCdcTriggers` issues `DROP TRIGGER IF EXISTS` so calling
|
|
62
|
+
* twice in succession produces the same state.
|
|
63
|
+
*
|
|
64
|
+
* Defence in depth: even after a `DROP TABLE`, the in-transaction trigger
|
|
65
|
+
* refresh path may have observed a freshly-created table of the same name
|
|
66
|
+
* (the DROP and CREATE happened inside the same transaction) and
|
|
67
|
+
* re-installed triggers compiled against the new schema. Those triggers
|
|
68
|
+
* are dropped here so the recreated table starts with a clean slate and
|
|
69
|
+
* the caller must explicitly `watch` it again.
|
|
70
|
+
*/
|
|
71
|
+
pruneDroppedTables(conn: SQLiteConnection, tables: readonly string[]): Promise<void>;
|
|
72
|
+
poll(conn: SQLiteConnection): Promise<ChangeEvent[]>;
|
|
73
|
+
advanceToLatest(conn: SQLiteConnection): Promise<void>;
|
|
74
|
+
cleanup(conn: SQLiteConnection): Promise<number>;
|
|
75
|
+
setPruneBoundary(seq: bigint): void;
|
|
76
|
+
clearPruneBoundary(): void;
|
|
77
|
+
get watchedTables(): ReadonlySet<string>;
|
|
78
|
+
private computeSeqBound;
|
|
79
|
+
private assertIdentifier;
|
|
80
|
+
private detectChangesTable;
|
|
81
|
+
private ensureChangesTable;
|
|
82
|
+
private getColumns;
|
|
83
|
+
private getPkColumns;
|
|
84
|
+
private dropTriggers;
|
|
85
|
+
private getStmt;
|
|
86
|
+
private installTriggers;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
export { ChangeTracker as C, type ChangeTrackerOptions as a };
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
import { QueryError } from './chunk-O7BHI3CF.mjs';
|
|
2
|
+
|
|
3
|
+
// src/core/query-executor.ts
|
|
4
|
+
var STATEMENT_CACHE_CAPACITY = 128;
|
|
5
|
+
var statementCaches = /* @__PURE__ */ new WeakMap();
|
|
6
|
+
async function getStatement(conn, sql) {
|
|
7
|
+
let cache = statementCaches.get(conn);
|
|
8
|
+
if (!cache) {
|
|
9
|
+
cache = /* @__PURE__ */ new Map();
|
|
10
|
+
statementCaches.set(conn, cache);
|
|
11
|
+
}
|
|
12
|
+
const cached = cache.get(sql);
|
|
13
|
+
if (cached) {
|
|
14
|
+
cache.delete(sql);
|
|
15
|
+
cache.set(sql, cached);
|
|
16
|
+
return cached;
|
|
17
|
+
}
|
|
18
|
+
const pending = conn.prepare(sql);
|
|
19
|
+
cache.set(sql, pending);
|
|
20
|
+
if (cache.size > STATEMENT_CACHE_CAPACITY) {
|
|
21
|
+
const oldest = cache.keys().next().value;
|
|
22
|
+
if (oldest !== void 0) {
|
|
23
|
+
cache.delete(oldest);
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
try {
|
|
27
|
+
return await pending;
|
|
28
|
+
} catch (err) {
|
|
29
|
+
cache.delete(sql);
|
|
30
|
+
throw err;
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
function bindParams(params) {
|
|
34
|
+
if (params === void 0) return [];
|
|
35
|
+
if (Array.isArray(params)) return params;
|
|
36
|
+
return [params];
|
|
37
|
+
}
|
|
38
|
+
async function query(conn, sql, params) {
|
|
39
|
+
try {
|
|
40
|
+
const stmt = await getStatement(conn, sql);
|
|
41
|
+
return await stmt.all(...bindParams(params));
|
|
42
|
+
} catch (err) {
|
|
43
|
+
throw new QueryError(err instanceof Error ? err.message : String(err), sql);
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
async function queryOne(conn, sql, params) {
|
|
47
|
+
try {
|
|
48
|
+
const stmt = await getStatement(conn, sql);
|
|
49
|
+
return await stmt.get(...bindParams(params));
|
|
50
|
+
} catch (err) {
|
|
51
|
+
throw new QueryError(err instanceof Error ? err.message : String(err), sql);
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
async function execute(conn, sql, params) {
|
|
55
|
+
try {
|
|
56
|
+
const stmt = await getStatement(conn, sql);
|
|
57
|
+
const result = await stmt.run(...bindParams(params));
|
|
58
|
+
return {
|
|
59
|
+
changes: result.changes,
|
|
60
|
+
lastInsertRowId: result.lastInsertRowId
|
|
61
|
+
};
|
|
62
|
+
} catch (err) {
|
|
63
|
+
throw new QueryError(err instanceof Error ? err.message : String(err), sql);
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
async function executeBatch(conn, sql, paramsBatch) {
|
|
67
|
+
try {
|
|
68
|
+
const stmt = await getStatement(conn, sql);
|
|
69
|
+
const results = [];
|
|
70
|
+
for (const params of paramsBatch) {
|
|
71
|
+
const result = await stmt.run(...bindParams(params));
|
|
72
|
+
results.push({
|
|
73
|
+
changes: result.changes,
|
|
74
|
+
lastInsertRowId: result.lastInsertRowId
|
|
75
|
+
});
|
|
76
|
+
}
|
|
77
|
+
return results;
|
|
78
|
+
} catch (err) {
|
|
79
|
+
throw new QueryError(err instanceof Error ? err.message : String(err), sql);
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
// src/core/transaction.ts
|
|
84
|
+
var Transaction = class _Transaction {
|
|
85
|
+
constructor(conn) {
|
|
86
|
+
this.conn = conn;
|
|
87
|
+
}
|
|
88
|
+
_lastInsertRowId = 0;
|
|
89
|
+
async query(sql, params) {
|
|
90
|
+
return query(this.conn, sql, params);
|
|
91
|
+
}
|
|
92
|
+
async execute(sql, params) {
|
|
93
|
+
const result = await execute(this.conn, sql, params);
|
|
94
|
+
this._lastInsertRowId = result.lastInsertRowId;
|
|
95
|
+
return result;
|
|
96
|
+
}
|
|
97
|
+
async executeBatch(sql, paramsBatch) {
|
|
98
|
+
const results = await executeBatch(this.conn, sql, paramsBatch);
|
|
99
|
+
if (results.length > 0) {
|
|
100
|
+
this._lastInsertRowId = results[results.length - 1].lastInsertRowId;
|
|
101
|
+
}
|
|
102
|
+
return results;
|
|
103
|
+
}
|
|
104
|
+
get lastInsertRowId() {
|
|
105
|
+
return this._lastInsertRowId;
|
|
106
|
+
}
|
|
107
|
+
static async run(conn, fn) {
|
|
108
|
+
return conn.transaction(async (txConn) => {
|
|
109
|
+
const tx = new _Transaction(txConn);
|
|
110
|
+
return fn(tx);
|
|
111
|
+
});
|
|
112
|
+
}
|
|
113
|
+
};
|
|
114
|
+
|
|
115
|
+
export { Transaction, execute, executeBatch, query, queryOne };
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
// src/replication/coordinator/compatibility.ts
|
|
2
|
+
function compatibilityAllowsPromotion(groupCompatibility, sessionCompatibility) {
|
|
3
|
+
if (!groupCompatibility) return true;
|
|
4
|
+
return compatibleMajorVersion(groupCompatibility.packageVersion, sessionCompatibility?.packageVersion) && compatibleMajorVersion(groupCompatibility.protocolVersion, sessionCompatibility?.protocolVersion) && compatibleMajorVersion(groupCompatibility.specVersion, sessionCompatibility?.specVersion);
|
|
5
|
+
}
|
|
6
|
+
function compatibleMajorVersion(required, candidate) {
|
|
7
|
+
if (!required) return true;
|
|
8
|
+
if (!candidate) return false;
|
|
9
|
+
const requiredMajor = parseMajorVersion(required);
|
|
10
|
+
const candidateMajor = parseMajorVersion(candidate);
|
|
11
|
+
if (requiredMajor === null || candidateMajor === null) {
|
|
12
|
+
return required === candidate;
|
|
13
|
+
}
|
|
14
|
+
return requiredMajor === candidateMajor;
|
|
15
|
+
}
|
|
16
|
+
function parseMajorVersion(version) {
|
|
17
|
+
const match = /^v?(\d+)/.exec(version);
|
|
18
|
+
if (!match) return null;
|
|
19
|
+
const major = match[1];
|
|
20
|
+
return major === void 0 ? null : Number(major);
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export { compatibilityAllowsPromotion };
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
// src/core/cdc/encoding.ts
|
|
2
|
+
var BLOB_TAG = "__sirannon_blob";
|
|
3
|
+
var INT_TAG = "__sirannon_int";
|
|
4
|
+
var SAFE_INT_BOUND_TEXT = "9007199254740991";
|
|
5
|
+
var HEX_BYTES_RE = /^[0-9a-fA-F]*$/;
|
|
6
|
+
function decodeTaggedValues(value) {
|
|
7
|
+
if (value === null || typeof value !== "object") return value;
|
|
8
|
+
if (isBinaryValue(value)) return value;
|
|
9
|
+
if (Array.isArray(value)) {
|
|
10
|
+
return value.map(decodeTaggedValues);
|
|
11
|
+
}
|
|
12
|
+
const obj = value;
|
|
13
|
+
const keys = Object.keys(obj);
|
|
14
|
+
if (keys.length === 1) {
|
|
15
|
+
const tag = keys[0];
|
|
16
|
+
const tagValue = obj[tag];
|
|
17
|
+
if (tag === BLOB_TAG && typeof tagValue === "string") {
|
|
18
|
+
return decodeHexBytes(tagValue);
|
|
19
|
+
}
|
|
20
|
+
if (tag === INT_TAG && typeof tagValue === "string") {
|
|
21
|
+
return BigInt(tagValue);
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
const out = {};
|
|
25
|
+
for (const k of keys) {
|
|
26
|
+
out[k] = decodeTaggedValues(obj[k]);
|
|
27
|
+
}
|
|
28
|
+
return out;
|
|
29
|
+
}
|
|
30
|
+
function isBinaryValue(value) {
|
|
31
|
+
return value instanceof Uint8Array || typeof Buffer !== "undefined" && Buffer.isBuffer(value);
|
|
32
|
+
}
|
|
33
|
+
function decodeHexBytes(hex) {
|
|
34
|
+
if (hex.length % 2 !== 0 || !HEX_BYTES_RE.test(hex)) {
|
|
35
|
+
throw new Error("Invalid CDC BLOB hex payload");
|
|
36
|
+
}
|
|
37
|
+
if (typeof Buffer !== "undefined") {
|
|
38
|
+
return Buffer.from(hex, "hex");
|
|
39
|
+
}
|
|
40
|
+
const bytes = new Uint8Array(hex.length / 2);
|
|
41
|
+
for (let i = 0; i < hex.length; i += 2) {
|
|
42
|
+
const byte = Number.parseInt(hex.slice(i, i + 2), 16);
|
|
43
|
+
if (Number.isNaN(byte)) {
|
|
44
|
+
throw new Error("Invalid CDC BLOB hex payload");
|
|
45
|
+
}
|
|
46
|
+
bytes[i / 2] = byte;
|
|
47
|
+
}
|
|
48
|
+
return bytes;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
export { SAFE_INT_BOUND_TEXT, decodeTaggedValues };
|
|
@@ -1,5 +1,67 @@
|
|
|
1
|
+
import { decodeTaggedValues, SAFE_INT_BOUND_TEXT } from './chunk-GS7T5YMI.mjs';
|
|
1
2
|
import { CDCError } from './chunk-O7BHI3CF.mjs';
|
|
2
3
|
|
|
4
|
+
// src/core/cdc/trigger-sql.ts
|
|
5
|
+
async function dropCdcTriggers(conn, table) {
|
|
6
|
+
await conn.exec(`DROP TRIGGER IF EXISTS "_sirannon_trg_${table}_insert"`);
|
|
7
|
+
await conn.exec(`DROP TRIGGER IF EXISTS "_sirannon_trg_${table}_update"`);
|
|
8
|
+
await conn.exec(`DROP TRIGGER IF EXISTS "_sirannon_trg_${table}_delete"`);
|
|
9
|
+
}
|
|
10
|
+
async function installCdcTriggers(conn, changesTable, table, columns, pkColumns, replication) {
|
|
11
|
+
const newJson = buildJsonObject(columns, "NEW");
|
|
12
|
+
const oldJson = buildJsonObject(columns, "OLD");
|
|
13
|
+
const newPk = buildPkRef(pkColumns, "NEW");
|
|
14
|
+
const oldPk = buildPkRef(pkColumns, "OLD");
|
|
15
|
+
const replCols = replication ? ", node_id, tx_id, hlc" : "";
|
|
16
|
+
const replVals = replication ? ", '', '', ''" : "";
|
|
17
|
+
await conn.exec(`
|
|
18
|
+
CREATE TRIGGER IF NOT EXISTS "_sirannon_trg_${table}_insert"
|
|
19
|
+
AFTER INSERT ON "${table}"
|
|
20
|
+
BEGIN
|
|
21
|
+
INSERT INTO "${changesTable}" (table_name, operation, row_id, new_data${replCols})
|
|
22
|
+
VALUES ('${table}', 'INSERT', ${newPk}, ${newJson}${replVals});
|
|
23
|
+
END
|
|
24
|
+
`);
|
|
25
|
+
await conn.exec(`
|
|
26
|
+
CREATE TRIGGER IF NOT EXISTS "_sirannon_trg_${table}_update"
|
|
27
|
+
AFTER UPDATE ON "${table}"
|
|
28
|
+
BEGIN
|
|
29
|
+
INSERT INTO "${changesTable}" (table_name, operation, row_id, old_data, new_data${replCols})
|
|
30
|
+
VALUES ('${table}', 'UPDATE', ${newPk}, ${oldJson}, ${newJson}${replVals});
|
|
31
|
+
END
|
|
32
|
+
`);
|
|
33
|
+
await conn.exec(`
|
|
34
|
+
CREATE TRIGGER IF NOT EXISTS "_sirannon_trg_${table}_delete"
|
|
35
|
+
AFTER DELETE ON "${table}"
|
|
36
|
+
BEGIN
|
|
37
|
+
INSERT INTO "${changesTable}" (table_name, operation, row_id, old_data${replCols})
|
|
38
|
+
VALUES ('${table}', 'DELETE', ${oldPk}, ${oldJson}${replVals});
|
|
39
|
+
END
|
|
40
|
+
`);
|
|
41
|
+
}
|
|
42
|
+
function buildPkRef(pkColumns, ref) {
|
|
43
|
+
if (pkColumns.length === 0) {
|
|
44
|
+
return `${ref}.rowid`;
|
|
45
|
+
}
|
|
46
|
+
if (pkColumns.length === 1) {
|
|
47
|
+
return `${ref}."${escId(pkColumns[0])}"`;
|
|
48
|
+
}
|
|
49
|
+
return pkColumns.map((col) => `${ref}."${escId(col)}"`).join(" || '-' || ");
|
|
50
|
+
}
|
|
51
|
+
function buildJsonObject(columns, ref) {
|
|
52
|
+
const pairs = columns.map((col) => {
|
|
53
|
+
const ident = `${ref}."${escId(col)}"`;
|
|
54
|
+
return `'${escStr(col)}', CASE typeof(${ident}) WHEN 'blob' THEN json(json_object('__sirannon_blob', hex(${ident}))) WHEN 'integer' THEN CASE WHEN ${ident} > ${SAFE_INT_BOUND_TEXT} OR ${ident} < -${SAFE_INT_BOUND_TEXT} THEN json(json_object('__sirannon_int', printf('%d', ${ident}))) ELSE ${ident} END ELSE ${ident} END`;
|
|
55
|
+
}).join(", ");
|
|
56
|
+
return `json_object(${pairs})`;
|
|
57
|
+
}
|
|
58
|
+
function escId(name) {
|
|
59
|
+
return name.replace(/"/g, '""');
|
|
60
|
+
}
|
|
61
|
+
function escStr(name) {
|
|
62
|
+
return name.replace(/'/g, "''");
|
|
63
|
+
}
|
|
64
|
+
|
|
3
65
|
// src/core/cdc/change-tracker.ts
|
|
4
66
|
var DEFAULT_RETENTION_MS = 36e5;
|
|
5
67
|
var DEFAULT_CHANGES_TABLE = "_sirannon_changes";
|
|
@@ -7,17 +69,20 @@ var DEFAULT_POLL_BATCH_SIZE = 1e3;
|
|
|
7
69
|
var IDENTIFIER_RE = /^[a-zA-Z_][a-zA-Z0-9_]*$/;
|
|
8
70
|
var ChangeTracker = class {
|
|
9
71
|
watched = /* @__PURE__ */ new Map();
|
|
10
|
-
lastSeq =
|
|
72
|
+
lastSeq = 0n;
|
|
11
73
|
retentionMs;
|
|
12
74
|
changesTable;
|
|
13
75
|
pollBatchSize;
|
|
76
|
+
replication;
|
|
14
77
|
changesTableReady = false;
|
|
15
78
|
watchedTablesCache = null;
|
|
16
79
|
stmtCache = /* @__PURE__ */ new WeakMap();
|
|
80
|
+
pruneBoundary = null;
|
|
17
81
|
constructor(options) {
|
|
18
82
|
this.retentionMs = options?.retention ?? DEFAULT_RETENTION_MS;
|
|
19
83
|
this.changesTable = options?.changesTable ?? DEFAULT_CHANGES_TABLE;
|
|
20
84
|
this.pollBatchSize = options?.pollBatchSize ?? DEFAULT_POLL_BATCH_SIZE;
|
|
85
|
+
this.replication = options?.replication ?? false;
|
|
21
86
|
this.assertIdentifier(this.changesTable, "changes table name");
|
|
22
87
|
}
|
|
23
88
|
async watch(conn, table) {
|
|
@@ -55,6 +120,92 @@ var ChangeTracker = class {
|
|
|
55
120
|
this.watched.delete(table);
|
|
56
121
|
this.watchedTablesCache = null;
|
|
57
122
|
}
|
|
123
|
+
/**
|
|
124
|
+
* Rebuilds CDC triggers for every watched table directly on the supplied
|
|
125
|
+
* connection, without opening a nested transaction.
|
|
126
|
+
*
|
|
127
|
+
* Required when a caller has already issued `BEGIN` on `conn` and runs a
|
|
128
|
+
* DDL statement that changes a watched table's column list: the next DML
|
|
129
|
+
* inside the same transaction must see triggers compiled against the new
|
|
130
|
+
* columns, otherwise CDC `new_data` silently omits them. Callers that are
|
|
131
|
+
* not inside an active transaction may also use this method; CREATE
|
|
132
|
+
* TRIGGER and DROP TRIGGER statements are committed individually by the
|
|
133
|
+
* driver in that case.
|
|
134
|
+
*
|
|
135
|
+
* Failure semantics:
|
|
136
|
+
* - If a watched table no longer exists (e.g. it was just dropped by the
|
|
137
|
+
* DDL), it is skipped. The watched-map entry is left alone so a separate
|
|
138
|
+
* cleanup path can handle it; throwing here would roll back the user's
|
|
139
|
+
* transaction over a benign condition.
|
|
140
|
+
* - If reading column metadata succeeds but the column list is unchanged,
|
|
141
|
+
* no triggers are touched.
|
|
142
|
+
* - If reading column metadata succeeds and the column list differs from
|
|
143
|
+
* the cached one, the existing triggers are dropped and reinstalled on
|
|
144
|
+
* `conn`. The cached column list is updated on success.
|
|
145
|
+
* - Any other error (driver failure, identifier validation failure) is
|
|
146
|
+
* rethrown so the caller's transaction can roll back deterministically.
|
|
147
|
+
*/
|
|
148
|
+
async refreshAllTriggersUsingConnection(conn) {
|
|
149
|
+
const tables = Array.from(this.watched.keys());
|
|
150
|
+
let anyMutated = false;
|
|
151
|
+
for (const table of tables) {
|
|
152
|
+
const existing = this.watched.get(table);
|
|
153
|
+
if (!existing) continue;
|
|
154
|
+
const columns = await this.getColumns(conn, table);
|
|
155
|
+
if (columns.length === 0) {
|
|
156
|
+
continue;
|
|
157
|
+
}
|
|
158
|
+
for (const col of columns) {
|
|
159
|
+
this.assertIdentifier(col, `column name in table '${table}'`);
|
|
160
|
+
}
|
|
161
|
+
const same = existing.columns.length === columns.length && existing.columns.every((col, i) => col === columns[i]);
|
|
162
|
+
if (same) {
|
|
163
|
+
continue;
|
|
164
|
+
}
|
|
165
|
+
const pkColumns = await this.getPkColumns(conn, table);
|
|
166
|
+
await this.dropTriggers(conn, table);
|
|
167
|
+
await this.installTriggers(conn, table, columns, pkColumns);
|
|
168
|
+
this.watched.set(table, { table, columns, pkColumns });
|
|
169
|
+
anyMutated = true;
|
|
170
|
+
}
|
|
171
|
+
if (anyMutated) {
|
|
172
|
+
this.watchedTablesCache = null;
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
/**
|
|
176
|
+
* Removes the supplied tables from the watched map and drops any leftover
|
|
177
|
+
* CDC triggers carrying their identifier on the supplied connection.
|
|
178
|
+
*
|
|
179
|
+
* Intended to be called after a DDL transaction that dropped one or more
|
|
180
|
+
* watched tables has committed. On rollback the caller must not invoke
|
|
181
|
+
* this method; the rollback semantics rely on the caller discarding its
|
|
182
|
+
* captured drop list before reaching this call.
|
|
183
|
+
*
|
|
184
|
+
* Idempotent: tables not currently in the watched map are silently
|
|
185
|
+
* skipped. `dropCdcTriggers` issues `DROP TRIGGER IF EXISTS` so calling
|
|
186
|
+
* twice in succession produces the same state.
|
|
187
|
+
*
|
|
188
|
+
* Defence in depth: even after a `DROP TABLE`, the in-transaction trigger
|
|
189
|
+
* refresh path may have observed a freshly-created table of the same name
|
|
190
|
+
* (the DROP and CREATE happened inside the same transaction) and
|
|
191
|
+
* re-installed triggers compiled against the new schema. Those triggers
|
|
192
|
+
* are dropped here so the recreated table starts with a clean slate and
|
|
193
|
+
* the caller must explicitly `watch` it again.
|
|
194
|
+
*/
|
|
195
|
+
async pruneDroppedTables(conn, tables) {
|
|
196
|
+
let mutated = false;
|
|
197
|
+
for (const table of tables) {
|
|
198
|
+
if (!this.watched.has(table)) {
|
|
199
|
+
continue;
|
|
200
|
+
}
|
|
201
|
+
await this.dropTriggers(conn, table);
|
|
202
|
+
this.watched.delete(table);
|
|
203
|
+
mutated = true;
|
|
204
|
+
}
|
|
205
|
+
if (mutated) {
|
|
206
|
+
this.watchedTablesCache = null;
|
|
207
|
+
}
|
|
208
|
+
}
|
|
58
209
|
async poll(conn) {
|
|
59
210
|
if (!this.changesTableReady) {
|
|
60
211
|
await this.detectChangesTable(conn);
|
|
@@ -71,7 +222,7 @@ var ChangeTracker = class {
|
|
|
71
222
|
ORDER BY seq ASC
|
|
72
223
|
LIMIT ?`
|
|
73
224
|
);
|
|
74
|
-
const rows = await stmt.all(this.lastSeq, this.pollBatchSize);
|
|
225
|
+
const rows = await stmt.all(this.lastSeq.toString(), this.pollBatchSize);
|
|
75
226
|
if (rows.length === 0) {
|
|
76
227
|
return [];
|
|
77
228
|
}
|
|
@@ -80,15 +231,33 @@ var ChangeTracker = class {
|
|
|
80
231
|
events.push({
|
|
81
232
|
type: row.operation.toLowerCase(),
|
|
82
233
|
table: row.table_name,
|
|
83
|
-
row: row.new_data ? JSON.parse(row.new_data) : {},
|
|
84
|
-
oldRow: row.old_data ? JSON.parse(row.old_data) : void 0,
|
|
234
|
+
row: row.new_data ? decodeTaggedValues(JSON.parse(row.new_data)) : {},
|
|
235
|
+
oldRow: row.old_data ? decodeTaggedValues(JSON.parse(row.old_data)) : void 0,
|
|
85
236
|
seq: BigInt(row.seq),
|
|
86
237
|
timestamp: row.changed_at
|
|
87
238
|
});
|
|
88
239
|
}
|
|
89
|
-
this.lastSeq = rows[rows.length - 1].seq;
|
|
240
|
+
this.lastSeq = BigInt(rows[rows.length - 1].seq);
|
|
90
241
|
return events;
|
|
91
242
|
}
|
|
243
|
+
async advanceToLatest(conn) {
|
|
244
|
+
if (!this.changesTableReady) {
|
|
245
|
+
await this.detectChangesTable(conn);
|
|
246
|
+
if (!this.changesTableReady) {
|
|
247
|
+
return;
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
const stmt = await this.getStmt(conn, "latest_seq", `SELECT MAX(seq) AS seq FROM "${this.changesTable}"`);
|
|
251
|
+
const row = await stmt.get();
|
|
252
|
+
const seq = row?.seq;
|
|
253
|
+
if (seq === void 0 || seq === null) {
|
|
254
|
+
return;
|
|
255
|
+
}
|
|
256
|
+
const latestSeq = typeof seq === "bigint" ? seq : BigInt(String(seq));
|
|
257
|
+
if (latestSeq > this.lastSeq) {
|
|
258
|
+
this.lastSeq = latestSeq;
|
|
259
|
+
}
|
|
260
|
+
}
|
|
92
261
|
async cleanup(conn) {
|
|
93
262
|
if (!this.changesTableReady) {
|
|
94
263
|
await this.detectChangesTable(conn);
|
|
@@ -97,25 +266,45 @@ var ChangeTracker = class {
|
|
|
97
266
|
}
|
|
98
267
|
}
|
|
99
268
|
const cutoff = Date.now() / 1e3 - this.retentionMs / 1e3;
|
|
100
|
-
|
|
269
|
+
const seqBound = this.computeSeqBound();
|
|
270
|
+
if (seqBound !== null) {
|
|
101
271
|
const stmt2 = await this.getStmt(
|
|
102
272
|
conn,
|
|
103
273
|
"cleanup_coordinated",
|
|
104
274
|
`DELETE FROM "${this.changesTable}" WHERE changed_at < ? AND seq <= ?`
|
|
105
275
|
);
|
|
106
|
-
const result2 = await stmt2.run(cutoff,
|
|
276
|
+
const result2 = await stmt2.run(cutoff, seqBound.toString());
|
|
107
277
|
return result2.changes;
|
|
108
278
|
}
|
|
109
279
|
const stmt = await this.getStmt(conn, "cleanup", `DELETE FROM "${this.changesTable}" WHERE changed_at < ?`);
|
|
110
280
|
const result = await stmt.run(cutoff);
|
|
111
281
|
return result.changes;
|
|
112
282
|
}
|
|
283
|
+
setPruneBoundary(seq) {
|
|
284
|
+
this.pruneBoundary = seq;
|
|
285
|
+
}
|
|
286
|
+
clearPruneBoundary() {
|
|
287
|
+
this.pruneBoundary = null;
|
|
288
|
+
}
|
|
113
289
|
get watchedTables() {
|
|
114
290
|
if (!this.watchedTablesCache) {
|
|
115
291
|
this.watchedTablesCache = new Set(this.watched.keys());
|
|
116
292
|
}
|
|
117
293
|
return this.watchedTablesCache;
|
|
118
294
|
}
|
|
295
|
+
computeSeqBound() {
|
|
296
|
+
const boundary = this.pruneBoundary;
|
|
297
|
+
if (this.lastSeq > 0n && boundary !== null) {
|
|
298
|
+
return this.lastSeq < boundary ? this.lastSeq : boundary;
|
|
299
|
+
}
|
|
300
|
+
if (boundary !== null) {
|
|
301
|
+
return boundary;
|
|
302
|
+
}
|
|
303
|
+
if (this.lastSeq > 0n) {
|
|
304
|
+
return this.lastSeq;
|
|
305
|
+
}
|
|
306
|
+
return null;
|
|
307
|
+
}
|
|
119
308
|
assertIdentifier(name, label) {
|
|
120
309
|
if (!IDENTIFIER_RE.test(name)) {
|
|
121
310
|
throw new CDCError(
|
|
@@ -134,7 +323,29 @@ var ChangeTracker = class {
|
|
|
134
323
|
if (this.changesTableReady) {
|
|
135
324
|
return;
|
|
136
325
|
}
|
|
137
|
-
|
|
326
|
+
if (this.replication) {
|
|
327
|
+
await conn.exec(`
|
|
328
|
+
CREATE TABLE IF NOT EXISTS "${this.changesTable}" (
|
|
329
|
+
seq INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
330
|
+
table_name TEXT NOT NULL,
|
|
331
|
+
operation TEXT NOT NULL,
|
|
332
|
+
row_id TEXT NOT NULL,
|
|
333
|
+
changed_at REAL NOT NULL DEFAULT (unixepoch('subsec')),
|
|
334
|
+
old_data TEXT,
|
|
335
|
+
new_data TEXT,
|
|
336
|
+
node_id TEXT NOT NULL DEFAULT '',
|
|
337
|
+
tx_id TEXT NOT NULL DEFAULT '',
|
|
338
|
+
hlc TEXT NOT NULL DEFAULT ''
|
|
339
|
+
)`);
|
|
340
|
+
await conn.exec(
|
|
341
|
+
`CREATE INDEX IF NOT EXISTS "idx_${this.changesTable}_changed_at" ON "${this.changesTable}" (changed_at)`
|
|
342
|
+
);
|
|
343
|
+
await conn.exec(
|
|
344
|
+
`CREATE INDEX IF NOT EXISTS "idx_${this.changesTable}_node_id" ON "${this.changesTable}" (node_id)`
|
|
345
|
+
);
|
|
346
|
+
await conn.exec(`CREATE INDEX IF NOT EXISTS "idx_${this.changesTable}_hlc" ON "${this.changesTable}" (hlc)`);
|
|
347
|
+
} else {
|
|
348
|
+
await conn.exec(`
|
|
138
349
|
CREATE TABLE IF NOT EXISTS "${this.changesTable}" (
|
|
139
350
|
seq INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
140
351
|
table_name TEXT NOT NULL,
|
|
@@ -144,9 +355,10 @@ CREATE TABLE IF NOT EXISTS "${this.changesTable}" (
|
|
|
144
355
|
old_data TEXT,
|
|
145
356
|
new_data TEXT
|
|
146
357
|
)`);
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
358
|
+
await conn.exec(
|
|
359
|
+
`CREATE INDEX IF NOT EXISTS "idx_${this.changesTable}_changed_at" ON "${this.changesTable}" (changed_at)`
|
|
360
|
+
);
|
|
361
|
+
}
|
|
150
362
|
this.changesTableReady = true;
|
|
151
363
|
}
|
|
152
364
|
async getColumns(conn, table) {
|
|
@@ -160,9 +372,7 @@ CREATE TABLE IF NOT EXISTS "${this.changesTable}" (
|
|
|
160
372
|
return info.filter((col) => col.pk > 0).sort((a, b) => a.pk - b.pk).map((col) => col.name);
|
|
161
373
|
}
|
|
162
374
|
async dropTriggers(conn, table) {
|
|
163
|
-
await conn
|
|
164
|
-
await conn.exec(`DROP TRIGGER IF EXISTS "_sirannon_trg_${table}_update"`);
|
|
165
|
-
await conn.exec(`DROP TRIGGER IF EXISTS "_sirannon_trg_${table}_delete"`);
|
|
375
|
+
await dropCdcTriggers(conn, table);
|
|
166
376
|
}
|
|
167
377
|
async getStmt(conn, key, sql) {
|
|
168
378
|
let stmts = this.stmtCache.get(conn);
|
|
@@ -181,54 +391,8 @@ CREATE TABLE IF NOT EXISTS "${this.changesTable}" (
|
|
|
181
391
|
throw err;
|
|
182
392
|
}
|
|
183
393
|
}
|
|
184
|
-
buildPkRef(pkColumns, ref) {
|
|
185
|
-
if (pkColumns.length === 0) {
|
|
186
|
-
return `${ref}.rowid`;
|
|
187
|
-
}
|
|
188
|
-
if (pkColumns.length === 1) {
|
|
189
|
-
return `${ref}."${this.escId(pkColumns[0])}"`;
|
|
190
|
-
}
|
|
191
|
-
return pkColumns.map((col) => `${ref}."${this.escId(col)}"`).join(" || '-' || ");
|
|
192
|
-
}
|
|
193
394
|
async installTriggers(conn, table, columns, pkColumns) {
|
|
194
|
-
|
|
195
|
-
const oldJson = this.buildJsonObject(columns, "OLD");
|
|
196
|
-
const newPk = this.buildPkRef(pkColumns, "NEW");
|
|
197
|
-
const oldPk = this.buildPkRef(pkColumns, "OLD");
|
|
198
|
-
await conn.exec(`
|
|
199
|
-
CREATE TRIGGER IF NOT EXISTS "_sirannon_trg_${table}_insert"
|
|
200
|
-
AFTER INSERT ON "${table}"
|
|
201
|
-
BEGIN
|
|
202
|
-
INSERT INTO "${this.changesTable}" (table_name, operation, row_id, new_data)
|
|
203
|
-
VALUES ('${table}', 'INSERT', ${newPk}, ${newJson});
|
|
204
|
-
END
|
|
205
|
-
`);
|
|
206
|
-
await conn.exec(`
|
|
207
|
-
CREATE TRIGGER IF NOT EXISTS "_sirannon_trg_${table}_update"
|
|
208
|
-
AFTER UPDATE ON "${table}"
|
|
209
|
-
BEGIN
|
|
210
|
-
INSERT INTO "${this.changesTable}" (table_name, operation, row_id, old_data, new_data)
|
|
211
|
-
VALUES ('${table}', 'UPDATE', ${newPk}, ${oldJson}, ${newJson});
|
|
212
|
-
END
|
|
213
|
-
`);
|
|
214
|
-
await conn.exec(`
|
|
215
|
-
CREATE TRIGGER IF NOT EXISTS "_sirannon_trg_${table}_delete"
|
|
216
|
-
AFTER DELETE ON "${table}"
|
|
217
|
-
BEGIN
|
|
218
|
-
INSERT INTO "${this.changesTable}" (table_name, operation, row_id, old_data)
|
|
219
|
-
VALUES ('${table}', 'DELETE', ${oldPk}, ${oldJson});
|
|
220
|
-
END
|
|
221
|
-
`);
|
|
222
|
-
}
|
|
223
|
-
buildJsonObject(columns, ref) {
|
|
224
|
-
const pairs = columns.map((col) => `'${this.escStr(col)}', ${ref}."${this.escId(col)}"`).join(", ");
|
|
225
|
-
return `json_object(${pairs})`;
|
|
226
|
-
}
|
|
227
|
-
escId(name) {
|
|
228
|
-
return name.replace(/"/g, '""');
|
|
229
|
-
}
|
|
230
|
-
escStr(name) {
|
|
231
|
-
return name.replace(/'/g, "''");
|
|
395
|
+
await installCdcTriggers(conn, this.changesTable, table, columns, pkColumns, this.replication);
|
|
232
396
|
}
|
|
233
397
|
};
|
|
234
398
|
|
|
@@ -331,9 +495,7 @@ function startPolling(conn, tracker, manager, intervalMs, onError) {
|
|
|
331
495
|
}
|
|
332
496
|
};
|
|
333
497
|
const interval = setInterval(tick, intervalMs);
|
|
334
|
-
|
|
335
|
-
interval.unref();
|
|
336
|
-
}
|
|
498
|
+
interval.unref?.();
|
|
337
499
|
const stop = () => {
|
|
338
500
|
clearInterval(interval);
|
|
339
501
|
};
|