@jarenjs/db 0.49.2 → 0.66.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +420 -71
- package/README.md +711 -79
- package/docs/HOSTS.md +269 -0
- package/docs/JOBS-FORMAT.md +309 -45
- package/docs/LIVE-FORMAT.md +156 -19
- package/docs/MIGRATION-FORMAT.md +247 -40
- package/docs/MODEL-FORMAT.md +968 -86
- package/package.json +21 -8
- package/schemas/jaren-migration.draft-07.schema.json +73 -0
- package/schemas/jaren-migration.schema.json +73 -0
- package/schemas/jaren-model.draft-07.schema.json +224 -162
- package/schemas/jaren-model.schema.json +224 -162
- package/src/algebra.js +227 -9
- package/src/backup.js +161 -0
- package/src/cancellation.js +48 -0
- package/src/capture.js +255 -44
- package/src/cli.js +337 -50
- package/src/cursor.js +411 -0
- package/src/dag-job.js +154 -21
- package/src/ddl.js +125 -11
- package/src/dialect.js +267 -112
- package/src/dialects/expression-read.js +158 -0
- package/src/dialects/postgres.js +618 -0
- package/src/dialects/rtree-ddl.js +129 -0
- package/src/dialects/sqlite.js +245 -12
- package/src/document-files.js +311 -0
- package/src/document-steps.js +422 -0
- package/src/documents.js +335 -0
- package/src/driver.js +503 -69
- package/src/drivers/bun.js +37 -1
- package/src/drivers/indexeddb-snapshot.js +149 -0
- package/src/drivers/node-pool.js +11 -0
- package/src/drivers/node-worker-endpoint.js +105 -0
- package/src/drivers/node-worker.js +204 -0
- package/src/drivers/node.js +41 -7
- package/src/drivers/postgres.js +331 -0
- package/src/drivers/wasm-oo1.js +97 -0
- package/src/drivers/wasm-session.js +67 -0
- package/src/drivers/wasm.js +18 -83
- package/src/drivers/worker-pool.js +183 -0
- package/src/drivers/worker-protocol.js +79 -0
- package/src/drivers/worker-queue.js +60 -0
- package/src/emit-model.js +14 -0
- package/src/emit.js +349 -51
- package/src/entity.js +102 -59
- package/src/errors.js +430 -2
- package/src/expression.js +284 -0
- package/src/graph.js +64 -8
- package/src/index.js +48 -19
- package/src/introspect.js +583 -0
- package/src/jobs.js +870 -99
- package/src/json-bytes.js +58 -0
- package/src/live-time.js +12 -3
- package/src/live.js +11 -1
- package/src/maintenance.js +175 -0
- package/src/migrate.js +606 -333
- package/src/model.js +241 -8
- package/src/plan.js +1238 -160
- package/src/pragmas.js +314 -0
- package/src/profile.js +151 -3
- package/src/query.js +1748 -312
- package/src/residual.js +17 -0
- package/src/series.js +12 -4
- package/src/store.js +1672 -276
- package/src/tracker.js +367 -68
- package/src/udf.js +88 -7
- package/types/index.d.ts +1246 -32
- package/types/node-pool.d.ts +28 -0
- package/types/node-worker.d.ts +54 -0
- package/types/node.d.ts +72 -3
- package/types/postgres.d.ts +46 -0
- package/types/typed.d.ts +81 -3
- package/types/wasm.d.ts +21 -0
- package/dist/types/algebra.d.ts +0 -230
- package/dist/types/app.d.ts +0 -49
- package/dist/types/capture.d.ts +0 -85
- package/dist/types/cli.d.ts +0 -2
- package/dist/types/dag-job.d.ts +0 -40
- package/dist/types/ddl.d.ts +0 -229
- package/dist/types/derive.d.ts +0 -250
- package/dist/types/dialect.d.ts +0 -154
- package/dist/types/dialects/sqlite.d.ts +0 -9
- package/dist/types/driver.d.ts +0 -110
- package/dist/types/drivers/bun.d.ts +0 -47
- package/dist/types/drivers/node.d.ts +0 -37
- package/dist/types/drivers/wasm.d.ts +0 -65
- package/dist/types/emit-model.d.ts +0 -44
- package/dist/types/emit.d.ts +0 -75
- package/dist/types/entity.d.ts +0 -23
- package/dist/types/errors.d.ts +0 -170
- package/dist/types/graph.d.ts +0 -28
- package/dist/types/index.d.ts +0 -37
- package/dist/types/jobs.d.ts +0 -140
- package/dist/types/knn.d.ts +0 -69
- package/dist/types/live-time.d.ts +0 -141
- package/dist/types/live.d.ts +0 -64
- package/dist/types/migrate.d.ts +0 -170
- package/dist/types/model.d.ts +0 -36
- package/dist/types/patch-sql.d.ts +0 -37
- package/dist/types/plan.d.ts +0 -142
- package/dist/types/profile.d.ts +0 -80
- package/dist/types/query.d.ts +0 -112
- package/dist/types/residual.d.ts +0 -64
- package/dist/types/series.d.ts +0 -227
- package/dist/types/store.d.ts +0 -60
- package/dist/types/tracker.d.ts +0 -43
- package/dist/types/typed.d.ts +0 -15
- package/dist/types/types.d.ts +0 -26
- package/dist/types/udf.d.ts +0 -75
- package/dist/types/window.d.ts +0 -52
package/src/drivers/bun.js
CHANGED
|
@@ -20,6 +20,7 @@
|
|
|
20
20
|
|
|
21
21
|
import { lazyOpen, openConnection } from '../driver.js';
|
|
22
22
|
import { sqliteDialect } from '../dialects/sqlite.js';
|
|
23
|
+
import { PRAGMA_NAMES } from '../pragmas.js';
|
|
23
24
|
|
|
24
25
|
/**
|
|
25
26
|
* Adapt an already-constructed `bun:sqlite` `Database` (or any object
|
|
@@ -30,12 +31,23 @@ import { sqliteDialect } from '../dialects/sqlite.js';
|
|
|
30
31
|
* @returns {any} a Connection, or a promise of one
|
|
31
32
|
*/
|
|
32
33
|
export function adaptBunDatabase(db, options) {
|
|
34
|
+
// Bun closes its query cache, but prepare() creates uncached statements
|
|
35
|
+
// that keep the SQLite file open until finalized. Track only weak
|
|
36
|
+
// references: a long-lived connection must not retain every past query.
|
|
37
|
+
/** @type {Set<WeakRef<any>>} */
|
|
38
|
+
const statements = new Set();
|
|
39
|
+
const collected = new FinalizationRegistry((ref) => statements.delete(ref));
|
|
33
40
|
const raw = {
|
|
34
41
|
/** @param {string} sql */
|
|
35
42
|
exec: (sql) => db.run(sql),
|
|
36
43
|
/** @param {string} sql */
|
|
37
44
|
prepare: (sql) => {
|
|
38
45
|
const statement = db.prepare(sql);
|
|
46
|
+
if (typeof statement.finalize === 'function') {
|
|
47
|
+
const ref = new WeakRef(statement);
|
|
48
|
+
statements.add(ref);
|
|
49
|
+
collected.register(statement, ref, ref);
|
|
50
|
+
}
|
|
39
51
|
return {
|
|
40
52
|
run: (params = []) => statement.run(...params),
|
|
41
53
|
// the driver contract says a missing row reads UNDEFINED;
|
|
@@ -51,7 +63,28 @@ export function adaptBunDatabase(db, options) {
|
|
|
51
63
|
: undefined),
|
|
52
64
|
};
|
|
53
65
|
},
|
|
54
|
-
close: () =>
|
|
66
|
+
close: () => {
|
|
67
|
+
const errors = [];
|
|
68
|
+
for (const ref of statements) {
|
|
69
|
+
collected.unregister(ref);
|
|
70
|
+
try {
|
|
71
|
+
ref.deref()?.finalize();
|
|
72
|
+
}
|
|
73
|
+
catch (error) {
|
|
74
|
+
errors.push(error);
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
statements.clear();
|
|
78
|
+
try {
|
|
79
|
+
db.close();
|
|
80
|
+
}
|
|
81
|
+
catch (error) {
|
|
82
|
+
errors.push(error);
|
|
83
|
+
}
|
|
84
|
+
if (errors.length === 1) throw errors[0];
|
|
85
|
+
if (errors.length > 1)
|
|
86
|
+
throw new AggregateError(errors, 'finalizing Bun statements and closing the database failed');
|
|
87
|
+
},
|
|
55
88
|
};
|
|
56
89
|
return openConnection(raw, {
|
|
57
90
|
dialect: sqliteDialect,
|
|
@@ -62,6 +95,9 @@ export function adaptBunDatabase(db, options) {
|
|
|
62
95
|
userFunctions: false,
|
|
63
96
|
deterministicIndexableFunctions: false,
|
|
64
97
|
aggregateFunctions: false,
|
|
98
|
+
// the configuration pragmas are the library's, not the binding's:
|
|
99
|
+
// bun:sqlite applies every one of the closed set
|
|
100
|
+
pragmas: PRAGMA_NAMES,
|
|
65
101
|
},
|
|
66
102
|
});
|
|
67
103
|
}
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
//@ts-check
|
|
2
|
+
/** Atomic, versioned SQLite snapshots. IndexedDB is storage, never an OPFS VFS. */
|
|
3
|
+
import { DbRuntimeError, sqliteResultError } from '../errors.js';
|
|
4
|
+
import { sqliteDialect } from '../dialects/sqlite.js';
|
|
5
|
+
import { adaptOo1Database } from './wasm-oo1.js';
|
|
6
|
+
|
|
7
|
+
/** @param {any} factory @param {string} name @returns {Promise<any>} */
|
|
8
|
+
export function openSnapshotStorage(factory, name) {
|
|
9
|
+
return new Promise((resolve, reject) => {
|
|
10
|
+
if (typeof factory?.open !== 'function') { reject(new Error('IndexedDB is unavailable')); return; }
|
|
11
|
+
const request = factory.open(name, 1);
|
|
12
|
+
let blocked = false;
|
|
13
|
+
request.onupgradeneeded = () => { request.result.createObjectStore('snapshots'); };
|
|
14
|
+
request.onerror = () => reject(request.error);
|
|
15
|
+
request.onblocked = () => { blocked = true; reject(new Error('the IndexedDB snapshot database upgrade is blocked')); };
|
|
16
|
+
request.onsuccess = () => {
|
|
17
|
+
const db = request.result;
|
|
18
|
+
if (blocked) { db.close(); return; }
|
|
19
|
+
db.onversionchange = () => db.close();
|
|
20
|
+
const transact = (key, write, bytes, expected) => new Promise((yes, no) => {
|
|
21
|
+
const tx = db.transaction('snapshots', write ? 'readwrite' : 'readonly');
|
|
22
|
+
const store = tx.objectStore('snapshots');
|
|
23
|
+
let result;
|
|
24
|
+
let failure;
|
|
25
|
+
const read = store.get(key);
|
|
26
|
+
read.onsuccess = () => {
|
|
27
|
+
try {
|
|
28
|
+
const old = read.result;
|
|
29
|
+
if (!write) { result = old ?? null; return; }
|
|
30
|
+
if ((old?.revision ?? 0) !== expected) {
|
|
31
|
+
failure = new Error('the durable snapshot changed in another connection');
|
|
32
|
+
tx.abort();
|
|
33
|
+
return;
|
|
34
|
+
}
|
|
35
|
+
result = expected + 1;
|
|
36
|
+
if (bytes === null) store.delete(key);
|
|
37
|
+
else store.put({ revision: result, bytes }, key);
|
|
38
|
+
}
|
|
39
|
+
catch (error) { failure = error; tx.abort(); }
|
|
40
|
+
};
|
|
41
|
+
tx.oncomplete = () => yes(result);
|
|
42
|
+
tx.onabort = () => no(failure ?? tx.error ?? new Error('the snapshot transaction was aborted'));
|
|
43
|
+
});
|
|
44
|
+
resolve({
|
|
45
|
+
read: (key) => transact(key, false),
|
|
46
|
+
write: (key, bytes, revision) => transact(key, true, bytes, revision),
|
|
47
|
+
remove: async (key) => { const old = await transact(key, false); return transact(key, true, null, old?.revision ?? 0); },
|
|
48
|
+
close: () => db.close(),
|
|
49
|
+
});
|
|
50
|
+
};
|
|
51
|
+
});
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/** Build an async binding over a synchronous wasm handle plus atomic storage.
|
|
55
|
+
* @param {any} sqlite3
|
|
56
|
+
* @param {{ openStorage: () => Promise<any>, maxBytes?: number }} options
|
|
57
|
+
* @returns {any}
|
|
58
|
+
*/
|
|
59
|
+
export function snapshotHandle(sqlite3, options) {
|
|
60
|
+
const maxBytes = options.maxBytes ?? 16 * 1024 * 1024;
|
|
61
|
+
if (!Number.isSafeInteger(maxBytes) || maxBytes < 1) throw new TypeError('snapshot maxBytes must be a positive integer');
|
|
62
|
+
return {
|
|
63
|
+
synchronous: false,
|
|
64
|
+
declares: { sessions: true, userFunctions: true, deterministicIndexableFunctions: true },
|
|
65
|
+
open: async (path, openOptions = {}) => {
|
|
66
|
+
const memory = path === ':memory:' || path === '';
|
|
67
|
+
const storage = memory ? null : await options.openStorage();
|
|
68
|
+
let db;
|
|
69
|
+
try {
|
|
70
|
+
const saved = await storage?.read(path);
|
|
71
|
+
if (saved != null && (!Number.isSafeInteger(saved.revision) || saved.revision < 1
|
|
72
|
+
|| !(saved.bytes instanceof Uint8Array) || saved.bytes.length < 1 || saved.bytes.length > maxBytes))
|
|
73
|
+
throw new DbRuntimeError('JD2094', 'the persisted SQLite snapshot is invalid or exceeds its declared byte bound');
|
|
74
|
+
let revision = saved?.revision ?? 0;
|
|
75
|
+
db = new sqlite3.oo1.DB(':memory:');
|
|
76
|
+
const { capi, wasm } = sqlite3;
|
|
77
|
+
if (saved?.bytes?.length > 0) {
|
|
78
|
+
if (!(saved.bytes instanceof Uint8Array) || saved.bytes.length > maxBytes)
|
|
79
|
+
throw new DbRuntimeError('JD2094', 'the persisted SQLite snapshot exceeds its declared byte bound');
|
|
80
|
+
const pointer = wasm.allocFromTypedArray(saved.bytes);
|
|
81
|
+
const rc = capi.sqlite3_deserialize(db.pointer, 'main', pointer,
|
|
82
|
+
BigInt(saved.bytes.length), BigInt(saved.bytes.length),
|
|
83
|
+
capi.SQLITE_DESERIALIZE_FREEONCLOSE | capi.SQLITE_DESERIALIZE_RESIZEABLE);
|
|
84
|
+
if (rc !== 0) { wasm.dealloc(pointer); throw sqliteResultError(rc, 'snapshot deserialize'); }
|
|
85
|
+
}
|
|
86
|
+
if (openOptions.readOnly === true) db.exec(sqliteDialect.pragma.set('query_only', 1));
|
|
87
|
+
const raw = adaptOo1Database(sqlite3, db);
|
|
88
|
+
let poisoned = null;
|
|
89
|
+
const requireUsable = () => { if (poisoned !== null) throw poisoned; };
|
|
90
|
+
const persist = async () => {
|
|
91
|
+
if (storage === null || openOptions.readOnly === true || !capi.sqlite3_get_autocommit(db.pointer)) return;
|
|
92
|
+
try {
|
|
93
|
+
const pages = db.selectValue(sqliteDialect.introspect.pragma('page_count'));
|
|
94
|
+
const pageSize = db.selectValue(sqliteDialect.introspect.pragma('page_size'));
|
|
95
|
+
if (pages * pageSize > maxBytes) throw new Error(`snapshot exceeds ${maxBytes} bytes`);
|
|
96
|
+
const bytes = capi.sqlite3_js_db_export(db.pointer);
|
|
97
|
+
if (bytes.length > maxBytes) throw new Error(`snapshot exceeds ${maxBytes} bytes`);
|
|
98
|
+
revision = await storage.write(path, bytes, revision);
|
|
99
|
+
}
|
|
100
|
+
catch (cause) {
|
|
101
|
+
poisoned = Object.assign(new DbRuntimeError('JD2094',
|
|
102
|
+
'the durable snapshot was not committed; this connection is invalid; reopen to read the last committed version',
|
|
103
|
+
{ cause }), { retryable: false, class: 'persistence' });
|
|
104
|
+
throw poisoned;
|
|
105
|
+
}
|
|
106
|
+
};
|
|
107
|
+
return {
|
|
108
|
+
...raw,
|
|
109
|
+
exec: async (sql) => { requireUsable(); raw.exec(sql); await persist(); },
|
|
110
|
+
prepare: (sql) => {
|
|
111
|
+
requireUsable();
|
|
112
|
+
const statement = raw.prepare(sql);
|
|
113
|
+
const invoke = (member, params) => {
|
|
114
|
+
requireUsable();
|
|
115
|
+
const value = statement[member](params);
|
|
116
|
+
return statement.readOnly ? value : persist().then(() => value);
|
|
117
|
+
};
|
|
118
|
+
return {
|
|
119
|
+
run: (params = []) => invoke('run', params),
|
|
120
|
+
get: (params = []) => invoke('get', params),
|
|
121
|
+
all: (params = []) => invoke('all', params),
|
|
122
|
+
iterate: (params = []) => {
|
|
123
|
+
requireUsable();
|
|
124
|
+
const iterator = statement.iterate(params);
|
|
125
|
+
return {
|
|
126
|
+
next: async () => { requireUsable(); const step = iterator.next(); if (!statement.readOnly) await persist(); return step; },
|
|
127
|
+
return: async () => { const step = iterator.return(); if (!statement.readOnly && poisoned === null) await persist(); return step; },
|
|
128
|
+
};
|
|
129
|
+
},
|
|
130
|
+
};
|
|
131
|
+
},
|
|
132
|
+
close: () => { try { raw.close(); } finally { storage?.close(); } },
|
|
133
|
+
};
|
|
134
|
+
}
|
|
135
|
+
catch (error) { db?.close(); storage?.close(); throw error; }
|
|
136
|
+
},
|
|
137
|
+
};
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
/** IndexedDB durability via bounded atomic snapshots, with async acknowledgements.
|
|
141
|
+
* @param {any} sqlite3
|
|
142
|
+
* @param {{ name?: string, indexedDB?: any, maxBytes?: number }} [options]
|
|
143
|
+
* @returns {any} an injected wasm handle
|
|
144
|
+
*/
|
|
145
|
+
export function indexedDbSnapshotHandle(sqlite3, options = {}) {
|
|
146
|
+
return snapshotHandle(sqlite3, { maxBytes: options.maxBytes,
|
|
147
|
+
openStorage: () => openSnapshotStorage(options.indexedDB ?? globalThis.indexedDB,
|
|
148
|
+
options.name ?? 'jaren-sqlite-snapshots') });
|
|
149
|
+
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
//@ts-check
|
|
2
|
+
import { nodeWorkerDriver } from './node-worker.js';
|
|
3
|
+
import { workerPoolDriver } from './worker-pool.js';
|
|
4
|
+
|
|
5
|
+
/** One writer and bounded read-only WAL workers behind the Connection contract.
|
|
6
|
+
* @param {{readers?:number, queueCapacity?:number, graceMs?:number, worker?:any}} [options]
|
|
7
|
+
* @returns {any}
|
|
8
|
+
*/
|
|
9
|
+
export function nodeWorkerPoolDriver(options = {}) {
|
|
10
|
+
return workerPoolDriver(options, nodeWorkerDriver);
|
|
11
|
+
}
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
//@ts-check
|
|
2
|
+
/** One worker owns one SQLite handle. Only bounded credit frames carry rows. */
|
|
3
|
+
import { parentPort, workerData } from 'node:worker_threads';
|
|
4
|
+
import { nodeDriver } from './node.js';
|
|
5
|
+
import { DbRuntimeError, cloneDriverError } from '../errors.js';
|
|
6
|
+
import { validRequest, generationFailure, rowBytes } from './worker-protocol.js';
|
|
7
|
+
|
|
8
|
+
const { generation, path, options, limits } = workerData;
|
|
9
|
+
let connection;
|
|
10
|
+
let sequence = 0;
|
|
11
|
+
const statements = new Map();
|
|
12
|
+
const cursors = new Map();
|
|
13
|
+
const frame = (kind, id, value) => ({ v: 1, generation, kind, id, ...value });
|
|
14
|
+
const bound = (reason) => new DbRuntimeError('JD2092', reason);
|
|
15
|
+
const release = (id) => {
|
|
16
|
+
const cursor = cursors.get(id);
|
|
17
|
+
if (cursor === undefined) return;
|
|
18
|
+
cursors.delete(id);
|
|
19
|
+
cursor.iterator.return?.();
|
|
20
|
+
if (cursor.ephemeral) statements.delete(cursor.statement);
|
|
21
|
+
};
|
|
22
|
+
const statementOf = (id) => {
|
|
23
|
+
const statement = statements.get(id);
|
|
24
|
+
if (statement === undefined) throw generationFailure(generation);
|
|
25
|
+
return statement;
|
|
26
|
+
};
|
|
27
|
+
const dispatch = (request) => {
|
|
28
|
+
switch (request.op) {
|
|
29
|
+
case 'exec': return connection.exec(request.sql);
|
|
30
|
+
case 'prepare': {
|
|
31
|
+
if (statements.size >= limits.statements) throw bound(`worker statement capacity ${limits.statements} exceeded`);
|
|
32
|
+
const id = ++sequence;
|
|
33
|
+
statements.set(id, { statement: connection.prepare(request.sql), ephemeral: request.ephemeral === true });
|
|
34
|
+
return id;
|
|
35
|
+
}
|
|
36
|
+
case 'get': {
|
|
37
|
+
const row = statementOf(request.statement).statement.get(request.params);
|
|
38
|
+
if (rowBytes(row) > limits.bytes) throw bound(`one worker row exceeds ${limits.bytes} bytes`);
|
|
39
|
+
return row;
|
|
40
|
+
}
|
|
41
|
+
case 'run': return statementOf(request.statement).statement.run(request.params);
|
|
42
|
+
case 'iterate': {
|
|
43
|
+
if (cursors.size >= limits.cursors) throw bound(`worker cursor capacity ${limits.cursors} exceeded`);
|
|
44
|
+
const record = statementOf(request.statement);
|
|
45
|
+
const id = ++sequence;
|
|
46
|
+
cursors.set(id, { iterator: record.statement.iterate(request.params),
|
|
47
|
+
statement: request.statement, ephemeral: record.ephemeral, buffered: null });
|
|
48
|
+
return id;
|
|
49
|
+
}
|
|
50
|
+
case 'next': {
|
|
51
|
+
const cursor = cursors.get(request.cursor);
|
|
52
|
+
if (cursor === undefined) throw generationFailure(generation);
|
|
53
|
+
if (request.rows > limits.rows || request.bytes > limits.bytes) throw bound('worker cursor credit exceeds the negotiated window');
|
|
54
|
+
const rows = [];
|
|
55
|
+
let bytes = 0;
|
|
56
|
+
try {
|
|
57
|
+
while (rows.length < request.rows) {
|
|
58
|
+
const step = cursor.buffered ?? cursor.iterator.next();
|
|
59
|
+
cursor.buffered = null;
|
|
60
|
+
if (step.done) { release(request.cursor); return { rows, bytes, done: true }; }
|
|
61
|
+
const size = rowBytes(step.value);
|
|
62
|
+
if (size > request.bytes) throw bound(`one worker row of ${size} bytes exceeds the ${request.bytes} byte window`);
|
|
63
|
+
if (bytes + size > request.bytes) { cursor.buffered = step; break; }
|
|
64
|
+
rows.push(step.value);
|
|
65
|
+
bytes += size;
|
|
66
|
+
}
|
|
67
|
+
return { rows, bytes, done: false };
|
|
68
|
+
}
|
|
69
|
+
catch (error) { release(request.cursor); throw error; }
|
|
70
|
+
}
|
|
71
|
+
case 'return': release(request.cursor); return undefined;
|
|
72
|
+
case 'finalize': {
|
|
73
|
+
for (const [id, cursor] of cursors) if (cursor.statement === request.statement) release(id);
|
|
74
|
+
statements.delete(request.statement);
|
|
75
|
+
return undefined;
|
|
76
|
+
}
|
|
77
|
+
case 'close': {
|
|
78
|
+
for (const id of cursors.keys()) release(id);
|
|
79
|
+
statements.clear();
|
|
80
|
+
return connection.close();
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
};
|
|
84
|
+
try {
|
|
85
|
+
connection = await nodeDriver().open(path, options);
|
|
86
|
+
// Functions cannot cross structured clone. Capture uses the existing
|
|
87
|
+
// journal path; a live query requires a synchronous connection.
|
|
88
|
+
parentPort.postMessage(frame('ready', 0, { capabilities: { ...connection.capabilities,
|
|
89
|
+
sessions: false, userFunctions: false, deterministicIndexableFunctions: false,
|
|
90
|
+
aggregateFunctions: false, backup: false, worker: true, pooling: false } }));
|
|
91
|
+
parentPort.on('message', (request) => {
|
|
92
|
+
try {
|
|
93
|
+
if (!validRequest(request)) throw new DbRuntimeError('JD2093', 'invalid worker protocol request');
|
|
94
|
+
if (request.generation !== generation) throw generationFailure(request.generation);
|
|
95
|
+
const value = dispatch(request);
|
|
96
|
+
parentPort.postMessage(frame('result', request.id, { value }));
|
|
97
|
+
if (request.op === 'close') parentPort.close();
|
|
98
|
+
}
|
|
99
|
+
catch (error) { parentPort.postMessage(frame('failure', request?.id ?? 0, { error: cloneDriverError(error) })); }
|
|
100
|
+
});
|
|
101
|
+
}
|
|
102
|
+
catch (error) {
|
|
103
|
+
parentPort.postMessage(frame('failure', 0, { error: cloneDriverError(error) }));
|
|
104
|
+
parentPort.close();
|
|
105
|
+
}
|
|
@@ -0,0 +1,204 @@
|
|
|
1
|
+
//@ts-check
|
|
2
|
+
/** A worker transport behind the ordinary Connection contract. */
|
|
3
|
+
import { finishConnection, lazyOpen } from '../driver.js';
|
|
4
|
+
import { sqliteDialect } from '../dialects/sqlite.js';
|
|
5
|
+
import { DbRuntimeError } from '../errors.js';
|
|
6
|
+
import { generationFailure, positiveOption, queueFailure, rowBytes, validResponse, validResult } from './worker-protocol.js';
|
|
7
|
+
|
|
8
|
+
/** Dedicated SQLite worker per connection. No function serialization or write replay.
|
|
9
|
+
* @param {{ windowRows?: number, windowBytes?: number, maxPending?: number,
|
|
10
|
+
* maxStatements?: number, maxCursors?: number, allMaxRows?: number,
|
|
11
|
+
* allMaxBytes?: number, closeTimeoutMs?: number, startupTimeoutMs?: number }} [configuration]
|
|
12
|
+
* @returns {any} a Driver
|
|
13
|
+
*/
|
|
14
|
+
export function nodeWorkerDriver(configuration = {}) {
|
|
15
|
+
const limits = {
|
|
16
|
+
rows: positiveOption('windowRows', configuration.windowRows, 64),
|
|
17
|
+
bytes: positiveOption('windowBytes', configuration.windowBytes, 1024 * 1024),
|
|
18
|
+
statements: positiveOption('maxStatements', configuration.maxStatements, 1024),
|
|
19
|
+
cursors: positiveOption('maxCursors', configuration.maxCursors, 64),
|
|
20
|
+
};
|
|
21
|
+
const maxPending = positiveOption('maxPending', configuration.maxPending, 64);
|
|
22
|
+
const allRows = positiveOption('allMaxRows', configuration.allMaxRows, 100000);
|
|
23
|
+
const allBytes = positiveOption('allMaxBytes', configuration.allMaxBytes, 16 * 1024 * 1024);
|
|
24
|
+
const closeMs = positiveOption('closeTimeoutMs', configuration.closeTimeoutMs, 5000);
|
|
25
|
+
const startupMs = positiveOption('startupTimeoutMs', configuration.startupTimeoutMs, 10000);
|
|
26
|
+
let generation = 0;
|
|
27
|
+
const driver = {
|
|
28
|
+
name: 'node-worker-sqlite', dialect: sqliteDialect,
|
|
29
|
+
open: (path = ':memory:', options = {}) => lazyOpen('node:worker_threads',
|
|
30
|
+
'Node worker threads are unavailable on this runtime', async ({ Worker }) => {
|
|
31
|
+
const epoch = ++generation;
|
|
32
|
+
const worker = new Worker(new URL('./node-worker-endpoint.js', import.meta.url), {
|
|
33
|
+
workerData: { path, options: { timeout: options.timeout, readOnly: options.readOnly }, generation: epoch, limits },
|
|
34
|
+
// Parent --test/--input-type/preloads do not describe the endpoint.
|
|
35
|
+
execArgv: ['--no-warnings=ExperimentalWarning'],
|
|
36
|
+
});
|
|
37
|
+
const pending = new Map();
|
|
38
|
+
let sequence = 0;
|
|
39
|
+
let failed = null;
|
|
40
|
+
let closing = false;
|
|
41
|
+
let closed = false;
|
|
42
|
+
let transactionDepth = 0;
|
|
43
|
+
let closePromise;
|
|
44
|
+
const metrics = { frames: 0, rows: 0, maxFrameRows: 0, maxFrameBytes: 0, maxPending: 0 };
|
|
45
|
+
let readyResolve;
|
|
46
|
+
let readyReject;
|
|
47
|
+
const ready = new Promise((resolve, reject) => { readyResolve = resolve; readyReject = reject; });
|
|
48
|
+
const lose = (cause) => {
|
|
49
|
+
if (failed !== null || closed) return;
|
|
50
|
+
failed = generationFailure(epoch, transactionDepth > 0, cause);
|
|
51
|
+
readyReject(failed);
|
|
52
|
+
for (const request of pending.values()) request.reject(failed);
|
|
53
|
+
pending.clear();
|
|
54
|
+
};
|
|
55
|
+
worker.on('error', lose);
|
|
56
|
+
worker.on('exit', (code) => { if (!closed) lose(new Error(`worker exited (${code})`)); });
|
|
57
|
+
worker.on('message', (message) => {
|
|
58
|
+
if (!validResponse(message, epoch)) { lose(new Error('invalid worker response')); return; }
|
|
59
|
+
if (message.kind === 'ready') { readyResolve(message.capabilities); return; }
|
|
60
|
+
if (message.kind === 'failure' && message.id === 0) {
|
|
61
|
+
readyReject(Object.assign(new Error(message.error.message), message.error)); return;
|
|
62
|
+
}
|
|
63
|
+
const request = pending.get(message.id);
|
|
64
|
+
if (request === undefined) return;
|
|
65
|
+
pending.delete(message.id);
|
|
66
|
+
if (message.kind === 'failure') {
|
|
67
|
+
const error = message.error;
|
|
68
|
+
request.reject(Object.assign(error.code?.startsWith('JD')
|
|
69
|
+
? new DbRuntimeError(error.code, error.message) : new Error(error.message), error));
|
|
70
|
+
}
|
|
71
|
+
else if (message.kind === 'result' && validResult(request.op, message.value, limits)) request.resolve(message.value);
|
|
72
|
+
else { request.reject(generationFailure(epoch, transactionDepth > 0)); lose(new Error('invalid worker response')); }
|
|
73
|
+
});
|
|
74
|
+
const timer = setTimeout(() => { lose(new Error('worker startup timed out')); worker.terminate(); }, startupMs);
|
|
75
|
+
let capabilities;
|
|
76
|
+
try { capabilities = await ready; }
|
|
77
|
+
catch (error) { await worker.terminate(); throw error; }
|
|
78
|
+
finally { clearTimeout(timer); }
|
|
79
|
+
const request = (op, data = {}, cleanup = false) => {
|
|
80
|
+
if (failed !== null) return Promise.reject(failed);
|
|
81
|
+
if (closed || (closing && !cleanup)) return Promise.reject(new DbRuntimeError('JD2063', 'the worker connection is closing or closed'));
|
|
82
|
+
if (!cleanup && pending.size >= maxPending)
|
|
83
|
+
return Promise.reject(queueFailure(`worker request capacity ${maxPending} exceeded`, pending.size));
|
|
84
|
+
// One return per live cursor and one close have reserved capacity.
|
|
85
|
+
if (cleanup && pending.size >= maxPending + limits.cursors + 1)
|
|
86
|
+
return Promise.reject(queueFailure('worker cleanup capacity exceeded', pending.size));
|
|
87
|
+
const id = ++sequence;
|
|
88
|
+
return new Promise((resolve, reject) => {
|
|
89
|
+
pending.set(id, { resolve, reject, op });
|
|
90
|
+
metrics.maxPending = Math.max(metrics.maxPending, pending.size);
|
|
91
|
+
try { worker.postMessage({ v: 1, generation: epoch, kind: 'request', id, op, ...data }); }
|
|
92
|
+
catch (error) { pending.delete(id); reject(error); }
|
|
93
|
+
});
|
|
94
|
+
};
|
|
95
|
+
const raw = {
|
|
96
|
+
closeDrainsIterators: true,
|
|
97
|
+
exec: (sql) => {
|
|
98
|
+
if (/^(?:SAVEPOINT|BEGIN)\b/.test(sql)) transactionDepth++;
|
|
99
|
+
return request('exec', { sql }).then((value) => {
|
|
100
|
+
if (/^RELEASE\b/.test(sql)) transactionDepth = Math.max(0, transactionDepth - 1);
|
|
101
|
+
if (/^(?:COMMIT|ROLLBACK(?! TO))\b/.test(sql)) transactionDepth = 0;
|
|
102
|
+
return value;
|
|
103
|
+
});
|
|
104
|
+
},
|
|
105
|
+
prepare: async (sql, metadata = {}) => {
|
|
106
|
+
const id = await request('prepare', { sql, ephemeral: metadata.ephemeral === true });
|
|
107
|
+
const iterate = async (params = []) => {
|
|
108
|
+
const cursor = await request('iterate', { statement: id, params });
|
|
109
|
+
let rows = [];
|
|
110
|
+
let at = 0;
|
|
111
|
+
let done = false;
|
|
112
|
+
let returned = false;
|
|
113
|
+
let pulling = Promise.resolve();
|
|
114
|
+
const next = async () => {
|
|
115
|
+
if (failed !== null) throw failed;
|
|
116
|
+
if (returned) return { done: true, value: undefined };
|
|
117
|
+
if (at < rows.length) return { done: false, value: rows[at++] };
|
|
118
|
+
if (done) return { done: true, value: undefined };
|
|
119
|
+
const batch = await request('next', { cursor, rows: limits.rows, bytes: limits.bytes });
|
|
120
|
+
metrics.frames++;
|
|
121
|
+
metrics.rows += batch.rows.length;
|
|
122
|
+
metrics.maxFrameRows = Math.max(metrics.maxFrameRows, batch.rows.length);
|
|
123
|
+
metrics.maxFrameBytes = Math.max(metrics.maxFrameBytes, batch.bytes);
|
|
124
|
+
if (returned) return { done: true, value: undefined };
|
|
125
|
+
rows = batch.rows;
|
|
126
|
+
at = 0;
|
|
127
|
+
done = batch.done;
|
|
128
|
+
return at < rows.length ? { done: false, value: rows[at++] } : { done: true, value: undefined };
|
|
129
|
+
};
|
|
130
|
+
return {
|
|
131
|
+
next: () => {
|
|
132
|
+
const result = pulling.then(next);
|
|
133
|
+
pulling = result.then(() => undefined, () => undefined);
|
|
134
|
+
return result;
|
|
135
|
+
},
|
|
136
|
+
return: async () => {
|
|
137
|
+
if (returned) return { done: true, value: undefined };
|
|
138
|
+
returned = true;
|
|
139
|
+
rows = [];
|
|
140
|
+
if (!done) await request('return', { cursor }, true);
|
|
141
|
+
return { done: true, value: undefined };
|
|
142
|
+
},
|
|
143
|
+
};
|
|
144
|
+
};
|
|
145
|
+
return {
|
|
146
|
+
run: (params = []) => request('run', { statement: id, params }),
|
|
147
|
+
get: (params = []) => request('get', { statement: id, params }),
|
|
148
|
+
iterate,
|
|
149
|
+
all: async (params = []) => {
|
|
150
|
+
const iterator = await iterate(params);
|
|
151
|
+
const rows = [];
|
|
152
|
+
let bytes = 0;
|
|
153
|
+
try {
|
|
154
|
+
for (;;) {
|
|
155
|
+
const step = await iterator.next();
|
|
156
|
+
if (step.done) return rows;
|
|
157
|
+
bytes += rowBytes(step.value);
|
|
158
|
+
if (rows.length >= allRows || bytes > allBytes)
|
|
159
|
+
throw new DbRuntimeError('JD2092', `worker all() exceeds its ${allRows} row / ${allBytes} byte bound; use a cursor`);
|
|
160
|
+
rows.push(step.value);
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
finally { await iterator.return(); }
|
|
164
|
+
},
|
|
165
|
+
};
|
|
166
|
+
},
|
|
167
|
+
close: () => {
|
|
168
|
+
if (closePromise !== undefined) return closePromise;
|
|
169
|
+
closing = true;
|
|
170
|
+
closePromise = new Promise((resolve, reject) => {
|
|
171
|
+
const timeout = setTimeout(() => {
|
|
172
|
+
lose(new Error('worker close timed out'));
|
|
173
|
+
// V8 termination cannot preempt a synchronous native SQLite
|
|
174
|
+
// call. Fence and detach now; never hold the caller's deadline
|
|
175
|
+
// hostage to that native call or claim it was rolled back.
|
|
176
|
+
worker.unref();
|
|
177
|
+
worker.terminate().catch(() => {});
|
|
178
|
+
reject(failed);
|
|
179
|
+
}, closeMs);
|
|
180
|
+
request('close', {}, true).then(() => {
|
|
181
|
+
closed = true;
|
|
182
|
+
clearTimeout(timeout);
|
|
183
|
+
worker.terminate().then(() => resolve(undefined), reject);
|
|
184
|
+
}, (error) => {
|
|
185
|
+
clearTimeout(timeout);
|
|
186
|
+
worker.unref();
|
|
187
|
+
worker.terminate().catch(() => {});
|
|
188
|
+
reject(error);
|
|
189
|
+
});
|
|
190
|
+
});
|
|
191
|
+
return closePromise;
|
|
192
|
+
},
|
|
193
|
+
};
|
|
194
|
+
const connection = finishConnection(raw, sqliteDialect, false, Object.freeze(capabilities), options.queueTimeout);
|
|
195
|
+
return Object.freeze({ ...connection,
|
|
196
|
+
get mustQueue() { return connection.mustQueue; },
|
|
197
|
+
generation: epoch,
|
|
198
|
+
metrics: () => Object.freeze({ ...metrics, pending: pending.size, generation: epoch, healthy: failed === null && !closed }),
|
|
199
|
+
restart: async () => { lose(new Error('worker restarted')); await worker.terminate(); return driver.open(path, options); },
|
|
200
|
+
});
|
|
201
|
+
}, []),
|
|
202
|
+
};
|
|
203
|
+
return Object.freeze(driver);
|
|
204
|
+
}
|
package/src/drivers/node.js
CHANGED
|
@@ -8,17 +8,23 @@
|
|
|
8
8
|
|
|
9
9
|
import { lazyOpen, openConnection } from '../driver.js';
|
|
10
10
|
import { sqliteDialect } from '../dialects/sqlite.js';
|
|
11
|
+
import { PRAGMA_NAMES } from '../pragmas.js';
|
|
11
12
|
|
|
12
13
|
/**
|
|
13
14
|
* Adapt an already-constructed `node:sqlite` `DatabaseSync` (or any
|
|
14
15
|
* object with its shape) into a probed connection. Exported so the
|
|
15
16
|
* adapter is exercisable without the builtin.
|
|
16
17
|
* @param {any} db - A `DatabaseSync`-shaped database
|
|
17
|
-
* @param {{ queueTimeout?: number
|
|
18
|
+
* @param {{ queueTimeout?: number,
|
|
19
|
+
* backup?: { copy: Function, rename: Function, remove: Function } }} [options]
|
|
20
|
+
* - `backup` is the online-backup primitive triple (the module-level
|
|
21
|
+
* `backup()` over this database, plus a rename and a removal); the
|
|
22
|
+
* connection declares the capability exactly when it is given
|
|
18
23
|
* @returns {any} a Connection, or a promise of one
|
|
19
24
|
*/
|
|
20
25
|
export function adaptNodeDatabase(db, options) {
|
|
21
26
|
const raw = {
|
|
27
|
+
...(options?.backup !== undefined ? { backup: options.backup } : undefined),
|
|
22
28
|
/** @param {string} sql */
|
|
23
29
|
exec: (sql) => db.exec(sql),
|
|
24
30
|
/** @param {string} sql */
|
|
@@ -32,11 +38,17 @@ export function adaptNodeDatabase(db, options) {
|
|
|
32
38
|
};
|
|
33
39
|
},
|
|
34
40
|
close: () => db.close(),
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
41
|
+
// each optional primitive is exposed only when the HANDLE has it, so
|
|
42
|
+
// a substitute that carries less than `node:sqlite` reports less —
|
|
43
|
+
// a declared capability the handle cannot honour is a TypeError at
|
|
44
|
+
// the first call instead of a `false` the planner can read
|
|
45
|
+
...(typeof db.function === 'function'
|
|
46
|
+
? { registerFunction: (name, options, fn) => db.function(name, options, fn) } : undefined),
|
|
47
|
+
...(typeof db.aggregate === 'function'
|
|
48
|
+
? { registerAggregate: (name, spec) => db.aggregate(name, spec) } : undefined),
|
|
49
|
+
...(typeof db.createSession === 'function'
|
|
50
|
+
? { session: (table) => (table === undefined
|
|
51
|
+
? db.createSession() : db.createSession({ table })) } : undefined),
|
|
40
52
|
};
|
|
41
53
|
return openConnection(raw, {
|
|
42
54
|
dialect: sqliteDialect,
|
|
@@ -47,6 +59,11 @@ export function adaptNodeDatabase(db, options) {
|
|
|
47
59
|
userFunctions: true,
|
|
48
60
|
deterministicIndexableFunctions: true,
|
|
49
61
|
aggregateFunctions: true,
|
|
62
|
+
// every configuration pragma of the closed set: a core SQLite
|
|
63
|
+
// library applies them all, and the read-back catches a build
|
|
64
|
+
// that compiled one out
|
|
65
|
+
pragmas: PRAGMA_NAMES,
|
|
66
|
+
backup: options?.backup !== undefined,
|
|
50
67
|
},
|
|
51
68
|
});
|
|
52
69
|
}
|
|
@@ -69,7 +86,18 @@ export function fromNodeModule(mod, path, options) {
|
|
|
69
86
|
const db = Object.keys(open).length > 0
|
|
70
87
|
? new mod.DatabaseSync(path, open)
|
|
71
88
|
: new mod.DatabaseSync(path);
|
|
72
|
-
|
|
89
|
+
// the online-backup primitives: the module's own `backup()` over this
|
|
90
|
+
// database, and the file system's rename and removal — imported
|
|
91
|
+
// lazily on first use, never at module scope, for the same reason the
|
|
92
|
+
// builtin itself is
|
|
93
|
+
const backup = typeof mod.backup === 'function'
|
|
94
|
+
? {
|
|
95
|
+
copy: (target, backupOptions) => mod.backup(db, target, backupOptions),
|
|
96
|
+
rename: (from, to) => import('node:fs/promises').then((fs) => fs.rename(from, to)),
|
|
97
|
+
remove: (target) => import('node:fs/promises').then((fs) => fs.rm(target, { force: true })),
|
|
98
|
+
}
|
|
99
|
+
: undefined;
|
|
100
|
+
return adaptNodeDatabase(db, backup === undefined ? options : { ...options, backup });
|
|
73
101
|
}
|
|
74
102
|
|
|
75
103
|
/**
|
|
@@ -91,3 +119,9 @@ export function nodeDriver() {
|
|
|
91
119
|
fromNodeModule, [path, options]),
|
|
92
120
|
});
|
|
93
121
|
}
|
|
122
|
+
|
|
123
|
+
export {
|
|
124
|
+
readDocuments, readJsonDocuments, readJsonlDocuments,
|
|
125
|
+
openAtomicTarget, openStreamTarget, openNullTarget,
|
|
126
|
+
formatOf, DOCUMENT_FORMATS,
|
|
127
|
+
} from '../document-files.js';
|