@jarenjs/db 0.84.3 → 0.86.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +27 -3
- package/README.md +35 -14
- package/docs/HOSTS.md +131 -4
- package/docs/MODEL-FORMAT.md +15 -4
- package/docs/NATIVE-PLANS.md +24 -7
- package/docs/SQLITE-RELATIONAL.md +190 -0
- package/package.json +24 -4
- package/schemas/jaren-model.authoring.schema.json +157 -0
- package/schemas/jaren-model.draft-07.schema.json +157 -0
- package/schemas/jaren-model.schema.json +157 -0
- package/src/capture.js +4 -2
- package/src/ddl.js +8 -3
- package/src/dialects/sqlite-relational.js +314 -0
- package/src/dialects/sqlite-schema.js +142 -0
- package/src/dialects/sqlite.js +17 -0
- package/src/driver.js +3 -0
- package/src/drivers/bun.js +26 -15
- package/src/drivers/node-process-endpoint.js +13 -0
- package/src/drivers/node-process.js +177 -0
- package/src/drivers/node-worker-endpoint.js +3 -103
- package/src/drivers/node-worker.js +6 -178
- package/src/drivers/node.js +3 -0
- package/src/drivers/snapshot.js +49 -0
- package/src/drivers/sqlite-endpoint.js +113 -0
- package/src/drivers/worker-client.js +185 -0
- package/src/drivers/worker-protocol.js +15 -0
- package/src/emit.js +37 -5
- package/src/engine-metadata.js +18 -0
- package/src/errors.js +1 -0
- package/src/index.js +4 -1
- package/src/introspect.js +1 -2
- package/src/jobs.js +4 -2
- package/src/migrate.js +12 -16
- package/src/model-api.js +4 -0
- package/src/model.js +12 -0
- package/src/mutation.js +66 -14
- package/src/physical.js +37 -7
- package/src/plan.js +66 -3
- package/src/query-api.js +5 -0
- package/src/query.js +39 -6
- package/src/relational-api.js +6 -0
- package/src/store.js +6 -4
- package/src/table-migration.js +158 -0
- package/types/bun.d.ts +3 -0
- package/types/entity.d.ts +1 -0
- package/types/index.d.ts +11 -2
- package/types/model.d.ts +1 -0
- package/types/node-process.d.ts +33 -0
- package/types/node.d.ts +3 -0
- package/types/query.d.ts +2 -0
- package/types/relational.d.ts +114 -0
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
//@ts-check
|
|
2
|
+
/** Supervised process ownership; response cancellation never implies SQL rollback. */
|
|
3
|
+
import { sqliteDialect } from '../dialects/sqlite.js';
|
|
4
|
+
import { sqlTokens } from '../dialects/check-read.js';
|
|
5
|
+
import { DbCompileError, DbRuntimeError } from '../errors.js';
|
|
6
|
+
import { createWorkerConnection } from './worker-client.js';
|
|
7
|
+
import { positiveOption, queueFailure, rowBytes, workerSettings, PROCESS_DEFAULTS } from './worker-protocol.js';
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* Finite owned SQLite processes behind the ordinary Driver contract.
|
|
11
|
+
* A quarantined process retains its admission credit until its exit event.
|
|
12
|
+
* @param {import('../../types/node-process.js').NodeProcessOptions} [configuration]
|
|
13
|
+
* @returns {import('../../types/node-process.js').NodeProcessDriver}
|
|
14
|
+
*/
|
|
15
|
+
export function nodeProcessDriver(configuration = {}) {
|
|
16
|
+
const maxOwners = positiveOption('maxOwners', configuration.maxOwners, PROCESS_DEFAULTS.maxOwners);
|
|
17
|
+
const timeoutMs = positiveOption('timeoutMs', configuration.timeoutMs, PROCESS_DEFAULTS.timeoutMs);
|
|
18
|
+
const maxRequestBytes = positiveOption('maxRequestBytes', configuration.maxRequestBytes, PROCESS_DEFAULTS.maxRequestBytes);
|
|
19
|
+
const settings = workerSettings(configuration, PROCESS_DEFAULTS);
|
|
20
|
+
const { limits } = settings;
|
|
21
|
+
const owners = new Set();
|
|
22
|
+
let generation = 0;
|
|
23
|
+
const driver = {
|
|
24
|
+
name: 'node-process-sqlite', dialect: sqliteDialect,
|
|
25
|
+
metrics: () => Object.freeze({ capacity: maxOwners, owners: owners.size,
|
|
26
|
+
quarantined: [...owners].filter((owner) => owner.status === 'quarantined').length,
|
|
27
|
+
healthy: [...owners].filter((owner) => owner.status === 'healthy').length }),
|
|
28
|
+
async open(path = ':memory:', options = {}) {
|
|
29
|
+
if (globalThis.process?.versions?.bun || globalThis.process?.release?.name !== 'node'
|
|
30
|
+
|| Number(process.versions.node.split('.')[0]) < 24)
|
|
31
|
+
throw new DbCompileError('JD0003', 'supervised SQLite processes require Node.js 24 or newer');
|
|
32
|
+
const [{ fork }, { resolve }] = await Promise.all([import('node:child_process'), import('node:path')]);
|
|
33
|
+
const identity = path === ':memory:' ? null : resolve(path);
|
|
34
|
+
if (owners.size >= maxOwners || identity && [...owners].some((owner) => owner.path === identity))
|
|
35
|
+
throw queueFailure('process owner capacity or database ownership is reserved', owners.size);
|
|
36
|
+
const epoch = ++generation;
|
|
37
|
+
const state = { path: identity, status: 'starting', transaction: 'none', generation: epoch, pid: null,
|
|
38
|
+
safeToReplace: false, exitCode: null, exitSignal: null };
|
|
39
|
+
owners.add(state);
|
|
40
|
+
let child;
|
|
41
|
+
try {
|
|
42
|
+
child = fork(new URL('./node-process-endpoint.js', import.meta.url), [], {
|
|
43
|
+
serialization: 'advanced', stdio: ['ignore', 'ignore', 'ignore', 'ipc'],
|
|
44
|
+
execArgv: ['--no-warnings=ExperimentalWarning'],
|
|
45
|
+
});
|
|
46
|
+
}
|
|
47
|
+
catch (error) { owners.delete(state); throw error; }
|
|
48
|
+
state.pid = child.pid ?? null;
|
|
49
|
+
let resolveExit, control, killPromise, active = false, closeAcknowledged = false;
|
|
50
|
+
let transactionOpen = false, ambiguous = false;
|
|
51
|
+
const prepared = new Map();
|
|
52
|
+
const cursors = new Map();
|
|
53
|
+
const report = () => Object.freeze({ ...state });
|
|
54
|
+
const exited = new Promise((resolve) => { resolveExit = resolve; });
|
|
55
|
+
child.once('exit', (code, signal) => {
|
|
56
|
+
state.status = 'exited'; state.safeToReplace = true;
|
|
57
|
+
state.exitCode = code; state.exitSignal = signal;
|
|
58
|
+
queueMicrotask(() => {
|
|
59
|
+
if (ambiguous) state.transaction = 'unknown';
|
|
60
|
+
else if (transactionOpen) state.transaction = 'rolled-back';
|
|
61
|
+
owners.delete(state); resolveExit(report());
|
|
62
|
+
});
|
|
63
|
+
});
|
|
64
|
+
// A failed spawn has no owner to quarantine and emits no exit event.
|
|
65
|
+
child.once('error', () => {
|
|
66
|
+
if (child.pid === undefined) {
|
|
67
|
+
state.status = 'exited'; state.safeToReplace = true;
|
|
68
|
+
owners.delete(state); resolveExit(report());
|
|
69
|
+
}
|
|
70
|
+
});
|
|
71
|
+
const terminate = () => {
|
|
72
|
+
if (!killPromise) {
|
|
73
|
+
if (state.status !== 'exited' && !closeAcknowledged) { state.status = 'quarantined'; child.kill('SIGKILL'); }
|
|
74
|
+
killPromise = exited;
|
|
75
|
+
}
|
|
76
|
+
return killPromise;
|
|
77
|
+
};
|
|
78
|
+
const transport = {
|
|
79
|
+
on: (name, fn) => child.on(name, fn),
|
|
80
|
+
postMessage: (message) => child.send(message, (error) => { if (error) control?.lose(error); }),
|
|
81
|
+
terminate, unref: () => {},
|
|
82
|
+
};
|
|
83
|
+
const info = (sql) => {
|
|
84
|
+
const tokens = sqlTokens(sql), words = tokens.filter((token) => token.kind === 'word').map((token) => token.value.toUpperCase());
|
|
85
|
+
const multiple = tokens.some((token, i) => token.kind === 'symbol' && token.value === ';' && i < tokens.length - 1);
|
|
86
|
+
return { multiple, end: multiple || ['COMMIT', 'END', 'RELEASE'].includes(words[0]),
|
|
87
|
+
rollback: words[0] === 'ROLLBACK' && !words.includes('TO'),
|
|
88
|
+
writes: multiple || !['SELECT', 'WITH'].includes(words[0]) || words.some((word) => ['INSERT', 'UPDATE', 'DELETE', 'REPLACE'].includes(word)) };
|
|
89
|
+
};
|
|
90
|
+
const unknownStatement = { multiple: true, writes: true, end: true };
|
|
91
|
+
const execution = (op, data) => op === 'exec' ? info(data.sql)
|
|
92
|
+
: ['next', 'return'].includes(op) ? cursors.get(data.cursor) ?? unknownStatement
|
|
93
|
+
: ['run', 'get', 'iterate'].includes(op) ? prepared.get(data.statement) ?? unknownStatement : null;
|
|
94
|
+
const observe = (op, data, open, failed = false, statement = execution(op, data)) => {
|
|
95
|
+
if (typeof open !== 'boolean') return;
|
|
96
|
+
const wasOpen = transactionOpen;
|
|
97
|
+
transactionOpen = open;
|
|
98
|
+
if (open) { ambiguous = false; state.transaction = 'active'; }
|
|
99
|
+
else if (statement?.multiple || [...cursors.values()].some((cursor) => cursor.writes)
|
|
100
|
+
|| op === 'return' && statement?.writes) { ambiguous = true; state.transaction = 'unknown'; }
|
|
101
|
+
else if (failed && wasOpen || statement?.rollback) {
|
|
102
|
+
ambiguous = false; state.transaction = 'rolled-back';
|
|
103
|
+
}
|
|
104
|
+
else if (!failed && statement && (wasOpen || ambiguous && statement.writes)) { ambiguous = false; state.transaction = 'committed'; }
|
|
105
|
+
};
|
|
106
|
+
const hooks = {
|
|
107
|
+
control: (value) => { control = value; },
|
|
108
|
+
lost: (error, pending) => {
|
|
109
|
+
const live = pending.map(({ op, data }) => execution(op, data)).filter(Boolean);
|
|
110
|
+
ambiguous = live.some((statement) => statement.end || statement.writes && !transactionOpen)
|
|
111
|
+
|| !transactionOpen && [...cursors.values()].some((statement) => statement.writes);
|
|
112
|
+
if (ambiguous) state.transaction = 'unknown';
|
|
113
|
+
if (ambiguous || transactionOpen) error.retryable = false;
|
|
114
|
+
if (state.status !== 'exited') { state.status = 'quarantined'; terminate(); }
|
|
115
|
+
},
|
|
116
|
+
request: (op, data) => {
|
|
117
|
+
if (data.params?.some((param) => param !== null && !['number', 'bigint', 'string'].includes(typeof param) && !(param instanceof Uint8Array)))
|
|
118
|
+
throw new DbRuntimeError('JD2093', 'process parameters must be SQLite scalars or byte arrays');
|
|
119
|
+
if (rowBytes(data) > maxRequestBytes) throw new DbRuntimeError('JD2092', 'process request exceeds its byte credit');
|
|
120
|
+
const statement = execution(op, data);
|
|
121
|
+
if (statement && (statement.end || statement.writes && !transactionOpen)) {
|
|
122
|
+
ambiguous = true; state.transaction = 'unknown';
|
|
123
|
+
}
|
|
124
|
+
},
|
|
125
|
+
failure: (op, data, open) => observe(op, data, open, true),
|
|
126
|
+
result: (op, data, value, open) => {
|
|
127
|
+
const statement = execution(op, data);
|
|
128
|
+
if (op === 'close') closeAcknowledged = true;
|
|
129
|
+
if (op === 'prepare') {
|
|
130
|
+
if (prepared.size >= limits.statements) prepared.delete(prepared.keys().next().value);
|
|
131
|
+
prepared.set(value, info(data.sql));
|
|
132
|
+
}
|
|
133
|
+
if (op === 'iterate') cursors.set(value, statement);
|
|
134
|
+
if (op === 'return' || op === 'next' && value.done) cursors.delete(data.cursor);
|
|
135
|
+
observe(op, data, open, false, statement);
|
|
136
|
+
},
|
|
137
|
+
};
|
|
138
|
+
const opening = createWorkerConnection(transport, { ...settings, epoch, options, hooks, awaitStartupExit: false,
|
|
139
|
+
reopen: () => driver.open(path, options) });
|
|
140
|
+
child.send({ generation: epoch, path, options: { timeout: options.timeout, readOnly: options.readOnly }, limits });
|
|
141
|
+
const connection = await opening;
|
|
142
|
+
state.status = 'healthy';
|
|
143
|
+
const cancel = (reason = 'caller cancelled') => {
|
|
144
|
+
const error = Object.assign(new DbRuntimeError('JD2097', String(reason)), {
|
|
145
|
+
generation: epoch, retryable: false, class: 'cancelled',
|
|
146
|
+
});
|
|
147
|
+
control.lose(error); terminate(); return error;
|
|
148
|
+
};
|
|
149
|
+
const result = Object.freeze({ ...connection,
|
|
150
|
+
get mustQueue() { return connection.mustQueue; },
|
|
151
|
+
capabilities: Object.freeze({ ...connection.capabilities, worker: false, process: true,
|
|
152
|
+
ownerTermination: true, cancellation: Object.freeze({ ...connection.capabilities.cancellation, midStatement: false }) }),
|
|
153
|
+
settlement: report, settled: () => exited, cancel,
|
|
154
|
+
metrics: () => Object.freeze({ ...connection.metrics(), owner: report(), supervised: active ? 1 : 0 }),
|
|
155
|
+
async supervise(body, request = {}) {
|
|
156
|
+
if (active) throw queueFailure('one supervised operation already owns this process', 1);
|
|
157
|
+
const bound = positiveOption('timeoutMs', request.timeoutMs, timeoutMs);
|
|
158
|
+
if (request.signal?.aborted) throw Object.assign(new DbRuntimeError('JD2097', 'cancelled before admission'), { generation: epoch, retryable: true, class: 'cancelled' });
|
|
159
|
+
if (state.status !== 'healthy') throw new DbRuntimeError('JD2090', 'process owner is not healthy');
|
|
160
|
+
active = true;
|
|
161
|
+
let timer, abort;
|
|
162
|
+
try {
|
|
163
|
+
return await new Promise((resolve, reject) => {
|
|
164
|
+
abort = () => { const error = cancel('supervised operation was cancelled'); reject(error); };
|
|
165
|
+
request.signal?.addEventListener('abort', abort, { once: true });
|
|
166
|
+
timer = setTimeout(() => { const error = cancel('supervised operation exceeded its response deadline'); reject(error); }, bound);
|
|
167
|
+
Promise.resolve().then(() => body(result)).then(resolve, reject);
|
|
168
|
+
});
|
|
169
|
+
}
|
|
170
|
+
finally { active = false; clearTimeout(timer); request.signal?.removeEventListener('abort', abort); }
|
|
171
|
+
},
|
|
172
|
+
});
|
|
173
|
+
return result;
|
|
174
|
+
},
|
|
175
|
+
};
|
|
176
|
+
return Object.freeze(driver);
|
|
177
|
+
}
|
|
@@ -1,105 +1,5 @@
|
|
|
1
1
|
//@ts-check
|
|
2
|
-
/**
|
|
2
|
+
/** A thread owns one SQLite endpoint and its bounded protocol frames. */
|
|
3
3
|
import { parentPort, workerData } from 'node:worker_threads';
|
|
4
|
-
import {
|
|
5
|
-
|
|
6
|
-
import { validRequest, generationFailure, rowBytes } from './worker-protocol.js';
|
|
7
|
-
|
|
8
|
-
const { generation, path, options, limits } = workerData;
|
|
9
|
-
let connection;
|
|
10
|
-
let sequence = 0;
|
|
11
|
-
const statements = new Map();
|
|
12
|
-
const cursors = new Map();
|
|
13
|
-
const frame = (kind, id, value) => ({ v: 1, generation, kind, id, ...value });
|
|
14
|
-
const bound = (reason) => new DbRuntimeError('JD2092', reason);
|
|
15
|
-
const release = (id) => {
|
|
16
|
-
const cursor = cursors.get(id);
|
|
17
|
-
if (cursor === undefined) return;
|
|
18
|
-
cursors.delete(id);
|
|
19
|
-
cursor.iterator.return?.();
|
|
20
|
-
if (cursor.ephemeral) statements.delete(cursor.statement);
|
|
21
|
-
};
|
|
22
|
-
const statementOf = (id) => {
|
|
23
|
-
const statement = statements.get(id);
|
|
24
|
-
if (statement === undefined) throw generationFailure(generation);
|
|
25
|
-
return statement;
|
|
26
|
-
};
|
|
27
|
-
const dispatch = (request) => {
|
|
28
|
-
switch (request.op) {
|
|
29
|
-
case 'exec': return connection.exec(request.sql);
|
|
30
|
-
case 'prepare': {
|
|
31
|
-
if (statements.size >= limits.statements) throw bound(`worker statement capacity ${limits.statements} exceeded`);
|
|
32
|
-
const id = ++sequence;
|
|
33
|
-
statements.set(id, { statement: connection.prepare(request.sql), ephemeral: request.ephemeral === true });
|
|
34
|
-
return id;
|
|
35
|
-
}
|
|
36
|
-
case 'get': {
|
|
37
|
-
const row = statementOf(request.statement).statement.get(request.params);
|
|
38
|
-
if (rowBytes(row) > limits.bytes) throw bound(`one worker row exceeds ${limits.bytes} bytes`);
|
|
39
|
-
return row;
|
|
40
|
-
}
|
|
41
|
-
case 'run': return statementOf(request.statement).statement.run(request.params);
|
|
42
|
-
case 'iterate': {
|
|
43
|
-
if (cursors.size >= limits.cursors) throw bound(`worker cursor capacity ${limits.cursors} exceeded`);
|
|
44
|
-
const record = statementOf(request.statement);
|
|
45
|
-
const id = ++sequence;
|
|
46
|
-
cursors.set(id, { iterator: record.statement.iterate(request.params),
|
|
47
|
-
statement: request.statement, ephemeral: record.ephemeral, buffered: null });
|
|
48
|
-
return id;
|
|
49
|
-
}
|
|
50
|
-
case 'next': {
|
|
51
|
-
const cursor = cursors.get(request.cursor);
|
|
52
|
-
if (cursor === undefined) throw generationFailure(generation);
|
|
53
|
-
if (request.rows > limits.rows || request.bytes > limits.bytes) throw bound('worker cursor credit exceeds the negotiated window');
|
|
54
|
-
const rows = [];
|
|
55
|
-
let bytes = 0;
|
|
56
|
-
try {
|
|
57
|
-
while (rows.length < request.rows) {
|
|
58
|
-
const step = cursor.buffered ?? cursor.iterator.next();
|
|
59
|
-
cursor.buffered = null;
|
|
60
|
-
if (step.done) { release(request.cursor); return { rows, bytes, done: true }; }
|
|
61
|
-
const size = rowBytes(step.value);
|
|
62
|
-
if (size > request.bytes) throw bound(`one worker row of ${size} bytes exceeds the ${request.bytes} byte window`);
|
|
63
|
-
if (bytes + size > request.bytes) { cursor.buffered = step; break; }
|
|
64
|
-
rows.push(step.value);
|
|
65
|
-
bytes += size;
|
|
66
|
-
}
|
|
67
|
-
return { rows, bytes, done: false };
|
|
68
|
-
}
|
|
69
|
-
catch (error) { release(request.cursor); throw error; }
|
|
70
|
-
}
|
|
71
|
-
case 'return': release(request.cursor); return undefined;
|
|
72
|
-
case 'finalize': {
|
|
73
|
-
for (const [id, cursor] of cursors) if (cursor.statement === request.statement) release(id);
|
|
74
|
-
statements.delete(request.statement);
|
|
75
|
-
return undefined;
|
|
76
|
-
}
|
|
77
|
-
case 'close': {
|
|
78
|
-
for (const id of cursors.keys()) release(id);
|
|
79
|
-
statements.clear();
|
|
80
|
-
return connection.close();
|
|
81
|
-
}
|
|
82
|
-
}
|
|
83
|
-
};
|
|
84
|
-
try {
|
|
85
|
-
connection = await nodeDriver().open(path, options);
|
|
86
|
-
// Functions cannot cross structured clone. Capture uses the existing
|
|
87
|
-
// journal path; a live query requires a synchronous connection.
|
|
88
|
-
parentPort.postMessage(frame('ready', 0, { capabilities: { ...connection.capabilities,
|
|
89
|
-
sessions: false, userFunctions: false, deterministicIndexableFunctions: false,
|
|
90
|
-
aggregateFunctions: false, backup: false, worker: true, pooling: false } }));
|
|
91
|
-
parentPort.on('message', (request) => {
|
|
92
|
-
try {
|
|
93
|
-
if (!validRequest(request)) throw new DbRuntimeError('JD2093', 'invalid worker protocol request');
|
|
94
|
-
if (request.generation !== generation) throw generationFailure(request.generation);
|
|
95
|
-
const value = dispatch(request);
|
|
96
|
-
parentPort.postMessage(frame('result', request.id, { value }));
|
|
97
|
-
if (request.op === 'close') parentPort.close();
|
|
98
|
-
}
|
|
99
|
-
catch (error) { parentPort.postMessage(frame('failure', request?.id ?? 0, { error: cloneDriverError(error) })); }
|
|
100
|
-
});
|
|
101
|
-
}
|
|
102
|
-
catch (error) {
|
|
103
|
-
parentPort.postMessage(frame('failure', 0, { error: cloneDriverError(error) }));
|
|
104
|
-
parentPort.close();
|
|
105
|
-
}
|
|
4
|
+
import { serveSqliteEndpoint } from './sqlite-endpoint.js';
|
|
5
|
+
await serveSqliteEndpoint(parentPort, workerData);
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
//@ts-check
|
|
2
2
|
/** A worker transport behind the ordinary Connection contract. */
|
|
3
|
-
import {
|
|
3
|
+
import { lazyOpen } from '../driver.js';
|
|
4
4
|
import { sqliteDialect } from '../dialects/sqlite.js';
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
5
|
+
import { createWorkerConnection } from './worker-client.js';
|
|
6
|
+
import { workerSettings } from './worker-protocol.js';
|
|
7
7
|
|
|
8
8
|
/** Dedicated SQLite worker per connection. No function serialization or write replay.
|
|
9
9
|
* @param {{ windowRows?: number, windowBytes?: number, maxPending?: number,
|
|
@@ -12,17 +12,7 @@ import { generationFailure, positiveOption, queueFailure, rowBytes, validRespons
|
|
|
12
12
|
* @returns {any} a Driver
|
|
13
13
|
*/
|
|
14
14
|
export function nodeWorkerDriver(configuration = {}) {
|
|
15
|
-
const limits =
|
|
16
|
-
rows: positiveOption('windowRows', configuration.windowRows, 64),
|
|
17
|
-
bytes: positiveOption('windowBytes', configuration.windowBytes, 1024 * 1024),
|
|
18
|
-
statements: positiveOption('maxStatements', configuration.maxStatements, 1024),
|
|
19
|
-
cursors: positiveOption('maxCursors', configuration.maxCursors, 64),
|
|
20
|
-
};
|
|
21
|
-
const maxPending = positiveOption('maxPending', configuration.maxPending, 64);
|
|
22
|
-
const allRows = positiveOption('allMaxRows', configuration.allMaxRows, 100000);
|
|
23
|
-
const allBytes = positiveOption('allMaxBytes', configuration.allMaxBytes, 16 * 1024 * 1024);
|
|
24
|
-
const closeMs = positiveOption('closeTimeoutMs', configuration.closeTimeoutMs, 5000);
|
|
25
|
-
const startupMs = positiveOption('startupTimeoutMs', configuration.startupTimeoutMs, 10000);
|
|
15
|
+
const { limits, maxPending, allRows, allBytes, closeMs, startupMs } = workerSettings(configuration);
|
|
26
16
|
let generation = 0;
|
|
27
17
|
const driver = {
|
|
28
18
|
name: 'node-worker-sqlite', dialect: sqliteDialect,
|
|
@@ -34,170 +24,8 @@ export function nodeWorkerDriver(configuration = {}) {
|
|
|
34
24
|
// Parent --test/--input-type/preloads do not describe the endpoint.
|
|
35
25
|
execArgv: ['--no-warnings=ExperimentalWarning'],
|
|
36
26
|
});
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
let failed = null;
|
|
40
|
-
let closing = false;
|
|
41
|
-
let closed = false;
|
|
42
|
-
let transactionDepth = 0;
|
|
43
|
-
let closePromise;
|
|
44
|
-
const metrics = { frames: 0, rows: 0, maxFrameRows: 0, maxFrameBytes: 0, maxPending: 0 };
|
|
45
|
-
let readyResolve;
|
|
46
|
-
let readyReject;
|
|
47
|
-
const ready = new Promise((resolve, reject) => { readyResolve = resolve; readyReject = reject; });
|
|
48
|
-
const lose = (cause) => {
|
|
49
|
-
if (failed !== null || closed) return;
|
|
50
|
-
failed = generationFailure(epoch, transactionDepth > 0, cause);
|
|
51
|
-
readyReject(failed);
|
|
52
|
-
for (const request of pending.values()) request.reject(failed);
|
|
53
|
-
pending.clear();
|
|
54
|
-
};
|
|
55
|
-
worker.on('error', lose);
|
|
56
|
-
worker.on('exit', (code) => { if (!closed) lose(new Error(`worker exited (${code})`)); });
|
|
57
|
-
worker.on('message', (message) => {
|
|
58
|
-
if (!validResponse(message, epoch)) { lose(new Error('invalid worker response')); return; }
|
|
59
|
-
if (message.kind === 'ready') { readyResolve(message.capabilities); return; }
|
|
60
|
-
if (message.kind === 'failure' && message.id === 0) {
|
|
61
|
-
readyReject(Object.assign(new Error(message.error.message), message.error)); return;
|
|
62
|
-
}
|
|
63
|
-
const request = pending.get(message.id);
|
|
64
|
-
if (request === undefined) return;
|
|
65
|
-
pending.delete(message.id);
|
|
66
|
-
if (message.kind === 'failure') {
|
|
67
|
-
const error = message.error;
|
|
68
|
-
request.reject(Object.assign(error.code?.startsWith('JD')
|
|
69
|
-
? new DbRuntimeError(error.code, error.message) : new Error(error.message), error));
|
|
70
|
-
}
|
|
71
|
-
else if (message.kind === 'result' && validResult(request.op, message.value, limits)) request.resolve(message.value);
|
|
72
|
-
else { request.reject(generationFailure(epoch, transactionDepth > 0)); lose(new Error('invalid worker response')); }
|
|
73
|
-
});
|
|
74
|
-
const timer = setTimeout(() => { lose(new Error('worker startup timed out')); worker.terminate(); }, startupMs);
|
|
75
|
-
let capabilities;
|
|
76
|
-
try { capabilities = await ready; }
|
|
77
|
-
catch (error) { await worker.terminate(); throw error; }
|
|
78
|
-
finally { clearTimeout(timer); }
|
|
79
|
-
const request = (op, data = {}, cleanup = false) => {
|
|
80
|
-
if (failed !== null) return Promise.reject(failed);
|
|
81
|
-
if (closed || (closing && !cleanup)) return Promise.reject(new DbRuntimeError('JD2063', 'the worker connection is closing or closed'));
|
|
82
|
-
if (!cleanup && pending.size >= maxPending)
|
|
83
|
-
return Promise.reject(queueFailure(`worker request capacity ${maxPending} exceeded`, pending.size));
|
|
84
|
-
// One return per live cursor and one close have reserved capacity.
|
|
85
|
-
if (cleanup && pending.size >= maxPending + limits.cursors + 1)
|
|
86
|
-
return Promise.reject(queueFailure('worker cleanup capacity exceeded', pending.size));
|
|
87
|
-
const id = ++sequence;
|
|
88
|
-
return new Promise((resolve, reject) => {
|
|
89
|
-
pending.set(id, { resolve, reject, op });
|
|
90
|
-
metrics.maxPending = Math.max(metrics.maxPending, pending.size);
|
|
91
|
-
try { worker.postMessage({ v: 1, generation: epoch, kind: 'request', id, op, ...data }); }
|
|
92
|
-
catch (error) { pending.delete(id); reject(error); }
|
|
93
|
-
});
|
|
94
|
-
};
|
|
95
|
-
const raw = {
|
|
96
|
-
closeDrainsIterators: true,
|
|
97
|
-
exec: (sql) => {
|
|
98
|
-
if (/^(?:SAVEPOINT|BEGIN)\b/.test(sql)) transactionDepth++;
|
|
99
|
-
return request('exec', { sql }).then((value) => {
|
|
100
|
-
if (/^RELEASE\b/.test(sql)) transactionDepth = Math.max(0, transactionDepth - 1);
|
|
101
|
-
if (/^(?:COMMIT|ROLLBACK(?! TO))\b/.test(sql)) transactionDepth = 0;
|
|
102
|
-
return value;
|
|
103
|
-
});
|
|
104
|
-
},
|
|
105
|
-
prepare: async (sql, metadata = {}) => {
|
|
106
|
-
const id = await request('prepare', { sql, ephemeral: metadata.ephemeral === true });
|
|
107
|
-
const iterate = async (params = []) => {
|
|
108
|
-
const cursor = await request('iterate', { statement: id, params });
|
|
109
|
-
let rows = [];
|
|
110
|
-
let at = 0;
|
|
111
|
-
let done = false;
|
|
112
|
-
let returned = false;
|
|
113
|
-
let pulling = Promise.resolve();
|
|
114
|
-
const next = async () => {
|
|
115
|
-
if (failed !== null) throw failed;
|
|
116
|
-
if (returned) return { done: true, value: undefined };
|
|
117
|
-
if (at < rows.length) return { done: false, value: rows[at++] };
|
|
118
|
-
if (done) return { done: true, value: undefined };
|
|
119
|
-
const batch = await request('next', { cursor, rows: limits.rows, bytes: limits.bytes });
|
|
120
|
-
metrics.frames++;
|
|
121
|
-
metrics.rows += batch.rows.length;
|
|
122
|
-
metrics.maxFrameRows = Math.max(metrics.maxFrameRows, batch.rows.length);
|
|
123
|
-
metrics.maxFrameBytes = Math.max(metrics.maxFrameBytes, batch.bytes);
|
|
124
|
-
if (returned) return { done: true, value: undefined };
|
|
125
|
-
rows = batch.rows;
|
|
126
|
-
at = 0;
|
|
127
|
-
done = batch.done;
|
|
128
|
-
return at < rows.length ? { done: false, value: rows[at++] } : { done: true, value: undefined };
|
|
129
|
-
};
|
|
130
|
-
return {
|
|
131
|
-
next: () => {
|
|
132
|
-
const result = pulling.then(next);
|
|
133
|
-
pulling = result.then(() => undefined, () => undefined);
|
|
134
|
-
return result;
|
|
135
|
-
},
|
|
136
|
-
return: async () => {
|
|
137
|
-
if (returned) return { done: true, value: undefined };
|
|
138
|
-
returned = true;
|
|
139
|
-
rows = [];
|
|
140
|
-
if (!done) await request('return', { cursor }, true);
|
|
141
|
-
return { done: true, value: undefined };
|
|
142
|
-
},
|
|
143
|
-
};
|
|
144
|
-
};
|
|
145
|
-
return {
|
|
146
|
-
run: (params = []) => request('run', { statement: id, params }),
|
|
147
|
-
get: (params = []) => request('get', { statement: id, params }),
|
|
148
|
-
iterate,
|
|
149
|
-
all: async (params = []) => {
|
|
150
|
-
const iterator = await iterate(params);
|
|
151
|
-
const rows = [];
|
|
152
|
-
let bytes = 0;
|
|
153
|
-
try {
|
|
154
|
-
for (;;) {
|
|
155
|
-
const step = await iterator.next();
|
|
156
|
-
if (step.done) return rows;
|
|
157
|
-
bytes += rowBytes(step.value);
|
|
158
|
-
if (rows.length >= allRows || bytes > allBytes)
|
|
159
|
-
throw new DbRuntimeError('JD2092', `worker all() exceeds its ${allRows} row / ${allBytes} byte bound; use a cursor`);
|
|
160
|
-
rows.push(step.value);
|
|
161
|
-
}
|
|
162
|
-
}
|
|
163
|
-
finally { await iterator.return(); }
|
|
164
|
-
},
|
|
165
|
-
};
|
|
166
|
-
},
|
|
167
|
-
close: () => {
|
|
168
|
-
if (closePromise !== undefined) return closePromise;
|
|
169
|
-
closing = true;
|
|
170
|
-
closePromise = new Promise((resolve, reject) => {
|
|
171
|
-
const timeout = setTimeout(() => {
|
|
172
|
-
lose(new Error('worker close timed out'));
|
|
173
|
-
// V8 termination cannot preempt a synchronous native SQLite
|
|
174
|
-
// call. Fence and detach now; never hold the caller's deadline
|
|
175
|
-
// hostage to that native call or claim it was rolled back.
|
|
176
|
-
worker.unref();
|
|
177
|
-
worker.terminate().catch(() => {});
|
|
178
|
-
reject(failed);
|
|
179
|
-
}, closeMs);
|
|
180
|
-
request('close', {}, true).then(() => {
|
|
181
|
-
closed = true;
|
|
182
|
-
clearTimeout(timeout);
|
|
183
|
-
worker.terminate().then(() => resolve(undefined), reject);
|
|
184
|
-
}, (error) => {
|
|
185
|
-
clearTimeout(timeout);
|
|
186
|
-
worker.unref();
|
|
187
|
-
worker.terminate().catch(() => {});
|
|
188
|
-
reject(error);
|
|
189
|
-
});
|
|
190
|
-
});
|
|
191
|
-
return closePromise;
|
|
192
|
-
},
|
|
193
|
-
};
|
|
194
|
-
const connection = finishConnection(raw, sqliteDialect, false, Object.freeze(capabilities), options.queueTimeout);
|
|
195
|
-
return Object.freeze({ ...connection,
|
|
196
|
-
get mustQueue() { return connection.mustQueue; },
|
|
197
|
-
generation: epoch,
|
|
198
|
-
metrics: () => Object.freeze({ ...metrics, pending: pending.size, generation: epoch, healthy: failed === null && !closed }),
|
|
199
|
-
restart: async () => { lose(new Error('worker restarted')); await worker.terminate(); return driver.open(path, options); },
|
|
200
|
-
});
|
|
27
|
+
return createWorkerConnection(worker, { epoch, options, limits, maxPending, allRows, allBytes,
|
|
28
|
+
closeMs, startupMs, reopen: () => driver.open(path, options) });
|
|
201
29
|
}, []),
|
|
202
30
|
};
|
|
203
31
|
return Object.freeze(driver);
|
package/src/drivers/node.js
CHANGED
|
@@ -38,6 +38,7 @@ export function adaptNodeDatabase(db, options) {
|
|
|
38
38
|
};
|
|
39
39
|
},
|
|
40
40
|
close: () => db.close(),
|
|
41
|
+
transactionState: () => typeof db.isTransaction === 'boolean' ? db.isTransaction : null,
|
|
41
42
|
// each optional primitive is exposed only when the HANDLE has it, so
|
|
42
43
|
// a substitute that carries less than `node:sqlite` reports less —
|
|
43
44
|
// a declared capability the handle cannot honour is a TypeError at
|
|
@@ -120,6 +121,8 @@ export function nodeDriver() {
|
|
|
120
121
|
});
|
|
121
122
|
}
|
|
122
123
|
|
|
124
|
+
export { snapshotDatabase } from './snapshot.js';
|
|
125
|
+
|
|
123
126
|
export {
|
|
124
127
|
readDocuments, readJsonDocuments, readJsonlDocuments, readCollectionBundle,
|
|
125
128
|
openAtomicTarget, openStreamTarget, openNullTarget,
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
//@ts-check
|
|
2
|
+
/** Disk-backed SQLite snapshots shared by the Node and Bun bindings. */
|
|
3
|
+
import { DbCompileError } from '../errors.js';
|
|
4
|
+
|
|
5
|
+
/** Reserve a new destination and fill it through SQLite, never a JS image.
|
|
6
|
+
* @param {string} target @param {() => any} write
|
|
7
|
+
* @param {{progress?:Function}} [options] @returns {Promise<number>} */
|
|
8
|
+
export async function writeSqliteSnapshot(target, write, options) {
|
|
9
|
+
if (typeof target !== 'string' || !target || target.includes('\0')) throw new TypeError('snapshot target must be a nonempty path');
|
|
10
|
+
const fs = await import('node:fs/promises');
|
|
11
|
+
const file = await fs.open(target, 'wx');
|
|
12
|
+
let complete = false;
|
|
13
|
+
try {
|
|
14
|
+
await write();
|
|
15
|
+
await file.sync();
|
|
16
|
+
const header = new Uint8Array(100);
|
|
17
|
+
const reader = await fs.open(target, 'r');
|
|
18
|
+
try { await reader.read(header, 0, header.length, 0); }
|
|
19
|
+
finally { await reader.close(); }
|
|
20
|
+
const encoded = (header[16] << 8) | header[17];
|
|
21
|
+
const pageSize = encoded === 1 ? 65536 : encoded;
|
|
22
|
+
const size = (await file.stat()).size;
|
|
23
|
+
const pages = size / pageSize;
|
|
24
|
+
options?.progress?.({ totalPages: pages, remainingPages: 0 });
|
|
25
|
+
complete = true;
|
|
26
|
+
return pages;
|
|
27
|
+
}
|
|
28
|
+
finally {
|
|
29
|
+
await file.close();
|
|
30
|
+
if (!complete) await fs.rm(target, { force: true });
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/** Create a consistent, bounded-memory snapshot on a new path, including
|
|
35
|
+
* committed WAL data. Does not open a model store or replace a target.
|
|
36
|
+
* SQLite's page caches bound working memory; no serialize/hex image is built.
|
|
37
|
+
* @param {any} connection @param {string} target
|
|
38
|
+
* @returns {Promise<{path:string,pages:number}>} */
|
|
39
|
+
export async function snapshotDatabase(connection, target) {
|
|
40
|
+
if (connection.dialect.name !== 'sqlite' || !connection.synchronous)
|
|
41
|
+
throw new DbCompileError('JD0038', 'snapshotDatabase requires a synchronous SQLite connection');
|
|
42
|
+
const write = () => connection.prepare('VACUUM INTO ?').run([target]);
|
|
43
|
+
// Acquire ownership before asynchronous filesystem work. Acquiring it
|
|
44
|
+
// afterwards would queue behind an outer transaction awaiting this call.
|
|
45
|
+
const snapshot = () => writeSqliteSnapshot(target, write);
|
|
46
|
+
const pages = await (typeof connection.exclusively === 'function'
|
|
47
|
+
? connection.exclusively(snapshot, 'a database snapshot') : snapshot());
|
|
48
|
+
return { path: target, pages };
|
|
49
|
+
}
|