@jarenjs/db 0.56.0 → 0.67.0

package/src/drivers/worker-pool.js

@@ -0,0 +1,183 @@
+//@ts-check
+/** One writer and bounded read-only WAL workers behind a Connection. */
+import { finishConnection } from '../driver.js';
+import { DbCompileError, DbRuntimeError } from '../errors.js';
+import { sqliteDialect } from '../dialects/sqlite.js';
+import { positiveOption } from './worker-protocol.js';
+import { workerQueue } from './worker-queue.js';
+
+/**
+ * Reads explicitly classified by their compiled operation may use a read worker.
+ * All other work uses the writer; an open transaction keeps one worker.
+ * @param {{ readers?: number, queueCapacity?: number, graceMs?: number,
+ *   worker?: any }} [configuration]
+ * @returns {any} a Driver
+ */
+export function workerPoolDriver(configuration, driverFactory) {
+  const readers = configuration.readers ?? 2;
+  const capacity = configuration.queueCapacity ?? 64;
+  if (!Number.isSafeInteger(readers) || readers < 0 || readers > 32) throw new TypeError('readers must be an integer from 0 through 32');
+  if (!Number.isSafeInteger(capacity) || capacity < 0) throw new TypeError('queueCapacity must be a nonnegative safe integer');
+  const graceMs = positiveOption('graceMs', configuration.graceMs, 5000);
+  return Object.freeze({
+    name: 'node-worker-pool-sqlite', dialect: sqliteDialect,
+    open: async (path = ':memory:', options = {}) => {
+      const driver = driverFactory(configuration.worker);
+      const slots = [];
+      let closed = false;
+      let transaction = null;
+      let depth = 0;
+      let committing = null;
+      let sequence = 0;
+      const memory = path === ':memory:' || path === '';
+      const makeSlot = async (readOnly) => {
+        const connection = await driver.open(path, { ...options, readOnly });
+        return { connection, readOnly, active: false, healthy: true, generation: connection.generation,
+          statements: new Map(), executions: 0 };
+      };
+      try {
+        slots.push(await makeSlot(options.readOnly === true));
+        if (!memory) {
+          const writer = slots[0].connection;
+          const statement = await writer.prepare(options.readOnly === true ? sqliteDialect.introspect.pragma('journal_mode') : sqliteDialect.pragma.set('journal_mode', 'WAL'));
+          const mode = await statement.get();
+          if (String(Object.values(mode)[0]).toLowerCase() !== 'wal')
+            throw new DbCompileError('JD0008', 'a worker pool requires a file in WAL mode');
+          for (let i = 0; i < readers; i++) slots.push(await makeSlot(true));
+        }
+      }
+      catch (error) { await Promise.allSettled(slots.map((slot) => slot.connection.close())); throw error; }
+      const queue = workerQueue(slots, capacity, () => performance.now());
+      const readLane = (readOnly) => options.readOnly === true || (readOnly && slots.length > 1);
+      const replace = async (slot) => {
+        if (closed || slot.healthy) return;
+        await slot.connection.close().catch(() => {});
+        try {
+          const fresh = await makeSlot(slot.readOnly);
+          Object.assign(slot, fresh);
+          queue.wake();
+        }
+        catch (error) { queue.stop(error); }
+      };
+      const execute = async (lease, fn) => {
+        try { lease.slot.executions++; return await fn(lease.slot); }
+        catch (error) { if (error?.code === 'JD2090') lease.slot.healthy = false; throw error; }
+      };
+      const withLease = async (readOnly, fn) => {
+        if (closed) throw new DbRuntimeError('JD2063', 'the worker pool is closed');
+        if (transaction !== null) return execute(transaction, fn);
+        const lease = await queue.acquire(readLane(readOnly));
+        try { return await execute(lease, fn); }
+        finally { lease.release(); await replace(lease.slot); }
+      };
+      const raw = {
+        closeDrainsIterators: true,
+        exec: async (sql) => {
+          const begin = /^(?:SAVEPOINT|BEGIN)\b/.test(sql);
+          const release = /^RELEASE\b/.test(sql);
+          const end = /^(?:COMMIT|ROLLBACK(?! TO))\b/.test(sql);
+          const rollback = /^ROLLBACK\b/.test(sql);
+          if (begin && transaction === null) transaction = await queue.acquire(options.readOnly === true);
+          if (begin) depth++;
+          const operation = withLease(false, (slot) => slot.connection.exec(sql));
+          if (end || (release && depth === 1)) committing = operation;
+          let succeeded = false;
+          try { const value = await operation; succeeded = true; return value; }
+          catch (error) {
+            // A failed begin owns no new savepoint. A lost generation
+            // cannot acknowledge rollback or the following release; unwind
+            // exactly this scope, leaving any outer owner pinned until its
+            // own rollback settles. A failed COMMIT keeps its lease for
+            // the normal rollback path.
+            if (begin || (rollback && error?.code === 'JD2090'))
+              depth = end ? 0 : Math.max(0, depth - 1);
+            throw error;
+          }
+          finally {
+            if (succeeded && release) depth = Math.max(0, depth - 1);
+            if (succeeded && end) depth = 0;
+            if (transaction !== null && depth === 0) {
+              const lease = transaction;
+              transaction = null;
+              lease.release();
+              await replace(lease.slot);
+            }
+            if (committing === operation) committing = null;
+          }
+        },
+        prepare: (sql, metadata = {}) => {
+          const id = ++sequence;
+          const statementFor = async (slot) => {
+            if (!slot.statements.has(id) || metadata.ephemeral) {
+              const statement = await slot.connection.prepare(sql, metadata);
+              if (metadata.ephemeral) return statement;
+              slot.statements.set(id, statement);
+            }
+            return slot.statements.get(id);
+          };
+          const call = (method, params) => withLease(metadata.readOnly === true,
+            async (slot) => (await statementFor(slot))[method](params));
+          return {
+            run: (params = []) => call('run', params),
+            get: (params = []) => call('get', params),
+            all: (params = []) => call('all', params),
+            iterate: async (params = []) => {
+              const pinned = transaction;
+              const lease = pinned ?? await queue.acquire(readLane(metadata.readOnly === true));
+              let iterator;
+              try { iterator = await execute(lease, async (slot) => (await statementFor(slot)).iterate(params)); }
+              catch (error) { if (pinned === null) { lease.release(); await replace(lease.slot); } throw error; }
+              let done = false;
+              const finish = async () => {
+                if (done) return;
+                done = true;
+                try { await iterator.return(); }
+                finally { if (pinned === null) { lease.release(); await replace(lease.slot); } }
+              };
+              return {
+                next: async () => {
+                  if (done) return { done: true, value: undefined };
+                  try {
+                    const step = await execute(lease, () => iterator.next());
+                    if (step.done) await finish();
+                    return step;
+                  }
+                  catch (error) { await finish().catch(() => {}); throw error; }
+                },
+                return: async () => { await finish(); return { done: true, value: undefined }; },
+              };
+            },
+          };
+        },
+        close: async () => {
+          if (closed) return;
+          closed = true;
+          queue.stop();
+          let timer;
+          await Promise.race([queue.drain(), new Promise((resolve) => { timer = setTimeout(resolve, graceMs); })]);
+          clearTimeout(timer);
+          // A commit already sent to SQLite owns its settlement. Closing
+          // never terminates that writer while its outcome is pending.
+          if (committing !== null) await committing.catch(() => {});
+          try { await Promise.all(slots.map((slot) => slot.connection.close())); }
+          finally {
+            for (const slot of slots) {
+              slot.healthy = false;
+              slot.active = false;
+              slot.statements.clear();
+            }
+          }
+        },
+      };
+      const capabilities = Object.freeze({ ...slots[0].connection.capabilities,
+        pooling: true, poolReaders: slots.filter((slot) => slot.readOnly).length,
+        poolWriters: slots.filter((slot) => !slot.readOnly).length });
+      const connection = finishConnection(raw, sqliteDialect, false, capabilities, options.queueTimeout);
+      return Object.freeze({ ...connection, get mustQueue() { return connection.mustQueue; },
+        metrics: () => Object.freeze({ ...queue.metrics(), workers: Object.freeze(slots.map((slot) =>
+          Object.freeze({ readOnly: slot.readOnly, healthy: slot.healthy, active: slot.active,
+            generation: slot.generation, executions: slot.executions }))) }),
+      });
+    },
+  });
+}

package/src/drivers/worker-protocol.js

@@ -0,0 +1,79 @@
+//@ts-check
+/** Structured-clone frames; identities belong to one connection generation. */
+import { DbRuntimeError } from '../errors.js';
+import { jsonStringBytes } from '../json-bytes.js';
+
+export const WORKER_PROTOCOL_VERSION = 1;
+const operations = new Set(['exec', 'prepare', 'run', 'get', 'iterate', 'next', 'return', 'finalize', 'close']);
+
+/** @param {number} generation @param {boolean} [transaction] @param {unknown} [cause] */
+export function generationFailure(generation, transaction = false, cause = undefined) {
+  return Object.assign(new DbRuntimeError('JD2090',
+    `driver generation ${generation} is no longer available; reopen the connection; no operation was replayed`,
+    { cause }), { class: 'generation', retryable: !transaction, generation });
+}
+
+/** @param {any} frame @returns {boolean} */
+export function validRequest(frame) {
+  if (frame === null || typeof frame !== 'object' || frame.v !== WORKER_PROTOCOL_VERSION
+    || frame.kind !== 'request' || !Number.isSafeInteger(frame.id) || frame.id < 1
+    || !Number.isSafeInteger(frame.generation) || frame.generation < 1 || !operations.has(frame.op)) return false;
+  if (frame.op === 'exec' || frame.op === 'prepare') return typeof frame.sql === 'string';
+  if (['run', 'get', 'iterate'].includes(frame.op))
+    return Number.isSafeInteger(frame.statement) && frame.statement > 0 && Array.isArray(frame.params);
+  if (frame.op === 'next') return Number.isSafeInteger(frame.cursor) && frame.cursor > 0
+    && Number.isSafeInteger(frame.rows) && frame.rows > 0 && Number.isSafeInteger(frame.bytes) && frame.bytes > 0;
+  if (frame.op === 'return') return Number.isSafeInteger(frame.cursor) && frame.cursor > 0;
+  if (frame.op === 'finalize') return Number.isSafeInteger(frame.statement) && frame.statement > 0;
+  return true;
+}
+
+/** Validate clone frames before dereferencing a remote payload.
+ * @param {any} frame @param {number} generation @returns {boolean}
+ */
+export function validResponse(frame, generation) {
+  if (frame === null || typeof frame !== 'object' || frame.v !== WORKER_PROTOCOL_VERSION
+    || frame.generation !== generation || !Number.isSafeInteger(frame.id) || frame.id < 0) return false;
+  if (frame.kind === 'ready') return frame.id === 0 && frame.capabilities !== null
+    && typeof frame.capabilities === 'object' && typeof frame.capabilities.version === 'string';
+  if (frame.kind === 'failure') return frame.error !== null && typeof frame.error === 'object'
+    && typeof frame.error.message === 'string';
+  return frame.kind === 'result' && frame.id > 0;
+}
+
+/** @param {string} op @param {any} value @param {{rows:number, bytes:number}} limits */
+export function validResult(op, value, limits) {
+  if (op === 'prepare' || op === 'iterate') return Number.isSafeInteger(value) && value > 0;
+  if (op !== 'next') return true;
+  return value !== null && typeof value === 'object' && Array.isArray(value.rows)
+    && value.rows.length <= limits.rows && typeof value.done === 'boolean'
+    && (value.rows.length > 0 || value.done) && Number.isSafeInteger(value.bytes)
+    && value.bytes >= 0 && value.bytes <= limits.bytes
+    && value.rows.reduce((sum, row) => sum + rowBytes(row), 0) === value.bytes;
+}
+
+/** A stable credit measure: JSON keys/scalars and raw blob bytes. Transport
+ * metadata is fixed per frame; no JSON string or copied blob is allocated.
+ * @param {any} value @returns {number}
+ */
+export function rowBytes(value) {
+  if (value === null || value === undefined) return 4;
+  if (ArrayBuffer.isView(value)) return value.byteLength;
+  if (typeof value === 'string') return jsonStringBytes(value);
+  if (typeof value !== 'object') return String(value).length;
+  let bytes = 2;
+  for (const key of Object.keys(value)) bytes += jsonStringBytes(key) + 2 + rowBytes(value[key]);
+  return bytes;
+}
+
+/** @param {string} name @param {any} value @param {number} fallback @returns {number} */
+export function positiveOption(name, value, fallback) {
+  if (value === undefined) return fallback;
+  if (!Number.isSafeInteger(value) || value < 1) throw new TypeError(`${name} must be a positive safe integer`);
+  return value;
+}
+
+/** @param {string} reason @param {number} depth */
+export function queueFailure(reason, depth) {
+  return Object.assign(new DbRuntimeError('JD2091', reason), { class: 'queue', retryable: true, depth });
+}

package/src/drivers/worker-queue.js

@@ -0,0 +1,60 @@
+//@ts-check
+/** Bounded FIFO admission, with a deterministic clock and inspectable leases. */
+import { DbRuntimeError } from '../errors.js';
+import { queueFailure } from './worker-protocol.js';
+
+/** @param {any[]} slots @param {number} capacity @param {() => number} now */
+export function workerQueue(slots, capacity, now) {
+  const waiting = [];
+  const waits = [];
+  const observers = new Set();
+  let stopped = null;
+  const idle = () => { if (slots.every((slot) => !slot.active)) for (const fn of observers) fn(); };
+  const choose = (readOnly) => slots.find((slot) => !slot.active && slot.healthy
+    && (readOnly ? slot.readOnly : !slot.readOnly));
+  const grant = (slot, entry) => {
+    slot.active = true;
+    const wait = Math.max(0, now() - entry.at);
+    if (waits.length === 1024) waits.shift();
+    waits.push(wait);
+    let released = false;
+    entry.resolve({ slot, release: () => {
+      if (released) return;
+      released = true;
+      slot.active = false;
+      pump();
+      idle();
+    } });
+  };
+  const pump = () => {
+    while (waiting.length > 0) {
+      const slot = choose(waiting[0].readOnly);
+      if (slot === undefined) break;
+      grant(slot, waiting.shift());
+    }
+  };
+  return {
+    acquire: (readOnly) => new Promise((resolve, reject) => {
+      if (stopped !== null) { reject(stopped); return; }
+      const entry = { readOnly, resolve, reject, at: now() };
+      const slot = waiting.length === 0 ? choose(readOnly) : undefined;
+      if (slot !== undefined) { grant(slot, entry); return; }
+      if (waiting.length >= capacity) { reject(queueFailure(`worker pool queue capacity ${capacity} exceeded`, waiting.length)); return; }
+      waiting.push(entry);
+    }),
+    stop(error = new DbRuntimeError('JD2063', 'the worker pool closed before the queued work ran')) {
+      stopped = error;
+      for (const entry of waiting.splice(0)) entry.reject(error);
+    },
+    drain: () => slots.every((slot) => !slot.active) ? Promise.resolve()
+      : new Promise((resolve) => { const done = () => { observers.delete(done); resolve(undefined); }; observers.add(done); }),
+    wake: pump,
+    metrics: () => {
+      const values = [...waits].sort((a, b) => a - b);
+      return Object.freeze({ active: slots.filter((s) => s.active).length,
+        idle: slots.filter((s) => !s.active && s.healthy).length, queued: waiting.length,
+        waitMs: Object.freeze({ p50: values[Math.floor(values.length * 0.5)] ?? 0,
+          p95: values[Math.min(values.length - 1, Math.floor(values.length * 0.95))] ?? 0 }) });
+    },
+  };
+}