@jarenjs/db 0.85.0 → 0.87.0

package/src/drivers/worker-client.js

@@ -0,0 +1,185 @@
+//@ts-check
+/** Shared bounded RPC Connection client for owned SQLite execution hosts. */
+import { finishConnection } from '../driver.js';
+import { sqliteDialect } from '../dialects/sqlite.js';
+import { DbRuntimeError } from '../errors.js';
+import { generationFailure, queueFailure, rowBytes, validResponse, validResult } from './worker-protocol.js';
+
+/** @param {any} worker @param {any} settings @returns {Promise<any>} */
+export async function createWorkerConnection(worker, settings) {
+  const { epoch, options, limits, maxPending, allRows, allBytes, closeMs, startupMs, reopen, hooks = {} } = settings;
+  const pending = new Map();
+  let sequence = 0;
+  let failed = null;
+  let closing = false;
+  let closed = false;
+  let transactionDepth = 0;
+  let closePromise;
+  const metrics = { frames: 0, rows: 0, maxFrameRows: 0, maxFrameBytes: 0, maxPending: 0 };
+  let readyResolve;
+  let readyReject;
+  const ready = new Promise((resolve, reject) => { readyResolve = resolve; readyReject = reject; });
+  const lose = (cause) => {
+    if (failed !== null || closed) return;
+    failed = generationFailure(epoch, transactionDepth > 0, cause);
+    hooks.lost?.(failed, [...pending.values()]);
+    readyReject(failed);
+    for (const request of pending.values()) request.reject(failed);
+    pending.clear();
+  };
+  hooks.control?.({ lose });
+  worker.on('error', lose);
+  worker.on('exit', (code) => { if (!closed) lose(new Error(`worker exited (${code})`)); });
+  worker.on('message', (message) => {
+    if (!validResponse(message, epoch)) { lose(new Error('invalid worker response')); return; }
+    if (message.kind === 'ready') { readyResolve(message.capabilities); return; }
+    if (message.kind === 'failure' && message.id === 0) {
+      readyReject(Object.assign(new Error(message.error.message), message.error)); return;
+    }
+    const request = pending.get(message.id);
+    if (request === undefined) return;
+    pending.delete(message.id);
+    if (message.kind === 'failure') {
+      hooks.failure?.(request.op, request.data, message.transaction);
+      const error = message.error;
+      request.reject(Object.assign(error.code?.startsWith('JD')
+        ? new DbRuntimeError(error.code, error.message) : new Error(error.message), error));
+    }
+    else if (message.kind === 'result' && validResult(request.op, message.value, limits)) {
+      hooks.result?.(request.op, request.data, message.value, message.transaction); request.resolve(message.value);
+    }
+    else { request.reject(generationFailure(epoch, transactionDepth > 0)); lose(new Error('invalid worker response')); }
+  });
+  const timer = setTimeout(() => { lose(new Error('worker startup timed out')); worker.terminate(); }, startupMs);
+  let capabilities;
+  try { capabilities = await ready; }
+  catch (error) {
+    const termination = worker.terminate();
+    if (settings.awaitStartupExit !== false) await termination;
+    else termination.catch(() => {});
+    throw error;
+  }
+  finally { clearTimeout(timer); }
+  const request = (op, data = {}, cleanup = false) => {
+    if (failed !== null) return Promise.reject(failed);
+    if (closed || (closing && !cleanup)) return Promise.reject(new DbRuntimeError('JD2063', 'the worker connection is closing or closed'));
+    if (!cleanup && pending.size >= maxPending)
+      return Promise.reject(queueFailure(`worker request capacity ${maxPending} exceeded`, pending.size));
+    // One return per live cursor and one close have reserved capacity.
+    if (cleanup && pending.size >= maxPending + limits.cursors + 1)
+      return Promise.reject(queueFailure('worker cleanup capacity exceeded', pending.size));
+    const id = ++sequence;
+    return new Promise((resolve, reject) => {
+      pending.set(id, { resolve, reject, op, data });
+      metrics.maxPending = Math.max(metrics.maxPending, pending.size);
+      try { hooks.request?.(op, data); worker.postMessage({ v: 1, generation: epoch, kind: 'request', id, op, ...data }); }
+      catch (error) { pending.delete(id); reject(error); }
+    });
+  };
+  const raw = {
+    closeDrainsIterators: true,
+    exec: (sql) => {
+      if (/^(?:SAVEPOINT|BEGIN)\b/.test(sql)) transactionDepth++;
+      return request('exec', { sql }).then((value) => {
+        if (/^RELEASE\b/.test(sql)) transactionDepth = Math.max(0, transactionDepth - 1);
+        if (/^(?:COMMIT|ROLLBACK(?! TO))\b/.test(sql)) transactionDepth = 0;
+        return value;
+      });
+    },
+    prepare: async (sql, metadata = {}) => {
+      const id = await request('prepare', { sql, ephemeral: metadata.ephemeral === true });
+      const iterate = async (params = []) => {
+        const cursor = await request('iterate', { statement: id, params });
+        let rows = [];
+        let at = 0;
+        let done = false;
+        let returned = false;
+        let pulling = Promise.resolve();
+        const next = async () => {
+          if (failed !== null) throw failed;
+          if (returned) return { done: true, value: undefined };
+          if (at < rows.length) return { done: false, value: rows[at++] };
+          if (done) return { done: true, value: undefined };
+          const batch = await request('next', { cursor, rows: limits.rows, bytes: limits.bytes });
+          metrics.frames++;
+          metrics.rows += batch.rows.length;
+          metrics.maxFrameRows = Math.max(metrics.maxFrameRows, batch.rows.length);
+          metrics.maxFrameBytes = Math.max(metrics.maxFrameBytes, batch.bytes);
+          if (returned) return { done: true, value: undefined };
+          rows = batch.rows;
+          at = 0;
+          done = batch.done;
+          return at < rows.length ? { done: false, value: rows[at++] } : { done: true, value: undefined };
+        };
+        return {
+          next: () => {
+            const result = pulling.then(next);
+            pulling = result.then(() => undefined, () => undefined);
+            return result;
+          },
+          return: async () => {
+            if (returned) return { done: true, value: undefined };
+            returned = true;
+            rows = [];
+            if (!done) await request('return', { cursor }, true);
+            return { done: true, value: undefined };
+          },
+        };
+      };
+      return {
+        run: (params = []) => request('run', { statement: id, params }),
+        get: (params = []) => request('get', { statement: id, params }),
+        iterate,
+        all: async (params = []) => {
+          const iterator = await iterate(params);
+          const rows = [];
+          let bytes = 0;
+          try {
+            for (;;) {
+              const step = await iterator.next();
+              if (step.done) return rows;
+              bytes += rowBytes(step.value);
+              if (rows.length >= allRows || bytes > allBytes)
+                throw new DbRuntimeError('JD2092', `worker all() exceeds its ${allRows} row / ${allBytes} byte bound; use a cursor`);
+              rows.push(step.value);
+            }
+          }
+          finally { await iterator.return(); }
+        },
+      };
+    },
+    close: () => {
+      if (closePromise !== undefined) return closePromise;
+      closing = true;
+      closePromise = new Promise((resolve, reject) => {
+        const timeout = setTimeout(() => {
+          lose(new Error('worker close timed out'));
+          // V8 termination cannot preempt a synchronous native SQLite
+          // call. Fence and detach now; never hold the caller's deadline
+          // hostage to that native call or claim it was rolled back.
+          worker.unref();
+          worker.terminate().catch(() => {});
+          reject(failed);
+        }, closeMs);
+        request('close', {}, true).then(() => {
+          closed = true;
+          clearTimeout(timeout);
+          worker.terminate().then(() => resolve(undefined), reject);
+        }, (error) => {
+          clearTimeout(timeout);
+          worker.unref();
+          worker.terminate().catch(() => {});
+          reject(error);
+        });
+      });
+      return closePromise;
+    },
+  };
+  const connection = finishConnection(raw, sqliteDialect, false, Object.freeze(capabilities), options.queueTimeout);
+  return Object.freeze({ ...connection,
+    get mustQueue() { return connection.mustQueue; },
+    generation: epoch,
+    metrics: () => Object.freeze({ ...metrics, pending: pending.size, generation: epoch, healthy: failed === null && !closed }),
+    restart: async () => { lose(new Error('worker restarted')); await worker.terminate(); return reopen(); },
+  });
+}

package/src/drivers/worker-protocol.js

@@ -4,6 +4,11 @@ import { DbRuntimeError } from '../errors.js';
 import { jsonStringBytes } from '../json-bytes.js';
 
 export const WORKER_PROTOCOL_VERSION = 1;
+export const WORKER_DEFAULTS = Object.freeze({ windowRows: 64, windowBytes: 1048576,
+  maxPending: 64, maxStatements: 1024, maxCursors: 64, allMaxRows: 100000,
+  allMaxBytes: 16777216, closeTimeoutMs: 5000, startupTimeoutMs: 10000 });
+export const PROCESS_DEFAULTS = Object.freeze({ ...WORKER_DEFAULTS, closeTimeoutMs: 1000,
+  maxOwners: 4, timeoutMs: 250, maxRequestBytes: 1048576 });
 const operations = new Set(['exec', 'prepare', 'run', 'get', 'iterate', 'next', 'return', 'finalize', 'close']);
 
 /** @param {number} generation @param {boolean} [transaction] @param {unknown} [cause] */
@@ -73,6 +78,16 @@ export function positiveOption(name, value, fallback) {
   return value;
 }
 
+/** One validation and spelling of shared transport credits.
+ * @param {any} configuration @param {typeof WORKER_DEFAULTS} [defaults] */
+export function workerSettings(configuration, defaults = WORKER_DEFAULTS) {
+  const values = Object.fromEntries(Object.keys(WORKER_DEFAULTS).map((name) =>
+    [name, positiveOption(name, configuration[name], defaults[name])]));
+  return { limits: { rows: values.windowRows, bytes: values.windowBytes, statements: values.maxStatements, cursors: values.maxCursors },
+    maxPending: values.maxPending, allRows: values.allMaxRows, allBytes: values.allMaxBytes,
+    closeMs: values.closeTimeoutMs, startupMs: values.startupTimeoutMs };
+}
+
 /** @param {string} reason @param {number} depth */
 export function queueFailure(reason, depth) {
   return Object.assign(new DbRuntimeError('JD2091', reason), { class: 'queue', retryable: true, depth });

package/src/errors.js

@@ -112,6 +112,7 @@ export const DB_CODES = Object.freeze({
   JD2094: 'the durable snapshot failed and the connection is invalid',
   JD2095: 'the trusted SQL or synchronous transaction authority was refused',
   JD2096: 'a persistence invariant rejected the mutation',
+  JD2097: 'the supervised operation was cancelled or exceeded its response deadline',
 });
 
 /**

package/src/index.js

@@ -105,4 +105,4 @@ export { planInvariants } from './ddl.js';
 export { planPhysicalMigration } from './migrate.js';
 export { sql, planRelational, relational } from './dialects/sqlite-relational.js';
 export { defineTable, planTable } from './dialects/sqlite-schema.js';
-export { planTableMigration, applyTableMigration, withForeignKeysSuspended } from './table-migration.js';
+export { planTableMigration, applyTableMigration, withForeignKeysSuspended, planSchemaChange, applySchemaChange } from './table-migration.js';

package/src/mutation.js

@@ -1,7 +1,7 @@
 //@ts-check
 /** Bounded column mutation documents, lowered through the entity's writer plan. */
 import { analyzeQuery } from '@jarenjs/json/query';
-import { canonicalizeJson } from '@jarenjs/json/canonical';
+import { createBoundedCache } from '@jarenjs/core/cache';
 import { chain, attempt } from './driver.js';
 import { DbCompileError, DbRuntimeError, wrapDriverError } from './errors.js';
 import { entityShape, planEntityPredicate } from './plan.js';
@@ -10,12 +10,13 @@ import { physicalSelection } from './physical.js';
 import { utf8Length } from './cursor.js';
 import { relationalEmitter } from './dialects/sqlite-relational.js';
 
-/** Compile once per document, execute within the existing guarded transaction.
+/** Bind each document, reuse its SQL statement within the guarded transaction.
  * @param {any} connection @param {any} entity @param {any} mapping @param {any} core */
 export function createEntityMutation(connection, entity, mapping, core) {
   const dialect = connection.dialect;
   const q = dialect.quoteIdentifier;
-  const plans = new Map();
+  // Cache executable structure, never a value-bearing document or its bindings.
+  const statements = createBoundedCache(64);
   const fail = (reason) => { throw new DbCompileError('JD0038', reason, entity.docPath); };
   const column = (name) => {
     const c = mapping.columns.find((entry) => entry.name === name);
@@ -56,15 +57,17 @@ export function createEntityMutation(connection, entity, mapping, core) {
     const param = (value) => { params.push(value); return dialect.parameterRef(params.length, 'v'); };
     const table = q(mapping.table);
     const sqlExpression = (expression, inline = false) => {
-      const mapped = (value) => {
-        if (Array.isArray(value)) return value.map(mapped);
+      const mapped = (value, depth = 0) => {
+        if (depth > 64) fail('SQL expression nesting exceeds 64');
+        const next = (child) => mapped(child, depth + 1);
+        if (Array.isArray(value)) return value.map(next);
         if (value === null || typeof value !== 'object') return value;
         if (value.$sql === 'value') return value;
         if (value.$sql === 'column') {
           if (value.table !== undefined && value.table !== 'it') fail('mutation column expressions refer to the current entity');
           return { $sql: 'column', name: column(value.name).physical };
         }
-        return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, mapped(item)]));
+        return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, next(item)]));
       };
       const emitter = relationalEmitter({ inline });
       const result = emitter.expr(mapped(expression));
@@ -187,19 +190,13 @@ export function createEntityMutation(connection, entity, mapping, core) {
       }
     }
     sql = prefix + sql + ` RETURNING ${physicalSelection(mapping, dialect)}`;
-    return { sql, params, returning, maxRows, maxBytes, statement: null };
+    return { sql, params, returning, maxRows, maxBytes };
   };
   return (document) => {
-    const key = canonicalizeJson(document);
-    let plan = plans.get(key);
-    if (!plan) {
-      plan = compile(document);
-      if (plans.size >= 64) plans.delete(plans.keys().next().value);
-      plans.set(key, plan);
-    }
+    const plan = compile(document);
     return connection.transaction(() => {
-      plan.statement ??= connection.prepare(plan.sql);
-      return chain(plan.statement, (statement) => chain(attempt(() => statement.all(plan.params),
+      const prepared = statements.getOrCreate(plan.sql, (text) => connection.prepare(text));
+      return chain(prepared, (statement) => chain(attempt(() => statement.all(plan.params),
         (error) => String(error?.message).includes('jaren-mutation-row-bound')
           ? new DbRuntimeError('JD2007', 'insert-select exceeded its source row bound', { cause: error })
           : wrapDriverError(error, { collection: entity.name, docPath: entity.docPath })), (rows) => {

package/src/relational-api.js

@@ -2,5 +2,5 @@
 /** Lightweight column-first SQLite authoring and execution. */
 export { sql, planRelational, relational } from './dialects/sqlite-relational.js';
 export { defineTable, planTable } from './dialects/sqlite-schema.js';
-export { planTableMigration, applyTableMigration, withForeignKeysSuspended } from './table-migration.js';
+export { planTableMigration, applyTableMigration, withForeignKeysSuspended, planSchemaChange, applySchemaChange } from './table-migration.js';
 export { sqliteDialect } from './dialects/sqlite.js';

package/src/table-migration.js

@@ -3,7 +3,7 @@
 import { hashContent } from '@jarenjs/core/string';
 import { canonicalizeJson } from '@jarenjs/json/canonical';
 import { DbCompileError } from './errors.js';
-import { defineTable, planTable } from './dialects/sqlite-schema.js';
+import { defineTable, planTable, schemaChangeSql } from './dialects/sqlite-schema.js';
 import { relationalEmitter, relationalIdentifier as q, sql } from './dialects/sqlite-relational.js';
 import { sqliteDialect as dialect, sqliteTableMigration } from './dialects/sqlite.js';
 import { sqlTokens } from './dialects/check-read.js';
@@ -17,6 +17,39 @@ const sync = (connection) => {
   if (connection.dialect.name !== 'sqlite' || !connection.synchronous || connection.mustQueue) refuse('table migration requires an available synchronous SQLite connection');
 };
 
+/** Connection settings that govern the meaning of an additive/drop/rename plan. */
+function schemaSettings(connection) {
+  return ['foreign_keys', 'legacy_alter_table', 'schema_version'].map((name) =>
+    connection.prepare(dialect.introspect.pragma(name)).get([])[name]);
+}
+
+/** Review one native main-schema change without changing the database.
+ * Plans describe one source snapshot; callers own durable migration receipts.
+ * @param {any} connection @param {any} operation */
+export function planSchemaChange(connection, operation) {
+  sync(connection);
+  const text = schemaChangeSql(operation);
+  const body = { version: 1, operation: structuredClone(operation), sql: text,
+    source: schema(connection), settings: schemaSettings(connection) };
+  return { ...body, checksum: fingerprint({ ...body, operation: text }) };
+}
+
+/** Apply a reviewed native change atomically, refusing schema/settings drift.
+ * Replan after any schema change; only explicit drop ifExists handles absence.
+ * @param {any} connection @param {ReturnType<typeof planSchemaChange>} plan */
+export function applySchemaChange(connection, plan) {
+  sync(connection);
+  const { checksum, ...body } = plan;
+  if (body.version !== 1 || checksum !== fingerprint({ ...body, operation: plan.sql }) || plan.sql !== schemaChangeSql(plan.operation)) refuse('schema change checksum or statement differs');
+  return connection.transaction(() => {
+    const before = schema(connection);
+    if (fingerprint(before) !== fingerprint(plan.source) || fingerprint(schemaSettings(connection)) !== fingerprint(plan.settings))
+      refuse('source schema or connection settings changed after planning');
+    connection.exec(plan.sql);
+    return { changed: fingerprint(before) === fingerprint(schema(connection)) ? 0 : 1 };
+  }, { mode: 'immediate' });
+}
+
 /** Inspect a live schema and generate a table plan without changing it.
  * Rebuilds require explicit opt-in. Unlisted indexes and triggers are preserved.
  * @param {any} connection @param {any} definition

package/types/index.d.ts

@@ -514,6 +514,8 @@ export interface StoreCapabilities {
   readonly sessions: boolean;
   readonly sessionReason: string | null;
   readonly worker: boolean;
+  readonly process: boolean;
+  readonly ownerTermination: boolean;
   readonly pooling: boolean;
   readonly poolReaders: number;
   readonly poolWriters: number;
@@ -2043,4 +2045,4 @@ export declare function planPhysicalMigration(connection: unknown, fromModel: un
   options: { id: string; steps: readonly unknown[]; dispositions: Readonly<Record<string, 'preserve' | 'replace' | 'drop'>>;
     assertions?: readonly { sql: string; params?: readonly unknown[]; expected: readonly unknown[] }[] }): unknown;
 
-export { sql, relational, planRelational, defineTable, planTable, planTableMigration, applyTableMigration, withForeignKeysSuspended } from './relational.js';
+export { sql, relational, planRelational, defineTable, planTable, planTableMigration, applyTableMigration, withForeignKeysSuspended, planSchemaChange, applySchemaChange } from './relational.js';

package/types/node-process.d.ts

@@ -0,0 +1,33 @@
+import type { NodeOpenOptions } from './node.js';
+import type { CancellationCapabilities, Driver } from './index.js';
+import type { NodeWorkerConnection, NodeWorkerOptions, WorkerMetrics } from './node-worker.js';
+
+export interface NodeProcessOptions extends NodeWorkerOptions { maxOwners?: number; timeoutMs?: number; maxRequestBytes?: number }
+export interface ProcessSettlement {
+  readonly path: string | null;
+  readonly generation: number;
+  readonly pid: number | null;
+  readonly status: 'starting' | 'healthy' | 'quarantined' | 'exited';
+  readonly transaction: 'none' | 'active' | 'committed' | 'rolled-back' | 'unknown';
+  readonly safeToReplace: boolean;
+  readonly exitCode: number | null;
+  readonly exitSignal: string | null;
+}
+export interface NodeProcessConnection extends NodeWorkerConnection {
+  readonly capabilities: Readonly<Record<string, unknown>> & {
+    readonly process: true; readonly ownerTermination: true;
+    readonly cancellation: CancellationCapabilities & {readonly midStatement: false};
+  };
+  supervise<T>(body: (connection: NodeProcessConnection) => T | Promise<T>, options?: {signal?: AbortSignal; timeoutMs?: number}): Promise<T>;
+  cancel(reason?: string): Error;
+  settlement(): ProcessSettlement;
+  /** Resolves only after the OS reports owner exit, never from a caller deadline. */
+  settled(): Promise<ProcessSettlement>;
+  metrics(): WorkerMetrics & {readonly owner: ProcessSettlement; readonly supervised: number};
+  restart(): Promise<NodeProcessConnection>;
+}
+export interface NodeProcessDriver extends Driver {
+  open(path?: string, options?: NodeOpenOptions): Promise<NodeProcessConnection>;
+  metrics(): Readonly<{capacity: number; owners: number; quarantined: number; healthy: number}>;
+}
+export declare function nodeProcessDriver(options?: NodeProcessOptions): NodeProcessDriver;

package/types/relational.d.ts

@@ -4,7 +4,7 @@ export type SqlInput = SqlValue | SqlExpression;
 export type SqlOperator = '=' | '<>' | '<' | '<=' | '>' | '>=' | 'IS' | 'IS NOT'
   | '+' | '-' | '*' | '/' | '%' | '||' | 'AND' | 'OR' | 'LIKE' | 'NOT LIKE' | 'GLOB';
 export type SqlFunction = 'coalesce' | 'nullif' | 'trim' | 'ltrim' | 'rtrim' | 'lower' | 'upper'
-  | 'length' | 'abs' | 'round' | 'typeof' | 'json_extract' | 'json_valid'
+  | 'length' | 'abs' | 'round' | 'typeof' | 'json_extract' | 'json_valid' | 'json_type'
   | 'count' | 'sum' | 'total' | 'avg' | 'min' | 'max'
   | 'date' | 'time' | 'datetime' | 'julianday' | 'unixepoch' | 'strftime';
 export type SqlType = 'INTEGER' | 'REAL' | 'TEXT' | 'BLOB' | 'NUMERIC';
@@ -78,8 +78,13 @@ export interface TableColumn {
   readonly default?: SqlInput; readonly collation?: SqlCollation;
   readonly identity?: 'rowid' | 'autoincrement'; readonly check?: SqlInput;
   readonly generated?: SqlInput; readonly stored?: boolean;
+  readonly references?: ColumnReference;
 }
 export type ForeignKeyAction = 'cascade' | 'restrict' | 'no action' | 'set null' | 'set default';
+export interface ColumnReference {
+  readonly table: string; readonly columns: readonly [string];
+  readonly onDelete?: ForeignKeyAction; readonly onUpdate?: ForeignKeyAction; readonly deferred?: boolean;
+}
 export type TableConstraint = { readonly name?: string } & (
   { readonly kind: 'unique'; readonly columns: readonly string[] }
   | { readonly kind: 'check'; readonly expression: SqlInput }
@@ -111,4 +116,17 @@ export interface TableMigrationPlan {
 export declare function planTableMigration(connection: unknown, definition: TableDefinition, options: TableMigrationOptions): TableMigrationPlan;
 export declare function applyTableMigration(connection: unknown, plan: TableMigrationPlan): { changed: number };
 export declare function withForeignKeysSuspended<T>(connection: unknown, fn: () => T): T;
+export type SchemaChange =
+  | { readonly op: 'addColumn'; readonly table: string; readonly column: TableColumn }
+  | { readonly op: 'dropIndex'; readonly name: string; readonly ifExists?: boolean }
+  | { readonly op: 'renameTable'; readonly table: string; readonly to: string }
+  | { readonly op: 'dropTable'; readonly table: string; readonly ifExists?: boolean };
+export interface SchemaChangePlan {
+  readonly version: 1; readonly operation: SchemaChange; readonly sql: string;
+  readonly source: readonly unknown[]; readonly settings: readonly number[]; readonly checksum: string;
+}
+/** Main-schema snapshot; does not execute SQL or infer replay/disposition policy. */
+export declare function planSchemaChange(connection: unknown, operation: SchemaChange): SchemaChangePlan;
+/** Refuses stale source/settings under an immediate transaction. */
+export declare function applySchemaChange(connection: unknown, plan: SchemaChangePlan): { changed: number };
 export { sqliteDialect } from './index.js';