@jarenjs/db 0.73.0 → 0.83.2

package/src/errors.js

@@ -46,6 +46,7 @@ export const DB_CODES = Object.freeze({
   JD0035: 'the continuation does not belong to this ordering',
   JD0036: 'a snapshot page needs an immutable ordering',
   JD0037: 'strictStreaming refused a plan that buffers',
+  JD0038: 'the native mutation document is unsupported or invalid',
   JD0040: 'the save spans a relation cycle',
   JD0050: 'live queries require change capture',
   JD0051: 'the demanded live mode is unavailable',
@@ -109,6 +110,8 @@ export const DB_CODES = Object.freeze({
   JD2092: 'a worker transport bound was exceeded',
   JD2093: 'the worker protocol frame is invalid',
   JD2094: 'the durable snapshot failed and the connection is invalid',
+  JD2095: 'the trusted SQL or synchronous transaction authority was refused',
+  JD2096: 'a persistence invariant rejected the mutation',
 });
 
 /**
@@ -554,6 +557,11 @@ export function wrapDriverError(error, details = {}) {
     generic.retryable = false;
     return generic;
   }
+  if (typeof error?.message === 'string' && error.message.includes('jaren invariant:')) {
+    const wrapped = new DbRuntimeError('JD2096', error.message, { ...details, cause: error });
+    wrapped.class = 'constraint'; wrapped.retryable = false;
+    return wrapped;
+  }
   const classified = classifyDriverError(error, details.unique);
   const message = error?.message ?? String(error);
   /** @type {any} */

package/src/graph.js

@@ -14,6 +14,8 @@
  * whole one.
  */
 
+import { columnCodec } from './physical.js';
+
 import { DbRuntimeError } from './errors.js';
 import { utf8Length } from './cursor.js';
 import { jsonBytes, decodeCountedJson } from './json-bytes.js';
@@ -73,9 +75,14 @@ function checkBounds(node, include, keyed, raw, sizes) {
  * @returns {any}
  */
 export function mergeEntityRow(entityMapping, row, docField = 'doc') {
-  const doc = JSON.parse(row[docField]);
+  const doc = entityMapping.document === false ? {} : JSON.parse(row[docField]);
   for (const column of entityMapping.columns) {
     if (column.source === 'epoch(document)') continue;
+    if (entityMapping.document === false && column.codec) {
+      const value = columnCodec(column).decode(row[column.physical ?? column.name]);
+      if (value !== undefined) Object.defineProperty(doc, column.name, { value, enumerable: true, writable: true, configurable: true });
+      continue;
+    }
     const value = row[column.name];
     if (value === null || value === undefined) continue;
     doc[column.name] = column.storage === 'boolean' ? value === 1 : value;

package/src/index.js

@@ -100,3 +100,6 @@ export {
 export { createDagJobRunner, RUN_IDENTITY_NODE } from './dag-job.js';
 export { REPLICATION_VERSION, REPLICATION_DEFAULTS, normalizeFrontier,
   normalizeReplication, normalizeReplicationSnapshot, encodeReplication, replicationIdentity } from './replication-format.js';
+
+export { planInvariants } from './ddl.js';
+export { planPhysicalMigration } from './migrate.js';

package/src/introspect.js

@@ -73,7 +73,7 @@ function loss(code, object, detail) {
  * @param {string} table
  * @returns {any} value-or-promise
  */
-function readTable(connection, table) {
+function readTable(connection, table, objects) {
   const dialect = connection.dialect;
   const all = (sql) => chain(connection.prepare(sql), (statement) => statement.all([]));
   return chain(all(dialect.introspect.columns(table)), (columnRows) =>
@@ -95,30 +95,46 @@ function readTable(connection, table) {
               (rows) => withColumns(i + 1, [...out, {
                 name: String(declared[i].name),
                 unique: Number(declared[i].uniq) !== 0,
-                columns: rows.map((row) => String(row.name)),
+                partial: Number(declared[i].partial ?? 0) !== 0,
+                columns: rows.map((row) => row.name == null ? null : String(row.name)),
+                sql: objects.find((o) => o.type === 'index' && o.name === String(declared[i].name))?.sql ?? null,
               }]));
           };
           return chain(withColumns(0, []), (indexes) =>
             chain(keyIndex === undefined
               ? []
-              : all(dialect.introspect.indexColumns(String(keyIndex.name))), (keyRows) => ({
+              : all(dialect.introspect.indexColumns(String(keyIndex.name))), (keyRows) =>
+              chain(dialect.introspect.checks === undefined ? []
+                : all(dialect.introspect.checks(table)), (checkRows) => ({
               name: table,
-              primaryKey: keyRows.map((row) => String(row.name)),
+              sql: objects.find((o) => o.type === 'table' && o.name === table)?.sql ?? null,
+              primaryKey: columnRows.some((row) => Number(row.pk) > 0)
+                ? columnRows.filter((row) => Number(row.pk) > 0)
+                  .sort((a, b) => Number(a.pk) - Number(b.pk)).map((row) => String(row.name))
+                : keyRows.map((row) => String(row.name)),
               columns: columnRows.map((row) => ({
                 name: String(row.name),
                 type: String(row.type),
                 generated: Number(row.hidden) !== 0,
+                primaryKeyOrdinal: Number(row.pk ?? 0),
+                nullable: Number(row.not_null ?? 0) === 0,
+                default: row.default_value ?? null,
               })),
               generated: dialect.readGenerated(generatedRows),
               indexes,
+              checks: dialect.readChecks?.(checkRows) ?? [],
               foreignKeys: fkRows.map((row) => ({
                 column: String(row.source_column),
                 target: String(row.target),
                 targetColumn: row.target_column === null || row.target_column === undefined
                   ? null : String(row.target_column),
                 onDelete: String(row.on_delete ?? 'NO ACTION').toUpperCase(),
+                onUpdate: String(row.on_update ?? 'NO ACTION').toUpperCase(),
+                group: row.id ?? null,
+                ordinal: Number(row.seq ?? 0),
+                match: String(row.match ?? 'NONE'),
               })),
-            })));
+            }))));
         }))));
 }
 
@@ -149,10 +165,23 @@ export function readSchema(connection, options = undefined) {
         if (String(row.type) === 'view') views.push(name);
         else names.push(name);
       }
-      const step = (i, out) => (i >= names.length
-        ? { tables: out, views }
-        : chain(readTable(connection, names[i]), (table) => step(i + 1, [...out, table])));
-      return step(0, []);
+      const readObjects = dialect.introspect.objects === undefined ? []
+        : chain(connection.prepare(dialect.introspect.objects()), (s) => s.all([]));
+      return chain(readObjects, (catalog) => {
+        const complete = [...catalog];
+        for (const row of rows) {
+          if (!complete.some((o) => o.type === row.type && o.name === row.name))
+            complete.push({ ...row, owner: row.name, sql: null });
+        }
+        const objects = complete.filter((row) => !engine.has(String(row.owner))
+          && (wanted === null || wanted.has(String(row.owner)) || wanted.has(String(row.name))))
+          .map((row) => ({ type: String(row.type), name: String(row.name),
+            owner: String(row.owner), sql: row.sql ?? null }));
+        const step = (i, out) => (i >= names.length
+          ? { tables: out, views, objects }
+          : chain(readTable(connection, names[i], objects), (table) => step(i + 1, [...out, table])));
+        return step(0, []);
+      });
     }));
 }
 
@@ -229,6 +258,11 @@ function deriveCollection(dialect, table, report, keys, functionNames) {
 
   const indexes = [];
   for (const index of table.indexes) {
+    if (index.partial || index.columns.some((column) => column === null)) {
+      report(loss('unmapped-index', `${table.name}.${index.name}`,
+        'a partial predicate or expression term cannot be represented by an unconditional member index'));
+      continue;
+    }
     if (index.columns.length === 1 && computed.has(index.columns[0])) {
       const declared = { name: indexName(table.name, index.name),
         expression: computed.get(index.columns[0]) };
@@ -416,9 +450,10 @@ function deriveEntity(dialect, table, report) {
   const uniqueColumns = new Set();
   const indexedColumns = new Set();
   for (const index of table.indexes) {
-    if (index.columns.length !== 1) {
+    if (index.partial || index.columns.some((column) => column === null)
+      || index.columns.length !== 1) {
       report(loss('unmapped-index', `${table.name}.${index.name}`,
-        'a composite index over an entity is not a property-level declaration'));
+        'a partial, expression or composite entity index is not a property-level declaration'));
       continue;
     }
     (index.unique ? uniqueColumns : indexedColumns).add(index.columns[0]);
@@ -461,6 +496,26 @@ function deriveEntity(dialect, table, report) {
     if (Object.keys(entity).length > 0) property['x-entity'] = entity;
     properties[column.name] = property;
   }
+  for (const check of table.checks) {
+    const property = properties[check.column];
+    const family = property?.type === 'integer' ? 'number'
+      : property?.type === 'string' ? 'string' : property?.type;
+    if (check.values === undefined || property === undefined
+      || check.values.some((value) => typeof value !== family
+        || (property.type === 'integer' && !Number.isInteger(value)))) {
+      report(loss('unmapped-constraint', `${table.name}.${check.name}`,
+        'the CHECK is not a complete scalar enum of the mapped column type'));
+      continue;
+    }
+    // Several enum CHECKs constrain the same column by intersection.
+    const values = property.enum === undefined ? check.values
+      : property.enum.filter((value) => check.values.includes(value));
+    if (values.length === 0) {
+      report(loss('unmapped-constraint', `${table.name}.${check.name}`,
+        'the enum CHECKs have an empty intersection, which a non-empty schema enum cannot declare'));
+    }
+    else property.enum = values;
+  }
   report(loss('document-members', table.name,
     'an entity\'s document column holds every property the mapping did not give a column, '
     + 'and those are not in the physical shape'));
@@ -517,6 +572,11 @@ export function introspectModel(connection, options = undefined) {
       add(loss('unmapped-view', view,
         'a view is not a shape a model document can declare'));
     }
+    for (const object of schema.objects) {
+      if (object.type === 'trigger') add({ ...loss('unmapped-object', object.name,
+        'an application-owned trigger program must be explicitly preserved; inspection grants no drop permission'),
+      sql: object.sql, owner: object.owner });
+    }
 
     const joinTables = new Set(schema.tables
       .filter((table) => looksLikeJoinTable(dialect, table))
@@ -530,6 +590,8 @@ export function introspectModel(connection, options = undefined) {
       if (joinTables.has(table.name)) continue;
       if (looksLikeCollection(dialect, table)) {
         collections[table.name] = deriveCollection(dialect, table, add, options?.keys, byName);
+        for (const check of table.checks) add(loss('unmapped-constraint', `${table.name}.${check.name}`,
+          'a CHECK over physical collection columns does not constrain the document schema'));
         continue;
       }
       const hasDocument = table.columns.some((column) => column.name === DOC_COLUMN);
@@ -578,6 +640,13 @@ export function introspectModel(connection, options = undefined) {
         + `thing(s) the model would — ${report.map((row) => `${row.code} (${row.object})`)
           .join(', ')}`);
     }
-    return { model, report: Object.freeze(report.map((row) => Object.freeze(row))) };
+    const inventory = schema.objects.map((object) => Object.freeze({ ...object,
+      disposition: object.type === 'trigger' || object.type === 'view'
+        || report.some((row) => row.object === object.name
+          || row.object === `${object.owner}.${object.name}`
+          || (row.code === 'unmapped-table' && row.object === object.owner))
+        ? 'preserve' : 'derived' }));
+    return { model, report: Object.freeze(report.map((row) => Object.freeze(row))),
+      inventory: Object.freeze(inventory) };
   });
 }

package/src/invariants.js

@@ -0,0 +1,45 @@
+//@ts-check
+/** Persistence rules compile through the existing query evaluator. SQL lowering
+ * accepts a bounded scalar subset and reports its writer population explicitly. */
+import { compileJsonQuery } from '@jarenjs/json/query';
+import { DbCompileError, DbRuntimeError } from './errors.js';
+
+/** @param {any} rules @param {string} path @returns {any[]} */
+export function normalizeInvariants(rules, path) {
+  if (rules === undefined) return [];
+  if (!Array.isArray(rules)) throw new DbCompileError('JD0005', 'invariants must be an array', path);
+  const names = new Set();
+  return rules.map((rule) => {
+    const fail = (reason) => { throw new DbCompileError('JD0005', reason, path); };
+    if (!rule || typeof rule !== 'object' || Array.isArray(rule)) fail('an invariant must be an object');
+    for (const key of Object.keys(rule))
+      if (!['name', 'on', 'assert', 'enforcement', 'audit'].includes(key)) fail(`unknown invariant member '${key}'`);
+    if (typeof rule.name !== 'string' || !/^[A-Za-z_][A-Za-z0-9_]*$/.test(rule.name) || names.has(rule.name)) fail('invariant names must be distinct identifiers');
+    names.add(rule.name);
+    if (!['database', 'store'].includes(rule.enforcement)) fail('invariant enforcement must be database or store');
+    if (!Array.isArray(rule.on) || !rule.on.length || new Set(rule.on).size !== rule.on.length
+      || rule.on.some((op) => !['insert', 'update', 'delete'].includes(op))) fail('invariant on must name distinct insert/update/delete operations');
+    if (rule.assert === undefined) fail('invariant assert is required');
+    if (rule.audit !== undefined && (rule.enforcement !== 'database' || !rule.audit
+      || typeof rule.audit.entity !== 'string' || !rule.audit.values || typeof rule.audit.values !== 'object'
+      || Array.isArray(rule.audit.values) || Object.keys(rule.audit).some((k) => !['entity', 'values'].includes(k)))) fail('audit is a database effect with an entity and values');
+    return { ...rule, evaluate: compileJsonQuery(rule.assert) };
+  });
+}
+
+/** @param {string} name @returns {DbRuntimeError} */
+export function invariantFailure(name) {
+  const error = new DbRuntimeError('JD2096', `persistence invariant '${name}' rejected the mutation`);
+  error.class = 'constraint';
+  error.retryable = false;
+  return error;
+}
+
+/** @param {any[]} rules @param {string} op @param {any} before @param {any} after */
+export function checkInvariants(rules, op, before, after) {
+  for (const rule of rules) {
+    if (rule.enforcement === 'store' && rule.on.includes(op)
+      && rule.evaluate({ old: before ?? null, new: after ?? null, op }) !== true)
+      throw invariantFailure(rule.name);
+  }
+}

package/src/jobs.js

@@ -194,7 +194,7 @@ function isLease(value) {
 /**
  * The queue engine over one open connection.
  * @param {{ connection: any, now?: () => number,
- *   random?: () => number,
+ *   random?: () => number, adopt?: boolean,
  *   gate?: (fn: () => any, what?: string, signal?: AbortSignal) => any,
  *   bracket?: (fn: () => any) => any,
  *   defaults?: Partial<typeof JOB_DEFAULTS>,
@@ -276,9 +276,22 @@ export function createJobEngine(options) {
 
   // the tables are created here, or refused here: a read-only store
   // leaked the driver's "attempt to write a readonly database"
-  const ready = attempt(() => bracket(() => chain(connection.exec(CREATE_JOBS), upgradeColumns)),
+  // Adoption accepts the current engine-owned schema without provisioning or
+  // upgrading it. The DDL remains the single schema declaration; differently
+  // shaped historical queues need an explicit upgrade before adoption.
+  const verifyExisting = () => chain(connection.prepare(connection.dialect.introspect.objects()), (statement) => chain(statement.all([]), (objects) => {
+    const normalize = (sql) => sql.replace(/'[^']*'|\bIF\s+NOT\s+EXISTS\b|[\s";]/g, (part) => part.startsWith("'") ? part : '');
+    for (const sql of CREATE_JOBS.split(';').filter((part) => part.trim())) {
+      const name = sql.match(/(?:TABLE|INDEX) IF NOT EXISTS "([^"]+)"/)[1];
+      const object = objects.find((value) => value.name === name);
+      if (!object?.sql || normalize(object.sql) !== normalize(sql))
+        throw new Error(`existing job object ${name} needs an explicit schema migration`);
+    }
+  }));
+  const ready = attempt(() => bracket(() => options.adopt === true
+    ? verifyExisting() : chain(connection.exec(CREATE_JOBS), upgradeColumns)),
     (error) => new DbCompileError('JD0002',
-      `the job tables could not be created (${error?.message ?? String(error)}) — `
+      `the job tables could not be opened (${error?.message ?? String(error)}) — `
       + 'a read-only store creates nothing; open it read-write once, or without jobs',
       '/jobs', error));
 
@@ -436,6 +449,21 @@ export function createJobEngine(options) {
       { docPath: '/jobs', collection: JOBS_TABLE });
   };
 
+  /** Verify current execution authority without renewing or writing a checkpoint.
+   * Inside tx.jobs this check and mapped record writes share the same lock.
+   * @param {any} lease */
+  const assertLease = (lease) => {
+    const misuse = requireLease(lease, 'assertLease()');
+    if (misuse !== null) throw misuse;
+    return chain(prepared('assertLease', `SELECT id FROM "${JOBS_TABLE}" WHERE id=? AND ${FENCE}`)
+      .get([lease.jobId, lease.token, now()]), (row) => {
+      if (row !== undefined) return true;
+      return chain(refuseSettlement(lease, 'assertLease()'), (error) => {
+        throw error ?? new DbRuntimeError('JD2065', 'the job is already settled', { docPath: '/jobs' });
+      });
+    });
+  };
+
   /** §4: the jittered exponential backoff. */
   const backoffOf = (attempts, workerDefaults) => {
     const base = workerDefaults?.backoffBase ?? defaults.backoffBase;
@@ -849,10 +877,11 @@ export function createJobEngine(options) {
    * @param {{ handlers: Record<string, Function>, concurrency?: number,
    *   pollInterval?: number, leaseMs?: number, owner?: string,
    *   backoffBase?: number, backoffCap?: number, renew?: boolean,
-   *   onOutcome?: (event: any) => void }} workerOptions
+   *   onOutcome?: (event: any) => void, effectSafety?: (job: any, context: any) => any }} workerOptions
    */
   const createWorker = (workerOptions) => {
     const handlers = workerOptions?.handlers;
+    if (workerOptions?.effectSafety !== undefined && typeof workerOptions.effectSafety !== 'function') throw new TypeError('effectSafety must be a function');
     if (handlers === null || typeof handlers !== 'object'
       || Object.keys(handlers).length === 0
       || Object.values(handlers).some((handler) => typeof handler !== 'function')) {
@@ -1143,8 +1172,11 @@ export function createJobEngine(options) {
       try {
         let result;
         try {
-          result = await handlers[job.kind](job.payload,
-            { job, checkpoints: attempt.checkpoints, signal: attempt.signal });
+          const context = { job, lease: () => attempt.lease, checkpoints: attempt.checkpoints, signal: attempt.signal,
+            pause: () => io(() => cancel(job.id, { lease: attempt.lease }), 'a durable effect pause') };
+          const admitted = workerOptions.effectSafety === undefined || await workerOptions.effectSafety(job, context) === true;
+          if (!admitted) { await context.pause(); recordCancelled(attempt); return; }
+          result = await handlers[job.kind](job.payload, context);
         }
         catch (error) {
           // past cancellation the store is closing: leave the leased
@@ -1330,6 +1362,7 @@ export function createJobEngine(options) {
     counts,
     claim,
     renew,
+    assertLease,
     complete,
     fail,
     checkpointsFor,

package/src/live-nested.js

@@ -1,5 +1,5 @@
 //@ts-check
-/** Two-level groups use bounded per-parent recomputation over maintained leaves. */
+/** Collection groups use bounded per-group recomputation over maintained leaves. */
 import { compileJsonQuery } from '@jarenjs/json/query';
 import { stableStringify } from '@jarenjs/core/object';
 import { DbRuntimeError } from './errors.js';
@@ -47,6 +47,8 @@ export function nestedGroupStrategy(description, context) {
   const keyOf = compileJsonQuery([{ $for: { [binding]: '$[*]' },
     ...(inner.$where === undefined ? {} : { $where: inner.$where }), $return: [key] }]);
   const evaluate = compileJsonQuery([inner]);
+  const aggregate = description.aggregate === undefined ? null
+    : compileJsonQuery([{ [description.aggregate]: '$[*]' }]);
   const docs = new Map();
   const groups = new Map();
   const results = new Map();
@@ -55,18 +57,25 @@ export function nestedGroupStrategy(description, context) {
   let resultCount = 0;
   const bytesOf = (value) => utf8Length(stableStringify(value));
   const stats = { refreshedGroups: 0, dependencyReads: 0, reruns: 0 };
-  const groupOf = (doc) => stableStringify(keyOf([doc], context.externals));
+  const groupOf = (doc) => {
+    const keys = keyOf([doc], context.externals);
+    return keys.length === 0 ? null : stableStringify(keys);
+  };
   const sorted = (keys) => [...keys].sort((a, b) => positions.get(a) - positions.get(b));
   const check = () => {
-    const entries = docs.size + resultCount;
+    const entries = docs.size + resultCount + (aggregate === null ? 0 : 1);
     if (entries > context.maxMaintained) throw new DbRuntimeError('JD2060', 'nested group dependencies exceed live.maxMaintained');
     if (maintainedBytes > context.maxBytes) throw new DbRuntimeError('JD2060', 'nested group dependencies exceed live.maxBytes');
     return entries;
   };
   const add = (token, doc) => {
     const group = groupOf(doc);
+    if (group === null) return null;
+    const bytes = bytesOf(doc);
+    if (docs.size + resultCount + 1 > context.maxMaintained || maintainedBytes + bytes > context.maxBytes)
+      throw new DbRuntimeError('JD2060', 'group dependencies exceed live state credits');
     docs.set(token, doc);
-    maintainedBytes += bytesOf(doc);
+    maintainedBytes += bytes;
     positions.set(token, context.rowPosition(token));
     if (!groups.has(group)) groups.set(group, new Set());
     groups.get(group).add(token);
@@ -87,20 +96,25 @@ export function nestedGroupStrategy(description, context) {
     stats.refreshedGroups++;
     check();
   };
-  const flatten = () => [...groups.keys()].sort((a, b) =>
-    positions.get(sorted(groups.get(a))[0]) - positions.get(sorted(groups.get(b))[0]))
-    .flatMap((group) => results.get(group) ?? []);
+  const flatten = () => {
+    const rows = [...groups.keys()].sort((a, b) =>
+      positions.get(sorted(groups.get(a))[0]) - positions.get(sorted(groups.get(b))[0]))
+      .flatMap((group) => results.get(group) ?? []);
+    return aggregate === null ? rows : aggregate(rows, context.externals);
+  };
   return {
     close() { docs.clear(); groups.clear(); results.clear(); positions.clear(); },
     init() {
-      const source = [{ $subsequence: [{ $for: { it: '$[*]' }, $return: '$it' }, 0, context.maxMaintained + 1] }];
+      const carrier = description.carrier ?? { $for: { [binding]: '$[*]' },
+        ...(inner.$where === undefined ? {} : { $where: inner.$where }), $return: `$${binding}` };
+      const source = [{ $subsequence: [carrier, 0, context.maxMaintained + 1] }];
       for (const doc of context.execute(source, { externals: context.externals })) add(context.keyOf(doc), doc);
       check();
       for (const group of groups.keys()) refresh(group);
       return flatten();
     }, entries: check, stats: () => ({ ...stats }),
     apply(record, previousRows) {
-      const changes = context.touchedKeys(record, { whole: true, members: new Set() });
+      const changes = context.touchedKeys(record, description.deps ?? { whole: true, members: new Set() });
       if (changes === null) return null;
       const affected = new Set();
       for (const token of changes.keys()) {
@@ -110,7 +124,10 @@ export function nestedGroupStrategy(description, context) {
           docs.delete(token); positions.delete(token); maintainedBytes -= bytesOf(previous);
         }
         const doc = context.readRow(token); stats.dependencyReads++;
-        if (doc !== undefined) affected.add(add(token, doc));
+        if (doc !== undefined) {
+          const group = add(token, doc);
+          if (group !== null) affected.add(group);
+        }
       }
       check();
       for (const group of affected) refresh(group);