@jarenjs/db 0.75.0 → 0.83.3

package/src/invariants.js

@@ -0,0 +1,45 @@
+//@ts-check
+/** Persistence rules compile through the existing query evaluator. SQL lowering
+ * accepts a bounded scalar subset and reports its writer population explicitly. */
+import { compileJsonQuery } from '@jarenjs/json/query';
+import { DbCompileError, DbRuntimeError } from './errors.js';
+
+/** @param {any} rules @param {string} path @returns {any[]} */
+export function normalizeInvariants(rules, path) {
+  if (rules === undefined) return [];
+  if (!Array.isArray(rules)) throw new DbCompileError('JD0005', 'invariants must be an array', path);
+  const names = new Set();
+  return rules.map((rule) => {
+    const fail = (reason) => { throw new DbCompileError('JD0005', reason, path); };
+    if (!rule || typeof rule !== 'object' || Array.isArray(rule)) fail('an invariant must be an object');
+    for (const key of Object.keys(rule))
+      if (!['name', 'on', 'assert', 'enforcement', 'audit'].includes(key)) fail(`unknown invariant member '${key}'`);
+    if (typeof rule.name !== 'string' || !/^[A-Za-z_][A-Za-z0-9_]*$/.test(rule.name) || names.has(rule.name)) fail('invariant names must be distinct identifiers');
+    names.add(rule.name);
+    if (!['database', 'store'].includes(rule.enforcement)) fail('invariant enforcement must be database or store');
+    if (!Array.isArray(rule.on) || !rule.on.length || new Set(rule.on).size !== rule.on.length
+      || rule.on.some((op) => !['insert', 'update', 'delete'].includes(op))) fail('invariant on must name distinct insert/update/delete operations');
+    if (rule.assert === undefined) fail('invariant assert is required');
+    if (rule.audit !== undefined && (rule.enforcement !== 'database' || !rule.audit
+      || typeof rule.audit.entity !== 'string' || !rule.audit.values || typeof rule.audit.values !== 'object'
+      || Array.isArray(rule.audit.values) || Object.keys(rule.audit).some((k) => !['entity', 'values'].includes(k)))) fail('audit is a database effect with an entity and values');
+    return { ...rule, evaluate: compileJsonQuery(rule.assert) };
+  });
+}
+
+/** @param {string} name @returns {DbRuntimeError} */
+export function invariantFailure(name) {
+  const error = new DbRuntimeError('JD2096', `persistence invariant '${name}' rejected the mutation`);
+  error.class = 'constraint';
+  error.retryable = false;
+  return error;
+}
+
+/** @param {any[]} rules @param {string} op @param {any} before @param {any} after */
+export function checkInvariants(rules, op, before, after) {
+  for (const rule of rules) {
+    if (rule.enforcement === 'store' && rule.on.includes(op)
+      && rule.evaluate({ old: before ?? null, new: after ?? null, op }) !== true)
+      throw invariantFailure(rule.name);
+  }
+}

package/src/jobs.js

@@ -194,7 +194,7 @@ function isLease(value) {
 /**
  * The queue engine over one open connection.
  * @param {{ connection: any, now?: () => number,
- *   random?: () => number,
+ *   random?: () => number, adopt?: boolean,
  *   gate?: (fn: () => any, what?: string, signal?: AbortSignal) => any,
  *   bracket?: (fn: () => any) => any,
  *   defaults?: Partial<typeof JOB_DEFAULTS>,
@@ -276,9 +276,22 @@ export function createJobEngine(options) {
 
   // the tables are created here, or refused here: a read-only store
   // leaked the driver's "attempt to write a readonly database"
-  const ready = attempt(() => bracket(() => chain(connection.exec(CREATE_JOBS), upgradeColumns)),
+  // Adoption accepts the current engine-owned schema without provisioning or
+  // upgrading it. The DDL remains the single schema declaration; differently
+  // shaped historical queues need an explicit upgrade before adoption.
+  const verifyExisting = () => chain(connection.prepare(connection.dialect.introspect.objects()), (statement) => chain(statement.all([]), (objects) => {
+    const normalize = (sql) => sql.replace(/'[^']*'|\bIF\s+NOT\s+EXISTS\b|[\s";]/g, (part) => part.startsWith("'") ? part : '');
+    for (const sql of CREATE_JOBS.split(';').filter((part) => part.trim())) {
+      const name = sql.match(/(?:TABLE|INDEX) IF NOT EXISTS "([^"]+)"/)[1];
+      const object = objects.find((value) => value.name === name);
+      if (!object?.sql || normalize(object.sql) !== normalize(sql))
+        throw new Error(`existing job object ${name} needs an explicit schema migration`);
+    }
+  }));
+  const ready = attempt(() => bracket(() => options.adopt === true
+    ? verifyExisting() : chain(connection.exec(CREATE_JOBS), upgradeColumns)),
     (error) => new DbCompileError('JD0002',
-      `the job tables could not be created (${error?.message ?? String(error)}) — `
+      `the job tables could not be opened (${error?.message ?? String(error)}) — `
       + 'a read-only store creates nothing; open it read-write once, or without jobs',
       '/jobs', error));
 
@@ -436,6 +449,21 @@ export function createJobEngine(options) {
       { docPath: '/jobs', collection: JOBS_TABLE });
   };
 
+  /** Verify current execution authority without renewing or writing a checkpoint.
+   * Inside tx.jobs this check and mapped record writes share the same lock.
+   * @param {any} lease */
+  const assertLease = (lease) => {
+    const misuse = requireLease(lease, 'assertLease()');
+    if (misuse !== null) throw misuse;
+    return chain(prepared('assertLease', `SELECT id FROM "${JOBS_TABLE}" WHERE id=? AND ${FENCE}`)
+      .get([lease.jobId, lease.token, now()]), (row) => {
+      if (row !== undefined) return true;
+      return chain(refuseSettlement(lease, 'assertLease()'), (error) => {
+        throw error ?? new DbRuntimeError('JD2065', 'the job is already settled', { docPath: '/jobs' });
+      });
+    });
+  };
+
   /** §4: the jittered exponential backoff. */
   const backoffOf = (attempts, workerDefaults) => {
     const base = workerDefaults?.backoffBase ?? defaults.backoffBase;
@@ -849,10 +877,11 @@ export function createJobEngine(options) {
    * @param {{ handlers: Record<string, Function>, concurrency?: number,
    *   pollInterval?: number, leaseMs?: number, owner?: string,
    *   backoffBase?: number, backoffCap?: number, renew?: boolean,
-   *   onOutcome?: (event: any) => void }} workerOptions
+   *   onOutcome?: (event: any) => void, effectSafety?: (job: any, context: any) => any }} workerOptions
    */
   const createWorker = (workerOptions) => {
     const handlers = workerOptions?.handlers;
+    if (workerOptions?.effectSafety !== undefined && typeof workerOptions.effectSafety !== 'function') throw new TypeError('effectSafety must be a function');
     if (handlers === null || typeof handlers !== 'object'
       || Object.keys(handlers).length === 0
       || Object.values(handlers).some((handler) => typeof handler !== 'function')) {
@@ -1143,8 +1172,11 @@ export function createJobEngine(options) {
       try {
         let result;
         try {
-          result = await handlers[job.kind](job.payload,
-            { job, checkpoints: attempt.checkpoints, signal: attempt.signal });
+          const context = { job, lease: () => attempt.lease, checkpoints: attempt.checkpoints, signal: attempt.signal,
+            pause: () => io(() => cancel(job.id, { lease: attempt.lease }), 'a durable effect pause') };
+          const admitted = workerOptions.effectSafety === undefined || await workerOptions.effectSafety(job, context) === true;
+          if (!admitted) { await context.pause(); recordCancelled(attempt); return; }
+          result = await handlers[job.kind](job.payload, context);
         }
         catch (error) {
           // past cancellation the store is closing: leave the leased
@@ -1330,6 +1362,7 @@ export function createJobEngine(options) {
     counts,
     claim,
     renew,
+    assertLease,
     complete,
     fail,
     checkpointsFor,

package/src/live.js

@@ -549,6 +549,7 @@ function rowsStrategy(description, context) {
         return flatten();
       }),
     entries: () => flatten().length,
+    close: () => itemsByKey.clear(),
     apply(record, previousRows) {
       const touched = touchedKeys(record, context.name, description.deps);
       if (touched === null) return null;
@@ -628,6 +629,7 @@ function windowStrategy(description, context) {
         return visibleRows();
       }),
     entries: () => sortedWindow.size(),
+    close: () => sortedWindow.clear(),
     apply(record, previousRows) {
       const touched = touchedKeys(record, context.name, description.deps);
       if (touched === null) return null;
@@ -707,6 +709,7 @@ function accumulatorStrategy(description, context) {
         return rowsOf();
       }),
     entries: () => contributions.size,
+    close: () => contributions.clear(),
     stats: () => ({ ...stats }),
     apply(record, previousRows) {
       const touched = touchedKeys(record, context.name, description.deps);
@@ -867,7 +870,7 @@ export function createLiveRegistry(bounds) {
             outcome = ops.length === 0 && !late ? null
               : { ops, rows, ...(late ? { late: outcome.late } : {}) };
           }
-          if (outcome !== null) checkBound(strategy.entries(outcome.rows));
+          checkBound(strategy.entries(outcome?.rows ?? state.result.rows));
         }
         catch (error) {
           state.status = 'errored';

package/src/migrate.js

@@ -42,6 +42,9 @@ import { normalizeEntities, explainMapping } from './model.js';
 import { derivedValue, memberAt, registerDeriveFunctions } from './derive.js';
 import { mergeEntityRow } from './graph.js';
 import { entityCore } from './entity.js';
+import { sqlTokens } from './dialects/check-read.js';
+import { readSchema } from './introspect.js';
+import { verifyPhysical, physicalSelection } from './physical.js';
 import {
   MIGRATION_VERSION, isPerDocumentAssertion, compileDocumentStep, checkMigrationDocument,
   normalizeAssertionBounds, ASSERTION_BOUNDS_DEFAULT, createAssertionBoundGuard,
@@ -195,6 +198,8 @@ export function planMigration(fromModel, toModel, options = undefined) {
   const dialect = options?.dialect ?? null;
   if (dialect === null || typeof dialect !== 'object')
     throw new TypeError('planMigration needs { dialect } (the store dialect renders the DDL)');
+  if ([fromModel, toModel].some((m) => Object.values(m.entities ?? {}).some((e) => e.physical !== undefined)))
+    throw refuse('JD0021', 'column layouts require planPhysicalMigration with explicit preservation dispositions');
   const mapping = { derived: options?.derived ?? 'virtual', rtree: options?.rtree !== false,
     // a model that declares an index EXPRESSION resolves its functions
     // here too: a plan is DDL, and DDL over a function this planner was
@@ -1054,6 +1059,18 @@ function walkRows(connection, table, batchSize, handle, keyed = true, entityMapp
   // the rest-document alone could not see `id` or `name` at all
   const dialect = connection.dialect;
   const q = dialect.quoteIdentifier;
+  if (entityMapping?.document === false) {
+    const order = entityMapping.keys.map((k) => q(entityMapping.columns.find((c) => c.name === k).physical)).join(', ');
+    const next = (offset) => {
+      check?.();
+      const sql = `SELECT ${physicalSelection(entityMapping, dialect)} FROM ${q(entityMapping.table)} ORDER BY ${order} ${dialect.limitClause(batchSize, offset)}`;
+      return chain(connection.prepare(sql), (statement) => chain(statement.all([]), (rows) => {
+        if (!rows.length) return null;
+        return chain(handle(rows.map((row, i) => ({ ...row, rid: offset + i }))), () => next(offset + rows.length));
+      }));
+    };
+    return next(0);
+  }
   const rid = dialect.rowIdentity();
   const keySelect = keyed ? `, ${q('key')} AS ${q('k')}` : '';
   const columnSelect = entityMapping === null ? '' : entityColumnsOf(entityMapping)
@@ -1118,6 +1135,8 @@ function runSteps(connection, migration, options) {
     // migration in flight back whole, as any step failure does
     if (options.check !== undefined) options.check();
     const current = migration.steps[i];
+    if (migration.physical && !['ddl', 'sql', 'rebuild'].includes(current.kind))
+      throw refuse('JD0021', 'physical preservation plans use explicit SQL/rebuild steps and preservation assertions');
     const fail = (reason, cause) => {
       throw refuse('JD0023',
         `migration '${migration.id}' step ${i} (${current.kind}) failed: ${reason}`,
@@ -1356,14 +1375,16 @@ function validateTargetState(connection, model, options) {
     const validate = options.compileSchema !== undefined
       ? options.compileSchema(entity.schema)
       : null;
-    if (validate === null) return verifyEntity(i + 1);
+    if (validate === null && entity.physical === null) return verifyEntity(i + 1);
     // the WHOLE document — mapped columns folded in — is what the target
     // schema judges; the rest-document alone failed every entity whose
     // required members are columns, so a pure widening could not land
     const entityMapping = explainMapping(model).entities[entity.name];
-    return chain(walkRows(connection, entity.name, options.batchSize, (rows) => {
+    return chain(entity.physical === null ? null : chain(readSchema(connection), (schema) =>
+      verifyPhysical(connection, planEntity(entity.name, entityMapping, explainMapping(model), dialect).physical, schema)), () =>
+      chain(walkRows(connection, entityMapping.table, options.batchSize, (rows) => {
       for (const row of rows) {
-        const outcome = validate(mergeEntityRow(entityMapping, row, 'doc'));
+        const outcome = validate === null ? true : validate(mergeEntityRow(entityMapping, row, 'doc'));
         const valid = outcome === true || outcome?.valid === true;
         if (!valid) {
           throw refuse('JD0021',
@@ -1371,7 +1392,7 @@ function validateTargetState(connection, model, options) {
             + 'validate against the target schema — a narrowing needs a data transform');
         }
       }
-    }, false, entityMapping), () => verifyEntity(i + 1));
+    }, false, entityMapping), () => verifyEntity(i + 1)));
   };
   const verifyNext = (i) => {
     if (i >= collections.length) return null;
@@ -1689,13 +1710,12 @@ export function migrate(target, migrations, options) {
         // table before reading it, a DRY RUN probes for it instead and
         // reads an absent one as an empty history — the promise a dry
         // run makes is the reason it is safe to point at production
-        const history = options.dryRun === true
-          ? chain(connection.prepare(dialect.introspect.tableExists()), (probe) =>
-            chain(probe.get([HISTORY_TABLE]), (row) => (row === undefined
-              ? []
-              : chain(connection.prepare(statements.select), (select) => select.all([])))))
-          : chain(connection.exec(statements.create), () =>
-            chain(connection.prepare(statements.select), (select) => select.all([])));
+        let historyExists = false;
+        const history = chain(connection.prepare(dialect.introspect.tableExists()), (probe) =>
+          chain(probe.get([HISTORY_TABLE]), (row) => {
+            historyExists = row !== undefined;
+            return !historyExists ? [] : chain(connection.prepare(statements.select), (select) => select.all([]));
+          }));
         work = chain(history, (appliedRows) => {
           // the list must agree with the history: same ids, same
           // order, same checksums — an edited applied migration is
@@ -1723,6 +1743,7 @@ export function migrate(target, migrations, options) {
           let expectedFrom = currentShape;
           for (const migration of pending) {
             checkMigrationDocument(migration);
+            checkPreservationPlan(migration);
             if (migration.from !== expectedFrom) {
               throw refuse('JD0020',
                 `migration '${migration.id}' expects shape '${migration.from}' but the `
@@ -1741,6 +1762,8 @@ export function migrate(target, migrations, options) {
             return { applied: [], skipped: appliedRows.map((row) => row.id), upToDate: true };
           }
 
+          if (pending.some((m) => m.physical) && options.shadow !== false)
+            throw refuse('JD0021', 'physical preservation plans require shadow:false; qualify against an explicit copy/fresh-target fixture');
           const shadowRun = options.shadow === false
             ? null
             : replayOnShadow(options.shadowDriver ?? target.driver,
@@ -1824,11 +1847,14 @@ export function migrate(target, migrations, options) {
                 bracket && dialect.capabilities.foreignKeysAlwaysOn !== true
                   ? connection.exec(dialect.pragma.foreignKeys(false)) : null,
                 () => chain(connection.exec(dialect.tx.beginImmediate), () => {
-                const body = () => chain(runSteps(connection, migration, runOptions), () =>
+                const body = () => chain(migration.physical ? verifyPreservation(connection, migration.physical, false) : null, () =>
+                  chain(historyExists ? null : connection.exec(statements.create), () =>
+                  chain(runSteps(connection, migration, runOptions), () =>
+                  chain(migration.physical ? verifyPreservation(connection, migration.physical, true) : null, () =>
                   chain(last && options.model !== undefined
                     ? chain(validateTargetState(connection, options.model,
                       { compileSchema: options.compileSchema, batchSize }),
-                    () => (normalizeEntities(options.model).size === 0 ? null
+                    () => (normalizeEntities(options.model).size === 0 || migration.physical ? null
                       : chain(compareShapeToModel(target.driver, connection,
                         options.model, options.registerFunctions), (difference) => {
                         if (difference !== null) {
@@ -1841,15 +1867,10 @@ export function migrate(target, migrations, options) {
                   () => chain(connection.prepare(statements.insert), (insert) =>
                     insert.run([migration.id, runtime.now(), migration.from,
                       migration.to, migrationChecksum(migration),
-                      migration.steps.length]))));
+                      migration.steps.length])))))));
                 const restore = () => (bracket
                   && dialect.capabilities.foreignKeysAlwaysOn !== true
                   ? connection.exec(dialect.pragma.foreignKeys(true)) : null);
-                const commit = () => chain(connection.exec(dialect.tx.commit), () =>
-                  chain(restore(), () => {
-                    applied.push(migration.id);
-                    return applyNext(i + 1);
-                  }));
                 const rollback = (error) =>
                   chain(connection.exec(dialect.tx.rollback), () =>
                     chain(restore(), () => { throw error; }));
@@ -1864,9 +1885,16 @@ export function migrate(target, migrations, options) {
                 catch (error) {
                   return rollback(error);
                 }
-                return outcome instanceof Promise
-                  ? outcome.then(commit, rollback)
-                  : commit();
+                const settle = () => {
+                  let result;
+                  try { result = connection.exec(dialect.tx.commit); }
+                  catch (error) { return rollback(error); }
+                  return result instanceof Promise ? result.then(published, rollback) : published();
+                };
+                const published = () => chain(restore(), () => {
+                  historyExists = true; applied.push(migration.id); return applyNext(i + 1);
+                });
+                return outcome instanceof Promise ? outcome.then(settle, rollback) : settle();
               }));
             };
             return chain(applyNext(0), () => ({
@@ -1886,3 +1914,89 @@ export function migrate(target, migrations, options) {
         : finish(work);
     })));
 }
+
+/** Plan an existing file's explicit preservation migration. Every source object
+ * needs a disposition; source/target assertions preserve application-owned facts.
+ * @param {any} connection @param {any} fromModel @param {any} toModel
+ * @param {{ id: string, steps: any[], dispositions: Record<string, 'preserve'|'replace'|'drop'>,
+ *   assertions?: { sql: string, params?: any[], expected: any[] }[] }} options @returns {any} */
+export function planPhysicalMigration(connection, fromModel, toModel, options) {
+  normalizeEntities(fromModel); normalizeEntities(toModel);
+  if (!options || typeof options.id !== 'string' || !options.id || !Array.isArray(options.steps))
+    throw refuse('JD0021', 'a physical plan requires id and explicit steps');
+  return chain(preservationSchemaOf(connection), (source) => {
+    const dispositions = options.dispositions ?? {};
+    const keys = source.map((o) => `${o.type}:${o.name}`);
+    if (Object.keys(dispositions).some((key) => !keys.includes(key)) || keys.some((key) => !['preserve', 'replace', 'drop'].includes(dispositions[key])))
+      throw refuse('JD0021', 'every physical source object must have an explicit preserve, replace or drop disposition');
+    const assertions = options.assertions ?? [];
+    for (const assertion of assertions) {
+      if (!assertion || typeof assertion.sql !== 'string' || !/^SELECT\b/i.test(assertion.sql.trim()) || !Array.isArray(assertion.expected))
+        throw refuse('JD0021', 'preservation assertions require a SELECT and expected rows');
+    }
+    const migration = { $migration: MIGRATION_VERSION, id: options.id, from: shapeHash(fromModel), to: shapeHash(toModel),
+      steps: options.steps, physical: { source, dispositions, assertions } };
+    checkMigrationDocument(migration);
+    checkPreservationPlan(migration);
+    return migration;
+  });
+}
+
+/** Validate saved plans again at execution, including SQL ownership boundaries. */
+function checkPreservationPlan(migration) {
+  const physical = migration.physical;
+  if (physical === undefined) return;
+  const fail = () => { throw refuse('JD0021', 'invalid physical source, dispositions, assertions or steps'); };
+  if (!physical || !Array.isArray(physical.source) || !physical.dispositions || !Array.isArray(physical.assertions)) fail();
+  const keys = physical.source.map((object) => {
+    if (!object || typeof object.name !== 'string' || !['table', 'view', 'index', 'trigger'].includes(object.type)) fail();
+    return `${object.type}:${object.name}`;
+  });
+  if (new Set(keys).size !== keys.length || Object.keys(physical.dispositions).some((key) => !keys.includes(key))
+    || keys.some((key) => !['preserve', 'replace', 'drop'].includes(physical.dispositions[key]))) fail();
+  for (const assertion of physical.assertions)
+    if (!assertion || typeof assertion.sql !== 'string' || !/^SELECT\b/i.test(assertion.sql.trim())
+      || !Array.isArray(assertion.expected) || (assertion.params !== undefined && !Array.isArray(assertion.params))) fail();
+  if (migration.steps.some((step) => !['ddl', 'sql', 'rebuild'].includes(step.kind))) fail();
+  const fragments = migration.steps.flatMap((step) => step.kind === 'rebuild'
+    ? [...(step.create ?? []), step.copy, ...(step.indexes ?? [])] : [step.sql]);
+  for (const sql of fragments) {
+    const tokens = typeof sql === 'string' ? sqlTokens(sql) : [];
+    const words = tokens.filter((t) => t.kind === 'word').map((t) => t.value.toUpperCase());
+    if (!['CREATE', 'ALTER', 'DROP', 'INSERT', 'UPDATE', 'DELETE'].includes(words[0])
+      || words.some((w) => /^(?:COMMIT|ROLLBACK|SAVEPOINT|RELEASE|ATTACH|DETACH|PRAGMA|VACUUM)$/.test(w)))
+      throw refuse('JD0021', 'physical steps cannot change transaction or connection ownership');
+  }
+}
+
+/** Preservation compares exact source programs, including whitespace in SQL literals. */
+function preservationSchemaOf(connection) {
+  return chain(readSchema(connection), (schema) => schema.objects
+    .filter((object) => !ENGINE_TABLES.has(object.name) && !ENGINE_TABLES.has(object.owner)));
+}
+
+/** Verify source identity before destructive steps, and every preserved object
+ * and fact before publication. The migration transaction owns all these reads. */
+function verifyPreservation(connection, physical, after) {
+  return chain(preservationSchemaOf(connection), (actual) => {
+    if (!after && canonicalizeJson(actual) !== canonicalizeJson(physical.source))
+      throw refuse('JD0020', 'the physical source schema changed after the plan was prepared');
+    if (after) {
+      for (const object of physical.source) {
+        const key = `${object.type}:${object.name}`;
+        const current = actual.find((o) => o.type === object.type && o.name === object.name);
+        if (physical.dispositions[key] === 'preserve' && canonicalizeJson(current ?? null) !== canonicalizeJson(object))
+          throw refuse('JD0023', `preserved object '${key}' was changed or lost`);
+        if (physical.dispositions[key] === 'drop' && current) throw refuse('JD0023', `declared drop '${key}' remains`);
+      }
+    }
+    const next = (i) => i >= physical.assertions.length ? null
+      : chain(connection.prepare(physical.assertions[i].sql, { readOnly: true }), (s) =>
+        chain(s.all(physical.assertions[i].params ?? []), (rows) => {
+          if (canonicalizeJson(rows) !== canonicalizeJson(physical.assertions[i].expected))
+            throw refuse('JD0023', `preservation assertion ${i} disagrees ${after ? 'after' : 'before'} migration`);
+          return next(i + 1);
+        }));
+    return next(0);
+  });
+}

package/src/model.js

@@ -27,6 +27,8 @@ import {
 import { compileJsonQuery } from '@jarenjs/json/query';
 
 import { DbCompileError } from './errors.js';
+import { normalizePhysical } from './physical.js';
+import { normalizeInvariants } from './invariants.js';
 
 /** The closed `x-entity` vocabulary; anything else is `JD0030`. */
 const ENTITY_MEMBERS = new Set(['key', 'unique', 'index', 'default', 'column', 'relation', 'version']);
@@ -343,9 +345,13 @@ export function normalizeEntities(model) {
       keys,
       relations,
       version: versions.length === 1 ? versions[0].name : null,
+      physical: normalizePhysical(spec.physical, properties, keys, docPath),
+      invariants: normalizeInvariants(spec.invariants, `${docPath}/invariants`),
     });
   }
 
+  for (const entity of entities.values())
+    if (entity.physical !== null && entity.relations.length) throw modelError('physical join tables are declared as entities; relation navigation is not qualified for column layouts', entity.docPath);
   resolveRelations(entities);
   return entities;
 }
@@ -556,6 +562,11 @@ export function explainMapping(model) {
   const mapping = { entities: {}, joinTables: {} };
 
   for (const entity of entities.values()) {
+    if (entity.physical !== null) {
+      mapping.entities[entity.name] = { ...entity.physical, document: false,
+        foreignKeys: [], indexes: [], version: entity.version, invariants: model.entities[entity.name].invariants ?? [] };
+      continue;
+    }
     const columns = [];
     const document = [];
     const indexes = [];
@@ -626,6 +637,7 @@ export function explainMapping(model) {
       indexes,
       document,
       version: entity.version ?? null,
+      ...(entity.invariants.length ? { invariants: model.entities[entity.name].invariants } : {}),
     };
   }
 

package/src/mutation.js

@@ -0,0 +1,165 @@
+//@ts-check
+/** Bounded column mutation documents, lowered through the entity's writer plan. */
+import { analyzeQuery } from '@jarenjs/json/query';
+import { canonicalizeJson } from '@jarenjs/json/canonical';
+import { chain, attempt } from './driver.js';
+import { DbCompileError, DbRuntimeError, wrapDriverError } from './errors.js';
+import { entityShape, planEntityPredicate } from './plan.js';
+import { createEntityPredicateEmitters } from './emit.js';
+import { physicalSelection } from './physical.js';
+import { utf8Length } from './cursor.js';
+
+/** Compile once per document, execute within the existing guarded transaction.
+ * @param {any} connection @param {any} entity @param {any} mapping @param {any} core */
+export function createEntityMutation(connection, entity, mapping, core) {
+  const dialect = connection.dialect;
+  const q = dialect.quoteIdentifier;
+  const plans = new Map();
+  const fail = (reason) => { throw new DbCompileError('JD0038', reason, entity.docPath); };
+  const column = (name) => {
+    const c = mapping.columns.find((entry) => entry.name === name);
+    if (!c) return fail(`unknown stored member '${name}'`);
+    return c;
+  };
+  const writableColumn = (name) => {
+    const c = column(name);
+    if (c.generated || core.plan.keys.includes(name) || name === entity.version)
+      fail(`'${name}' is an identity, revision or generated member`);
+    return c;
+  };
+  const comparison = (c, prefix = '') => {
+    const value = prefix + q(c.physical);
+    return c.storage === 'string' ? dialect.codepoint(value) : value;
+  };
+  const compile = (document) => {
+    if (entity.physical == null || dialect.name !== 'sqlite') fail('native mutations require a declared SQLite column layout');
+    core.plan.writable();
+    if (!document || typeof document !== 'object' || Array.isArray(document)) fail('a mutation is an object');
+    const allowed = { update: ['key', 'expectedRevision', 'set'], upsert: ['values', 'conflict', 'update'],
+      'insert-select': ['source', 'where', 'select', 'conflict', 'onConflict'] }[document.op];
+    if (!allowed) fail('op must be update, upsert or insert-select');
+    for (const key of Object.keys(document))
+      if (!['op', 'returning', 'maxRows', 'maxBytes', ...allowed].includes(key)) fail(`unknown mutation member '${key}'`);
+    const maxRows = document.maxRows ?? 100;
+    const maxBytes = document.maxBytes ?? 1_048_576;
+    if (![maxRows, maxBytes].every((n) => Number.isSafeInteger(n) && n > 0 && n < Number.MAX_SAFE_INTEGER)) fail('mutation row and byte bounds must be positive safe integers');
+    const returning = document.returning ?? mapping.columns.map((c) => c.name);
+    if (!Array.isArray(returning) || !returning.length || new Set(returning).size !== returning.length) fail('returning is a nonempty distinct member list');
+    returning.forEach(column);
+    // A before/after invariant needs the preimage. These plans promise one
+    // data statement; database invariants retain their trigger enforcement.
+    if (entity.invariants.some((rule) => rule.enforcement === 'store')) fail('store invariants require the entity writer with before/after images');
+    const params = [];
+    const param = (value) => { params.push(value); return dialect.parameterRef(params.length, 'v'); };
+    const table = q(mapping.table);
+    let sql;
+    let prefix = '';
+    if (document.op === 'update') {
+      if (!document.set || typeof document.set !== 'object' || Array.isArray(document.set) || !Object.keys(document.set).length) fail('update needs a nonempty set object');
+      const assignments = Object.entries(document.set).map(([name, value]) => {
+        const c = writableColumn(name);
+        const encoded = core.plan.encodeColumn(name, value);
+        return { name: q(c.physical), compare: comparison(c), value: param(encoded), encoded };
+      });
+      const parts = core.normalizeKey(document.key);
+      const where = core.plan.keys.map((key, i) => `${q(column(key).physical)} = ${param(core.plan.encodeColumn(key, parts[i]))}`);
+      if (entity.version !== null) {
+        if (!Number.isSafeInteger(document.expectedRevision) || document.expectedRevision < 0) fail('a versioned update needs expectedRevision');
+        where.push(`${q(column(entity.version).physical)} = ${param(document.expectedRevision)}`);
+      }
+      else if (document.expectedRevision !== undefined) fail('expectedRevision needs a declared version member');
+      where.push(`(${assignments.map((a) => `${a.compare} IS NOT ${param(a.encoded)}`).join(' OR ')})`);
+      const sets = assignments.map((a) => `${a.name} = ${a.value}`);
+      if (entity.version !== null) sets.push(`${q(column(entity.version).physical)} = ${q(column(entity.version).physical)} + 1`);
+      sql = `UPDATE ${table} SET ${sets.join(', ')} WHERE ${where.join(' AND ')}`;
+    }
+    else {
+      if (JSON.stringify(document.conflict) !== JSON.stringify(core.plan.keys)) fail('conflict must name the complete ordered primary key');
+      let names;
+      let source;
+      if (document.op === 'upsert') {
+        if (!document.values || typeof document.values !== 'object' || Array.isArray(document.values)) fail('upsert needs values');
+        const complete = core.complete(document.values, { updating: false });
+        const split = core.plan.split(complete);
+        names = split.values.map((v) => v.name);
+        if (core.plan.keys.some((key) => !names.includes(key))) fail('upsert requires every primary-key value');
+        source = `VALUES (${split.values.map((v) => param(v.value)).join(', ')})`;
+      }
+      else {
+        if (document.source !== entity.name || document.onConflict !== 'nothing') fail('insert-select supports same-entity sources with onConflict nothing');
+        if (!document.select || typeof document.select !== 'object' || Array.isArray(document.select)) fail('insert-select needs a projection');
+        names = Object.keys(document.select);
+        if (!names.length || core.plan.keys.some((key) => !names.includes(key))) fail('insert-select projects every primary-key member');
+        const values = names.map((name) => {
+          const target = column(name);
+          if (target.generated || name === entity.version) fail('insert-select leaves generated values and revisions database-owned');
+          const value = document.select[name];
+          if (typeof value === 'string' && value.startsWith('$it.')) {
+            const from = column(value.slice(4));
+            if (from.codec !== target.codec || from.null !== target.null) fail('insert-select requires identical source and target codecs');
+            return `${q('s')}.${q(from.physical)}`;
+          }
+          if (value && typeof value === 'object' && !Array.isArray(value) && Object.keys(value).length === 1 && Object.hasOwn(value, '$literal'))
+            return param(core.plan.encodeColumn(name, value.$literal));
+          return fail('insert-select values are singular member paths or $literal values');
+        });
+        const analyzed = analyzeQuery({ $for: { it: '$[*]' }, $where: document.where ?? true, $return: '$it' });
+        const predicate = planEntityPredicate(analyzed.root.where, analyzed.root.forBindings[0].slot, entityShape(entity, mapping));
+        if ('refusal' in predicate) fail(predicate.refusal.reason);
+        const emitter = createEntityPredicateEmitters(dialect, (slot) => {
+          if (!Object.hasOwn(slot, 'literal')) return fail('mutation predicates use literal values');
+          return param(slot.literal);
+        });
+        const where = emitter.emitPred(q('s'), `${q('s')}.${q('doc')}`, predicate.pred);
+        source = `SELECT ${values.join(', ')} FROM ${table} AS ${q('s')} WHERE ${where}`
+          + ` ORDER BY ${core.plan.keys.map((key) => `${q('s')}.${q(column(key).physical)}`).join(', ')}`
+          + ` LIMIT ${maxRows + 1}`;
+        const aliases = names.map((name, i) => q(`v${i}`));
+        prefix = `WITH ${q('_jaren_source')} (${aliases.join(', ')}) AS MATERIALIZED (${source}) `;
+        source = `SELECT ${aliases.join(', ')} FROM ${q('_jaren_source')} WHERE `
+          + dialect.mutationRowGuard(`(SELECT COUNT(*) FROM ${q('_jaren_source')})`, maxRows);
+      }
+      sql = `INSERT INTO ${table} (${names.map((name) => q(column(name).physical)).join(', ')}) ${source}`
+        + ` ON CONFLICT (${core.plan.keys.map((key) => q(column(key).physical)).join(', ')})`;
+      if (document.op === 'insert-select') sql += ' DO NOTHING';
+      else {
+        if (!Array.isArray(document.update) || !document.update.length || new Set(document.update).size !== document.update.length) fail('upsert update names distinct stored members');
+        const changes = document.update.map((name) => {
+          const c = writableColumn(name);
+          if (!names.includes(name)) fail('an upsert update member must be supplied in values');
+          return { name: q(c.physical), compare: comparison(c, `${table}.`) };
+        });
+        const sets = changes.map(({ name }) => `${name} = excluded.${name}`);
+        if (entity.version !== null) sets.push(`${q(column(entity.version).physical)} = ${table}.${q(column(entity.version).physical)} + 1`);
+        sql += ` DO UPDATE SET ${sets.join(', ')} WHERE ${changes.map(({ name, compare }) => `${compare} IS NOT excluded.${name}`).join(' OR ')}`;
+      }
+    }
+    sql = prefix + sql + ` RETURNING ${physicalSelection(mapping, dialect)}`;
+    return { sql, params, returning, maxRows, maxBytes, statement: null };
+  };
+  return (document) => {
+    const key = canonicalizeJson(document);
+    let plan = plans.get(key);
+    if (!plan) {
+      plan = compile(document);
+      if (plans.size >= 64) plans.delete(plans.keys().next().value);
+      plans.set(key, plan);
+    }
+    return connection.transaction(() => {
+      plan.statement ??= connection.prepare(plan.sql);
+      return chain(plan.statement, (statement) => chain(attempt(() => statement.all(plan.params),
+        (error) => String(error?.message).includes('jaren-mutation-row-bound')
+          ? new DbRuntimeError('JD2007', 'insert-select exceeded its source row bound', { cause: error })
+          : wrapDriverError(error, { collection: entity.name, docPath: entity.docPath })), (rows) => {
+        const stored = rows.map(core.plan.merge);
+        if (rows.length > plan.maxRows || utf8Length(JSON.stringify(stored)) > plan.maxBytes)
+          throw new DbRuntimeError('JD2007', 'native mutation exceeded its returned row or byte bound');
+        stored.forEach(core.validateOnly);
+        const returned = stored.map((row) => Object.fromEntries(plan.returning
+          .filter((name) => Object.hasOwn(row, name)).map((name) => [name, row[name]])));
+        return { mode: 'native', affected: rows.length, rows: returned,
+          admitted: { statements: 1, rows: rows.length, bytes: utf8Length(JSON.stringify(stored)) } };
+      }));
+    });
+  };
+}