@jarenjs/db 0.73.0 → 0.83.2

package/src/live.js

@@ -112,11 +112,11 @@ function documentsSource(binding, where) {
 
 /**
  * Recognise the canonical single-level group form (§7): one binding
- * over `$[*]`, optional `$where`, `$groupby` with ONE binding, and a
+ * over `$[*]`, optional `$where`, one or more group keys, and a
  * `$return` object whose members are the group key (`$g` plain or
  * defaulted) or a single-member aggregate over the group sequence.
  * @param {any} inner
- * @returns {null | { binding: string, group: string, groupExpr: any,
+ * @returns {null | { binding: string, groups: string[], key: any,
  *   where: any, members: { name: string, kind: 'key' | 'aggregate',
  *   fn?: string, operand?: any, defaulted?: boolean }[] }}
  */
@@ -126,18 +126,30 @@ function recogniseGroupForm(inner) {
   const allowed = new Set(['$for', '$where', '$groupby', '$return']);
   if (!Object.keys(inner).every((key) => allowed.has(key))) return null;
   const groupNames = Object.keys(inner.$groupby);
-  if (groupNames.length !== 1) return null;
-  const group = groupNames[0];
+  if (groupNames.length === 0) return null;
+  // Subset evaluation must never read the global collection root.
+  const local = (node) => {
+    if (typeof node === 'string') return node !== '$' && !/^\$(?:\.|\[)/.test(node);
+    if (Array.isArray(node)) return node.every(local);
+    if (!isJsonObject(node) || Object.hasOwn(node, '$const')) return true;
+    return Object.values(node).every(local);
+  };
+  if (![inner.$where, inner.$groupby, inner.$return].every(local)) return null;
+  const scalar = groupNames.some((name) => inner.$return === `$${name}`)
+    || (isJsonObject(inner.$return) && Object.keys(inner.$return).length === 1
+      && AGGREGATE_MEMBERS.has(Object.keys(inner.$return)[0]));
+  if (scalar) return { binding, groups: groupNames,
+    key: groupNames.map((name) => [inner.$groupby[name]]), where: inner.$where, members: [] };
   if (!isJsonObject(inner.$return)) return null;
   const members = [];
   for (const name of Object.keys(inner.$return)) {
     const expr = inner.$return[name];
-    if (expr === `$${group}`) {
+    if (groupNames.some((group) => expr === `$${group}`)) {
       members.push({ name, kind: 'key', defaulted: false });
       continue;
     }
     if (isJsonObject(expr) && Array.isArray(expr.$default)
-      && expr.$default.length === 2 && expr.$default[0] === `$${group}`
+      && expr.$default.length === 2 && groupNames.some((group) => expr.$default[0] === `$${group}`)
       && expr.$default[1] === null && Object.keys(expr).length === 1) {
       members.push({ name, kind: 'key', defaulted: true });
       continue;
@@ -150,7 +162,8 @@ function recogniseGroupForm(inner) {
     }
     return null;
   }
-  return { binding, group, groupExpr: inner.$groupby[group], where: inner.$where, members };
+  return { binding, groups: groupNames, key: groupNames.map((name) => [inner.$groupby[name]]),
+    where: inner.$where, members };
 }
 
 /**
@@ -276,10 +289,16 @@ export function classifyLiveQuery(document, queryShape, keyed, eventTime = null)
 
   if (aggregate !== null) {
     if (windowed) return rerun('a windowed aggregate maintains no accumulator');
+    if (recogniseGroupForm(inner) !== null) {
+      const grouped = classifyLiveQuery(inner, queryShape, keyed, eventTime);
+      if (grouped.strategy === 'group') return { ...grouped, aggregate: aggregate.name };
+    }
     const planned = planQuery({ [aggregate.name]: inner }, queryShape, {});
     if (planned.mode !== 'native' || planned.plan.aggregate === null) {
       return rerun(refinedOnly(planned, false) ? SPATIAL_RERUN.aggregate : plannerReason(planned));
     }
+    if (planned.plan.group !== null || planned.plan.bucket !== null)
+      return rerun('an aggregate over groups needs group-level maintenance');
     if (!keyed) return rerun('rows without a document key cannot be tracked');
     return {
       strategy: 'accumulator',
@@ -300,19 +319,9 @@ export function classifyLiveQuery(document, queryShape, keyed, eventTime = null)
         : `the group filter did not translate: ${plannerReason(planned)}`);
     }
     if (!keyed) return rerun('rows without a document key cannot be tracked');
-    const rowDocument = {
-      $for: { [group.binding]: '$[*]' },
-      ...(group.where !== undefined ? { $where: group.where } : {}),
-      $return: {
-        k: [group.groupExpr],
-        ...Object.fromEntries(group.members
-          .filter((member) => member.kind === 'aggregate')
-          .map((member) => [member.name, [member.operand]])),
-      },
-    };
     return {
-      strategy: 'group', group, carrier, rowDocument,
-      deps: memberDeps(analyzeQuery([rowDocument]).root),
+      strategy: 'group', inner, binding: group.binding, key: group.key, carrier,
+      deps: memberDeps(analyzeQuery(inner).root),
     };
   }
 
@@ -324,6 +333,16 @@ export function classifyLiveQuery(document, queryShape, keyed, eventTime = null)
     return rerun('a k-nearest ranking re-runs (the vector column cuts the candidates and the engine orders them)');
 
   const planned = planQuery(inner, queryShape, {});
+  const distinct = isJsonObject(inner) ? inner.$distinct : null;
+  if (!windowed && keyed && planned.mode === 'native' && isJsonObject(distinct)
+    && distinct.$orderby === undefined && bindingNameOf(distinct) !== null) {
+    const binding = bindingNameOf(distinct);
+    const name = binding === '_distinct' ? '_distinctKey' : '_distinct';
+    const grouped = { ...distinct, $groupby: { [name]: distinct.$return }, $return: `$${name}` };
+    return { strategy: 'distinct', inner: grouped, binding, key: [distinct.$return],
+      carrier: documentsSource(binding, distinct.$where),
+      deps: memberDeps(analyzeQuery(grouped).root) };
+  }
   if (planned.mode === 'set') {
     // §7's spatial rows: the fetch is SQL-narrowed by the pushed box or
     // cell range and the exact predicate is what per-row re-evaluation
@@ -346,6 +365,13 @@ export function classifyLiveQuery(document, queryShape, keyed, eventTime = null)
   }
   if (!keyed) return rerun('rows without a document key cannot be tracked');
 
+  // SQL can group a complete selection without proving that evaluating
+  // the document once per changed row maintains its result. Canonical
+  // maintained groups were handled above; the remaining groups and
+  // distinct projections need set-level invalidation.
+  if (planned.plan.group !== null || planned.plan.bucket !== null)
+    return rerun('this grouping or distinct projection needs set-level maintenance');
+
   if (planned.plan.order !== null) {
     if (offset > 0 || (planned.plan.window?.offset ?? 0) > 0) {
       return rerun('an offset window re-runs');
@@ -523,6 +549,7 @@ function rowsStrategy(description, context) {
         return flatten();
       }),
     entries: () => flatten().length,
+    close: () => itemsByKey.clear(),
     apply(record, previousRows) {
       const touched = touchedKeys(record, context.name, description.deps);
       if (touched === null) return null;
@@ -602,6 +629,7 @@ function windowStrategy(description, context) {
         return visibleRows();
       }),
     entries: () => sortedWindow.size(),
+    close: () => sortedWindow.clear(),
     apply(record, previousRows) {
       const touched = touchedKeys(record, context.name, description.deps);
       if (touched === null) return null;
@@ -681,6 +709,7 @@ function accumulatorStrategy(description, context) {
         return rowsOf();
       }),
     entries: () => contributions.size,
+    close: () => contributions.clear(),
     stats: () => ({ ...stats }),
     apply(record, previousRows) {
       const touched = touchedKeys(record, context.name, description.deps);
@@ -713,121 +742,6 @@ function accumulatorStrategy(description, context) {
   };
 }
 
-/**
- * Canonical single-level `groupBy` with aggregate returns: §7's
- * per-group deltas — the accumulator machinery once per group, groups
- * in first-appearance order.
- * @param {any} description
- * @param {any} context
- */
-function groupStrategy(description, context) {
-  const { group, rowDocument, carrier } = description;
-  const evaluate = compileJsonQuery([rowDocument]);
-
-  /** @type {Map<string, { key: any[], rows: Map<string, any>, row: any }>}
-   * group token → per-row contributions, in first-appearance order */
-  const groups = new Map();
-
-  const contributionOf = (doc) => {
-    const evaluated = /** @type {any[]} */ (evaluate([doc], context.externals));
-    return evaluated.length === 0 ? undefined : evaluated[0];
-  };
-  const buildRow = (entry) => {
-    /** @type {any} */
-    const row = {};
-    for (const member of group.members) {
-      if (member.kind === 'key') {
-        if (entry.key.length > 0) row[member.name] = entry.key[0];
-        else if (member.defaulted) row[member.name] = null;
-        continue;
-      }
-      let sum = 0;
-      let count = 0;
-      let extreme;
-      let n = 0;
-      for (const contribution of entry.rows.values()) {
-        const items = /** @type {any[]} */ (contribution[member.name] ?? []);
-        n += items.length;
-        for (const value of items) {
-          sum += value;
-          count += 1;
-          if (extreme === undefined
-            || (member.fn === 'min' ? value < extreme : value > extreme)) extreme = value;
-        }
-      }
-      if (member.fn === 'count') row[member.name] = n;
-      else if (member.fn === 'sum') row[member.name] = sum;
-      else if (count > 0) row[member.name] = member.fn === 'avg' ? sum / count : extreme;
-      // an empty avg/min/max leaves the member absent, the engine's
-      // empty-sequence rule
-    }
-    return row;
-  };
-  const rowsOf = () => [...groups.values()].map((entry) => entry.row);
-
-  const placeRow = (token, doc, changedGroups) => {
-    const contribution = doc === undefined ? undefined : contributionOf(doc);
-    for (const [groupToken, entry] of groups) {
-      if (!entry.rows.has(token)) continue;
-      if (contribution !== undefined
-        && stableStringify(entry.rows.get(token)) === stableStringify(contribution)) {
-        return; // unchanged in place
-      }
-      entry.rows.delete(token);
-      changedGroups.add(groupToken);
-      break;
-    }
-    if (contribution === undefined) return;
-    const groupToken = stableStringify(contribution.k) ?? '';
-    let entry = groups.get(groupToken);
-    if (entry === undefined) {
-      entry = { key: contribution.k, rows: new Map(), row: null };
-      groups.set(groupToken, entry);
-    }
-    entry.rows.set(token, contribution);
-    changedGroups.add(groupToken);
-  };
-  const settle = (changedGroups) => {
-    for (const groupToken of changedGroups) {
-      const entry = groups.get(groupToken);
-      if (entry === undefined) continue;
-      if (entry.rows.size === 0) groups.delete(groupToken);
-      else entry.row = sharedRow(entry.row, buildRow(entry));
-    }
-  };
-
-  return {
-    init: () => chain(context.execute([carrier], { externals: context.externals }),
-      (docs) => {
-        const changedGroups = new Set();
-        for (const doc of /** @type {any[]} */ (docs)) {
-          placeRow(context.keyOf(doc), doc, changedGroups);
-        }
-        settle(changedGroups);
-        return rowsOf();
-      }),
-    entries: () => {
-      let total = groups.size;
-      for (const entry of groups.values()) total += entry.rows.size;
-      return total;
-    },
-    apply(record, previousRows) {
-      const touched = touchedKeys(record, context.name, description.deps);
-      if (touched === null) return null;
-      const changedGroups = new Set();
-      for (const [token, change] of touched) {
-        const doc = change.kind === 'delete'
-          ? undefined
-          : change.kind === 'insert' ? change.doc : context.readRow(token);
-        placeRow(token, doc, changedGroups);
-      }
-      if (changedGroups.size === 0) return null;
-      settle(changedGroups);
-      return diffAgainst(previousRows, rowsOf());
-    },
-  };
-}
-
 /**
  * Everything outside the table: re-run the WHOLE query on
  * invalidation and diff against the previous result with value-equal
@@ -903,7 +817,8 @@ export function createLiveRegistry(bounds) {
     };
     const STRATEGIES = {
       rows: rowsStrategy, window: windowStrategy, accumulator: accumulatorStrategy,
-      group: groupStrategy, bucket: bucketStrategy, rolling: rollingStrategy, join: joinStrategy, graph: joinStrategy,
+      group: nestedGroupStrategy, distinct: nestedGroupStrategy,
+      bucket: bucketStrategy, rolling: rollingStrategy, join: joinStrategy, graph: joinStrategy,
       'nested-group': nestedGroupStrategy,
     };
     const strategy = (STRATEGIES[classification.strategy] ?? rerunStrategy)(
@@ -955,7 +870,7 @@ export function createLiveRegistry(bounds) {
             outcome = ops.length === 0 && !late ? null
               : { ops, rows, ...(late ? { late: outcome.late } : {}) };
           }
-          if (outcome !== null) checkBound(strategy.entries(outcome.rows));
+          checkBound(strategy.entries(outcome?.rows ?? state.result.rows));
         }
         catch (error) {
           state.status = 'errored';

package/src/migrate.js

@@ -42,6 +42,9 @@ import { normalizeEntities, explainMapping } from './model.js';
 import { derivedValue, memberAt, registerDeriveFunctions } from './derive.js';
 import { mergeEntityRow } from './graph.js';
 import { entityCore } from './entity.js';
+import { sqlTokens } from './dialects/check-read.js';
+import { readSchema } from './introspect.js';
+import { verifyPhysical, physicalSelection } from './physical.js';
 import {
   MIGRATION_VERSION, isPerDocumentAssertion, compileDocumentStep, checkMigrationDocument,
   normalizeAssertionBounds, ASSERTION_BOUNDS_DEFAULT, createAssertionBoundGuard,
@@ -195,6 +198,8 @@ export function planMigration(fromModel, toModel, options = undefined) {
   const dialect = options?.dialect ?? null;
   if (dialect === null || typeof dialect !== 'object')
     throw new TypeError('planMigration needs { dialect } (the store dialect renders the DDL)');
+  if ([fromModel, toModel].some((m) => Object.values(m.entities ?? {}).some((e) => e.physical !== undefined)))
+    throw refuse('JD0021', 'column layouts require planPhysicalMigration with explicit preservation dispositions');
   const mapping = { derived: options?.derived ?? 'virtual', rtree: options?.rtree !== false,
     // a model that declares an index EXPRESSION resolves its functions
     // here too: a plan is DDL, and DDL over a function this planner was
@@ -1054,6 +1059,18 @@ function walkRows(connection, table, batchSize, handle, keyed = true, entityMapp
   // the rest-document alone could not see `id` or `name` at all
   const dialect = connection.dialect;
   const q = dialect.quoteIdentifier;
+  if (entityMapping?.document === false) {
+    const order = entityMapping.keys.map((k) => q(entityMapping.columns.find((c) => c.name === k).physical)).join(', ');
+    const next = (offset) => {
+      check?.();
+      const sql = `SELECT ${physicalSelection(entityMapping, dialect)} FROM ${q(entityMapping.table)} ORDER BY ${order} ${dialect.limitClause(batchSize, offset)}`;
+      return chain(connection.prepare(sql), (statement) => chain(statement.all([]), (rows) => {
+        if (!rows.length) return null;
+        return chain(handle(rows.map((row, i) => ({ ...row, rid: offset + i }))), () => next(offset + rows.length));
+      }));
+    };
+    return next(0);
+  }
   const rid = dialect.rowIdentity();
   const keySelect = keyed ? `, ${q('key')} AS ${q('k')}` : '';
   const columnSelect = entityMapping === null ? '' : entityColumnsOf(entityMapping)
@@ -1118,6 +1135,8 @@ function runSteps(connection, migration, options) {
     // migration in flight back whole, as any step failure does
     if (options.check !== undefined) options.check();
     const current = migration.steps[i];
+    if (migration.physical && !['ddl', 'sql', 'rebuild'].includes(current.kind))
+      throw refuse('JD0021', 'physical preservation plans use explicit SQL/rebuild steps and preservation assertions');
     const fail = (reason, cause) => {
       throw refuse('JD0023',
         `migration '${migration.id}' step ${i} (${current.kind}) failed: ${reason}`,
@@ -1356,14 +1375,16 @@ function validateTargetState(connection, model, options) {
     const validate = options.compileSchema !== undefined
       ? options.compileSchema(entity.schema)
       : null;
-    if (validate === null) return verifyEntity(i + 1);
+    if (validate === null && entity.physical === null) return verifyEntity(i + 1);
     // the WHOLE document — mapped columns folded in — is what the target
     // schema judges; the rest-document alone failed every entity whose
     // required members are columns, so a pure widening could not land
     const entityMapping = explainMapping(model).entities[entity.name];
-    return chain(walkRows(connection, entity.name, options.batchSize, (rows) => {
+    return chain(entity.physical === null ? null : chain(readSchema(connection), (schema) =>
+      verifyPhysical(connection, planEntity(entity.name, entityMapping, explainMapping(model), dialect).physical, schema)), () =>
+      chain(walkRows(connection, entityMapping.table, options.batchSize, (rows) => {
       for (const row of rows) {
-        const outcome = validate(mergeEntityRow(entityMapping, row, 'doc'));
+        const outcome = validate === null ? true : validate(mergeEntityRow(entityMapping, row, 'doc'));
         const valid = outcome === true || outcome?.valid === true;
         if (!valid) {
           throw refuse('JD0021',
@@ -1371,7 +1392,7 @@ function validateTargetState(connection, model, options) {
             + 'validate against the target schema — a narrowing needs a data transform');
         }
       }
-    }, false, entityMapping), () => verifyEntity(i + 1));
+    }, false, entityMapping), () => verifyEntity(i + 1)));
   };
   const verifyNext = (i) => {
     if (i >= collections.length) return null;
@@ -1689,13 +1710,12 @@ export function migrate(target, migrations, options) {
         // table before reading it, a DRY RUN probes for it instead and
         // reads an absent one as an empty history — the promise a dry
         // run makes is the reason it is safe to point at production
-        const history = options.dryRun === true
-          ? chain(connection.prepare(dialect.introspect.tableExists()), (probe) =>
-            chain(probe.get([HISTORY_TABLE]), (row) => (row === undefined
-              ? []
-              : chain(connection.prepare(statements.select), (select) => select.all([])))))
-          : chain(connection.exec(statements.create), () =>
-            chain(connection.prepare(statements.select), (select) => select.all([])));
+        let historyExists = false;
+        const history = chain(connection.prepare(dialect.introspect.tableExists()), (probe) =>
+          chain(probe.get([HISTORY_TABLE]), (row) => {
+            historyExists = row !== undefined;
+            return !historyExists ? [] : chain(connection.prepare(statements.select), (select) => select.all([]));
+          }));
         work = chain(history, (appliedRows) => {
           // the list must agree with the history: same ids, same
           // order, same checksums — an edited applied migration is
@@ -1723,6 +1743,7 @@ export function migrate(target, migrations, options) {
           let expectedFrom = currentShape;
           for (const migration of pending) {
             checkMigrationDocument(migration);
+            checkPreservationPlan(migration);
             if (migration.from !== expectedFrom) {
               throw refuse('JD0020',
                 `migration '${migration.id}' expects shape '${migration.from}' but the `
@@ -1741,6 +1762,8 @@ export function migrate(target, migrations, options) {
             return { applied: [], skipped: appliedRows.map((row) => row.id), upToDate: true };
           }
 
+          if (pending.some((m) => m.physical) && options.shadow !== false)
+            throw refuse('JD0021', 'physical preservation plans require shadow:false; qualify against an explicit copy/fresh-target fixture');
           const shadowRun = options.shadow === false
             ? null
             : replayOnShadow(options.shadowDriver ?? target.driver,
@@ -1824,11 +1847,14 @@ export function migrate(target, migrations, options) {
                 bracket && dialect.capabilities.foreignKeysAlwaysOn !== true
                   ? connection.exec(dialect.pragma.foreignKeys(false)) : null,
                 () => chain(connection.exec(dialect.tx.beginImmediate), () => {
-                const body = () => chain(runSteps(connection, migration, runOptions), () =>
+                const body = () => chain(migration.physical ? verifyPreservation(connection, migration.physical, false) : null, () =>
+                  chain(historyExists ? null : connection.exec(statements.create), () =>
+                  chain(runSteps(connection, migration, runOptions), () =>
+                  chain(migration.physical ? verifyPreservation(connection, migration.physical, true) : null, () =>
                   chain(last && options.model !== undefined
                     ? chain(validateTargetState(connection, options.model,
                       { compileSchema: options.compileSchema, batchSize }),
-                    () => (normalizeEntities(options.model).size === 0 ? null
+                    () => (normalizeEntities(options.model).size === 0 || migration.physical ? null
                       : chain(compareShapeToModel(target.driver, connection,
                         options.model, options.registerFunctions), (difference) => {
                         if (difference !== null) {
@@ -1841,15 +1867,10 @@ export function migrate(target, migrations, options) {
                   () => chain(connection.prepare(statements.insert), (insert) =>
                     insert.run([migration.id, runtime.now(), migration.from,
                       migration.to, migrationChecksum(migration),
-                      migration.steps.length]))));
+                      migration.steps.length])))))));
                 const restore = () => (bracket
                   && dialect.capabilities.foreignKeysAlwaysOn !== true
                   ? connection.exec(dialect.pragma.foreignKeys(true)) : null);
-                const commit = () => chain(connection.exec(dialect.tx.commit), () =>
-                  chain(restore(), () => {
-                    applied.push(migration.id);
-                    return applyNext(i + 1);
-                  }));
                 const rollback = (error) =>
                   chain(connection.exec(dialect.tx.rollback), () =>
                     chain(restore(), () => { throw error; }));
@@ -1864,9 +1885,16 @@ export function migrate(target, migrations, options) {
                 catch (error) {
                   return rollback(error);
                 }
-                return outcome instanceof Promise
-                  ? outcome.then(commit, rollback)
-                  : commit();
+                const settle = () => {
+                  let result;
+                  try { result = connection.exec(dialect.tx.commit); }
+                  catch (error) { return rollback(error); }
+                  return result instanceof Promise ? result.then(published, rollback) : published();
+                };
+                const published = () => chain(restore(), () => {
+                  historyExists = true; applied.push(migration.id); return applyNext(i + 1);
+                });
+                return outcome instanceof Promise ? outcome.then(settle, rollback) : settle();
               }));
             };
             return chain(applyNext(0), () => ({
@@ -1886,3 +1914,89 @@ export function migrate(target, migrations, options) {
         : finish(work);
     })));
 }
+
+/** Plan an existing file's explicit preservation migration. Every source object
+ * needs a disposition; source/target assertions preserve application-owned facts.
+ * @param {any} connection @param {any} fromModel @param {any} toModel
+ * @param {{ id: string, steps: any[], dispositions: Record<string, 'preserve'|'replace'|'drop'>,
+ *   assertions?: { sql: string, params?: any[], expected: any[] }[] }} options @returns {any} */
+export function planPhysicalMigration(connection, fromModel, toModel, options) {
+  normalizeEntities(fromModel); normalizeEntities(toModel);
+  if (!options || typeof options.id !== 'string' || !options.id || !Array.isArray(options.steps))
+    throw refuse('JD0021', 'a physical plan requires id and explicit steps');
+  return chain(preservationSchemaOf(connection), (source) => {
+    const dispositions = options.dispositions ?? {};
+    const keys = source.map((o) => `${o.type}:${o.name}`);
+    if (Object.keys(dispositions).some((key) => !keys.includes(key)) || keys.some((key) => !['preserve', 'replace', 'drop'].includes(dispositions[key])))
+      throw refuse('JD0021', 'every physical source object must have an explicit preserve, replace or drop disposition');
+    const assertions = options.assertions ?? [];
+    for (const assertion of assertions) {
+      if (!assertion || typeof assertion.sql !== 'string' || !/^SELECT\b/i.test(assertion.sql.trim()) || !Array.isArray(assertion.expected))
+        throw refuse('JD0021', 'preservation assertions require a SELECT and expected rows');
+    }
+    const migration = { $migration: MIGRATION_VERSION, id: options.id, from: shapeHash(fromModel), to: shapeHash(toModel),
+      steps: options.steps, physical: { source, dispositions, assertions } };
+    checkMigrationDocument(migration);
+    checkPreservationPlan(migration);
+    return migration;
+  });
+}
+
+/** Validate saved plans again at execution, including SQL ownership boundaries. */
+function checkPreservationPlan(migration) {
+  const physical = migration.physical;
+  if (physical === undefined) return;
+  const fail = () => { throw refuse('JD0021', 'invalid physical source, dispositions, assertions or steps'); };
+  if (!physical || !Array.isArray(physical.source) || !physical.dispositions || !Array.isArray(physical.assertions)) fail();
+  const keys = physical.source.map((object) => {
+    if (!object || typeof object.name !== 'string' || !['table', 'view', 'index', 'trigger'].includes(object.type)) fail();
+    return `${object.type}:${object.name}`;
+  });
+  if (new Set(keys).size !== keys.length || Object.keys(physical.dispositions).some((key) => !keys.includes(key))
+    || keys.some((key) => !['preserve', 'replace', 'drop'].includes(physical.dispositions[key]))) fail();
+  for (const assertion of physical.assertions)
+    if (!assertion || typeof assertion.sql !== 'string' || !/^SELECT\b/i.test(assertion.sql.trim())
+      || !Array.isArray(assertion.expected) || (assertion.params !== undefined && !Array.isArray(assertion.params))) fail();
+  if (migration.steps.some((step) => !['ddl', 'sql', 'rebuild'].includes(step.kind))) fail();
+  const fragments = migration.steps.flatMap((step) => step.kind === 'rebuild'
+    ? [...(step.create ?? []), step.copy, ...(step.indexes ?? [])] : [step.sql]);
+  for (const sql of fragments) {
+    const tokens = typeof sql === 'string' ? sqlTokens(sql) : [];
+    const words = tokens.filter((t) => t.kind === 'word').map((t) => t.value.toUpperCase());
+    if (!['CREATE', 'ALTER', 'DROP', 'INSERT', 'UPDATE', 'DELETE'].includes(words[0])
+      || words.some((w) => /^(?:COMMIT|ROLLBACK|SAVEPOINT|RELEASE|ATTACH|DETACH|PRAGMA|VACUUM)$/.test(w)))
+      throw refuse('JD0021', 'physical steps cannot change transaction or connection ownership');
+  }
+}
+
+/** Preservation compares exact source programs, including whitespace in SQL literals. */
+function preservationSchemaOf(connection) {
+  return chain(readSchema(connection), (schema) => schema.objects
+    .filter((object) => !ENGINE_TABLES.has(object.name) && !ENGINE_TABLES.has(object.owner)));
+}
+
+/** Verify source identity before destructive steps, and every preserved object
+ * and fact before publication. The migration transaction owns all these reads. */
+function verifyPreservation(connection, physical, after) {
+  return chain(preservationSchemaOf(connection), (actual) => {
+    if (!after && canonicalizeJson(actual) !== canonicalizeJson(physical.source))
+      throw refuse('JD0020', 'the physical source schema changed after the plan was prepared');
+    if (after) {
+      for (const object of physical.source) {
+        const key = `${object.type}:${object.name}`;
+        const current = actual.find((o) => o.type === object.type && o.name === object.name);
+        if (physical.dispositions[key] === 'preserve' && canonicalizeJson(current ?? null) !== canonicalizeJson(object))
+          throw refuse('JD0023', `preserved object '${key}' was changed or lost`);
+        if (physical.dispositions[key] === 'drop' && current) throw refuse('JD0023', `declared drop '${key}' remains`);
+      }
+    }
+    const next = (i) => i >= physical.assertions.length ? null
+      : chain(connection.prepare(physical.assertions[i].sql, { readOnly: true }), (s) =>
+        chain(s.all(physical.assertions[i].params ?? []), (rows) => {
+          if (canonicalizeJson(rows) !== canonicalizeJson(physical.assertions[i].expected))
+            throw refuse('JD0023', `preservation assertion ${i} disagrees ${after ? 'after' : 'before'} migration`);
+          return next(i + 1);
+        }));
+    return next(0);
+  });
+}

package/src/model.js

@@ -27,6 +27,8 @@ import {
 import { compileJsonQuery } from '@jarenjs/json/query';
 
 import { DbCompileError } from './errors.js';
+import { normalizePhysical } from './physical.js';
+import { normalizeInvariants } from './invariants.js';
 
 /** The closed `x-entity` vocabulary; anything else is `JD0030`. */
 const ENTITY_MEMBERS = new Set(['key', 'unique', 'index', 'default', 'column', 'relation', 'version']);
@@ -343,9 +345,13 @@ export function normalizeEntities(model) {
       keys,
       relations,
       version: versions.length === 1 ? versions[0].name : null,
+      physical: normalizePhysical(spec.physical, properties, keys, docPath),
+      invariants: normalizeInvariants(spec.invariants, `${docPath}/invariants`),
     });
   }
 
+  for (const entity of entities.values())
+    if (entity.physical !== null && entity.relations.length) throw modelError('physical join tables are declared as entities; relation navigation is not qualified for column layouts', entity.docPath);
   resolveRelations(entities);
   return entities;
 }
@@ -556,6 +562,11 @@ export function explainMapping(model) {
   const mapping = { entities: {}, joinTables: {} };
 
   for (const entity of entities.values()) {
+    if (entity.physical !== null) {
+      mapping.entities[entity.name] = { ...entity.physical, document: false,
+        foreignKeys: [], indexes: [], version: entity.version, invariants: model.entities[entity.name].invariants ?? [] };
+      continue;
+    }
     const columns = [];
     const document = [];
     const indexes = [];
@@ -626,6 +637,7 @@ export function explainMapping(model) {
       indexes,
       document,
       version: entity.version ?? null,
+      ...(entity.invariants.length ? { invariants: model.entities[entity.name].invariants } : {}),
     };
   }