@jarenjs/db 0.49.2 → 0.66.1

package/src/migrate.js

@@ -23,15 +23,28 @@
 
 import { canonicalizeJson } from '@jarenjs/json/canonical';
 import { hashContent } from '@jarenjs/core/string';
+import { resolveRuntime } from '@jarenjs/core/runtime';
+import { refuseCancelled } from './cancellation.js';
+import { setObjectMember } from '@jarenjs/core/object';
 import { compileJsonQuery } from '@jarenjs/json/query';
 import { compileJsltStylesheet } from '@jarenjs/json/jslt';
 
 import { DbCompileError } from './errors.js';
 import { chain, toPromise } from './driver.js';
 import { normalizeModel } from './store.js';
+import { CHANGES_TABLE, CHANGES_STATE_TABLE } from './capture.js';
+import { JOBS_TABLE, JOB_CHECKPOINTS_TABLE } from './jobs.js';
 import { planCollection, verifyShape, planEntity, planJoinTable } from './ddl.js';
 import { normalizeEntities, explainMapping } from './model.js';
 import { derivedValue, memberAt, registerDeriveFunctions } from './derive.js';
+import { mergeEntityRow } from './graph.js';
+import { entityCore } from './entity.js';
+import {
+  MIGRATION_VERSION, isPerDocumentAssertion, compileDocumentStep, checkMigrationDocument,
+  normalizeAssertionBounds, ASSERTION_BOUNDS_DEFAULT, createAssertionBoundGuard,
+} from './document-steps.js';
+
+export { MIGRATION_VERSION, isPerDocumentAssertion, ASSERTION_BOUNDS_DEFAULT };
 
 /**
  * The physical mapping a connection's driver imposes on derived index
@@ -41,22 +54,30 @@ import { derivedValue, memberAt, registerDeriveFunctions } from './derive.js';
  * @param {any} connection
  * @returns {{ derived: 'virtual' | 'stored', rtree: boolean }}
  */
-function mappingFor(connection) {
+function mappingFor(connection, expressions = undefined) {
+  const registered = connection.capabilities?.deterministicIndexableFunctions === true;
   return {
-    derived: connection.capabilities?.deterministicIndexableFunctions === true
-      ? 'virtual' : 'stored',
+    derived: registered ? 'virtual' : 'stored',
     // the same reasoning for the R*Tree mapping: a build without the
     // module plans (and verifies) the B-tree shape
     rtree: connection.capabilities?.rtree === true,
+    // and the same for a declared index expression: this connection
+    // either computes it or calls the engine's own immutable function
+    expressions,
+    registered,
   };
 }
 
-/** The migration format version. */
-export const MIGRATION_VERSION = '0.1';
-
 /** The history table name (outside the model's identifier namespace
  * conventions on purpose — a collection cannot collide with it). */
 export const HISTORY_TABLE = '_jaren_migrations';
+/** The tables the engine owns beside a model's: never a shape-drift finding. */
+/** The tables this package owns. A model never declared one, so one
+ * found in a database is the engine's own bookkeeping rather than
+ * anybody's drift — the drift check skips them and the introspector
+ * does not derive them. */
+export const ENGINE_TABLES = new Set([HISTORY_TABLE, CHANGES_TABLE, CHANGES_STATE_TABLE,
+  JOBS_TABLE, JOB_CHECKPOINTS_TABLE]);
 
 /**
  * The signature-grade identity of a model SHAPE.
@@ -64,7 +85,41 @@ export const HISTORY_TABLE = '_jaren_migrations';
  * @returns {string}
  */
 export function shapeHash(model) {
-  return hashContent(canonicalizeJson(model));
+  return hashContent(canonicalizeJson(withoutRenameHints(model)));
+}
+
+/**
+ * The model without its `x-rename` hints. A hint is a PLANNING
+ * instruction, not shape: two models that differ only by the hint
+ * describe the same database, and hashing the hint made an empty
+ * migration necessary just to move the recorded shape once the hint
+ * was removed.
+ * @param {any} model
+ * @returns {any}
+ */
+function withoutRenameHints(model) {
+  if (model === null || typeof model !== 'object') return model;
+  const out = {};
+  for (const key of Object.keys(model)) setObjectMember(out, key, model[key]);
+  for (const member of ['collections', 'entities']) {
+    const declared = model[member];
+    if (declared === null || typeof declared !== 'object' || Array.isArray(declared)) continue;
+    const stripped = {};
+    for (const name of Object.keys(declared)) {
+      const spec = declared[name];
+      if (spec !== null && typeof spec === 'object' && !Array.isArray(spec)
+        && Object.hasOwn(spec, 'x-rename')) {
+        const copy = { ...spec };
+        delete copy['x-rename'];
+        setObjectMember(stripped, name, copy);
+      }
+      else {
+        setObjectMember(stripped, name, spec);
+      }
+    }
+    out[member] = stripped;
+  }
+  return out;
 }
 
 /**
@@ -125,7 +180,8 @@ function deriveStep(collection, plan, columnNames, note) {
  * @param {any} fromModel
  * @param {any} toModel
  * @param {{ id?: string, dialect?: any,
- *   derived?: 'virtual' | 'stored', rtree?: boolean }} [options]
+ *   derived?: 'virtual' | 'stored', rtree?: boolean,
+ *   expressions?: Record<string, any> }} [options]
  * @returns {{ migration: any, report: {
  *   renamed: { from: string, to: string }[],
  *   added: string[], removed: string[],
@@ -136,9 +192,13 @@ export function planMigration(fromModel, toModel, options = undefined) {
   const dialect = options?.dialect ?? null;
   if (dialect === null || typeof dialect !== 'object')
     throw new TypeError('planMigration needs { dialect } (the store dialect renders the DDL)');
-  const mapping = { derived: options?.derived ?? 'virtual', rtree: options?.rtree !== false };
-  const fromCollections = normalizeModel(fromModel);
-  const toCollections = normalizeModel(toModel);
+  const mapping = { derived: options?.derived ?? 'virtual', rtree: options?.rtree !== false,
+    // a model that declares an index EXPRESSION resolves its functions
+    // here too: a plan is DDL, and DDL over a function this planner was
+    // not told about is DDL nobody can apply
+    expressions: options?.expressions, registered: options?.derived !== 'stored' };
+  const fromCollections = normalizeModel(fromModel, options?.expressions);
+  const toCollections = normalizeModel(toModel, options?.expressions);
 
   const steps = [];
   const report = {
@@ -153,6 +213,10 @@ export function planMigration(fromModel, toModel, options = undefined) {
     const hint = toModel.collections[name]?.['x-rename'];
     if (hint === undefined) continue;
     if (!fromCollections.has(hint)) {
+      // the hint's work is done once the from-model already declares the
+      // target and no longer the source: planning a model against itself
+      // must yield nothing, not refuse the hint it still carries
+      if (fromCollections.has(name)) continue;
       throw new TypeError(
         `x-rename on '${name}' names '${hint}', which the from-model does not declare`);
     }
@@ -423,6 +487,7 @@ function planEntityChanges(fromModel, toModel, dialect, steps, report) {
     const hint = toModel.entities[name]?.['x-rename'];
     if (hint === undefined) continue;
     if (!fromEntities.has(hint)) {
+      if (fromEntities.has(name)) continue; // a satisfied hint (see the collections)
       throw new TypeError(
         `x-rename on entity '${name}' names '${hint}', which the from-model does not declare`);
     }
@@ -434,30 +499,62 @@ function planEntityChanges(fromModel, toModel, dialect, steps, report) {
     report.renamed.push({ from: hint, to: name });
     steps.push({ kind: 'ddl', sql: dialect.ddl.renameTable(hint, name),
       note: `rename entity '${hint}' to '${name}'` });
-    for (const joinName of Object.keys(fromMapping.joinTables)) {
-      const pair = joinName.split('_');
-      if (!pair.includes(hint)) continue;
-      const renamedPair = pair.map((part) => (part === hint ? name : part)).sort();
-      const newJoin = renamedPair.join('_');
+  }
+  const consumedOldEntities = new Set(renamedFrom.values());
+  // a join table's endpoints come from the MAPPING, never from splitting
+  // its name: an entity named with an underscore, or a `through` name,
+  // does not split into its endpoints — and a split that guessed wrong
+  // renamed the table to a name nothing declares, created the declared
+  // one empty, and dropped the memberships as "destructive"
+  const implicitJoinName = (join) => [join.left.entity, join.right.entity].sort().join('_');
+  const renamedEntity = (entityName) => {
+    for (const [to, from] of renamedFrom) if (from === entityName) return to;
+    return entityName;
+  };
+  const renamedJoinName = (joinName) => {
+    const join = fromMapping.joinTables[joinName];
+    if (joinName !== implicitJoinName(join)) return joinName; // a `through` name stays
+    return [renamedEntity(join.left.entity), renamedEntity(join.right.entity)].sort().join('_');
+  };
+  for (const [joinName, join] of Object.entries(fromMapping.joinTables)) {
+    if (![join.left, join.right].some((side) => renamedEntity(side.entity) !== side.entity)) continue;
+    const newJoin = renamedJoinName(joinName);
+    const target = toMapping.joinTables[newJoin];
+    if (target === undefined) continue; // the relation is gone: the drop below names it
+    // the fresh build orders the endpoint columns by the SORTED entity
+    // names, and a rename can flip that order — then the primary key's
+    // column order would differ from a fresh build's and the shape
+    // check would refuse the migrated database, so the table is rebuilt
+    // in the target order with its rows copied; when the order holds,
+    // renaming the table and the column is enough
+    const renamedColumns = [join.left, join.right]
+      .map((side) => ({ from: side.column, to: `${renamedEntity(side.entity)}_key` }));
+    const targetOrder = [target.left.column, target.right.column];
+    const sameOrder = renamedColumns.every((column, i) => column.to === targetOrder[i]);
+    if (sameOrder) {
       if (newJoin !== joinName) {
         steps.push({ kind: 'ddl', sql: dialect.ddl.renameTable(joinName, newJoin),
           note: `rename join table '${joinName}' with its endpoint` });
-        steps.push({ kind: 'ddl',
-          sql: dialect.ddl.renameColumn(newJoin, `${hint}_key`, `${name}_key`),
-          note: `rename the endpoint column '${hint}_key' with its entity` });
       }
+      for (const column of renamedColumns) {
+        if (column.from === column.to) continue;
+        steps.push({ kind: 'ddl', sql: dialect.ddl.renameColumn(newJoin, column.from, column.to),
+          note: `rename the endpoint column '${column.from}' with its entity` });
+      }
+      continue;
     }
+    const q = dialect.quoteIdentifier;
+    const sourceOf = (toColumn) => renamedColumns.find((column) => column.to === toColumn).from;
+    for (const sql of planJoinTable(newJoin, target, toMapping, dialect).createSql) {
+      steps.push({ kind: 'ddl', sql, note: `rebuild join table '${joinName}' as '${newJoin}' in its endpoint order` });
+    }
+    steps.push({ kind: 'sql',
+      sql: `INSERT INTO ${q(newJoin)} (${targetOrder.map(q).join(', ')}) `
+        + `SELECT ${targetOrder.map((column) => q(sourceOf(column))).join(', ')} FROM ${q(joinName)}`,
+      note: `copy the memberships of '${joinName}' into '${newJoin}'` });
+    steps.push({ kind: 'ddl', sql: dialect.ddl.dropTable(joinName),
+      note: `drop '${joinName}' — its memberships now live in '${newJoin}'` });
   }
-  const consumedOldEntities = new Set(renamedFrom.values());
-  const renamedJoinName = (joinName) => {
-    const pair = joinName.split('_');
-    return pair
-      .map((part) => {
-        for (const [to, from] of renamedFrom) if (from === part) return to;
-        return part;
-      })
-      .sort().join('_');
-  };
 
   // ————— per-entity strategies —————
   for (const [name, toEntity] of toEntities) {
@@ -589,9 +686,26 @@ function planEntityChanges(fromModel, toModel, dialect, steps, report) {
     }
   }
 
-  // ————— dropped entities —————
-  for (const [name] of fromEntities) {
-    if (toEntities.has(name) || consumedOldEntities.has(name)) continue;
+  // ————— dropped entities, children before parents —————
+  // foreign keys are enforced while a migration runs (node:sqlite has
+  // them on by default), so a parent with RESTRICT children cannot go
+  // first: the dropped set is ordered so that every entity referencing
+  // another dropped entity is dropped before it
+  const dropped = [...fromEntities.keys()]
+    .filter((name) => !toEntities.has(name) && !consumedOldEntities.has(name));
+  const droppedSet = new Set(dropped);
+  const references = (name) => new Set(fromMapping.entities[name].foreignKeys
+    .map((fk) => fk.references).filter((target) => droppedSet.has(target) && target !== name));
+  const dropOrder = [];
+  const placed = new Set();
+  while (dropOrder.length < dropped.length) {
+    // ready: every dropped entity that no other UNPLACED dropped entity references
+    const ready = dropped.filter((name) => !placed.has(name)
+      && !dropped.some((other) => !placed.has(other) && other !== name && references(other).has(name)));
+    if (ready.length === 0) { dropOrder.push(...dropped.filter((name) => !placed.has(name))); break; }
+    for (const name of ready) { placed.add(name); dropOrder.push(name); }
+  }
+  for (const name of dropOrder) {
     report.removed.push(name);
     report.destructive = true;
     steps.push({ kind: 'ddl', sql: dialect.ddl.dropTable(name),
@@ -650,6 +764,12 @@ function renderRebuild(name, fromName, fm, tm, fromMapping, toMapping, dialect,
   const sources = [];
   let docExpr = q('doc');
   const lost = [];
+  // a SQL NULL is an ABSENT member (§9.3): folding it in as JSON null
+  // turned every row without the value into a narrowing the target
+  // schema refused
+  const foldColumn = (expression, columnName, fold) =>
+    `CASE WHEN ${q(columnName)} IS NULL THEN ${expression} `
+    + `ELSE ${dialect.jsonSet(expression, pathText(columnName), fold)} END`;
   for (const { name: columnName, column } of ordered) {
     targets.push(q(columnName));
     if (fromHas(columnName)) {
@@ -659,9 +779,10 @@ function renderRebuild(name, fromName, fm, tm, fromMapping, toMapping, dialect,
       if (column !== null && column.source === 'epoch(document)'
         && fromColumn !== undefined && fromColumn.source !== 'epoch(document)') {
         // plain text column becomes a derived instant: derive from the
-        // old column and keep the string in the document
+        // old column and keep the string in the document — an absent
+        // string stays absent (§9.3), never a JSON null
         sources.push(dialect.epochFromRfc3339(q(columnName)));
-        docExpr = dialect.jsonSet(docExpr, pathText(columnName), q(columnName));
+        docExpr = foldColumn(docExpr, columnName, q(columnName));
       }
       else if (sameStorage) {
         sources.push(q(columnName));
@@ -686,12 +807,25 @@ function renderRebuild(name, fromName, fm, tm, fromMapping, toMapping, dialect,
       const fold = fromColumn.storage === 'boolean'
         ? dialect.jsonEncode(`CASE WHEN ${q(columnName)} = 1 THEN 'true' ELSE 'false' END`)
         : q(columnName);
-      docExpr = dialect.jsonSet(docExpr, pathText(columnName), fold);
+      docExpr = foldColumn(docExpr, columnName, fold);
     }
     else if (fromColumn.source !== 'epoch(document)') {
       lost.push(columnName);
     }
   }
+  // an INFERRED foreign-key column (no property of its own) that the
+  // target no longer carries: its values are lost unless the target
+  // declares the property, in which case they fold into the document —
+  // walking the mapped columns alone dropped it without a word
+  for (const columnName of fromFkOnly) {
+    if (ordered.some((entry) => entry.name === columnName)) continue;
+    if (toMapping.entities[name] !== undefined && tm.document.includes(columnName)) {
+      docExpr = foldColumn(docExpr, columnName, q(columnName));
+    }
+    else {
+      lost.push(columnName);
+    }
+  }
   // properties that moved INTO columns leave the document
   for (const { name: columnName, column } of ordered) {
     if (!fromHas(columnName) && (column === null || column.source !== 'epoch(document)'))
@@ -718,15 +852,19 @@ function renderRebuild(name, fromName, fm, tm, fromMapping, toMapping, dialect,
  * equality compares against, and the tests.
  * @param {any} connection
  * @param {any} model
+ * @param {Record<string, any>} [expressions] - the host's declared
+ *   index-expression functions, resolved into the DDL the same way the
+ *   open path resolves them
  * @returns {any} value-or-promise
  */
-export function createModelShape(connection, model) {
+export function createModelShape(connection, model, expressions = undefined) {
   const dialect = connection.dialect;
   /** @type {string[]} */
   const statements = [];
-  for (const collection of normalizeModel(model).values()) {
+  for (const collection of normalizeModel(model, expressions).values()) {
     statements.push(
-      ...planCollection(collection.name, collection, dialect, mappingFor(connection)).createSql);
+      ...planCollection(collection.name, collection, dialect,
+        mappingFor(connection, expressions)).createSql);
   }
   const entities = normalizeEntities(model);
   if (entities.size > 0) {
@@ -759,7 +897,9 @@ export function schemaShapeOf(connection) {
   const dialect = connection.dialect;
   return chain(connection.prepare(dialect.introspect.schemaDump()), (statement) =>
     chain(statement.all([]), (rows) => rows
-      .filter((row) => row.name !== HISTORY_TABLE && row.owner !== HISTORY_TABLE)
+      // the engine's own tables — history, the change log and its state
+      // row, the job queue — are never a model's drift
+      .filter((row) => !ENGINE_TABLES.has(String(row.name)) && !ENGINE_TABLES.has(String(row.owner)))
       .map((row) => ({
         type: String(row.type),
         name: String(row.name),
@@ -824,14 +964,22 @@ function normalizeSchemaSql(sql) {
  * @param {((connection: any) => any) | undefined} registerFunctions
  * @returns {any} value-or-promise of `string | null`
  */
-export function compareShapeToModel(driver, connection, model, registerFunctions) {
+export function compareShapeToModel(driver, connection, model, registerFunctions, expressions) {
+  // The comparison IS a text comparison: it builds the model's shape in
+  // a reference database and compares the two engines' stored CREATE
+  // statements. An engine that keeps none has nothing to compare, and
+  // says so here rather than reading an undefined statement — the
+  // structural drift check (columns, indexes, foreign-key tuples) runs
+  // per collection and per entity either way, and `declaredSqlText` is
+  // what tells a reader which half they got.
+  if (connection.dialect.capabilities.declaredSqlText !== true) return null;
   return chain(driver.open(':memory:', {}), (reference) =>
     chain(chain(registerDeriveFunctions(reference),
       () => (registerFunctions !== undefined ? registerFunctions(reference) : null)), () => {
       const finish = (result) => chain(reference.close(), () => result);
       let outcome;
       try {
-        outcome = chain(createModelShape(reference, model), () =>
+        outcome = chain(createModelShape(reference, model, expressions), () =>
           chain(schemaShapeOf(reference), (wanted) =>
             chain(schemaShapeOf(connection), (actual) => {
               const wantedText = JSON.stringify(wanted);
@@ -864,54 +1012,6 @@ export function compareShapeToModel(driver, connection, model, registerFunctions
     }));
 }
 
-const STEP_KINDS = new Set(['ddl', 'jslt', 'query', 'sql', 'rebuild', 'derive']);
-
-/**
- * Structural validation of one migration document, including the
- * draft refusal (`JD0021`).
- * @param {any} migration
- */
-function checkMigrationDocument(migration) {
-  if (migration === null || typeof migration !== 'object'
-    || migration.$migration !== MIGRATION_VERSION
-    || typeof migration.id !== 'string' || migration.id === ''
-    || typeof migration.from !== 'string' || typeof migration.to !== 'string'
-    || !Array.isArray(migration.steps)) {
-    throw refuse('JD0023',
-      `migration '${migration?.id ?? '<unknown>'}' is not a valid ${MIGRATION_VERSION} migration document`);
-  }
-  for (let i = 0; i < migration.steps.length; i++) {
-    const step = migration.steps[i];
-    if (step === null || typeof step !== 'object' || !STEP_KINDS.has(step.kind)) {
-      throw refuse('JD0023',
-        `migration '${migration.id}' step ${i} has no recognised kind`);
-    }
-    if (step.kind === 'rebuild'
-      && (typeof step.table !== 'string' || !Array.isArray(step.create)
-        || typeof step.copy !== 'string' || !Array.isArray(step.indexes))) {
-      throw refuse('JD0023',
-        `migration '${migration.id}' step ${i} is a rebuild without its rendered `
-        + 'table/create/copy/indexes');
-    }
-    if (step.kind === 'sql' && typeof step.sql !== 'string') {
-      throw refuse('JD0023',
-        `migration '${migration.id}' step ${i} is a sql step without sql text`);
-    }
-    if (step.kind === 'derive'
-      && (typeof step.collection !== 'string' || !Array.isArray(step.columns)
-        || step.columns.length === 0)) {
-      throw refuse('JD0023',
-        `migration '${migration.id}' step ${i} is a derive backfill without its columns`);
-    }
-    if (step.kind === 'jslt' && step.draft === true) {
-      throw refuse('JD0021',
-        `migration '${migration.id}' step ${i} is a DRAFT transform for collection `
-        + `'${step.collection}' — the planner cannot infer a data transform; fill in `
-        + 'the stylesheet (or delete the step for a pure widening) and remove "draft"');
-    }
-  }
-}
-
 /**
  * The batched row walk shared by transforms and post-validation:
  * `SELECT rowid, json(doc) ... WHERE rowid > ? ORDER BY rowid LIMIT ?`
@@ -921,38 +1021,67 @@ function checkMigrationDocument(migration) {
  * @param {number} batchSize
  * @param {(rows: { rid: any, doc: string, key: any }[]) => any} handle
  *   value-or-promise per batch
+ * @param {boolean} [keyed]
+ * @param {any} [entityMapping]
+ * @param {() => void} [check] - run before every batch: the
+ *   cancellation boundary a data step has between its batches
  * @returns {any}
  */
-function walkRows(connection, table, batchSize, handle, keyed = true) {
+function walkRows(connection, table, batchSize, handle, keyed = true, entityMapping = null,
+  check = undefined) {
   // entity tables carry no 'key' column — the transform walk goes by
-  // row identity alone; only the collection walks select the key
+  // row identity alone; only the collection walks select the key. An
+  // entity's mapped columns ride beside the document so the row can be
+  // read WHOLE (`mergeEntityRow`): a transform or an assertion that saw
+  // the rest-document alone could not see `id` or `name` at all
   const dialect = connection.dialect;
   const q = dialect.quoteIdentifier;
   const rid = dialect.rowIdentity();
   const keySelect = keyed ? `, ${q('key')} AS ${q('k')}` : '';
-  const sql = `SELECT ${rid} AS ${q('rid')}, ${dialect.jsonText(q('doc'))} AS ${q('doc')}`
-    + `${keySelect} FROM ${q(table)} WHERE ${rid} > ${dialect.parameterRef(1, 'after')} `
-    + `ORDER BY ${rid} ${dialect.limitClause(batchSize, undefined)}`;
-  return chain(connection.prepare(sql), (statement) => {
-    const nextBatch = (after) =>
-      chain(statement.all([after]), (rows) => {
+  const columnSelect = entityMapping === null ? '' : entityColumnsOf(entityMapping)
+    .map((column) => `, ${q(column)}`).join('');
+  const select = `SELECT ${rid} AS ${q('rid')}, ${dialect.jsonText(q('doc'))} AS ${q('doc')}`
+    + `${keySelect}${columnSelect} FROM ${q(table)}`;
+  const ordered = ` ORDER BY ${rid} ${dialect.limitClause(batchSize, undefined)}`;
+  // An INTEGER PRIMARY KEY can be negative. The first batch has no
+  // lower bound; subsequent batches seek from a row actually read.
+  return chain(connection.prepare(select + ordered), (first) => chain(connection.prepare(
+    `${select} WHERE ${rid} > ${dialect.parameterRef(1, 'after')}${ordered}`), (statement) => {
+    const nextBatch = (after) => {
+      if (check !== undefined) check();
+      return chain(after === undefined ? first.all([]) : statement.all([after]), (rows) => {
         if (rows.length === 0) return null;
         return chain(handle(rows), () =>
           nextBatch(rows[rows.length - 1].rid));
       });
-    return nextBatch(-1);
-  });
+    };
+    return nextBatch(undefined);
+  }));
 }
 
-/** All documents of a collection (the assertion steps' working set —
- * a documented whole-collection read). */
-function allDocs(connection, table) {
-  const dialect = connection.dialect;
-  const q = dialect.quoteIdentifier;
-  const sql = `SELECT ${dialect.jsonText(q('doc'))} AS ${q('doc')} FROM ${q(table)} `
-    + `ORDER BY ${dialect.rowIdentity()}`;
-  return chain(connection.prepare(sql), (statement) =>
-    chain(statement.all([]), (rows) => rows.map((row) => JSON.parse(row.doc))));
+/** The physical columns an entity row carries beside its document. */
+function entityColumnsOf(entityMapping) {
+  const names = new Set(entityMapping.columns.map((column) => column.name));
+  for (const fk of entityMapping.foreignKeys) names.add(fk.column);
+  return [...names];
+}
+
+/**
+ * The entity mapping a migration step over `table` runs under, or
+ * `null` for a collection. The TARGET model maps the table: a chain's
+ * intermediate shapes are hashes only, so an entity transform belongs
+ * to the last migration of a chain (MIGRATION-FORMAT §9); without a
+ * target model the step sees the rest-document, as it always did.
+ * @param {any} options
+ * @param {string} table
+ * @returns {{ entity: any, mapping: any } | null}
+ */
+function entityStepMapping(options, table) {
+  if (options.model === undefined) return null;
+  const entities = normalizeEntities(options.model);
+  const entity = entities.get(table);
+  if (entity === undefined) return null;
+  return { entity, mapping: explainMapping(options.model).entities[table] };
 }
 
 /**
@@ -967,6 +1096,9 @@ function runSteps(connection, migration, options) {
   const q = dialect.quoteIdentifier;
   const step = (i) => {
     if (i >= migration.steps.length) return null;
+    // the cancellation boundary between steps: a refusal here rolls the
+    // migration in flight back whole, as any step failure does
+    if (options.check !== undefined) options.check();
     const current = migration.steps[i];
     const fail = (reason, cause) => {
       throw refuse('JD0023',
@@ -1001,6 +1133,7 @@ function runSteps(connection, migration, options) {
         ];
         const runNext = (j) => {
           if (j >= statements.length) {
+            if (dialect.capabilities.foreignKeysAlwaysOn === true) return null;
             return chain(connection.prepare(dialect.pragma.foreignKeyCheck()),
               (checkStatement) => chain(checkStatement.all([]), (violations) => {
                 if (violations.length > 0) {
@@ -1046,16 +1179,48 @@ function runSteps(connection, migration, options) {
               collection: current.collection,
               derived: derivedRows,
             });
-          }, false), () => derivedRows));
+          }, false, null, options.check), () => derivedRows));
       }
       if (current.kind === 'jslt') {
-        let transform;
-        try {
-          transform = compileJsltStylesheet(current.stylesheet);
-        }
-        catch (cause) {
-          return fail(`the stylesheet does not compile: ${/** @type {Error} */ (cause).message}`,
-            /** @type {Error} */ (cause));
+        const stepEntity = entityStepMapping(options, current.collection);
+        const operation = compileDocumentStep(current, i, {
+          migrationId: migration.id,
+          compileJslt: compileJsltStylesheet,
+          compileQuery: compileJsonQuery,
+          keys: stepEntity === null ? [] : stepEntity.mapping.keys,
+        });
+        if (stepEntity !== null) {
+          // an entity row is transformed WHOLE: the mapped columns fold in
+          // before the stylesheet and split out after it, through the
+          // entity's own split — a column-mapped member the stylesheet
+          // wrote used to land in the document and be shadowed on read
+          const core = entityCore(connection, stepEntity.entity, stepEntity.mapping, null,
+            options.runtime);
+          const columns = entityColumnsOf(stepEntity.mapping);
+          const assignments = [
+            ...columns.map((column, i) => `${q(column)} = ${dialect.parameterRef(i + 1, 'v')}`),
+            `${q('doc')} = ${dialect.jsonEncode(dialect.parameterRef(columns.length + 1, 'doc'))}`,
+          ];
+          const updateSql = `UPDATE ${q(current.collection)} SET ${assignments.join(', ')} `
+            + `WHERE ${dialect.rowIdentity()} = ${dialect.parameterRef(columns.length + 2, 'rid')}`;
+          let transformed = 0;
+          return chain(connection.prepare(updateSql), (update) =>
+            chain(walkRows(connection, current.collection, options.batchSize, (rows) => {
+              for (const row of rows) {
+                const whole = mergeEntityRow(stepEntity.mapping, row, 'doc');
+                const next = operation.apply(whole, row.rid);
+                const { values, rest } = core.plan.split(next);
+                const byName = new Map(values.map((value) => [value.name, value.value]));
+                update.run([...columns.map((column) => byName.get(column) ?? null),
+                  JSON.stringify(rest), row.rid]);
+                transformed++;
+              }
+              options.onProgress?.({
+                migration: migration.id,
+                collection: current.collection,
+                transformed,
+              });
+            }, false, stepEntity.mapping, options.check), () => transformed));
         }
         const updateSql = `UPDATE ${q(current.collection)} SET ${q('doc')} = `
           + `${dialect.jsonEncode(dialect.parameterRef(1, 'doc'))} `
@@ -1065,9 +1230,7 @@ function runSteps(connection, migration, options) {
         return chain(connection.prepare(updateSql), (update) =>
           chain(walkRows(connection, current.collection, options.batchSize, (rows) => {
             for (const row of rows) {
-              const next = transform(JSON.parse(row.doc));
-              if (next === null || typeof next !== 'object' || Array.isArray(next))
-                fail(`the transform produced a non-document for row ${row.rid}`);
+              const next = operation.apply(JSON.parse(row.doc), row.rid);
               update.run([JSON.stringify(next), row.rid]);
               transformed++;
             }
@@ -1076,29 +1239,66 @@ function runSteps(connection, migration, options) {
               collection: current.collection,
               transformed,
             });
-          }, keyed), () => transformed));
+          }, keyed, null, options.check), () => transformed));
       }
       // kind === 'query': the assertion step
-      let compiled;
-      try {
-        compiled = compileJsonQuery(current.assert);
+      const assertionMapping = entityStepMapping(options, current.collection)?.mapping ?? null;
+      const operation = compileDocumentStep(current, i, {
+        migrationId: migration.id,
+        compileJslt: compileJsltStylesheet,
+        compileQuery: compileJsonQuery,
+      });
+      const assertOver = operation.assert;
+      const readDoc = (row) => (assertionMapping === null
+        ? JSON.parse(row.doc)
+        : mergeEntityRow(assertionMapping, row, 'doc'));
+
+      if (operation.fold !== null) {
+        // an associative aggregate: each batch is answered by the engine
+        // and the partial answers combine, so the collection is never
+        // held. The whole-collection read this replaces is the one
+        // statement a cross-document assertion used to cost.
+        let accumulated = operation.fold.start();
+        let folded = 0;
+        return chain(walkRows(connection, current.collection, options.batchSize, (rows) => {
+          accumulated = operation.fold.combine(accumulated, rows.map(readDoc));
+          folded += rows.length;
+          options.onProgress?.({
+            migration: migration.id,
+            collection: current.collection,
+            asserted: folded,
+          });
+        }, false, assertionMapping, options.check), () => operation.fold.finish(accumulated));
       }
-      catch (cause) {
-        return fail(`the assertion does not compile: ${/** @type {Error} */ (cause).message}`,
-          /** @type {Error} */ (cause));
+
+      if (!operation.perDocument) {
+        // materializing: the answer needs every document at once. That is
+        // a cost, so it is bounded and the bound is crossed BEFORE the
+        // excess is held — the walk stops at the row that would break it
+        const guard = createAssertionBoundGuard(options.assertionBounds, current.collection,
+          `the assertion of migration '${migration.id}' step ${i}`);
+        const gathered = [];
+        return chain(walkRows(connection, current.collection, options.batchSize, (rows) => {
+          for (const row of rows) {
+            const doc = readDoc(row);
+            guard.admit(doc, typeof row.doc === 'string' ? row.doc : undefined);
+            gathered.push(doc);
+          }
+        }, false, assertionMapping, options.check), () => assertOver(gathered));
       }
-      return chain(allDocs(connection, current.collection), (docs) => {
-        if (current.expect === 'ebv') {
-          if (!compiled.ebv(docs)) fail('the EBV assertion answered false');
-          return null;
-        }
-        const result = compiled(docs);
-        if (result !== undefined) {
-          const count = Array.isArray(result) ? result.length : 1;
-          fail(`the assertion expected an empty sequence, got ${count} item(s)`);
-        }
-        return null;
-      });
+      // per-document: walk in keyset batches like every other step,
+      // failing fast at the first batch that violates
+      let asserted = 0;
+      return walkRows(connection, current.collection, options.batchSize, (rows) => {
+        const docs = rows.map(readDoc);
+        assertOver(docs);
+        asserted += docs.length;
+        options.onProgress?.({
+          migration: migration.id,
+          collection: current.collection,
+          asserted,
+        });
+      }, false, assertionMapping, options.check);
     }), () => step(i + 1));
   };
   return step(0);
@@ -1116,10 +1316,9 @@ function runSteps(connection, migration, options) {
  * @returns {any} value-or-promise
  */
 function validateTargetState(connection, model, options) {
-  const collections = [...normalizeModel(model).values()];
+  const collections = [...normalizeModel(model, options.expressions).values()];
   const entities = [...normalizeEntities(model).values()];
   const dialect = connection.dialect;
-  const q = dialect.quoteIdentifier;
   // entity tables carry no 'key' column; the batched walk goes by row
   // identity and validates every stored document against the target
   const verifyEntity = (i) => {
@@ -1129,32 +1328,27 @@ function validateTargetState(connection, model, options) {
       ? options.compileSchema(entity.schema)
       : null;
     if (validate === null) return verifyEntity(i + 1);
-    const rid = dialect.rowIdentity();
-    const sql = `SELECT ${rid} AS ${q('rid')}, ${dialect.jsonText(q('doc'))} AS ${q('doc')} `
-      + `FROM ${q(entity.name)} WHERE ${rid} > ${dialect.parameterRef(1, 'after')} `
-      + `ORDER BY ${rid} ${dialect.limitClause(options.batchSize, undefined)}`;
-    return chain(connection.prepare(sql), (statement) => {
-      const nextBatch = (after) =>
-        chain(statement.all([after]), (rows) => {
-          if (rows.length === 0) return null;
-          for (const row of rows) {
-            const outcome = validate(JSON.parse(row.doc));
-            const valid = outcome === true || outcome?.valid === true;
-            if (!valid) {
-              throw refuse('JD0021',
-                `entity '${entity.name}': a stored document (row ${row.rid}) does not `
-                + 'validate against the target schema — a narrowing needs a data transform');
-            }
-          }
-          return nextBatch(rows[rows.length - 1].rid);
-        });
-      return nextBatch(-1);
-    });
+    // the WHOLE document — mapped columns folded in — is what the target
+    // schema judges; the rest-document alone failed every entity whose
+    // required members are columns, so a pure widening could not land
+    const entityMapping = explainMapping(model).entities[entity.name];
+    return chain(walkRows(connection, entity.name, options.batchSize, (rows) => {
+      for (const row of rows) {
+        const outcome = validate(mergeEntityRow(entityMapping, row, 'doc'));
+        const valid = outcome === true || outcome?.valid === true;
+        if (!valid) {
+          throw refuse('JD0021',
+            `entity '${entity.name}': a stored document (row ${row.rid}) does not `
+            + 'validate against the target schema — a narrowing needs a data transform');
+        }
+      }
+    }, false, entityMapping), () => verifyEntity(i + 1));
   };
   const verifyNext = (i) => {
     if (i >= collections.length) return null;
     const collection = collections[i];
-    const plan = planCollection(collection.name, collection, dialect, mappingFor(connection));
+    const plan = planCollection(collection.name, collection, dialect,
+      mappingFor(connection, options.expressions));
     const validate = options.compileSchema !== undefined
       ? options.compileSchema(collection.schema)
       : null;
@@ -1212,8 +1406,13 @@ function replayOnShadow(driver, shadowPath, baseline, migrations, model, options
       (options.registerFunctions !== undefined ? options.registerFunctions(shadow) : null));
     const apply = (i) => {
       if (i >= migrations.length) return null;
+      // a rebuild moves rows between tables while their keys point at
+      // the old one, so the switch comes off around it — on an engine
+      // that HAS a switch. One that always enforces cannot rebuild that
+      // way, and `alterTableFull` is why it never has to
       const bracket = migrations[i].steps.some(
-        (candidate) => candidate.kind === 'rebuild');
+        (candidate) => candidate.kind === 'rebuild')
+        && shadow.dialect.capabilities.foreignKeysAlwaysOn !== true;
       return chain(
         bracket ? shadow.exec(shadow.dialect.pragma.foreignKeys(false)) : null,
         () => chain(runSteps(shadow, migrations[i], options), () =>
@@ -1221,13 +1420,13 @@ function replayOnShadow(driver, shadowPath, baseline, migrations, model, options
             () => apply(i + 1))));
     };
     const run = () => chain(registered, () =>
-      chain(createModelShape(shadow, baseline), () => chain(apply(0), () => {
+      chain(createModelShape(shadow, baseline, options.expressions), () => chain(apply(0), () => {
         if (model === undefined) return null;
-        const target = [...normalizeModel(model).values()];
+        const target = [...normalizeModel(model, options.expressions).values()];
         const verifyNext = (i) => {
           if (i >= target.length) return null;
           const plan = planCollection(target[i].name, target[i], shadow.dialect,
-            mappingFor(shadow));
+            mappingFor(shadow, options.expressions));
           return chain(
             verifyShape(shadow, plan, target[i].name, target[i].docPath),
             () => verifyNext(i + 1));
@@ -1289,18 +1488,33 @@ function historyStatements(dialect) {
 }
 
 /**
- * Report a database's migration state without touching it: what is
- * applied, what is pending, whether an applied migration was edited,
- * and — once the chain is fully applied — whether the physical shape
- * DRIFTED from the model (someone changed the database by hand, §12).
+ * Report a database's migration state without touching it — the
+ * history table is probed, never created, so a fresh file stays byte
+ * for byte what it was: what is applied, what is pending, whether an
+ * applied migration was edited, and — once the chain is fully applied —
+ * whether the physical shape DRIFTED from the model (someone changed
+ * the database by hand, §12).
  * @param {{ driver: any, path?: string }} target
  * @param {any[]} migrations - the full ordered list
- * @param {{ baseline: any, model?: any,
- *   registerFunctions?: (connection: any) => any }} options
+ * @param {{ model?: any, registerFunctions?: (connection: any) => any,
+ *   signal?: AbortSignal, deadline?: number,
+ *   runtime?: Partial<import('@jarenjs/core/runtime').Runtime> }} options
+ *   - `signal`/`deadline` refuse a call already cancelled (`JD2080`) or
+ *   past its deadline (`JD2075`) on the runtime record's clock
  * @returns {Promise<{ applied: string[], pending: string[],
  *   drift: string | null, upToDate: boolean }>}
  */
-export function migrationStatus(target, migrations, options) {
+export function migrationStatus(target, migrations, options = {}) {
+  // a call already cancelled, or past its deadline on the caller's
+  // clock, opens nothing
+  try {
+    refuseCancelled({ signal: options.signal, deadline: options.deadline },
+      resolveRuntime(options.runtime).now,
+      { abortCode: 'JD2080', aborted: 'it ran', passed: 'the status read ran', ran: 'no step ran' });
+  }
+  catch (error) {
+    return Promise.reject(error);
+  }
   return toPromise(chain(
     target.driver.open(target.path ?? ':memory:', {}),
     (connection) => {
@@ -1310,10 +1524,15 @@ export function migrationStatus(target, migrations, options) {
       const failClosed = (error) => chain(connection.close(), () => { throw error; });
       let work;
       try {
+        // §6's "writes NOTHING" holds for a status read too: the history
+        // table is probed, never created, and an absent one reads as an
+        // empty history — the same promise the dry run makes
         work = chain(registerDeriveFunctions(connection), () =>
-          chain(connection.exec(statements.create), () =>
-          chain(connection.prepare(statements.select), (select) =>
-            chain(select.all([]), (rows) => {
+          chain(chain(connection.prepare(dialect.introspect.tableExists()), (probe) =>
+            chain(probe.get([HISTORY_TABLE]), (present) => (present === undefined
+              ? []
+              : chain(connection.prepare(statements.select), (select) => select.all([]))))),
+          (rows) => chain(rows, () => {
               for (let i = 0; i < rows.length; i++) {
                 const doc = migrations[i];
                 if (doc === undefined || doc.id !== rows[i].id
@@ -1335,7 +1554,7 @@ export function migrationStatus(target, migrations, options) {
                 (difference) => ({
                   applied, pending, drift: difference, upToDate: difference === null,
                 }));
-            }))));
+            })));
       }
       catch (error) {
         return failClosed(error);
@@ -1360,7 +1579,17 @@ export function migrationStatus(target, migrations, options) {
  * @param {any[]} migrations
  * @param {{ baseline: any, model?: any, compileSchema?: Function,
  *   dryRun?: boolean, batchSize?: number, onProgress?: Function,
- *   shadow?: boolean, shadowPath?: string }} options
+ *   shadow?: boolean, shadowPath?: string, shadowDriver?: any,
+ *   signal?: AbortSignal, deadline?: number,
+ *   runtime?: Partial<import('@jarenjs/core/runtime').Runtime> }} options
+ *   `signal` and `deadline` cancel between migrations, steps and
+ *   batches (`JD2080` / `JD2075`, the deadline read against `runtime`'s
+ *   clock); a cancelled migration rolls back whole and the completed
+ *   ones stand.
+ *   `runtime` is the host's runtime record: the clock every applied
+ *   migration is stamped with, and the clock and identifiers an entity
+ *   step's `default: 'now'` / `default: 'uuid'` fill; the platform's own
+ *   when absent
  * @returns {Promise<any>}
  */
 export function migrate(target, migrations, options) {
@@ -1375,11 +1604,40 @@ export function migrate(target, migrations, options) {
       'migrate needs { baseline }: the model the store was first created with '
       + '(the chain anchor and the shadow starting shape)');
   const batchSize = options.batchSize ?? 500;
+  const runtime = resolveRuntime(options.runtime);
+  /**
+   * The cancellation boundary: between migrations, between steps and
+   * between the batches of a data step, on the runtime record's clock.
+   * Nothing interrupts a statement that has started; a refusal inside a
+   * migration rolls that migration back whole and the completed ones
+   * stand, so a rerun resumes from the recorded position.
+   */
+  const check = () => refuseCancelled({ signal: options.signal, deadline: options.deadline }, runtime.now, {
+    abortCode: 'JD2080', aborted: 'its next step', passed: 'its next step',
+    ran: 'no further step ran; a migration in flight rolled back whole and a rerun resumes '
+      + 'from the recorded position',
+  });
   const runOptions = {
     batchSize,
+    assertionBounds: normalizeAssertionBounds(options.assertionBounds),
     onProgress: options.onProgress,
     registerFunctions: options.registerFunctions,
+    // the host's declared index-expression functions ride to every
+    // planner and every connection this run opens — the shadow's
+    // baseline, the reference database and the real store all resolve a
+    // declared expression against the same declarations the open path
+    // was given, or they would plan DDL nobody can apply
+    expressions: options.expressions,
+    model: options.model,
+    runtime,
+    check,
   };
+  try {
+    check();
+  }
+  catch (error) {
+    return Promise.reject(error);
+  }
 
   return toPromise(chain(
     target.driver.open(target.path ?? ':memory:', { timeout: target.busyTimeout ?? 5000 }),
@@ -1395,174 +1653,189 @@ export function migrate(target, migrations, options) {
 
       let work;
       try {
-        work = chain(connection.exec(statements.create), () =>
-        chain(connection.prepare(statements.select), (select) =>
-          chain(select.all([]), (appliedRows) => {
-            // the list must agree with the history: same ids, same
-            // order, same checksums — an edited applied migration is
-            // always a bug worth failing on
-            for (let i = 0; i < appliedRows.length; i++) {
-              const row = appliedRows[i];
-              const doc = migrations[i];
-              if (doc === undefined || doc.id !== row.id) {
-                throw refuse('JD0022',
-                  `history position ${i} records '${row.id}' but the migration list has `
-                  + `'${doc?.id ?? '<nothing>'}' — the list must contain every applied `
-                  + 'migration, in order');
-              }
-              if (migrationChecksum(doc) !== row.checksum) {
-                throw refuse('JD0022',
-                  `migration '${row.id}' differs from the document recorded in the `
-                  + 'history — an applied migration must never be edited');
-              }
+        // §6's "writes NOTHING": an apply creates the empty history
+        // table before reading it, a DRY RUN probes for it instead and
+        // reads an absent one as an empty history — the promise a dry
+        // run makes is the reason it is safe to point at production
+        const history = options.dryRun === true
+          ? chain(connection.prepare(dialect.introspect.tableExists()), (probe) =>
+            chain(probe.get([HISTORY_TABLE]), (row) => (row === undefined
+              ? []
+              : chain(connection.prepare(statements.select), (select) => select.all([])))))
+          : chain(connection.exec(statements.create), () =>
+            chain(connection.prepare(statements.select), (select) => select.all([])));
+        work = chain(history, (appliedRows) => {
+          // the list must agree with the history: same ids, same
+          // order, same checksums — an edited applied migration is
+          // always a bug worth failing on
+          for (let i = 0; i < appliedRows.length; i++) {
+            const row = appliedRows[i];
+            const doc = migrations[i];
+            if (doc === undefined || doc.id !== row.id) {
+              throw refuse('JD0022',
+                `history position ${i} records '${row.id}' but the migration list has `
+                + `'${doc?.id ?? '<nothing>'}' — the list must contain every applied `
+                + 'migration, in order');
             }
-            const pending = migrations.slice(appliedRows.length);
-            const currentShape = appliedRows.length > 0
-              ? appliedRows[appliedRows.length - 1].to_hash
-              : shapeHash(options.baseline);
-
-            let expectedFrom = currentShape;
-            for (const migration of pending) {
-              checkMigrationDocument(migration);
-              if (migration.from !== expectedFrom) {
-                throw refuse('JD0020',
-                  `migration '${migration.id}' expects shape '${migration.from}' but the `
-                  + `database is at '${expectedFrom}' — refusing to run against the wrong shape`);
-              }
-              expectedFrom = migration.to;
+            if (migrationChecksum(doc) !== row.checksum) {
+              throw refuse('JD0022',
+                `migration '${row.id}' differs from the document recorded in the `
+                + 'history — an applied migration must never be edited');
             }
-            if (options.model !== undefined && pending.length > 0
-              && expectedFrom !== shapeHash(options.model)) {
+          }
+          const pending = migrations.slice(appliedRows.length);
+          const currentShape = appliedRows.length > 0
+            ? appliedRows[appliedRows.length - 1].to_hash
+            : shapeHash(options.baseline);
+
+          let expectedFrom = currentShape;
+          for (const migration of pending) {
+            checkMigrationDocument(migration);
+            if (migration.from !== expectedFrom) {
               throw refuse('JD0020',
-                "the last migration's to-hash is not the target model's shape — the "
-                + 'migration chain and the code disagree about where this ends');
+                `migration '${migration.id}' expects shape '${migration.from}' but the `
+                + `database is at '${expectedFrom}' — refusing to run against the wrong shape`);
             }
+            expectedFrom = migration.to;
+          }
+          if (options.model !== undefined && pending.length > 0
+            && expectedFrom !== shapeHash(options.model)) {
+            throw refuse('JD0020',
+              "the last migration's to-hash is not the target model's shape — the "
+              + 'migration chain and the code disagree about where this ends');
+          }
 
-            if (pending.length === 0) {
-              return { applied: [], skipped: appliedRows.map((row) => row.id), upToDate: true };
-            }
+          if (pending.length === 0) {
+            return { applied: [], skipped: appliedRows.map((row) => row.id), upToDate: true };
+          }
 
-            const shadowRun = options.shadow === false
-              ? null
-              : replayOnShadow(target.driver, options.shadowPath ?? ':memory:',
-                options.baseline, migrations, options.model, runOptions);
-
-            return chain(shadowRun, () => {
-              if (options.dryRun === true) {
-                const rendered = [];
-                const counts = {};
-                const collect = (i) => {
-                  if (i >= pending.length) return null;
-                  const migration = pending[i];
-                  for (const migrationStep of migration.steps) {
-                    if (migrationStep.kind === 'ddl') rendered.push(migrationStep.sql);
-                    else if (migrationStep.kind === 'sql') {
-                      rendered.push(`-- data step (sql): ${migrationStep.note ?? ''}`);
-                      rendered.push(migrationStep.sql);
-                    }
-                    else if (migrationStep.kind === 'rebuild') {
-                      rendered.push(`-- rebuild '${migrationStep.table}' (§10 procedure)`);
-                      rendered.push(...migrationStep.create, migrationStep.copy,
-                        dialect.ddl.dropTable(migrationStep.table),
-                        dialect.ddl.renameTable(`${migrationStep.table}__rebuild`,
-                          migrationStep.table),
-                        ...migrationStep.indexes,
-                        dialect.pragma.foreignKeyCheck());
-                    }
-                    else if (migrationStep.kind === 'jslt')
-                      rendered.push(`-- jslt transform over '${migrationStep.collection}'`);
-                    else rendered.push(`-- assert over '${migrationStep.collection}'`);
-                  }
-                  const jsltCollections = [...new Set(migration.steps
-                    .filter((s) => s.kind === 'jslt').map((s) => s.collection))];
-                  const count = (j) => {
-                    if (j >= jsltCollections.length) return null;
-                    const table = jsltCollections[j];
-                    const countSql = `SELECT COUNT(*) AS ${dialect.quoteIdentifier('n')} `
-                      + `FROM ${dialect.quoteIdentifier(table)}`;
-                    return chain(connection.prepare(countSql), (statement) =>
-                      chain(statement.get([]), (row) => {
-                        counts[table] = row.n;
-                        return count(j + 1);
-                      }));
-                  };
-                  return chain(count(0), () => collect(i + 1));
-                };
-                return chain(collect(0), () => ({
-                  dryRun: true,
-                  pending: pending.map((migration) => migration.id),
-                  statements: rendered,
-                  counts,
-                  shadowValidated: options.shadow !== false,
-                }));
-              }
+          const shadowRun = options.shadow === false
+            ? null
+            : replayOnShadow(options.shadowDriver ?? target.driver,
+              options.shadowPath ?? ':memory:',
+              options.baseline, migrations, options.model, runOptions);
 
-              // the real run: one exclusive transaction per migration
-              const applied = [];
-              const applyNext = (i) => {
+          return chain(shadowRun, () => {
+            if (options.dryRun === true) {
+              const rendered = [];
+              const counts = {};
+              const collect = (i) => {
                 if (i >= pending.length) return null;
                 const migration = pending[i];
-                const last = i === pending.length - 1;
-                // the §10 procedure's pragma bracket, literally: the
-                // foreign_keys pragma is a no-op inside a transaction,
-                // and node:sqlite enables enforcement BY DEFAULT — a
-                // parent-table rebuild could not even DROP without this
-                const bracket = migration.steps.some(
-                  (candidate) => candidate.kind === 'rebuild');
-                return chain(
-                  bracket ? connection.exec(dialect.pragma.foreignKeys(false)) : null,
-                  () => chain(connection.exec(dialect.tx.beginImmediate), () => {
-                  const body = () => chain(runSteps(connection, migration, runOptions), () =>
-                    chain(last && options.model !== undefined
-                      ? chain(validateTargetState(connection, options.model,
-                        { compileSchema: options.compileSchema, batchSize }),
-                      () => (normalizeEntities(options.model).size === 0 ? null
-                        : chain(compareShapeToModel(target.driver, connection,
-                          options.model, options.registerFunctions), (difference) => {
-                          if (difference !== null) {
-                            throw refuse('JD0023',
-                              `the migrated shape does not equal the target model's: ${difference}`);
-                          }
-                          return null;
-                        })))
-                      : null,
-                    () => chain(connection.prepare(statements.insert), (insert) =>
-                      insert.run([migration.id, Date.now(), migration.from,
-                        migration.to, migrationChecksum(migration),
-                        migration.steps.length]))));
-                  const restore = () => (bracket
-                    ? connection.exec(dialect.pragma.foreignKeys(true)) : null);
-                  const commit = () => chain(connection.exec(dialect.tx.commit), () =>
-                    chain(restore(), () => {
-                      applied.push(migration.id);
-                      return applyNext(i + 1);
-                    }));
-                  const rollback = (error) =>
-                    chain(connection.exec(dialect.tx.rollback), () =>
-                      chain(restore(), () => { throw error; }));
-                  // only body() may route to this migration's rollback:
-                  // commit() chains the NEXT migration, whose failure
-                  // rolls ITSELF back — catching it here would roll
-                  // back a transaction that already committed
-                  let outcome;
-                  try {
-                    outcome = body();
+                for (const migrationStep of migration.steps) {
+                  if (migrationStep.kind === 'ddl') rendered.push(migrationStep.sql);
+                  else if (migrationStep.kind === 'sql') {
+                    rendered.push(`-- data step (sql): ${migrationStep.note ?? ''}`);
+                    rendered.push(migrationStep.sql);
                   }
-                  catch (error) {
-                    return rollback(error);
+                  else if (migrationStep.kind === 'rebuild') {
+                    rendered.push(`-- rebuild '${migrationStep.table}' (§10 procedure)`);
+                    rendered.push(...migrationStep.create, migrationStep.copy,
+                      dialect.ddl.dropTable(migrationStep.table),
+                      dialect.ddl.renameTable(`${migrationStep.table}__rebuild`,
+                        migrationStep.table),
+                      ...migrationStep.indexes);
+                    if (dialect.capabilities.foreignKeysAlwaysOn !== true)
+                      rendered.push(dialect.pragma.foreignKeyCheck());
                   }
-                  return outcome instanceof Promise
-                    ? outcome.then(commit, rollback)
-                    : commit();
-                }));
+                  else if (migrationStep.kind === 'jslt')
+                    rendered.push(`-- jslt transform over '${migrationStep.collection}'`);
+                  else rendered.push(`-- assert over '${migrationStep.collection}'`);
+                }
+                const jsltCollections = [...new Set(migration.steps
+                  .filter((s) => s.kind === 'jslt').map((s) => s.collection))];
+                const count = (j) => {
+                  if (j >= jsltCollections.length) return null;
+                  const table = jsltCollections[j];
+                  const countSql = `SELECT COUNT(*) AS ${dialect.quoteIdentifier('n')} `
+                    + `FROM ${dialect.quoteIdentifier(table)}`;
+                  return chain(connection.prepare(countSql), (statement) =>
+                    chain(statement.get([]), (row) => {
+                      counts[table] = row.n;
+                      return count(j + 1);
+                    }));
+                };
+                return chain(count(0), () => collect(i + 1));
               };
-              return chain(applyNext(0), () => ({
-                applied,
-                skipped: appliedRows.map((row) => row.id),
-                shape: expectedFrom,
+              return chain(collect(0), () => ({
+                dryRun: true,
+                pending: pending.map((migration) => migration.id),
+                statements: rendered,
+                counts,
+                shadowValidated: options.shadow !== false,
               }));
-            });
-          })));
+            }
+
+            // the real run: one exclusive transaction per migration
+            const applied = [];
+            const applyNext = (i) => {
+              if (i >= pending.length) return null;
+              // the boundary between migrations
+              check();
+              const migration = pending[i];
+              const last = i === pending.length - 1;
+              // the §10 procedure's pragma bracket, literally: the
+              // foreign_keys pragma is a no-op inside a transaction,
+              // and node:sqlite enables enforcement BY DEFAULT — a
+              // parent-table rebuild could not even DROP without this
+              const bracket = migration.steps.some(
+                (candidate) => candidate.kind === 'rebuild');
+              return chain(
+                bracket && dialect.capabilities.foreignKeysAlwaysOn !== true
+                  ? connection.exec(dialect.pragma.foreignKeys(false)) : null,
+                () => chain(connection.exec(dialect.tx.beginImmediate), () => {
+                const body = () => chain(runSteps(connection, migration, runOptions), () =>
+                  chain(last && options.model !== undefined
+                    ? chain(validateTargetState(connection, options.model,
+                      { compileSchema: options.compileSchema, batchSize }),
+                    () => (normalizeEntities(options.model).size === 0 ? null
+                      : chain(compareShapeToModel(target.driver, connection,
+                        options.model, options.registerFunctions), (difference) => {
+                        if (difference !== null) {
+                          throw refuse('JD0023',
+                            `the migrated shape does not equal the target model's: ${difference}`);
+                        }
+                        return null;
+                      })))
+                    : null,
+                  () => chain(connection.prepare(statements.insert), (insert) =>
+                    insert.run([migration.id, runtime.now(), migration.from,
+                      migration.to, migrationChecksum(migration),
+                      migration.steps.length]))));
+                const restore = () => (bracket
+                  && dialect.capabilities.foreignKeysAlwaysOn !== true
+                  ? connection.exec(dialect.pragma.foreignKeys(true)) : null);
+                const commit = () => chain(connection.exec(dialect.tx.commit), () =>
+                  chain(restore(), () => {
+                    applied.push(migration.id);
+                    return applyNext(i + 1);
+                  }));
+                const rollback = (error) =>
+                  chain(connection.exec(dialect.tx.rollback), () =>
+                    chain(restore(), () => { throw error; }));
+                // only body() may route to this migration's rollback:
+                // commit() chains the NEXT migration, whose failure
+                // rolls ITSELF back — catching it here would roll
+                // back a transaction that already committed
+                let outcome;
+                try {
+                  outcome = body();
+                }
+                catch (error) {
+                  return rollback(error);
+                }
+                return outcome instanceof Promise
+                  ? outcome.then(commit, rollback)
+                  : commit();
+              }));
+            };
+            return chain(applyNext(0), () => ({
+              applied,
+              skipped: appliedRows.map((row) => row.id),
+              shape: expectedFrom,
+            }));
+          });
+        });
 
       }
       catch (error) {