@jarenjs/db 0.56.0 → 0.67.0

package/src/migrate.js

@@ -23,6 +23,8 @@
 
 import { canonicalizeJson } from '@jarenjs/json/canonical';
 import { hashContent } from '@jarenjs/core/string';
+import { resolveRuntime } from '@jarenjs/core/runtime';
+import { refuseCancelled } from './cancellation.js';
 import { setObjectMember } from '@jarenjs/core/object';
 import { compileJsonQuery } from '@jarenjs/json/query';
 import { compileJsltStylesheet } from '@jarenjs/json/jslt';
@@ -30,11 +32,19 @@ import { compileJsltStylesheet } from '@jarenjs/json/jslt';
 import { DbCompileError } from './errors.js';
 import { chain, toPromise } from './driver.js';
 import { normalizeModel } from './store.js';
+import { CHANGES_TABLE, CHANGES_STATE_TABLE } from './capture.js';
+import { JOBS_TABLE, JOB_CHECKPOINTS_TABLE } from './jobs.js';
 import { planCollection, verifyShape, planEntity, planJoinTable } from './ddl.js';
 import { normalizeEntities, explainMapping } from './model.js';
 import { derivedValue, memberAt, registerDeriveFunctions } from './derive.js';
 import { mergeEntityRow } from './graph.js';
 import { entityCore } from './entity.js';
+import {
+  MIGRATION_VERSION, isPerDocumentAssertion, compileDocumentStep, checkMigrationDocument,
+  normalizeAssertionBounds, ASSERTION_BOUNDS_DEFAULT, createAssertionBoundGuard,
+} from './document-steps.js';
+
+export { MIGRATION_VERSION, isPerDocumentAssertion, ASSERTION_BOUNDS_DEFAULT };
 
 /**
  * The physical mapping a connection's driver imposes on derived index
@@ -44,22 +54,30 @@ import { entityCore } from './entity.js';
  * @param {any} connection
  * @returns {{ derived: 'virtual' | 'stored', rtree: boolean }}
  */
-function mappingFor(connection) {
+function mappingFor(connection, expressions = undefined) {
+  const registered = connection.capabilities?.deterministicIndexableFunctions === true;
   return {
-    derived: connection.capabilities?.deterministicIndexableFunctions === true
-      ? 'virtual' : 'stored',
+    derived: registered ? 'virtual' : 'stored',
     // the same reasoning for the R*Tree mapping: a build without the
     // module plans (and verifies) the B-tree shape
     rtree: connection.capabilities?.rtree === true,
+    // and the same for a declared index expression: this connection
+    // either computes it or calls the engine's own immutable function
+    expressions,
+    registered,
   };
 }
 
-/** The migration format version. */
-export const MIGRATION_VERSION = '0.1';
-
 /** The history table name (outside the model's identifier namespace
  * conventions on purpose — a collection cannot collide with it). */
 export const HISTORY_TABLE = '_jaren_migrations';
+/** The tables the engine owns beside a model's: never a shape-drift finding. */
+/** The tables this package owns. A model never declared one, so one
+ * found in a database is the engine's own bookkeeping rather than
+ * anybody's drift — the drift check skips them and the introspector
+ * does not derive them. */
+export const ENGINE_TABLES = new Set([HISTORY_TABLE, CHANGES_TABLE, CHANGES_STATE_TABLE,
+  JOBS_TABLE, JOB_CHECKPOINTS_TABLE]);
 
 /**
  * The signature-grade identity of a model SHAPE.
@@ -162,7 +180,8 @@ function deriveStep(collection, plan, columnNames, note) {
  * @param {any} fromModel
  * @param {any} toModel
  * @param {{ id?: string, dialect?: any,
- *   derived?: 'virtual' | 'stored', rtree?: boolean }} [options]
+ *   derived?: 'virtual' | 'stored', rtree?: boolean,
+ *   expressions?: Record<string, any> }} [options]
  * @returns {{ migration: any, report: {
  *   renamed: { from: string, to: string }[],
  *   added: string[], removed: string[],
@@ -173,9 +192,13 @@ export function planMigration(fromModel, toModel, options = undefined) {
   const dialect = options?.dialect ?? null;
   if (dialect === null || typeof dialect !== 'object')
     throw new TypeError('planMigration needs { dialect } (the store dialect renders the DDL)');
-  const mapping = { derived: options?.derived ?? 'virtual', rtree: options?.rtree !== false };
-  const fromCollections = normalizeModel(fromModel);
-  const toCollections = normalizeModel(toModel);
+  const mapping = { derived: options?.derived ?? 'virtual', rtree: options?.rtree !== false,
+    // a model that declares an index EXPRESSION resolves its functions
+    // here too: a plan is DDL, and DDL over a function this planner was
+    // not told about is DDL nobody can apply
+    expressions: options?.expressions, registered: options?.derived !== 'stored' };
+  const fromCollections = normalizeModel(fromModel, options?.expressions);
+  const toCollections = normalizeModel(toModel, options?.expressions);
 
   const steps = [];
   const report = {
@@ -829,15 +852,19 @@ function renderRebuild(name, fromName, fm, tm, fromMapping, toMapping, dialect,
  * equality compares against, and the tests.
  * @param {any} connection
  * @param {any} model
+ * @param {Record<string, any>} [expressions] - the host's declared
+ *   index-expression functions, resolved into the DDL the same way the
+ *   open path resolves them
  * @returns {any} value-or-promise
  */
-export function createModelShape(connection, model) {
+export function createModelShape(connection, model, expressions = undefined) {
   const dialect = connection.dialect;
   /** @type {string[]} */
   const statements = [];
-  for (const collection of normalizeModel(model).values()) {
+  for (const collection of normalizeModel(model, expressions).values()) {
     statements.push(
-      ...planCollection(collection.name, collection, dialect, mappingFor(connection)).createSql);
+      ...planCollection(collection.name, collection, dialect,
+        mappingFor(connection, expressions)).createSql);
   }
   const entities = normalizeEntities(model);
   if (entities.size > 0) {
@@ -870,7 +897,9 @@ export function schemaShapeOf(connection) {
   const dialect = connection.dialect;
   return chain(connection.prepare(dialect.introspect.schemaDump()), (statement) =>
     chain(statement.all([]), (rows) => rows
-      .filter((row) => row.name !== HISTORY_TABLE && row.owner !== HISTORY_TABLE)
+      // the engine's own tables — history, the change log and its state
+      // row, the job queue — are never a model's drift
+      .filter((row) => !ENGINE_TABLES.has(String(row.name)) && !ENGINE_TABLES.has(String(row.owner)))
       .map((row) => ({
         type: String(row.type),
         name: String(row.name),
@@ -935,14 +964,22 @@ function normalizeSchemaSql(sql) {
  * @param {((connection: any) => any) | undefined} registerFunctions
  * @returns {any} value-or-promise of `string | null`
  */
-export function compareShapeToModel(driver, connection, model, registerFunctions) {
+export function compareShapeToModel(driver, connection, model, registerFunctions, expressions) {
+  // The comparison IS a text comparison: it builds the model's shape in
+  // a reference database and compares the two engines' stored CREATE
+  // statements. An engine that keeps none has nothing to compare, and
+  // says so here rather than reading an undefined statement — the
+  // structural drift check (columns, indexes, foreign-key tuples) runs
+  // per collection and per entity either way, and `declaredSqlText` is
+  // what tells a reader which half they got.
+  if (connection.dialect.capabilities.declaredSqlText !== true) return null;
   return chain(driver.open(':memory:', {}), (reference) =>
     chain(chain(registerDeriveFunctions(reference),
       () => (registerFunctions !== undefined ? registerFunctions(reference) : null)), () => {
       const finish = (result) => chain(reference.close(), () => result);
       let outcome;
       try {
-        outcome = chain(createModelShape(reference, model), () =>
+        outcome = chain(createModelShape(reference, model, expressions), () =>
           chain(schemaShapeOf(reference), (wanted) =>
             chain(schemaShapeOf(connection), (actual) => {
               const wantedText = JSON.stringify(wanted);
@@ -975,54 +1012,6 @@ export function compareShapeToModel(driver, connection, model, registerFunctions
     }));
 }
 
-const STEP_KINDS = new Set(['ddl', 'jslt', 'query', 'sql', 'rebuild', 'derive']);
-
-/**
- * Structural validation of one migration document, including the
- * draft refusal (`JD0021`).
- * @param {any} migration
- */
-function checkMigrationDocument(migration) {
-  if (migration === null || typeof migration !== 'object'
-    || migration.$migration !== MIGRATION_VERSION
-    || typeof migration.id !== 'string' || migration.id === ''
-    || typeof migration.from !== 'string' || typeof migration.to !== 'string'
-    || !Array.isArray(migration.steps)) {
-    throw refuse('JD0023',
-      `migration '${migration?.id ?? '<unknown>'}' is not a valid ${MIGRATION_VERSION} migration document`);
-  }
-  for (let i = 0; i < migration.steps.length; i++) {
-    const step = migration.steps[i];
-    if (step === null || typeof step !== 'object' || !STEP_KINDS.has(step.kind)) {
-      throw refuse('JD0023',
-        `migration '${migration.id}' step ${i} has no recognised kind`);
-    }
-    if (step.kind === 'rebuild'
-      && (typeof step.table !== 'string' || !Array.isArray(step.create)
-        || typeof step.copy !== 'string' || !Array.isArray(step.indexes))) {
-      throw refuse('JD0023',
-        `migration '${migration.id}' step ${i} is a rebuild without its rendered `
-        + 'table/create/copy/indexes');
-    }
-    if (step.kind === 'sql' && typeof step.sql !== 'string') {
-      throw refuse('JD0023',
-        `migration '${migration.id}' step ${i} is a sql step without sql text`);
-    }
-    if (step.kind === 'derive'
-      && (typeof step.collection !== 'string' || !Array.isArray(step.columns)
-        || step.columns.length === 0)) {
-      throw refuse('JD0023',
-        `migration '${migration.id}' step ${i} is a derive backfill without its columns`);
-    }
-    if (step.kind === 'jslt' && step.draft === true) {
-      throw refuse('JD0021',
-        `migration '${migration.id}' step ${i} is a DRAFT transform for collection `
-        + `'${step.collection}' — the planner cannot infer a data transform; fill in `
-        + 'the stylesheet (or delete the step for a pure widening) and remove "draft"');
-    }
-  }
-}
-
 /**
  * The batched row walk shared by transforms and post-validation:
  * `SELECT rowid, json(doc) ... WHERE rowid > ? ORDER BY rowid LIMIT ?`
@@ -1032,9 +1021,14 @@ function checkMigrationDocument(migration) {
  * @param {number} batchSize
  * @param {(rows: { rid: any, doc: string, key: any }[]) => any} handle
  *   value-or-promise per batch
+ * @param {boolean} [keyed]
+ * @param {any} [entityMapping]
+ * @param {() => void} [check] - run before every batch: the
+ *   cancellation boundary a data step has between its batches
  * @returns {any}
  */
-function walkRows(connection, table, batchSize, handle, keyed = true, entityMapping = null) {
+function walkRows(connection, table, batchSize, handle, keyed = true, entityMapping = null,
+  check = undefined) {
   // entity tables carry no 'key' column — the transform walk goes by
   // row identity alone; only the collection walks select the key. An
   // entity's mapped columns ride beside the document so the row can be
@@ -1046,18 +1040,23 @@ function walkRows(connection, table, batchSize, handle, keyed = true, entityMapp
   const keySelect = keyed ? `, ${q('key')} AS ${q('k')}` : '';
   const columnSelect = entityMapping === null ? '' : entityColumnsOf(entityMapping)
     .map((column) => `, ${q(column)}`).join('');
-  const sql = `SELECT ${rid} AS ${q('rid')}, ${dialect.jsonText(q('doc'))} AS ${q('doc')}`
-    + `${keySelect}${columnSelect} FROM ${q(table)} WHERE ${rid} > ${dialect.parameterRef(1, 'after')} `
-    + `ORDER BY ${rid} ${dialect.limitClause(batchSize, undefined)}`;
-  return chain(connection.prepare(sql), (statement) => {
-    const nextBatch = (after) =>
-      chain(statement.all([after]), (rows) => {
+  const select = `SELECT ${rid} AS ${q('rid')}, ${dialect.jsonText(q('doc'))} AS ${q('doc')}`
+    + `${keySelect}${columnSelect} FROM ${q(table)}`;
+  const ordered = ` ORDER BY ${rid} ${dialect.limitClause(batchSize, undefined)}`;
+  // An INTEGER PRIMARY KEY can be negative. The first batch has no
+  // lower bound; subsequent batches seek from a row actually read.
+  return chain(connection.prepare(select + ordered), (first) => chain(connection.prepare(
+    `${select} WHERE ${rid} > ${dialect.parameterRef(1, 'after')}${ordered}`), (statement) => {
+    const nextBatch = (after) => {
+      if (check !== undefined) check();
+      return chain(after === undefined ? first.all([]) : statement.all([after]), (rows) => {
         if (rows.length === 0) return null;
         return chain(handle(rows), () =>
           nextBatch(rows[rows.length - 1].rid));
       });
-    return nextBatch(-1);
-  });
+    };
+    return nextBatch(undefined);
+  }));
 }
 
 /** The physical columns an entity row carries beside its document. */
@@ -1085,21 +1084,6 @@ function entityStepMapping(options, table) {
   return { entity, mapping: explainMapping(options.model).entities[table] };
 }
 
-/** All documents of a collection or entity (the assertion steps' working
- * set — a documented whole-collection read), entities read WHOLE. */
-function allDocs(connection, table, entityMapping = null) {
-  const dialect = connection.dialect;
-  const q = dialect.quoteIdentifier;
-  const columnSelect = entityMapping === null ? '' : entityColumnsOf(entityMapping)
-    .map((column) => `, ${q(column)}`).join('');
-  const sql = `SELECT ${dialect.jsonText(q('doc'))} AS ${q('doc')}${columnSelect} FROM ${q(table)} `
-    + `ORDER BY ${dialect.rowIdentity()}`;
-  return chain(connection.prepare(sql), (statement) =>
-    chain(statement.all([]), (rows) => rows.map((row) => (entityMapping === null
-      ? JSON.parse(row.doc)
-      : mergeEntityRow(entityMapping, row, 'doc')))));
-}
-
 /**
  * Run one migration's steps against a connection.
  * @param {any} connection
@@ -1112,6 +1096,9 @@ function runSteps(connection, migration, options) {
   const q = dialect.quoteIdentifier;
   const step = (i) => {
     if (i >= migration.steps.length) return null;
+    // the cancellation boundary between steps: a refusal here rolls the
+    // migration in flight back whole, as any step failure does
+    if (options.check !== undefined) options.check();
     const current = migration.steps[i];
     const fail = (reason, cause) => {
       throw refuse('JD0023',
@@ -1146,6 +1133,7 @@ function runSteps(connection, migration, options) {
         ];
         const runNext = (j) => {
           if (j >= statements.length) {
+            if (dialect.capabilities.foreignKeysAlwaysOn === true) return null;
             return chain(connection.prepare(dialect.pragma.foreignKeyCheck()),
               (checkStatement) => chain(checkStatement.all([]), (violations) => {
                 if (violations.length > 0) {
@@ -1191,24 +1179,23 @@ function runSteps(connection, migration, options) {
               collection: current.collection,
               derived: derivedRows,
             });
-          }, false), () => derivedRows));
+          }, false, null, options.check), () => derivedRows));
       }
       if (current.kind === 'jslt') {
-        let transform;
-        try {
-          transform = compileJsltStylesheet(current.stylesheet);
-        }
-        catch (cause) {
-          return fail(`the stylesheet does not compile: ${/** @type {Error} */ (cause).message}`,
-            /** @type {Error} */ (cause));
-        }
         const stepEntity = entityStepMapping(options, current.collection);
+        const operation = compileDocumentStep(current, i, {
+          migrationId: migration.id,
+          compileJslt: compileJsltStylesheet,
+          compileQuery: compileJsonQuery,
+          keys: stepEntity === null ? [] : stepEntity.mapping.keys,
+        });
         if (stepEntity !== null) {
           // an entity row is transformed WHOLE: the mapped columns fold in
           // before the stylesheet and split out after it, through the
           // entity's own split — a column-mapped member the stylesheet
           // wrote used to land in the document and be shadowed on read
-          const core = entityCore(connection, stepEntity.entity, stepEntity.mapping, null);
+          const core = entityCore(connection, stepEntity.entity, stepEntity.mapping, null,
+            options.runtime);
           const columns = entityColumnsOf(stepEntity.mapping);
           const assignments = [
             ...columns.map((column, i) => `${q(column)} = ${dialect.parameterRef(i + 1, 'v')}`),
@@ -1221,19 +1208,7 @@ function runSteps(connection, migration, options) {
             chain(walkRows(connection, current.collection, options.batchSize, (rows) => {
               for (const row of rows) {
                 const whole = mergeEntityRow(stepEntity.mapping, row, 'doc');
-                const next = transform(whole);
-                if (next === null || typeof next !== 'object' || Array.isArray(next))
-                  fail(`the transform produced a non-document for row ${row.rid}`);
-                // the key is the row's identity, not the document's to
-                // change: a body that leaves it out keeps it, a body
-                // that rewrites it is refused, as a collection's key is
-                for (const key of stepEntity.mapping.keys) {
-                  if (next[key] === undefined) next[key] = whole[key];
-                  else if (next[key] !== whole[key]) {
-                    fail(`the transform changed the key member '${key}' of row ${row.rid} — `
-                      + `key changes are not supported in ${MIGRATION_VERSION}`);
-                  }
-                }
+                const next = operation.apply(whole, row.rid);
                 const { values, rest } = core.plan.split(next);
                 const byName = new Map(values.map((value) => [value.name, value.value]));
                 update.run([...columns.map((column) => byName.get(column) ?? null),
@@ -1245,7 +1220,7 @@ function runSteps(connection, migration, options) {
                 collection: current.collection,
                 transformed,
               });
-            }, false, stepEntity.mapping), () => transformed));
+            }, false, stepEntity.mapping, options.check), () => transformed));
         }
         const updateSql = `UPDATE ${q(current.collection)} SET ${q('doc')} = `
           + `${dialect.jsonEncode(dialect.parameterRef(1, 'doc'))} `
@@ -1255,9 +1230,7 @@ function runSteps(connection, migration, options) {
         return chain(connection.prepare(updateSql), (update) =>
           chain(walkRows(connection, current.collection, options.batchSize, (rows) => {
             for (const row of rows) {
-              const next = transform(JSON.parse(row.doc));
-              if (next === null || typeof next !== 'object' || Array.isArray(next))
-                fail(`the transform produced a non-document for row ${row.rid}`);
+              const next = operation.apply(JSON.parse(row.doc), row.rid);
               update.run([JSON.stringify(next), row.rid]);
               transformed++;
             }
@@ -1266,30 +1239,66 @@ function runSteps(connection, migration, options) {
               collection: current.collection,
               transformed,
             });
-          }, keyed), () => transformed));
+          }, keyed, null, options.check), () => transformed));
       }
       // kind === 'query': the assertion step
-      let compiled;
-      try {
-        compiled = compileJsonQuery(current.assert);
+      const assertionMapping = entityStepMapping(options, current.collection)?.mapping ?? null;
+      const operation = compileDocumentStep(current, i, {
+        migrationId: migration.id,
+        compileJslt: compileJsltStylesheet,
+        compileQuery: compileJsonQuery,
+      });
+      const assertOver = operation.assert;
+      const readDoc = (row) => (assertionMapping === null
+        ? JSON.parse(row.doc)
+        : mergeEntityRow(assertionMapping, row, 'doc'));
+
+      if (operation.fold !== null) {
+        // an associative aggregate: each batch is answered by the engine
+        // and the partial answers combine, so the collection is never
+        // held. The whole-collection read this replaces is the one
+        // statement a cross-document assertion used to cost.
+        let accumulated = operation.fold.start();
+        let folded = 0;
+        return chain(walkRows(connection, current.collection, options.batchSize, (rows) => {
+          accumulated = operation.fold.combine(accumulated, rows.map(readDoc));
+          folded += rows.length;
+          options.onProgress?.({
+            migration: migration.id,
+            collection: current.collection,
+            asserted: folded,
+          });
+        }, false, assertionMapping, options.check), () => operation.fold.finish(accumulated));
       }
-      catch (cause) {
-        return fail(`the assertion does not compile: ${/** @type {Error} */ (cause).message}`,
-          /** @type {Error} */ (cause));
+
+      if (!operation.perDocument) {
+        // materializing: the answer needs every document at once. That is
+        // a cost, so it is bounded and the bound is crossed BEFORE the
+        // excess is held — the walk stops at the row that would break it
+        const guard = createAssertionBoundGuard(options.assertionBounds, current.collection,
+          `the assertion of migration '${migration.id}' step ${i}`);
+        const gathered = [];
+        return chain(walkRows(connection, current.collection, options.batchSize, (rows) => {
+          for (const row of rows) {
+            const doc = readDoc(row);
+            guard.admit(doc, typeof row.doc === 'string' ? row.doc : undefined);
+            gathered.push(doc);
+          }
+        }, false, assertionMapping, options.check), () => assertOver(gathered));
       }
-      return chain(allDocs(connection, current.collection,
-        entityStepMapping(options, current.collection)?.mapping ?? null), (docs) => {
-        if (current.expect === 'ebv') {
-          if (!compiled.ebv(docs)) fail('the EBV assertion answered false');
-          return null;
-        }
-        const result = compiled(docs);
-        if (result !== undefined) {
-          const count = Array.isArray(result) ? result.length : 1;
-          fail(`the assertion expected an empty sequence, got ${count} item(s)`);
-        }
-        return null;
-      });
+      // per-document: walk in keyset batches like every other step,
+      // failing fast at the first batch that violates
+      let asserted = 0;
+      return walkRows(connection, current.collection, options.batchSize, (rows) => {
+        const docs = rows.map(readDoc);
+        assertOver(docs);
+        asserted += docs.length;
+        options.onProgress?.({
+          migration: migration.id,
+          collection: current.collection,
+          asserted,
+        });
+      }, false, assertionMapping, options.check);
     }), () => step(i + 1));
   };
   return step(0);
@@ -1307,10 +1316,9 @@ function runSteps(connection, migration, options) {
  * @returns {any} value-or-promise
  */
 function validateTargetState(connection, model, options) {
-  const collections = [...normalizeModel(model).values()];
+  const collections = [...normalizeModel(model, options.expressions).values()];
   const entities = [...normalizeEntities(model).values()];
   const dialect = connection.dialect;
-  const q = dialect.quoteIdentifier;
   // entity tables carry no 'key' column; the batched walk goes by row
   // identity and validates every stored document against the target
   const verifyEntity = (i) => {
@@ -1324,33 +1332,23 @@ function validateTargetState(connection, model, options) {
     // schema judges; the rest-document alone failed every entity whose
     // required members are columns, so a pure widening could not land
     const entityMapping = explainMapping(model).entities[entity.name];
-    const rid = dialect.rowIdentity();
-    const columnSelect = entityColumnsOf(entityMapping).map((column) => `, ${q(column)}`).join('');
-    const sql = `SELECT ${rid} AS ${q('rid')}, ${dialect.jsonText(q('doc'))} AS ${q('doc')}${columnSelect} `
-      + `FROM ${q(entity.name)} WHERE ${rid} > ${dialect.parameterRef(1, 'after')} `
-      + `ORDER BY ${rid} ${dialect.limitClause(options.batchSize, undefined)}`;
-    return chain(connection.prepare(sql), (statement) => {
-      const nextBatch = (after) =>
-        chain(statement.all([after]), (rows) => {
-          if (rows.length === 0) return null;
-          for (const row of rows) {
-            const outcome = validate(mergeEntityRow(entityMapping, row, 'doc'));
-            const valid = outcome === true || outcome?.valid === true;
-            if (!valid) {
-              throw refuse('JD0021',
-                `entity '${entity.name}': a stored document (row ${row.rid}) does not `
-                + 'validate against the target schema — a narrowing needs a data transform');
-            }
-          }
-          return nextBatch(rows[rows.length - 1].rid);
-        });
-      return nextBatch(-1);
-    });
+    return chain(walkRows(connection, entity.name, options.batchSize, (rows) => {
+      for (const row of rows) {
+        const outcome = validate(mergeEntityRow(entityMapping, row, 'doc'));
+        const valid = outcome === true || outcome?.valid === true;
+        if (!valid) {
+          throw refuse('JD0021',
+            `entity '${entity.name}': a stored document (row ${row.rid}) does not `
+            + 'validate against the target schema — a narrowing needs a data transform');
+        }
+      }
+    }, false, entityMapping), () => verifyEntity(i + 1));
   };
   const verifyNext = (i) => {
     if (i >= collections.length) return null;
     const collection = collections[i];
-    const plan = planCollection(collection.name, collection, dialect, mappingFor(connection));
+    const plan = planCollection(collection.name, collection, dialect,
+      mappingFor(connection, options.expressions));
     const validate = options.compileSchema !== undefined
       ? options.compileSchema(collection.schema)
       : null;
@@ -1408,8 +1406,13 @@ function replayOnShadow(driver, shadowPath, baseline, migrations, model, options
       (options.registerFunctions !== undefined ? options.registerFunctions(shadow) : null));
     const apply = (i) => {
       if (i >= migrations.length) return null;
+      // a rebuild moves rows between tables while their keys point at
+      // the old one, so the switch comes off around it — on an engine
+      // that HAS a switch. One that always enforces cannot rebuild that
+      // way, and `alterTableFull` is why it never has to
       const bracket = migrations[i].steps.some(
-        (candidate) => candidate.kind === 'rebuild');
+        (candidate) => candidate.kind === 'rebuild')
+        && shadow.dialect.capabilities.foreignKeysAlwaysOn !== true;
       return chain(
         bracket ? shadow.exec(shadow.dialect.pragma.foreignKeys(false)) : null,
         () => chain(runSteps(shadow, migrations[i], options), () =>
@@ -1417,13 +1420,13 @@ function replayOnShadow(driver, shadowPath, baseline, migrations, model, options
             () => apply(i + 1))));
     };
     const run = () => chain(registered, () =>
-      chain(createModelShape(shadow, baseline), () => chain(apply(0), () => {
+      chain(createModelShape(shadow, baseline, options.expressions), () => chain(apply(0), () => {
         if (model === undefined) return null;
-        const target = [...normalizeModel(model).values()];
+        const target = [...normalizeModel(model, options.expressions).values()];
         const verifyNext = (i) => {
           if (i >= target.length) return null;
           const plan = planCollection(target[i].name, target[i], shadow.dialect,
-            mappingFor(shadow));
+            mappingFor(shadow, options.expressions));
           return chain(
             verifyShape(shadow, plan, target[i].name, target[i].docPath),
             () => verifyNext(i + 1));
@@ -1485,18 +1488,33 @@ function historyStatements(dialect) {
 }
 
 /**
- * Report a database's migration state without touching it: what is
- * applied, what is pending, whether an applied migration was edited,
- * and — once the chain is fully applied — whether the physical shape
- * DRIFTED from the model (someone changed the database by hand, §12).
+ * Report a database's migration state without touching it — the
+ * history table is probed, never created, so a fresh file stays byte
+ * for byte what it was: what is applied, what is pending, whether an
+ * applied migration was edited, and — once the chain is fully applied —
+ * whether the physical shape DRIFTED from the model (someone changed
+ * the database by hand, §12).
  * @param {{ driver: any, path?: string }} target
  * @param {any[]} migrations - the full ordered list
- * @param {{ baseline: any, model?: any,
- *   registerFunctions?: (connection: any) => any }} options
+ * @param {{ model?: any, registerFunctions?: (connection: any) => any,
+ *   signal?: AbortSignal, deadline?: number,
+ *   runtime?: Partial<import('@jarenjs/core/runtime').Runtime> }} options
+ *   - `signal`/`deadline` refuse a call already cancelled (`JD2080`) or
+ *   past its deadline (`JD2075`) on the runtime record's clock
  * @returns {Promise<{ applied: string[], pending: string[],
  *   drift: string | null, upToDate: boolean }>}
  */
-export function migrationStatus(target, migrations, options) {
+export function migrationStatus(target, migrations, options = {}) {
+  // a call already cancelled, or past its deadline on the caller's
+  // clock, opens nothing
+  try {
+    refuseCancelled({ signal: options.signal, deadline: options.deadline },
+      resolveRuntime(options.runtime).now,
+      { abortCode: 'JD2080', aborted: 'it ran', passed: 'the status read ran', ran: 'no step ran' });
+  }
+  catch (error) {
+    return Promise.reject(error);
+  }
   return toPromise(chain(
     target.driver.open(target.path ?? ':memory:', {}),
     (connection) => {
@@ -1506,10 +1524,15 @@ export function migrationStatus(target, migrations, options) {
       const failClosed = (error) => chain(connection.close(), () => { throw error; });
       let work;
       try {
+        // §6's "writes NOTHING" holds for a status read too: the history
+        // table is probed, never created, and an absent one reads as an
+        // empty history — the same promise the dry run makes
         work = chain(registerDeriveFunctions(connection), () =>
-          chain(connection.exec(statements.create), () =>
-          chain(connection.prepare(statements.select), (select) =>
-            chain(select.all([]), (rows) => {
+          chain(chain(connection.prepare(dialect.introspect.tableExists()), (probe) =>
+            chain(probe.get([HISTORY_TABLE]), (present) => (present === undefined
+              ? []
+              : chain(connection.prepare(statements.select), (select) => select.all([]))))),
+          (rows) => chain(rows, () => {
               for (let i = 0; i < rows.length; i++) {
                 const doc = migrations[i];
                 if (doc === undefined || doc.id !== rows[i].id
@@ -1531,7 +1554,7 @@ export function migrationStatus(target, migrations, options) {
                 (difference) => ({
                   applied, pending, drift: difference, upToDate: difference === null,
                 }));
-            }))));
+            })));
       }
       catch (error) {
         return failClosed(error);
@@ -1556,7 +1579,17 @@ export function migrationStatus(target, migrations, options) {
  * @param {any[]} migrations
  * @param {{ baseline: any, model?: any, compileSchema?: Function,
  *   dryRun?: boolean, batchSize?: number, onProgress?: Function,
- *   shadow?: boolean, shadowPath?: string }} options
+ *   shadow?: boolean, shadowPath?: string, shadowDriver?: any,
+ *   signal?: AbortSignal, deadline?: number,
+ *   runtime?: Partial<import('@jarenjs/core/runtime').Runtime> }} options
+ *   `signal` and `deadline` cancel between migrations, steps and
+ *   batches (`JD2080` / `JD2075`, the deadline read against `runtime`'s
+ *   clock); a cancelled migration rolls back whole and the completed
+ *   ones stand.
+ *   `runtime` is the host's runtime record: the clock every applied
+ *   migration is stamped with, and the clock and identifiers an entity
+ *   step's `default: 'now'` / `default: 'uuid'` fill; the platform's own
+ *   when absent
  * @returns {Promise<any>}
  */
 export function migrate(target, migrations, options) {
@@ -1571,12 +1604,40 @@ export function migrate(target, migrations, options) {
       'migrate needs { baseline }: the model the store was first created with '
       + '(the chain anchor and the shadow starting shape)');
   const batchSize = options.batchSize ?? 500;
+  const runtime = resolveRuntime(options.runtime);
+  /**
+   * The cancellation boundary: between migrations, between steps and
+   * between the batches of a data step, on the runtime record's clock.
+   * Nothing interrupts a statement that has started; a refusal inside a
+   * migration rolls that migration back whole and the completed ones
+   * stand, so a rerun resumes from the recorded position.
+   */
+  const check = () => refuseCancelled({ signal: options.signal, deadline: options.deadline }, runtime.now, {
+    abortCode: 'JD2080', aborted: 'its next step', passed: 'its next step',
+    ran: 'no further step ran; a migration in flight rolled back whole and a rerun resumes '
+      + 'from the recorded position',
+  });
   const runOptions = {
     batchSize,
+    assertionBounds: normalizeAssertionBounds(options.assertionBounds),
     onProgress: options.onProgress,
     registerFunctions: options.registerFunctions,
+    // the host's declared index-expression functions ride to every
+    // planner and every connection this run opens — the shadow's
+    // baseline, the reference database and the real store all resolve a
+    // declared expression against the same declarations the open path
+    // was given, or they would plan DDL nobody can apply
+    expressions: options.expressions,
     model: options.model,
+    runtime,
+    check,
   };
+  try {
+    check();
+  }
+  catch (error) {
+    return Promise.reject(error);
+  }
 
   return toPromise(chain(
     target.driver.open(target.path ?? ':memory:', { timeout: target.busyTimeout ?? 5000 }),
@@ -1650,7 +1711,8 @@ export function migrate(target, migrations, options) {
 
           const shadowRun = options.shadow === false
             ? null
-            : replayOnShadow(target.driver, options.shadowPath ?? ':memory:',
+            : replayOnShadow(options.shadowDriver ?? target.driver,
+              options.shadowPath ?? ':memory:',
               options.baseline, migrations, options.model, runOptions);
 
           return chain(shadowRun, () => {
@@ -1672,8 +1734,9 @@ export function migrate(target, migrations, options) {
                       dialect.ddl.dropTable(migrationStep.table),
                       dialect.ddl.renameTable(`${migrationStep.table}__rebuild`,
                         migrationStep.table),
-                      ...migrationStep.indexes,
-                      dialect.pragma.foreignKeyCheck());
+                      ...migrationStep.indexes);
+                    if (dialect.capabilities.foreignKeysAlwaysOn !== true)
+                      rendered.push(dialect.pragma.foreignKeyCheck());
                   }
                   else if (migrationStep.kind === 'jslt')
                     rendered.push(`-- jslt transform over '${migrationStep.collection}'`);
@@ -1707,6 +1770,8 @@ export function migrate(target, migrations, options) {
             const applied = [];
             const applyNext = (i) => {
               if (i >= pending.length) return null;
+              // the boundary between migrations
+              check();
               const migration = pending[i];
               const last = i === pending.length - 1;
               // the §10 procedure's pragma bracket, literally: the
@@ -1716,7 +1781,8 @@ export function migrate(target, migrations, options) {
               const bracket = migration.steps.some(
                 (candidate) => candidate.kind === 'rebuild');
               return chain(
-                bracket ? connection.exec(dialect.pragma.foreignKeys(false)) : null,
+                bracket && dialect.capabilities.foreignKeysAlwaysOn !== true
+                  ? connection.exec(dialect.pragma.foreignKeys(false)) : null,
                 () => chain(connection.exec(dialect.tx.beginImmediate), () => {
                 const body = () => chain(runSteps(connection, migration, runOptions), () =>
                   chain(last && options.model !== undefined
@@ -1733,10 +1799,11 @@ export function migrate(target, migrations, options) {
                       })))
                     : null,
                   () => chain(connection.prepare(statements.insert), (insert) =>
-                    insert.run([migration.id, Date.now(), migration.from,
+                    insert.run([migration.id, runtime.now(), migration.from,
                       migration.to, migrationChecksum(migration),
                       migration.steps.length]))));
                 const restore = () => (bracket
+                  && dialect.capabilities.foreignKeysAlwaysOn !== true
                   ? connection.exec(dialect.pragma.foreignKeys(true)) : null);
                 const commit = () => chain(connection.exec(dialect.tx.commit), () =>
                   chain(restore(), () => {