@quereus/quereus 4.3.2 → 4.4.1

package/dist/src/vtab/memory/layer/manager.js

@@ -1,4 +1,5 @@
 import { buildColumnIndexMap, columnDefToSchema, resolvePkDefaultConflict, resolveNamedConstraintClass, validateCollationForType } from '../../../schema/table.js';
+import { keyParts } from '../types.js';
 import { BTree } from 'inheritree';
 import { StatusCode } from '../../../common/types.js';
 import { BaseLayer, iteratePrimaryRows, populateIndexFromRows, populateIndexFromRowsAsync } from './base.js';
@@ -8,19 +9,83 @@ import { MemoryVirtualTableConnection } from '../connection.js';
 import { QuereusError } from '../../../common/errors.js';
 import { ConflictResolution } from '../../../common/constants.js';
 import { buildUniqueConstraintSchema, buildForeignKeyConstraintSchema, buildCheckConstraintSchema, validateForeignKeyOverExistingRows, maintainedTableUniqueViolationError } from '../../../schema/constraint-builder.js';
-import { indexEnforcesUnique, uniqueEnforcementCollations } from '../../../schema/unique-enforcement.js';
-import { compareSqlValues, compareSqlValuesFast, rowsValueIdentical, normalizeCollationName } from '../../../util/comparison.js';
+import { indexEnforcesUnique, uniqueEnforcementCollations, uniqueEnforcementComparators } from '../../../schema/unique-enforcement.js';
+import { generateIndexDDL, generateDropIndexDDL } from '../../../schema/ddl-generator.js';
+import { compareSqlValues, rowsValueIdentical, normalizeCollationName, comparisonSemanticsDiffer } from '../../../util/comparison.js';
 import { scanLayer as scanLayerImpl } from './scan-layer.js';
-import { createPrimaryKeyFunctions, buildPrimaryKeyFromValues } from '../utils/primary-key.js';
+import { createPrimaryKeyFunctions, buildPrimaryKeyFromValues, primaryKeyArity } from '../utils/primary-key.js';
 import { createMemoryTableLoggers } from '../utils/logging.js';
 import { tryFoldLiteral } from '../../../parser/utils.js';
-import { validateAndParse } from '../../../types/validation.js';
+import { validateAndParse, coerceRowToSchema } from '../../../types/validation.js';
 import { inferType } from '../../../types/registry.js';
 import { compilePredicate } from '../utils/predicate.js';
 import { renameColumnInIndexPredicates } from '../../../schema/rename-rewriter.js';
 import { MemoryIndex } from '../index.js';
+import { convertRowAtIndex, mapRows, mapRowsAsync } from './row-convert.js';
 let tableManagerCounter = 0;
 const logger = createMemoryTableLoggers('layer:manager');
+/** A copy of `items` with `value` inserted at `at` (`at === items.length` ⇒ append). */
+function insertValueAt(items, at, value) {
+    const out = items.slice();
+    out.splice(at, 0, value);
+    return out;
+}
+/** Index of a column after a new one is inserted at `at`. */
+const shiftForInsert = (index, at) => index >= at ? index + 1 : index;
+/**
+ * The index-bearing fields of `schema` renumbered for a column inserted at `at`, to be
+ * spread over the schema alongside the new `columns` array. Only reached when `at` is a
+ * real insert point (an append leaves every existing index valid), and the mirror image
+ * of the shift `dropColumn` performs — except that an insert can never empty an index or
+ * a constraint, so none of `dropColumn`'s prune/filter logic has a counterpart here.
+ *
+ * `checkConstraints` and the manager's implicit covering structures are name/AST-keyed
+ * and need nothing. `statistics.columnStats` is keyed by column name. `columnIndexMap` is
+ * rebuilt from the new column list by the caller.
+ *
+ * A foreign key's `columns` are indices into THIS table and always shift. Its
+ * `referencedColumns` are NOT touched: every constructor of a foreign key leaves that
+ * array empty (`constraint-builder.ts`, `schema/manager.ts`) and enforcement resolves the
+ * parent indices on demand from `referencedColumnNames` against the parent's CURRENT
+ * schema (`resolveReferencedColumns`) — so even a self-referential key needs no shift here.
+ *
+ * `derivation.logicalKey` / `derivation.ordering` also hold indices into this table and are
+ * NOT shifted, in either direction: they exist only on a maintained table (a materialized
+ * view's backing table), whose shape is defined by its body, and the emitter rejects every
+ * structural ALTER on one before dispatch.
+ */
+function shiftSchemaIndicesForInsert(schema, at) {
+    const shift = (index) => shiftForInsert(index, at);
+    const primaryKeyDefinition = schema.primaryKeyDefinition.map(def => ({ ...def, index: shift(def.index) }));
+    const indexes = schema.indexes?.map(idx => ({
+        ...idx,
+        columns: idx.columns.map(ic => ({ ...ic, index: shift(ic.index) })),
+    }));
+    const uniqueConstraints = schema.uniqueConstraints?.map(uc => ({
+        ...uc,
+        columns: Object.freeze(uc.columns.map(shift)),
+    }));
+    const foreignKeys = schema.foreignKeys?.map(fk => ({
+        ...fk,
+        columns: Object.freeze(fk.columns.map(shift)),
+    }));
+    // Generated-column bookkeeping is keyed BY column index and holds column indices.
+    // Nothing inside the memory module reads it — the engine recomputes the graph after
+    // its own ADD COLUMN — but a wrapper driving this API directly gets the schema back
+    // verbatim, so leaving it unshifted would hand out a schema whose generated-column
+    // edges point at the wrong columns.
+    const generatedColumnDependencies = schema.generatedColumnDependencies
+        && new Map(Array.from(schema.generatedColumnDependencies, ([col, deps]) => [shift(col), deps.map(shift)]));
+    const generatedColumnTopoOrder = schema.generatedColumnTopoOrder?.map(shift);
+    return {
+        primaryKeyDefinition: Object.freeze(primaryKeyDefinition),
+        ...(indexes && { indexes: Object.freeze(indexes) }),
+        ...(uniqueConstraints && { uniqueConstraints: Object.freeze(uniqueConstraints) }),
+        ...(foreignKeys && { foreignKeys: Object.freeze(foreignKeys) }),
+        ...(generatedColumnDependencies && { generatedColumnDependencies }),
+        ...(generatedColumnTopoOrder && { generatedColumnTopoOrder: Object.freeze(generatedColumnTopoOrder) }),
+    };
+}
 export class MemoryTableManager {
     managerId;
     db;
@@ -305,10 +370,10 @@ export class MemoryTableManager {
         // `baseLayer` by ALTER TABLE) or carries an out-of-date schema, leave
         // it alone — committing such a layer would supplant the schema-aware
         // committed head with stale data.
+        let refusedSnapshotWrap = false;
         if (!connection.pendingTransactionLayer
             && connection.readLayer !== this._currentCommittedLayer
-            && connection.readLayer instanceof TransactionLayer
-            && connection.readLayer.getSchema() === this.tableSchema) {
+            && connection.readLayer instanceof TransactionLayer) {
             let walker = connection.readLayer.getParent();
             let isAhead = false;
             while (walker) {
@@ -319,17 +384,32 @@ export class MemoryTableManager {
                 walker = walker.getParent();
             }
             if (isAhead) {
-                connection.pendingTransactionLayer = new TransactionLayer(connection.readLayer);
-                if (this.eventEmitter?.hasDataListeners?.()) {
-                    connection.pendingTransactionLayer.enableChangeTracking();
+                if (connection.readLayer.getSchema() === this.tableSchema) {
+                    connection.pendingTransactionLayer = new TransactionLayer(connection.readLayer);
+                    if (this.eventEmitter?.hasDataListeners?.()) {
+                        connection.pendingTransactionLayer.enableChangeTracking();
+                    }
+                }
+                else {
+                    // A snapshot AHEAD of the head whose schema object nonetheless predates the
+                    // manager's: some schema-mutating arm failed to hand its new TableSchema to
+                    // the open layers (`adoptSchemaOnOpenLayers` and friends). Refusing the wrap
+                    // is still the safe call, but it discards every row the transaction staged,
+                    // so flag it for the warning below rather than let COMMIT report success over
+                    // a silent data loss.
+                    refusedSnapshotWrap = true;
                 }
             }
         }
         const pendingLayer = connection.pendingTransactionLayer;
         if (!pendingLayer) {
+            if (refusedSnapshotWrap) {
+                logger.warn('Commit Transaction', this._tableName, 'Discarding staged rows: savepoint snapshot carries a schema object older than the table\'s. A schema-mutating arm did not adopt its new schema on the open transaction layers.', { schemaName: this.schemaName, connectionId: connection.connectionId });
+            }
             // No pending — refresh readLayer to the current committed head so a
             // stale ancestor (post-schema-change) doesn't leak into the next
-            // statement's view.
+            // statement's view. (The ordinary read-only / no-writes commit lands
+            // here too, which is why the warning above is gated.)
             connection.readLayer = this._currentCommittedLayer;
             return;
         }
@@ -382,6 +462,15 @@ export class MemoryTableManager {
                     // since the pending layer forked. Replaying stale-schema rows onto
                     // the new-schema head is unsafe — abort with BUSY rather than corrupt
                     // the committed head (mirrors the stale-ancestor caution above).
+                    //
+                    // NOTE: `adoptSchemaOnOpenLayers` only walks the DDL connection's own chain,
+                    // so a SIBLING connection's pending layer keeps its creation-time schema
+                    // object even after a metadata-only change like `RENAME TO` that moves no
+                    // rows — which would make this arm fire spuriously. Not reachable from SQL
+                    // today: `renameTable` never calls `ensureSchemaChangeSafety`, and a single
+                    // Database runs one transaction at a time. If sibling connections ever hold
+                    // concurrent pending layers over one manager, compare column shape here
+                    // rather than schema-object identity.
                     if (pendingLayer.getSchema() !== this.tableSchema) {
                         connection.pendingTransactionLayer = null;
                         connection.clearSavepoints();
@@ -415,12 +504,21 @@ export class MemoryTableManager {
             connection.clearSavepoints();
             // Emit data change events after successful commit
             if (changes.length > 0 && this.eventEmitter?.emitDataChange) {
+                // The event contract carries the PK as a component array. Derive the stored
+                // key's shape from the PK arity, never from `Array.isArray` — a JSON PK whose
+                // value is a document like `[1]` is a SCALAR key that happens to be a JS
+                // array, and sniffing it would emit `[1]` where the contract wants `[[1]]`.
+                // NOTE: on the tuple path this hands listeners the STORED key array itself
+                // (`VTableDataChangeEvent.key` is mutable `SqlValue[]`, as it always was).
+                // If a listener is ever seen mutating `event.key`, copy here — an in-place
+                // edit would re-order the primary tree under the comparator.
+                const eventKeyIsTuple = primaryKeyArity(this.tableSchema) !== 1;
                 for (const change of changes) {
                     const event = {
                         type: change.type,
                         schemaName: this.schemaName,
                         tableName: this._tableName,
-                        key: Array.isArray(change.pk) ? change.pk : [change.pk],
+                        key: keyParts(change.pk, eventKeyIsTuple),
                         oldRow: change.oldRow,
                         newRow: change.newRow,
                     };
@@ -640,7 +738,7 @@ export class MemoryTableManager {
         const result = primaryTree.get(key);
         return result === undefined ? null : result;
     }
-    async performMutation(connection, operation, values, oldKeyValues, onConflict) {
+    async performMutation(connection, operation, values, oldKeyValues, onConflict, preCoerced) {
         this.validateMutationPermissions(operation);
         const wasExplicitTransaction = connection.explicitTransaction;
         this.ensureTransactionLayer(connection);
@@ -648,10 +746,10 @@ export class MemoryTableManager {
         let result;
         switch (operation) {
             case 'insert':
-                result = await this.performInsert(targetLayer, values, onConflict);
+                result = await this.performInsert(targetLayer, values, onConflict, preCoerced);
                 break;
             case 'update':
-                result = await this.performUpdate(targetLayer, values, oldKeyValues, onConflict);
+                result = await this.performUpdate(targetLayer, values, oldKeyValues, onConflict, preCoerced);
                 break;
             case 'delete':
                 result = await this.performDelete(targetLayer, oldKeyValues);
@@ -692,20 +790,23 @@ export class MemoryTableManager {
             // transaction is auto-created (autocommit mode).  Leave explicitTransaction flag as-is.
         }
     }
-    async performInsert(targetLayer, values, onConflict) {
+    async performInsert(targetLayer, values, onConflict, preCoerced) {
         if (!values) {
             throw new QuereusError("INSERT requires values.", StatusCode.MISUSE);
         }
-        // Validate and parse values according to column types
+        // Validate and parse values according to column types. Skip when the caller
+        // already converted to declared form and says so via preCoerced — the DML
+        // executor (whose emitters convert by static type at the top of the pipeline)
+        // and the isolation overlay's flush/tombstone writes. JSON conversion is not
+        // idempotent, so re-running it can throw or silently change the value. Direct
+        // API callers leave the flag unset and this conversion remains their contract.
+        // NOTE: skipping it also skips that helper's "too many values" width guard. Every
+        // preCoerced caller today builds its row from this same schema's column order
+        // (the DML builders project the source into full table-column shape), so the
+        // width is structural; if an externally-shaped row ever reaches here preCoerced,
+        // hoist the width check out of coerceRowToSchema and run it unconditionally.
         const schema = targetLayer.getSchema();
-        const validatedRow = values.map((value, index) => {
-            if (index >= schema.columns.length) {
-                throw new QuereusError(`Too many values for INSERT into ${this._tableName}: expected ${schema.columns.length}, got ${values.length}`, StatusCode.ERROR);
-            }
-            const column = schema.columns[index];
-            return validateAndParse(value, column.logicalType, column.name);
-        });
-        const newRowData = validatedRow;
+        const newRowData = preCoerced ? values : coerceRowToSchema(values, schema.columns, `INSERT into ${this._tableName}`);
         const primaryKey = this.primaryKeyFromRow(newRowData);
         const existingRow = this.lookupEffectiveRow(primaryKey, targetLayer);
         if (existingRow !== null) {
@@ -736,20 +837,14 @@ export class MemoryTableManager {
         targetLayer.recordUpsert(primaryKey, newRowData, null);
         return { status: 'ok', row: newRowData, evictedRows: evicted.length > 0 ? evicted : undefined };
     }
-    async performUpdate(targetLayer, values, oldKeyValues, onConflict) {
+    async performUpdate(targetLayer, values, oldKeyValues, onConflict, preCoerced) {
         if (!values || !oldKeyValues) {
             throw new QuereusError("UPDATE requires new values and old key values.", StatusCode.MISUSE);
         }
-        // Validate and parse values according to column types
+        // Validate and parse values according to column types. Skip when preCoerced
+        // (see performInsert).
         const schema = targetLayer.getSchema();
-        const validatedRow = values.map((value, index) => {
-            if (index >= schema.columns.length) {
-                throw new QuereusError(`Too many values for UPDATE on ${this._tableName}: expected ${schema.columns.length}, got ${values.length}`, StatusCode.ERROR);
-            }
-            const column = schema.columns[index];
-            return validateAndParse(value, column.logicalType, column.name);
-        });
-        const newRowData = validatedRow;
+        const newRowData = preCoerced ? values : coerceRowToSchema(values, schema.columns, `UPDATE on ${this._tableName}`);
         const targetPrimaryKey = buildPrimaryKeyFromValues(oldKeyValues, schema.primaryKeyDefinition);
         const oldRowData = this.lookupEffectiveRow(targetPrimaryKey, targetLayer);
         if (!oldRowData) {
@@ -833,6 +928,14 @@ export class MemoryTableManager {
      * Returns true if any column covered by a UNIQUE constraint changed between
      * old and new rows, or if any column referenced by a partial-UNIQUE predicate
      * changed (which may transition the row into or out of the predicate's scope).
+     *
+     * NOTE: the per-column test is byte-level `compareSqlValues`, not the enforcement
+     * comparator, so it OVER-triggers for a semantic-ordering column: rewriting a
+     * TIMESPAN 'PT1H' to 'PT60M' reports "changed" and re-runs the UNIQUE check, which
+     * then excludes the row's own primary key and passes. Correct — this only gates
+     * whether to re-check — just not minimal. If UPDATE-heavy workloads over
+     * semantic-ordering UNIQUE columns ever show the redundant re-check as hot, route
+     * this through `uniqueEnforcementComparators` too.
      */
     uniqueColumnsChanged(schema, oldRow, newRow) {
         if (!schema.uniqueConstraints)
@@ -988,8 +1091,11 @@ export class MemoryTableManager {
         if (existingPKs.length === 0)
             return null;
         // Resolve the per-column enforcement collations once, ahead of the candidate loop
-        // (which collation governs, and why, is spelled out at the compare below).
+        // (which collation governs, and why, is spelled out at the compare below), then
+        // build the per-column comparators from them through the shared helper the store
+        // and isolation re-validators also use.
         const enforcementCollations = uc.columns.map((col, i) => this.collationResolver(index.specColumns[i]?.collation ?? schema.columns[col].collation ?? 'BINARY'));
+        const compares = uniqueEnforcementComparators(schema.columns, uc.columns, enforcementCollations);
         for (const existingPK of existingPKs) {
             if (this.comparePrimaryKeys(newPrimaryKey, existingPK) === 0)
                 continue;
@@ -1010,7 +1116,12 @@ export class MemoryTableManager {
             // the enforcing structure, and an explicit `create unique index …
             // (col collate nocase)` may declare a coarser collation than the
             // column — re-checking under the column's collation would skip the
-            // case-variant candidates the index legitimately unifies.
+            // case-variant candidates the index legitimately unifies. A
+            // semantic-ordering column (TIMESPAN, JSON) compares through its
+            // declared type's `compare` instead, so the re-check agrees with the
+            // typed BTree that produced the candidate — without it a 'PT60M' probe
+            // found the 'PT1H' row's PK and then discarded it as unequal, and the
+            // duplicate was admitted.
             // This is the authoritative LIVE-index source for the per-column
             // enforcement collation. The shared `uniqueEnforcementCollations(schema,
             // uc)` helper (which store/isolation import, and checkUniqueViaMaterializedView
@@ -1028,7 +1139,7 @@ export class MemoryTableManager {
             const conflictingRow = this.lookupEffectiveRow(existingPK, targetLayer);
             if (!conflictingRow)
                 continue;
-            if (!uc.columns.every((col, i) => compareSqlValuesFast(newRowData[col], conflictingRow[col], enforcementCollations[i]) === 0))
+            if (!uc.columns.every((col, i) => compares[i](newRowData[col], conflictingRow[col]) === 0))
                 continue;
             if (index.predicate && !index.rowMatchesPredicate(conflictingRow))
                 continue;
@@ -1072,7 +1183,13 @@ export class MemoryTableManager {
      * backing delete still lands within this statement.
      */
     async checkUniqueViaMaterializedView(targetLayer, schema, uc, mv, newRowData, newPrimaryKey, onConflict, evicted) {
-        const newSourcePk = Array.isArray(newPrimaryKey) ? newPrimaryKey : [newPrimaryKey];
+        // Component-array view of the source PK. Shape comes from the PK arity, never
+        // from `Array.isArray` — a single-column JSON PK holding a document like `[1]`
+        // is a scalar key that is also a JS array (see the `BTreeKey` invariant).
+        // A wrong shape here only WIDENS the candidate set (the row fails its own
+        // self-exclusion), and the loop below re-excludes self through the real PK
+        // comparator — which is why no query result can observe this line.
+        const newSourcePk = keyParts(newPrimaryKey, primaryKeyArity(schema) !== 1);
         const conflicts = await this.db._lookupCoveringConflicts(mv, uc, newRowData, newSourcePk);
         // Re-validate under each column's enforcement collation — the index's per-column
         // COLLATE for an index-derived UNIQUE, else the declared column collation
@@ -1084,7 +1201,11 @@ export class MemoryTableManager {
         // index-derived UNIQUE whose declared candidate set could be a subset is declined
         // upstream by findRowTimeCoveringStructure's collation gate, so only BINARY-floor
         // or equal-collation MVs ever reach here.
+        // A semantic-ordering column (TIMESPAN, JSON) re-checks through its declared
+        // type's `compare` rather than the collation — same rule the index path and both
+        // out-of-package re-validators follow, via the shared helper.
         const collations = uniqueEnforcementCollations(schema, uc).map(name => this.collationResolver(name ?? 'BINARY'));
+        const compares = uniqueEnforcementComparators(schema.columns, uc.columns, collations);
         for (const conflict of conflicts) {
             const existingPK = buildPrimaryKeyFromValues(conflict.pk, schema.primaryKeyDefinition);
             if (this.comparePrimaryKeys(newPrimaryKey, existingPK) === 0)
@@ -1093,7 +1214,7 @@ export class MemoryTableManager {
             const conflictingRow = this.lookupEffectiveRow(existingPK, targetLayer);
             if (!conflictingRow)
                 continue;
-            if (!uc.columns.every((col, i) => compareSqlValuesFast(newRowData[col], conflictingRow[col], collations[i]) === 0))
+            if (!uc.columns.every((col, i) => compares[i](newRowData[col], conflictingRow[col]) === 0))
                 continue;
             if (onConflict === ConflictResolution.IGNORE) {
                 return { status: 'ok', row: undefined };
@@ -1123,8 +1244,19 @@ export class MemoryTableManager {
         const predicate = uc.predicate
             ? compilePredicate(uc.predicate, schema.columns, schema.name)
             : undefined;
-        // One resolve per column, not per scanned row.
+        // One resolve per column, not per scanned row. A semantic-ordering column
+        // (TIMESPAN, JSON) compares through its declared type's `compare` instead of the
+        // collation — the shared rule across all three backends.
+        //
+        // NOTE: the collations come from the DECLARED column collation, not from
+        // `uniqueEnforcementCollations(schema, uc)` as the index and MV paths use. For an
+        // index-derived UNIQUE (`create unique index … collate x`) the two disagree — this
+        // path would enforce under the column's collation instead of the index's. Harmless
+        // today: this fallback fires only when no covering structure resolves at all, and
+        // an index-derived UC always resolves its own index. If a shape ever reaches here
+        // with `uc.derivedFromIndex` set, switch this line to the shared resolver.
         const collations = uc.columns.map(colIdx => this.collationResolver(schema.columns[colIdx].collation ?? 'BINARY'));
+        const compares = uniqueEnforcementComparators(schema.columns, uc.columns, collations);
         for (const path of primaryTree.ascending(primaryTree.first())) {
             const existingRow = primaryTree.at(path);
             const existingPK = this.primaryKeyFromRow(existingRow);
@@ -1132,7 +1264,7 @@ export class MemoryTableManager {
                 continue;
             if (predicate && predicate.evaluate(existingRow) !== true)
                 continue;
-            const allMatch = uc.columns.every((colIdx, i) => compareSqlValuesFast(newRowData[colIdx], existingRow[colIdx], collations[i]) === 0);
+            const allMatch = uc.columns.every((colIdx, i) => compares[i](newRowData[colIdx], existingRow[colIdx]) === 0);
             if (!allMatch)
                 continue;
             if (onConflict === ConflictResolution.IGNORE) {
@@ -1156,10 +1288,25 @@ export class MemoryTableManager {
     }
     renameTable(newName) {
         logger.operation('Rename Table', this._tableName, { newName });
-        this._tableName = newName;
         const renamed = Object.freeze({ ...this.tableSchema, name: newName });
+        // Re-key the Database connection registry FIRST, while it still answers to the old
+        // name. Every registry lookup below (`adoptSchemaOnOpenLayers` walks
+        // `openTransactionLayersOldestFirst` -> `ddlConnection` -> `registeredConnections`)
+        // keys on `<schema>.<current name>`, so a sweep run after `_tableName` moves would
+        // match nothing and silently no-op.
+        this.rekeyRegisteredConnections(this._tableName, newName);
+        this._tableName = newName;
         this.tableSchema = renamed;
         this.baseLayer.tableSchema = renamed;
+        // Hand the renamed schema to the open transaction's layers, like every other
+        // schema-mutating arm. A rename rebuilds no `IndexSchema`, so `adoptSchema` is the
+        // right (cheapest) level — each layer keeps its `MemoryIndex` and only swaps the
+        // schema pointer; the reshape pair ADD/DROP COLUMN need would be pure overhead.
+        // Skipping it leaves an eager savepoint snapshot on its frozen pre-rename schema
+        // object, which fails `commitTransaction`'s snapshot-wrap identity check
+        // (`readLayer.getSchema() === this.tableSchema`) and drops the transaction's staged
+        // rows at COMMIT.
+        this.adoptSchemaOnOpenLayers(renamed);
         // Emit schema change event
         this.eventEmitter?.emitSchemaChange?.({
             type: 'alter',
@@ -1453,7 +1600,15 @@ export class MemoryTableManager {
         }
     }
     // --- Schema Operations (simplified with inherited BTrees) ---
-    async addColumn(columnDefAst, backfillEvaluator) {
+    /**
+     * ADD COLUMN. `insertAtIndex` — when supplied — puts the new column at that slot
+     * instead of appending; every existing column at or after it shifts right by one,
+     * and each index-bearing schema field is renumbered to match (see
+     * {@link shiftSchemaIndicesForInsert}). Omitted ⇒ append, which is what SQL
+     * `alter table … add column` always asks for and which leaves the schema rebuild
+     * on exactly the path it took before positions existed.
+     */
+    async addColumn(columnDefAst, backfillEvaluator, insertAtIndex) {
         if (this.isReadOnly)
             throw new QuereusError(`Table '${this._tableName}' is read-only`, StatusCode.READONLY);
         const lockKey = `MemoryTable.SchemaChange:${this.schemaName}.${this._tableName}`;
@@ -1498,20 +1653,76 @@ export class MemoryTableManager {
                 throw new QuereusError(`Cannot add NOT NULL column '${newColumnSchema.name}' to non-empty table `
                     + `'${this.schemaName}.${this._tableName}' without a DEFAULT value`, StatusCode.CONSTRAINT);
             }
-            const updatedColumnsSchema = Object.freeze([...this.tableSchema.columns, newColumnSchema]);
+            const appendIndex = this.tableSchema.columns.length;
+            const targetIndex = insertAtIndex ?? appendIndex;
+            if (!Number.isInteger(targetIndex) || targetIndex < 0 || targetIndex > appendIndex) {
+                throw new QuereusError(`Cannot add column '${newColumnSchema.name}' at position ${insertAtIndex}: `
+                    + `expected an integer in [0, ${appendIndex}]`, StatusCode.MISUSE);
+            }
+            const updatedColumnsSchema = Object.freeze(insertValueAt(this.tableSchema.columns, targetIndex, newColumnSchema));
+            // Appending never invalidates an existing column index, so the append path keeps
+            // carrying every index-bearing field through the spread unchanged; only a real
+            // insert pays for the renumber.
+            const shiftedIndexFields = targetIndex < appendIndex
+                ? shiftSchemaIndicesForInsert(this.tableSchema, targetIndex)
+                : undefined;
             const finalNewTableSchema = Object.freeze({
                 ...this.tableSchema,
+                ...shiftedIndexFields,
                 columns: updatedColumnsSchema,
                 columnIndexMap: buildColumnIndexMap(updatedColumnsSchema),
             });
+            // Phase 1 of reaching the open transaction's own pending rows: compute their
+            // post-ADD shape before ANY mutation, since the per-row backfill below can throw
+            // on a pending row (and there is no undo for a half-reshaped layer chain).
+            //
+            // The NOT NULL check mirrors the base backfill's per-row check in
+            // `BaseLayer.recreatePrimaryTreeWithNewColumn`, extended to pending rows. It is
+            // deliberately NOT gated on `backfillEvaluator`, which today makes the
+            // literal-default arm redundant: `validateNotNullBackfill`
+            // (runtime/emit/alter-table.ts) already rejects NOT NULL without a usable DEFAULT
+            // by querying the DDL connection's EFFECTIVE rows — pending ones included — so the
+            // only violation that reaches here is a per-row evaluator yielding NULL. Kept
+            // ungated so the module enforces its own invariant instead of relying on that gate.
+            const reshapePlans = await this.prepareReshapeOnOpenLayers(async (row) => {
+                const value = backfillEvaluator ? await backfillEvaluator(row) : defaultValue;
+                if (newColumnSchema.notNull && value === null) {
+                    throw new QuereusError(`NOT NULL constraint failed: adding column '${this._tableName}.${newColumnSchema.name}' would leave NULL in a row pending in the open transaction`, StatusCode.CONSTRAINT);
+                }
+                return insertValueAt(row, targetIndex, value);
+            }, 
+            // Event-log variant: same map applied to the log's historical images (oldRow
+            // included — the evaluator is a function of a row and the pre-image is a row),
+            // but BEST-EFFORT: an image the evaluator (or its CHECKs) rejects gets NULL —
+            // the honest "column did not exist yet" placeholder — and the NOT NULL throw
+            // above is omitted, so no historical image can abort the ALTER.
+            async (row) => {
+                let value = defaultValue;
+                if (backfillEvaluator) {
+                    try {
+                        value = await backfillEvaluator(row);
+                    }
+                    catch {
+                        value = null;
+                    }
+                }
+                return insertValueAt(row, targetIndex, value);
+            });
             this.baseLayer.updateSchema(finalNewTableSchema);
             // A non-foldable DEFAULT (e.g. `new.<col>`) backfills each existing row from
             // its own value via the engine-supplied evaluator; a literal/NULL default
             // uses the single folded `defaultValue` for every row.
-            await this.baseLayer.addColumnToBase(newColumnSchema, defaultValue, backfillEvaluator);
+            await this.baseLayer.addColumnToBase(newColumnSchema, defaultValue, backfillEvaluator, targetIndex);
             this.tableSchema = finalNewTableSchema;
             this.initializePrimaryKeyFunctions();
-            // Emit schema change event
+            this.installReshapeOnOpenLayers(finalNewTableSchema, reshapePlans);
+            // Emit schema change event.
+            // NOTE: the event names the added column but not its position, which is fine while
+            // the only non-append caller is an in-process wrapper that rebuilds its view from the
+            // post-change schema. An append never invalidates an existing column index; an insert
+            // does. If a position ever becomes SQL-reachable, anything that caches a column index
+            // across this swap (planner caches, prepared statements, a wrapper's own schema
+            // snapshot) needs to learn the insert point — so the event would have to carry it.
             this.eventEmitter?.emitSchemaChange?.({
                 type: 'alter',
                 objectType: 'column',
@@ -1583,6 +1794,28 @@ export class MemoryTableManager {
                     .filter(ic => ic.index !== colIndex)
                     .map(ic => ({ ...ic, index: ic.index > colIndex ? ic.index - 1 : ic.index }))
             })).filter(idx => idx.columns.length > 0);
+            // Shift or prune the table's own foreign keys. `foreignKeys[].columns` are indices
+            // into THIS table, so they renumber over the removed slot exactly as the PK and the
+            // UNIQUE constraints above do. A foreign key that loses ANY of its child columns is
+            // removed outright, the same call the UNIQUE pruning makes: a key missing one of its
+            // child columns is a different constraint against the parent's key, not a narrowed
+            // one. Both halves matter — left unshifted, a surviving key's index either dangles
+            // past the end of the column array or, worse, silently slides onto an unrelated
+            // column and starts enforcing against the parent there.
+            //
+            // `referencedColumns` is not touched: enforcement resolves the parent indices on
+            // demand from `referencedColumnNames` against the parent's current schema. See
+            // `shiftSchemaIndicesForInsert`, the ADD-COLUMN-at-a-position mirror of this block.
+            const remainingForeignKeys = (this.tableSchema.foreignKeys ?? [])
+                .filter(fk => !fk.columns.includes(colIndex))
+                .map(fk => ({ ...fk, columns: Object.freeze(fk.columns.map(i => i > colIndex ? i - 1 : i)) }));
+            // NOTE: the generated-column bookkeeping (`generatedColumnDependencies` /
+            // `generatedColumnTopoOrder`, also index-keyed) is likewise unshifted, but the
+            // engine recomputes it from column names right after this returns
+            // (`withGeneratedColumnGraph` in `runDropColumn`), so only a caller driving the
+            // module API directly ever observes the stale map.
+            //
+            // `primaryKey` below is not a TableSchema field at all — nothing reads it.
             const finalNewTableSchema = Object.freeze({
                 ...this.tableSchema,
                 columns: Object.freeze(updatedColumnsSchema),
@@ -1593,7 +1826,16 @@ export class MemoryTableManager {
                 uniqueConstraints: remainingUniqueConstraints.length > 0
                     ? Object.freeze(remainingUniqueConstraints)
                     : undefined,
+                foreignKeys: remainingForeignKeys.length > 0
+                    ? Object.freeze(remainingForeignKeys)
+                    : undefined,
             });
+            // Phase 1 of reaching the open transaction's own pending rows. The rewrite here is
+            // a pure filter and cannot throw — the split exists for ADD COLUMN's fallible
+            // backfill — but running it before any mutation keeps the two paths uniform. The
+            // same filter reshapes the pending-change event log (second arg).
+            const dropSlot = (row) => row.filter((_, idx) => idx !== colIndex);
+            const reshapePlans = await this.prepareReshapeOnOpenLayers(dropSlot, dropSlot);
             this.baseLayer.updateSchema(finalNewTableSchema);
             await this.baseLayer.dropColumnFromBase(colIndex);
             this.tableSchema = finalNewTableSchema;
@@ -1602,6 +1844,7 @@ export class MemoryTableManager {
             for (const key of droppedUcKeys)
                 this.implicitCoveringStructures.delete(key);
             this.initializePrimaryKeyFunctions();
+            this.installReshapeOnOpenLayers(finalNewTableSchema, reshapePlans);
             // Emit schema change event
             this.eventEmitter?.emitSchemaChange?.({
                 type: 'drop',
@@ -1685,6 +1928,20 @@ export class MemoryTableManager {
             await this.baseLayer.handleColumnRename();
             this.tableSchema = finalNewTableSchema;
             this.initializePrimaryKeyFunctions();
+            // The DDL transaction's own layers froze their schema at creation; hand them the
+            // renamed one, mirroring `handleColumnRename`'s base-side secondary rebuild —
+            // `updatedIndexes` above rebuilt every `IndexSchema` object, which is exactly the
+            // identity discriminator `adoptSchema` rebuilds a layer's `MemoryIndex` on.
+            //
+            // A rename touches neither the column set nor the primary key, so `adoptSchema` is
+            // the right level (not the `prepareReshapeOnOpenLayers` / `installReshapeOnOpenLayers`
+            // pair `add column` / `drop column` need) — but it is just as mandatory: without it an
+            // eager savepoint snapshot keeps its frozen pre-rename schema, fails
+            // `commitTransaction`'s snapshot-wrap identity check
+            // (`readLayer.getSchema() === this.tableSchema`), and the transaction's staged rows are
+            // silently dropped at COMMIT. Covered by
+            // `test/logic/41.8-alter-savepoint-staged-rows.sqllogic`.
+            this.adoptSchemaOnOpenLayers(finalNewTableSchema);
             // Emit schema change event
             this.eventEmitter?.emitSchemaChange?.({
                 type: 'alter',
@@ -1738,12 +1995,23 @@ export class MemoryTableManager {
             // A collation change re-keys any PK / UNIQUE / index that orders by this
             // column, so it needs the structure re-sort + uniqueness re-validation below.
             let collationChanged = false;
+            // Set when a SET DATA TYPE moves to a type that ORDERS values differently
+            // (comparisonSemanticsDiffer). Same consequence as a collation change — every
+            // structure keyed by the column has to be re-sorted and its uniqueness re-judged —
+            // but a separate flag, because the index-column collation propagation and the
+            // `pkColumnRekeyed` primary-tree path below belong to SET COLLATE alone. Usually
+            // coincides with `valueConvert` (every non-alias retype sets that); the rebuild
+            // chains below test `valueConvert` FIRST, whose full-rebuild path subsumes the
+            // comparator-only re-sort.
+            let comparatorChanged = false;
             // Set to the per-value conversion function when a physical value rewrite is needed:
-            // SET DATA TYPE (retype every value) or SET NOT NULL backfill (map null → DEFAULT).
-            // Both REPLACE the base primary tree with a freshly-built one (never an in-place
-            // mutation, which inheritree forbids while the open transaction's layers derive from
-            // it) and convert the transaction's own-written rows in every open layer, after
-            // `updateSchema` (see the rebuild section and convertColumnOnOpenLayers).
+            // SET DATA TYPE (every non-alias retype — values normalize to the new type's
+            // canonical spelling even when the storage class is unchanged) or SET NOT NULL
+            // backfill (map null → DEFAULT). Both REPLACE the base primary tree with a
+            // freshly-built one (never an in-place mutation, which inheritree forbids while the
+            // open transaction's layers derive from it) and convert the transaction's
+            // own-written rows in every open layer, after `updateSchema` (see the rebuild
+            // section and convertColumnOnOpenLayers).
             let valueConvert = null;
             // SET NOT NULL backfill also routes NULL values through `valueConvert`; SET DATA TYPE
             // leaves nulls untouched. Selects the null-including path in convertBaseRows/convertColumn.
@@ -1821,19 +2089,40 @@ export class MemoryTableManager {
             }
             else if (change.setDataType !== undefined) {
                 const newLogicalType = inferType(change.setDataType);
-                if (newLogicalType.physicalType === oldCol.logicalType.physicalType) {
-                    newCol = { ...oldCol, logicalType: newLogicalType };
-                }
-                else {
-                    // Retyping a PRIMARY KEY column changes the physical key bytes. The in-place
-                    // base rewrite below cannot re-key the primary tree (an upsert of a row whose
-                    // PK value changed physical type lands at a NEW key, orphaning the old entry),
-                    // and the transaction-layer conversion deliberately leaves every layer's primary
-                    // tree keyed as-is. Reject rather than corrupt it — the type-change analogue of
-                    // the SET COLLATE primary-key carve-out (`alter-collate-pk-in-transaction`).
+                // Gate on logical-type IDENTITY, not the physical storage class: `inferType`
+                // flattens aliases to the shared type object (`varchar(50)` IS `TEXT_TYPE`,
+                // `bigint` IS `INTEGER_TYPE`), so an alias retype is schema-only — no scan, no
+                // rewrite, no re-key. Every OTHER retype validates AND normalizes every value,
+                // whether or not the storage class changes: a same-class move (text → date)
+                // must reject values the new type refuses ('hello') and rewrite the rest to
+                // the new type's canonical spelling ('2024-06-05T00:00:00Z' → '2024-06-05'),
+                // or the column holds values no INSERT could have produced — unfindable by
+                // equality (DATE compares BINARY over the stored text) and invisible to UNIQUE.
+                // NOTE: the rewrite is unconditional once the types differ, even when every
+                // converted value comes back byte-identical (already-canonical ISO durations on a
+                // text → timespan). That costs a full base-tree + secondary-index rebuild here, and
+                // a physical rewrite of every row on the store leg. If ALTER on large tables ever
+                // shows up as slow, have the convert pre-pass record whether any value actually
+                // moved and, when none did, fall through to the comparator-only re-key.
+                if (newLogicalType !== oldCol.logicalType) {
+                    // Retyping a PRIMARY KEY column moves the key bytes (a class change), the
+                    // tree's ordering (a comparator move), or at minimum the key values'
+                    // canonical spelling — and neither the base rewrite below nor the
+                    // transaction-layer conversion re-keys the primary tree. The engine already
+                    // refuses SET DATA TYPE on any PK column (runtime/emit/alter-table.ts);
+                    // this guards direct module calls. Reject rather than corrupt — the
+                    // type-change analogue of the SET COLLATE primary-key carve-out
+                    // (`alter-collate-pk-in-transaction`).
                     if (this.tableSchema.primaryKeyDefinition.some(def => def.index === colIndex)) {
                         throw new QuereusError(`Cannot change the data type of primary key column '${change.columnName}' of table '${this._tableName}'.`, StatusCode.CONSTRAINT);
                     }
+                    // Every MemoryIndex builds its comparator from the column's LOGICAL type
+                    // (`createTypedComparator(columnSchema.logicalType, …)` in vtab/memory/index.ts),
+                    // so a move to a differently-ordering type (text ↔ timespan: 'PT1H' ≡ 'PT60M';
+                    // text ↔ date/time/datetime: binary, ignoring the column's collation) re-sorts
+                    // every structure keyed by the column. Tracked separately from the value rewrite
+                    // because SET COLLATE shares the re-key machinery without rewriting values.
+                    comparatorChanged = comparisonSemanticsDiffer(oldCol.logicalType, newLogicalType);
                     const convert = (v) => {
                         try {
                             return validateAndParse(v, newLogicalType, change.columnName);
@@ -1868,8 +2157,8 @@ export class MemoryTableManager {
                     // derive from that tree (`MutatedBaseError`). `valueConvert` drives both the base
                     // replacement and the transaction-layer conversion.
                     valueConvert = convert;
-                    newCol = { ...oldCol, logicalType: newLogicalType };
                 }
+                newCol = { ...oldCol, logicalType: newLogicalType };
             }
             else if (change.setDefault !== undefined) {
                 newCol = { ...oldCol, defaultValue: change.setDefault };
@@ -1878,15 +2167,30 @@ export class MemoryTableManager {
                 throw new QuereusError('ALTER COLUMN requires an attribute to change', StatusCode.INTERNAL);
             }
             const updatedCols = this.tableSchema.columns.map((c, i) => i === colIndex ? newCol : c);
+            // The changes that move the COMPARATOR the structures are keyed by: SET COLLATE
+            // (values untouched) and a retype into a differently-ordering type (which, unless it
+            // is an alias retype, ALSO rewrites values — `valueConvert` — and its full-rebuild
+            // path below subsumes the comparator-only re-sort). Both need fresh IndexSchema
+            // objects and, when no rewrite runs, the re-validate / re-sort / adopt sequence.
+            const structuresRekeyed = collationChanged || comparatorChanged;
             // Propagate a collation change into every PK-definition entry and index
             // column that orders by this column, so their comparators re-key under it.
             const updatedPkDef = collationChanged
                 ? this.tableSchema.primaryKeyDefinition.map(def => def.index === colIndex ? { ...def, collation: newCol.collation } : def)
                 : this.tableSchema.primaryKeyDefinition;
-            const updatedIndexes = (collationChanged && this.tableSchema.indexes)
+            // Every re-keying change also REBUILDS the IndexSchema objects, because their identity is
+            // the discriminator `TransactionLayer.adoptSchema` uses to decide an open layer's
+            // MemoryIndex must be replaced rather than kept (an index it holds under an object the new
+            // schema still carries is left alone). SET COLLATE has a field to write — the index
+            // column's own collation. A comparator-only retype has none: an index column carries no
+            // logical type, it reads the column's from `newSchema.columns`, so the fresh object IS the
+            // whole signal. Without it the base rebuilt its trees under the new comparator while the
+            // DDL transaction's own layers went on comparing the old way — and shadowed the base's
+            // rebuilt trees once the pending layer became the committed head.
+            const updatedIndexes = (structuresRekeyed && this.tableSchema.indexes)
                 ? this.tableSchema.indexes.map(idx => ({
                     ...idx,
-                    columns: idx.columns.map(ic => ic.index === colIndex ? { ...ic, collation: newCol.collation } : ic),
+                    columns: idx.columns.map(ic => (ic.index === colIndex && collationChanged) ? { ...ic, collation: newCol.collation } : ic),
                 }))
                 : this.tableSchema.indexes;
             const finalNewTableSchema = Object.freeze({
@@ -1898,14 +2202,49 @@ export class MemoryTableManager {
             });
             const pkColumnRekeyed = collationChanged && updatedPkDef.some(def => def.index === colIndex);
             // Validate BEFORE any mutation, over the DDL transaction's EFFECTIVE rows: a pair
-            // the transaction inserted that collides under the NEW collation must reject the
+            // the transaction inserted that collides under the NEW schema must reject the
             // change, and one it has deleted must not block it. A throw here leaves the schema,
             // the base layer and the index map exactly as they were.
             //
-            // The primary key gets a stricter pre-pass (`validateRekeyedPrimaryKey`): no layer in
-            // the chain, base included, may hold a collision. That is what lets every step below
-            // succeed unconditionally, and what `TransactionLayer.rekeyPrimaryKey` relies on.
-            if (collationChanged) {
+            // A value-REWRITING change (`valueConvert`: SET DATA TYPE, or a SET NOT NULL null →
+            // DEFAULT backfill) probes the CONVERTED rows — the same per-row conversion the base
+            // rewrite below applies — under the NEW schema's comparators, so one call covers both
+            // ways an ALTER can collapse two previously-distinct rows: the VALUES converge
+            // (`'1'`/`'01'` → 1; two NULLs → one DEFAULT; '2024-06-05T00:00:00Z' → '2024-06-05')
+            // and/or the COMPARATOR moves (text → timespan makes 'PT1H'/'PT60M' one value). A
+            // same-storage-class retype sets `valueConvert` AND `structuresRekeyed`, so the
+            // rewrite arm is tested FIRST — its converted-row probe subsumes the comparator-only
+            // one. The rebuild that follows is deliberately non-enforcing, so this is the only
+            // guard.
+            //
+            // The comparator-MOVING arm without a rewrite has one trigger left: SET COLLATE. The
+            // values are untouched — hence no `mapRow` — and only the comparator the structures
+            // are keyed by moves. `newSchema` carries the new collations, so the probe indexes
+            // compare as the rebuilt ones will.
+            //
+            // The primary key: SET COLLATE on a PK member gets a stricter pre-pass
+            // (`validateRekeyedPrimaryKey`): no layer in the chain, base included, may hold a
+            // collision. That is what lets every step below succeed unconditionally, and what
+            // `TransactionLayer.rekeyPrimaryKey` relies on. The rewrite arm needs no PK pre-pass
+            // and cannot shadow one: a retype of a PK column is rejected upstream, the backfill
+            // only fires on a column holding NULLs (which a PK member cannot — the engine
+            // enforces NOT NULL on every PK member regardless of the declared nullability), and
+            // `pkColumnRekeyed` is set only by SET COLLATE, which never sets `valueConvert` — so
+            // whenever `pkColumnRekeyed` is true, the `structuresRekeyed` arm still runs.
+            // NOTE: if PK members ever become genuinely nullable, the backfill arm gains a PK
+            // collision path (two NULL keys → one DEFAULT) and needs `validateRekeyedPrimaryKey`
+            // plus a primary-tree re-key of its own.
+            if (valueConvert) {
+                const convert = valueConvert;
+                const nulls = convertNulls;
+                await this.validateRekeyedUniqueStructures(finalNewTableSchema, colIndex, rows, 
+                // Unlike `convertBaseRows`, a conversion failure here is NOT swallowed. Every row
+                // the probe sees is visible to the transaction, and the setDataType pre-pass above
+                // already proved each one convertible, so a throw is unreachable; surfacing it as
+                // MISMATCH still beats probing a stale value that could mask a collision.
+                row => convertRowAtIndex(row, colIndex, convert, nulls));
+            }
+            else if (structuresRekeyed) {
                 await this.validateRekeyedUniqueStructures(finalNewTableSchema, colIndex, rows);
                 // NOTE: `validateRekeyedPrimaryKey` deliberately ignores `rows`. It asserts that no
                 // LAYER of this manager's own chain holds a PK collision under the new comparator —
@@ -1919,27 +2258,23 @@ export class MemoryTableManager {
                     this.validateRekeyedPrimaryKey(finalNewTableSchema);
             }
             this.baseLayer.updateSchema(finalNewTableSchema);
-            // A collation change re-sorts the structures that order by the column. The
-            // secondary rebuild is NON-enforcing (the pre-pass above owns uniqueness, and the
-            // base's rows are not a subset of the effective rows); the primary tree rebuild is
-            // strict, since a PK collision cannot be represented at all — the pre-pass has
-            // already proved the base collision-free, so the strict rebuild is a live invariant
-            // check, not the enforcement path. It runs LAST so its throw leaves the live tree
-            // intact for the catch's rollback.
-            if (collationChanged) {
-                this.baseLayer.rebuildAllSecondaryIndexes();
-                if (pkColumnRekeyed) {
-                    basePrimaryTreeBeforeRekey = this.baseLayer.primaryTree;
-                    this.baseLayer.rebuildPrimaryTreeStrict();
-                }
-            }
-            else if (valueConvert) {
+            // Both arms rebuild the base's structures under the schema `updateSchema` above just
+            // swapped in — each MemoryIndex rebuilds its comparator from the CURRENT schema's
+            // collation and logical type. The secondary rebuilds are NON-enforcing (the pre-pass
+            // above owns uniqueness, and the base's rows are not a subset of the effective rows);
+            // the SET COLLATE PK rebuild is strict, since a PK collision cannot be represented at
+            // all — the pre-pass has already proved the base collision-free, so the strict
+            // rebuild is a live invariant check, not the enforcement path. It runs LAST so its
+            // throw leaves the live tree intact for the catch's rollback.
+            if (valueConvert) {
                 // SET DATA TYPE / SET NOT NULL backfill: convert the committed base rows and REPLACE
                 // the primary tree with a fresh one holding them (not an in-place upsert — inheritree
                 // forbids mutating a base while the open transaction's layers derive from it).
                 // `rebuildPrimaryTreeFromRows` also rebuilds every secondary index from the converted
-                // rows, so index-backed lookups see the new values. The old tree is saved for the
-                // catch's rollback, exactly as the PK-rekey path does.
+                // rows under the new comparators, so index-backed lookups see the new values — which
+                // is why this arm runs first and subsumes `rebuildAllSecondaryIndexes` when a
+                // same-storage-class retype sets `structuresRekeyed` too. The old tree is saved for
+                // the catch's rollback, exactly as the PK-rekey path does.
                 // NOTE: rebuilds EVERY secondary index, not only those covering the altered column;
                 // if a wide-index table ever shows this as slow, filter to indexes whose columns
                 // include colIndex. Mirrors the collationChanged path's unconditional rebuild.
@@ -1954,31 +2289,56 @@ export class MemoryTableManager {
                 basePrimaryTreeBeforeRekey = this.baseLayer.primaryTree;
                 this.baseLayer.rebuildPrimaryTreeFromRows(convertedBaseRows);
             }
+            else if (structuresRekeyed) {
+                this.baseLayer.rebuildAllSecondaryIndexes();
+                if (pkColumnRekeyed) {
+                    basePrimaryTreeBeforeRekey = this.baseLayer.primaryTree;
+                    this.baseLayer.rebuildPrimaryTreeStrict();
+                }
+            }
             this.tableSchema = finalNewTableSchema;
             this.initializePrimaryKeyFunctions();
             // The base rebuild handed every secondary index a fresh tree under the new
-            // collation; the DDL transaction's own layers still inherit the old ones and froze
+            // comparator; the DDL transaction's own layers still inherit the old ones and froze
             // the old schema at construction. Re-key them, or the rest of the transaction — and
             // everything after the pending layer becomes the committed head at commit — keeps
-            // comparing under the old collation.
+            // comparing under the old collation / old logical type.
             //
             // When the altered column is part of the primary key, `rebuildPrimaryTreeStrict` also
             // swapped the base primary tree object out from under those layers' copy-on-write
             // bases and invalidated their `pkFunctions`; `rekeyPrimaryKey` rebuilds both, plus
             // every secondary index (each derives its PK comparator/encoder from the PK
             // definition). Outside a transaction there are no open layers and both are no-ops.
-            if (collationChanged) {
-                this.adoptSchemaOnOpenLayers(finalNewTableSchema, pkColumnRekeyed);
-            }
-            else if (valueConvert) {
+            if (valueConvert) {
                 // SET DATA TYPE / SET NOT NULL backfill converted the base above; the DDL
                 // transaction's own open layers still froze the old schema and hold their own-written
                 // rows unconverted. Swap the schema and convert those values so the rest of the
                 // transaction — and the committed head at commit — read the new value. For SET NOT
                 // NULL this fills the transaction's OWN pending NULL rows (which live in the pending
-                // layer, never reached by the old in-place base upsert). No-op in autocommit.
+                // layer, never reached by the old in-place base upsert). `convertColumn` installs the
+                // new schema and rebuilds every one of the layer's secondary indexes from it, so it
+                // subsumes `adoptSchema` when a same-storage-class retype moved the comparators too.
+                // No-op in autocommit.
                 this.convertColumnOnOpenLayers(finalNewTableSchema, colIndex, valueConvert, convertNulls);
             }
+            else if (structuresRekeyed) {
+                this.adoptSchemaOnOpenLayers(finalNewTableSchema, pkColumnRekeyed);
+            }
+            else {
+                // The remaining changes (SET DEFAULT, DROP NOT NULL, a SET NOT NULL that needed no
+                // backfill, a retype between aliases of one logical type) touch nothing a layer
+                // DERIVES — its index set, its `uniqueConstraints` enforcement and its MemoryIndex
+                // comparators all stay put, and DEFAULT application / NOT NULL enforcement happen
+                // above the module, off the catalog schema. But the frozen schema OBJECT is itself
+                // observable: `commitTransaction` only wraps a swapped-in savepoint snapshot back
+                // into the committed chain when `readLayer.getSchema() === this.tableSchema`. Left
+                // stale, a `rollback to savepoint` taken across one of these ALTERs restores a
+                // snapshot the wrap then skips, and the transaction's staged rows are silently
+                // dropped at COMMIT. Adoption is cheap here — every IndexSchema object is carried
+                // over unchanged, so `adoptSchema` keeps each layer's MemoryIndex and only swaps
+                // the schema reference.
+                this.adoptSchemaOnOpenLayers(finalNewTableSchema);
+            }
             this.eventEmitter?.emitSchemaChange?.({
                 type: 'alter',
                 objectType: 'column',
@@ -2067,12 +2427,15 @@ export class MemoryTableManager {
             // The DDL transaction's own layers froze their schema at creation; hand them the
             // new one so the rest of the transaction scans and enforces the new index.
             this.adoptSchemaOnOpenLayers(finalNewTableSchema);
-            // Emit schema change event
+            // Emit schema change event. The `ddl` is what a sync peer re-executes to
+            // replicate the index; rendered against the POST-create schema so it matches
+            // what a receiving peer regenerates when comparing definitions.
             this.eventEmitter?.emitSchemaChange?.({
                 type: 'create',
                 objectType: 'index',
                 schemaName: this.schemaName,
                 objectName: indexName,
+                ddl: generateIndexDDL(newIndexSchemaEntry, finalNewTableSchema),
             });
             logger.operation('Create Index', this._tableName, { indexName });
         }
@@ -2135,6 +2498,7 @@ export class MemoryTableManager {
                 objectType: 'index',
                 schemaName: this.schemaName,
                 objectName: indexName,
+                ddl: generateDropIndexDDL(this.schemaName, indexName),
             });
             logger.operation('Drop Index', this._tableName, { indexName });
         }
@@ -2285,6 +2649,14 @@ export class MemoryTableManager {
                 this.baseLayer.rebuildAllSecondaryIndexes();
             this.tableSchema = newSchema;
             this.initializePrimaryKeyFunctions();
+            // Hand the new schema to the DDL transaction's own frozen layers, as `dropConstraint`
+            // does: a renamed covering index has to move under its new key in each layer's index
+            // map (the old key is dropped, the new one rebuilt over the parent's already-renamed
+            // tree — hence oldest-first), and even the pure schema-only classes (CHECK / FK) must
+            // re-freeze the new schema OBJECT, or `commitTransaction`'s snapshot-wrap identity
+            // check (`readLayer.getSchema() === this.tableSchema`) skips a savepoint snapshot
+            // restored across this ALTER and drops the transaction's staged rows at COMMIT.
+            this.adoptSchemaOnOpenLayers(newSchema);
             this.eventEmitter?.emitSchemaChange?.({
                 type: 'alter',
                 objectType: 'table',
@@ -2529,6 +2901,19 @@ export class MemoryTableManager {
         // bug. A detached connection always has `pendingTransactionLayer === null` (disconnect
         // defers while a pending layer is uncommitted), so this never discards in-flight writes.
         this.repointRegisteredConnections();
+        // Re-pointing `readLayer` is not enough: a savepoint taken while a connection held no
+        // pending layer stored the layer it was READING as the view to restore. Consolidation
+        // has just drained that layer into the base, and the caller is about to reshape the
+        // base's rows — so the stored reference is a pre-change snapshot of the committed rows.
+        // Left alone, the transaction's `rollback to savepoint` reinstates it and the old column
+        // shape commits over the new one.
+        //
+        // Every connection, not just the re-pointed ones: the DDL issuer is exempt from the
+        // sweeps above (its read view holds its own uncommitted rows) but its own lazy markers
+        // are just as stale.
+        for (const connection of this.knownConnections()) {
+            connection.repointLazySavepointsToCommittedHead(this.baseLayer);
+        }
         logger.debugLog(`Schema change safety check passed for ${this._tableName}. Current committed layer is base.`);
     }
     /**
@@ -2548,6 +2933,34 @@ export class MemoryTableManager {
             mc.readLayer = this.baseLayer;
         }
     }
+    /**
+     * Moves this manager's Database-registered connections from `<schema>.<oldName>` to
+     * `<schema>.<newName>`, for `ALTER TABLE ... RENAME TO`.
+     *
+     * `MemoryVirtualTableConnection.tableName` is set once at registration, and
+     * `Database.getConnectionsForTable` matches on exactly that string — so without this
+     * sweep a rename makes the transaction's own connection invisible to
+     * {@link registeredConnections} and everything built on it: {@link ddlConnection},
+     * {@link knownConnections}, {@link repointRegisteredConnections},
+     * {@link openTransactionLayersOldestFirst}. The visible symptom is a second ALTER in
+     * the same transaction failing with BUSY — {@link ensureSchemaChangeSafety} exempts
+     * `ddlConnection()` from its "nobody else may hold open work" sweep, and with the
+     * registry stale that is `undefined`, so the transaction's own connection is judged a
+     * stranger. `MemoryTable.ensureConnection` would likewise stop reusing it and register
+     * a second connection for the same table.
+     *
+     * Must run BEFORE `_tableName` moves — it looks the connections up under the old name.
+     */
+    rekeyRegisteredConnections(oldName, newName) {
+        const newQualifiedName = `${this.schemaName}.${newName}`;
+        for (const c of this.db.getConnectionsForTable(`${this.schemaName}.${oldName}`)) {
+            if (!(c instanceof MemoryVirtualTableConnection))
+                continue;
+            if (c.getMemoryConnection().tableManager !== this)
+                continue;
+            c.rename(newQualifiedName);
+        }
+    }
     /** Every Database-registered {@link MemoryTableConnection} backed by this manager. */
     *registeredConnections() {
         const qualifiedName = `${this.schemaName}.${this._tableName}`;
@@ -2625,20 +3038,12 @@ export class MemoryTableManager {
     convertBaseRows(colIndex, convert, convertNulls = false) {
         const out = [];
         for (const row of iteratePrimaryRows(this.baseLayer.primaryTree)) {
-            const oldVal = row[colIndex];
-            if (oldVal === null && !convertNulls) {
-                out.push(row);
-                continue;
-            }
-            let newVal;
             try {
-                newVal = convert(oldVal);
+                out.push(convertRowAtIndex(row, colIndex, convert, convertNulls));
             }
             catch {
                 out.push(row); // shadowed unconvertible value — keep as-is
-                continue;
             }
-            out.push(row.map((v, i) => i === colIndex ? newVal : v));
         }
         return out;
     }
@@ -2652,31 +3057,47 @@ export class MemoryTableManager {
      * the transaction's pending rows outside this manager, so only it can name the rows the
      * issuing connection actually sees; when it supplies them, they are the judged set and this
      * manager's own committed rows are ignored. See `vtab/module.ts` {@link EffectiveRowSource}.
+     *
+     * `mapRow` judges the rows in a form they do not yet have on disk: an `alter column` that
+     * REWRITES values passes the per-row conversion here, so the probe sees the post-ALTER
+     * values. Applied to whichever stream is used; the mapping is a read-side wrap and mutates
+     * nothing.
      */
-    async validateUniqueOverEffectiveRows(indexSchema, schema, rows) {
+    async validateUniqueOverEffectiveRows(indexSchema, schema, rows, mapRow) {
         const probe = new MemoryIndex(indexSchema, schema.columns, this.collationResolver, this.primaryKeyFunctions.compare, this.primaryKeyFunctions.encode, schema.name);
         if (rows) {
-            await populateIndexFromRowsAsync(rows(), probe, this.primaryKeyFunctions.extractFromRow, true, this._tableName, schema.columns);
+            const stream = mapRow ? mapRowsAsync(rows(), mapRow) : rows();
+            await populateIndexFromRowsAsync(stream, probe, this.primaryKeyFunctions.extractFromRow, true, this._tableName, schema.columns);
             return;
         }
-        populateIndexFromRows(this.effectiveDdlRows(), probe, this.primaryKeyFunctions.extractFromRow, true, this._tableName, schema.columns);
+        const own = this.effectiveDdlRows();
+        populateIndexFromRows(mapRow ? mapRows(own, mapRow) : own, probe, this.primaryKeyFunctions.extractFromRow, true, this._tableName, schema.columns);
     }
     /**
-     * `ALTER COLUMN … SET COLLATE` arm of {@link validateUniqueOverEffectiveRows}: rejects the
-     * change when it makes two of the DDL transaction's effective rows collide under any
-     * uniqueness-enforcing structure that orders by the altered column. Runs before anything is
-     * mutated, so a rejection leaves the table and schema untouched and the transaction usable.
+     * `ALTER COLUMN` arm of {@link validateUniqueOverEffectiveRows}: rejects the change when it
+     * makes two of the DDL transaction's effective rows collide under any uniqueness-enforcing
+     * structure that covers the altered column. Runs before anything is mutated, so a rejection
+     * leaves the table and schema untouched and the transaction usable.
+     *
+     * Serves both families of change that can collapse two distinct values onto one:
+     *
+     *  - `SET COLLATE` re-keys the structures under a new comparator; the stored values are
+     *    untouched, so no `mapRow` is passed.
+     *  - a value REWRITE (`SET DATA TYPE`, or a `SET NOT NULL` null → DEFAULT backfill) leaves
+     *    the comparators alone but changes the values; the caller passes a `mapRow` that applies
+     *    the same per-row conversion the base rewrite will, so the probe judges the post-ALTER
+     *    values.
      *
-     * `newSchema` carries the post-change per-column collations, so each probe index compares
-     * exactly as the rebuilt structure will. Indexes that do not mention the column keep their
-     * keys and need no re-check.
+     * `newSchema` carries the post-change per-column collations AND logical types, so each probe
+     * index compares exactly as the rebuilt structure will (it matters for e.g. `text → real`).
+     * Indexes that do not mention the column keep their keys and need no re-check.
      *
      * NOTE: walks `schema.indexes`, so a UNIQUE constraint covered by a row-time materialized
      * view rather than its auto-index (`findIndexForConstraint` prefers the MV) is not re-checked
      * here. The auto-index always exists alongside, so the structure is still validated — but if
      * an MV-only covering shape ever becomes reachable, this walk must follow it.
      */
-    async validateRekeyedUniqueStructures(newSchema, alteredColumnIndex, rows) {
+    async validateRekeyedUniqueStructures(newSchema, alteredColumnIndex, rows, mapRow) {
         // NOTE: the probe index carries the manager's PRE-change `primaryKeyFunctions` (the new
         // ones cannot exist before the schema swaps). Only the probe's per-entry PK bookkeeping
         // uses them, and every effective row already has a distinct PK under the old encoder, so
@@ -2688,7 +3109,7 @@ export class MemoryTableManager {
             if (!indexEnforcesUnique(newSchema, indexSchema))
                 continue;
             // `rows` is re-callable, so each structure gets a fresh stream.
-            await this.validateUniqueOverEffectiveRows(indexSchema, newSchema, rows);
+            await this.validateUniqueOverEffectiveRows(indexSchema, newSchema, rows, mapRow);
         }
     }
     /**
@@ -2792,7 +3213,7 @@ export class MemoryTableManager {
      * `convertNulls` routes null own-values through `convert` (the SET NOT NULL null → DEFAULT map);
      * SET DATA TYPE leaves them untouched. No-op in autocommit.
      *
-     * PRIMARY-KEY columns never reach here — a physical retype of a key column is rejected before
+     * PRIMARY-KEY columns never reach here — any retype of a key column is rejected before
      * any mutation, and SET NOT NULL leaves the key bytes unchanged — so the primary tree's keys are
      * stable and no re-key is needed.
      */
@@ -2801,14 +3222,58 @@ export class MemoryTableManager {
             layer.convertColumn(colIndex, convert, newSchema, convertNulls);
         }
     }
+    /**
+     * Phase 1 of propagating `add column` / `drop column` into every open transaction layer:
+     * computes each layer's net own-writes rewritten to the new column set, WITHOUT mutating
+     * anything (see {@link TransactionLayer.prepareReshapedColumns}). Runs BEFORE the base is
+     * touched, because `reshapeRow` can throw — ADD COLUMN's per-row backfill evaluates an
+     * engine-supplied `default (new.<col>)` expression against each pending row, and enforces
+     * NOT NULL on the result — and a failure part-way through a mutation pass would leave the
+     * base and the already-visited layers at the new arity with the rest at the old, with no
+     * undo. Failing here instead rejects the ALTER with every structure untouched, exactly like
+     * the pre-mutation validation passes of `alterColumn`.
+     *
+     * The returned plans pin the layer set: {@link installReshapeOnOpenLayers} walks the plans,
+     * not a fresh chain snapshot, so the two phases cannot disagree about which layers exist.
+     * Reading each layer's effective rows here — before the base rebuild — yields the same rows
+     * the single-pass prototype read after it: a layer's pre-reshape tree resolves against the
+     * parent's OLD tree either way (the base rebuild REPLACES the base's tree object; it never
+     * mutates the one the layers inherit from).
+     */
+    async prepareReshapeOnOpenLayers(reshapeRow, reshapeEventRow) {
+        const plans = [];
+        for (const layer of this.openTransactionLayersOldestFirst()) {
+            plans.push({ layer, prepared: await layer.prepareReshapedColumns(reshapeRow, reshapeEventRow) });
+        }
+        return plans;
+    }
+    /**
+     * Phase 2: installs the prepared reshape into each open transaction layer, oldest-first
+     * (the plans were gathered in that order), after the base has been rebuilt at the new arity
+     * — each layer's rebuilt tree inherits copy-on-write from its parent's NEW one. Synchronous
+     * and mutation-only; everything fallible ran in {@link prepareReshapeOnOpenLayers}.
+     *
+     * Beyond keeping the transaction's own reads correct, this is what makes the
+     * `commitTransaction` snapshot-wrap identity check (`readLayer.getSchema() ===
+     * this.tableSchema`) pass after a `rollback to savepoint`: without it the eager savepoint
+     * snapshot keeps its frozen pre-ALTER schema, the wrap is skipped, and the snapshot's rows
+     * are silently dropped at commit.
+     */
+    installReshapeOnOpenLayers(newSchema, plans) {
+        for (const { layer, prepared } of plans) {
+            layer.installReshapedColumns(newSchema, prepared);
+        }
+    }
     /**
      * The DDL connection's open {@link TransactionLayer}s, oldest-first, base excluded — its
      * pending layer and every savepoint snapshot below it, the layers on the view's parent chain
      * above the base. Empty in autocommit (no connection, or the view IS the base).
      *
-     * Both schema adoption ({@link adoptSchemaOnOpenLayers}) and value conversion
-     * ({@link convertColumnOnOpenLayers}) MUST apply oldest-first: each layer rebuilds its
-     * structures over its parent's, so the parent's must already be the new/converted ones.
+     * Schema adoption ({@link adoptSchemaOnOpenLayers}), value conversion
+     * ({@link convertColumnOnOpenLayers}), and column-set reshape
+     * ({@link prepareReshapeOnOpenLayers} / {@link installReshapeOnOpenLayers}) MUST all apply
+     * oldest-first: each layer rebuilds its structures over its parent's, so the parent's must
+     * already be the new/converted/reshaped ones.
      *
      * NOTE: the walk takes every TransactionLayer below the view, which normally means the pending
      * layer and its savepoint snapshots. A committed layer already drained into the base by
@@ -2841,6 +3306,17 @@ export class MemoryTableManager {
                 logger.debugLog(`[Consolidate] Copying data from transaction layer ${transactionLayer.getLayerId()} to base layer for ${this._tableName}`);
                 // Copy all data from the transaction layer to the base layer
                 await this.copyTransactionDataToBase(transactionLayer);
+                // The drained chain's pending-change event logs were already delivered when
+                // each layer committed, but the layers can remain in the DDL connection's own
+                // open-transaction parent chain. Once the base becomes the committed head, the
+                // commit-time collection (`collectPendingChanges`) walks that chain down to
+                // the base and would re-deliver them — clear the logs so consolidation is
+                // event-neutral. (Savepoint snapshots of the OPEN transaction sit above the
+                // committed head, not in this chain, so their un-emitted events are kept.)
+                for (let cur = transactionLayer; cur && cur !== this.baseLayer; cur = cur.getParent()) {
+                    if (cur instanceof TransactionLayer)
+                        cur.clearPendingChanges();
+                }
                 // Force all connections to read from the base layer
                 for (const conn of this.connections.values()) {
                     if (conn.readLayer === transactionLayer) {