@quereus/isolation 4.4.1 → 4.5.0

package/dist/src/isolation-module.js

@@ -1,4 +1,4 @@
-import { MemoryTableModule, PhysicalType, QuereusError, StatusCode, tryFoldLiteral, columnDefToSchema, isConstraintViolation, inferType, validateAndParse } from '@quereus/quereus';
+import { MemoryTableModule, PhysicalType, QuereusError, StatusCode, tryFoldLiteral, columnDefToSchema, isConstraintViolation, inferType, validateAndParse, validateCollationForType, formatKeyValue } from '@quereus/quereus';
 import { IsolatedTable } from './isolated-table.js';
 import { applyOverlayToUnderlying } from './flush.js';
 import { makeFullScanFilterInfo } from './filter-info.js';
@@ -286,7 +286,8 @@ export class IsolationModule {
      * overlay module's table registry rather than leaking there for the life of the
      * `Database`. This is the single sink every overlay-discard path funnels through —
      * without it, `MemoryTableModule.tables` accumulates one dead `_overlay_<table>_<id>`
-     * entry per writing transaction (and one more per rebuild), unbounded.
+     * entry per writing transaction (and one more per `alterPrimaryKey` overlay swap — the
+     * only DDL path left that replaces an overlay rather than adopting in place), unbounded.
      *
      * `MemoryTableManager.destroy` rolls back the overlay's own pending layer and clears its
      * connections; a later db-side teardown of the (now-detached) `MemoryVirtualTableConnection`
@@ -790,7 +791,7 @@ export class IsolationModule {
      * covers the data conditions the pre-validation pass rejects before any overlay is touched;
      * this one covers a UNIQUE (or any other) violation raised by the in-place adoption itself.
      */
-    buildRebuildPoisonMessage(schemaName, tableName, ddlDescription, cause) {
+    buildInPlaceAdoptPoisonMessage(schemaName, tableName, ddlDescription, cause) {
         return `Another connection's ${ddlDescription} on '${schemaName}.${tableName}' declared a constraint this connection's `
             + `uncommitted rows violate (${cause}); roll back this transaction.`;
     }
@@ -907,8 +908,7 @@ export class IsolationModule {
         // Take the index back off the underlying's refreshed schema rather than reusing the
         // caller's object: that is the canonical post-create form (resolved column indices,
         // any normalization the underlying applied), and the overlay's copy is derived from it.
-        const updatedSchema = this.assertIndexPresent(state.underlyingTable, schemaName, tableName, indexSchema.name);
-        const created = updatedSchema.indexes.find(idx => idx.name.toLowerCase() === indexSchema.name.toLowerCase());
+        const { schema: updatedSchema, index: created } = this.assertIndexPresent(state.underlyingTable, schemaName, tableName, indexSchema.name);
         await this.applyIndexChangeToOverlays(db, schemaName, tableName, `create index '${indexSchema.name}'`, overlayState => this.createOverlayIndex(overlayState, updatedSchema.name, created));
     }
     /**
@@ -935,29 +935,34 @@ export class IsolationModule {
         await overlayTable.createIndex(this.createOverlayIndexSchema(indexSchema, baseName, overlaySchema.name));
     }
     /**
-     * Reads back the underlying table instance's post-`createIndex` schema, asserting it now
-     * carries the new index.
+     * Reads back the underlying table instance's post-`createIndex` schema plus the entry it now
+     * carries for the new index — the canonical post-create form each overlay's copy is derived
+     * from ({@link createOverlayIndex}).
      *
      * Both bundled underlyings refresh the instance's cached `tableSchema` (memory through
-     * `MemoryTable.createIndex`, the store through `StoreTable.updateSchema`), and the overlay
-     * rebuild below copies its index/constraint set from it. A third-party underlying that
-     * refreshed only its module-level schema would silently rebuild overlays under the PRE-index
-     * schema, re-opening the very hole this method exists to close — so assert rather than assume.
+     * `MemoryTable.createIndex`, the store through `StoreTable.updateSchema`). A third-party
+     * underlying that refreshed only its module-level schema would leave every overlay without
+     * the index — silently re-opening the very hole `createIndex`'s forward exists to close — so
+     * assert rather than assume.
      */
     assertIndexPresent(underlyingTable, schemaName, tableName, indexName) {
-        const updatedSchema = underlyingTable.tableSchema;
-        const present = updatedSchema && this.schemaHasIndex(updatedSchema, indexName);
-        if (!updatedSchema || !present) {
+        const schema = underlyingTable.tableSchema;
+        const index = schema && this.findSchemaIndex(schema, indexName);
+        if (!schema || !index) {
             throw new QuereusError(`Isolation layer: underlying table '${schemaName}.${tableName}' did not refresh its cached tableSchema after `
                 + `creating index '${indexName}'. The per-connection overlays cannot adopt an index the underlying does not `
                 + `report; the underlying module must refresh VirtualTable.tableSchema in createIndex.`, StatusCode.INTERNAL);
         }
-        return updatedSchema;
+        return { schema, index };
+    }
+    /** `schema`'s index named `indexName` (case-insensitively), if it declares one. */
+    findSchemaIndex(schema, indexName) {
+        const lower = indexName.toLowerCase();
+        return schema.indexes?.find(idx => idx.name.toLowerCase() === lower);
     }
     /** Case-insensitive "does `schema` declare an index named `indexName`". */
     schemaHasIndex(schema, indexName) {
-        const lower = indexName.toLowerCase();
-        return schema.indexes?.some(idx => idx.name.toLowerCase() === lower) ?? false;
+        return this.findSchemaIndex(schema, indexName) !== undefined;
     }
     /**
      * Drops an index on the underlying table, then drops it from every per-connection overlay.
@@ -967,6 +972,11 @@ export class IsolationModule {
      * so MemoryTable.tableSchema stays fresh), prefer that. Otherwise fall back
      * to the module-level dropIndex (e.g. StoreModule, which refreshes the
      * StoreTable's cached tableSchema and tears down the index store).
+     *
+     * Mirrors createIndex's early return when neither hook exists too: an underlying that
+     * declares indexes at CREATE TABLE time but cannot drop one still has them, and dropping
+     * from the overlays alone would leave each overlay enforcing LESS than the base it flushes
+     * into — a silent divergence surfacing only as a commit-time rejection.
      */
     async dropIndex(db, schemaName, tableName, indexName) {
         const state = this.getUnderlyingState(schemaName, tableName);
@@ -976,6 +986,9 @@ export class IsolationModule {
         else if (this.underlying.dropIndex) {
             await this.underlying.dropIndex(db, schemaName, tableName, indexName);
         }
+        else {
+            return; // underlying cannot drop indexes; nothing was dropped, nothing to forward
+        }
         await this.applyIndexChangeToOverlays(db, schemaName, tableName, `drop index '${indexName}'`, overlayState => this.dropOverlayIndex(overlayState, indexName));
     }
     /**
@@ -1032,7 +1045,7 @@ export class IsolationModule {
      * judged by the DDL's own validation pass, so a rejection here means validation and
      * migration have drifted — {@link issuerOverlayDriftError}); a foreign overlay →
      * poison and leave it untouched so its owner errors and rolls back
-     * ({@link buildRebuildPoisonMessage}). Any non-CONSTRAINT failure is a layer-invariant
+     * ({@link buildInPlaceAdoptPoisonMessage}). Any non-CONSTRAINT failure is a layer-invariant
      * violation, not a data condition, and rethrows for everyone.
      *
      * Shared by the index paths ({@link applyIndexChangeToOverlays}) and the ALTER TABLE
@@ -1043,11 +1056,24 @@ export class IsolationModule {
             await apply();
         }
         catch (e) {
-            if (!(e instanceof QuereusError) || e.code !== StatusCode.CONSTRAINT)
+            // NOTE: a rethrow here abandons the rest of the caller's overlay walk, and the DDL has
+            // already landed on the shared underlying — so overlays not yet visited neither adopt
+            // the change nor get poisoned. Safe today: every rethrown source is a layer-invariant
+            // violation (INTERNAL), which no overlay could have satisfied anyway. The two
+            // per-overlay DATA conditions — CONSTRAINT, and the BUSY the overlay module's own
+            // primary-key re-key representability check raises — are routed below instead.
+            if (!(e instanceof QuereusError) || (e.code !== StatusCode.CONSTRAINT && e.code !== StatusCode.BUSY))
                 throw e;
-            if (isIssuer)
+            if (isIssuer) {
+                // A BUSY out of the issuer's own overlay is a representability refusal its
+                // validation pass never claimed to judge — not drift. The tier-2 pre-flight
+                // surfaces it before the underlying mutates for the bundled overlay module, so
+                // this rethrow is reachable only through a host-injected overlay module.
+                if (e.code === StatusCode.BUSY)
+                    throw e;
                 throw this.issuerOverlayDriftError(schemaName, tableName, ddlDescription, e);
-            overlayState.poison = { message: this.buildRebuildPoisonMessage(schemaName, tableName, ddlDescription, e.message) };
+            }
+            overlayState.poison = { message: this.buildInPlaceAdoptPoisonMessage(schemaName, tableName, ddlDescription, e.message) };
         }
     }
     /**
@@ -1065,9 +1091,12 @@ export class IsolationModule {
      *   backfills each staged row exactly as the committed path does (literal default,
      *   per-row `new.<col>` evaluator, or NULL); tombstone rows get NULL.
      * - ALTER COLUMN: {@link forwardAlterColumnToOverlay}. `set data type` / `set collate`
-     *   / `set default` forward straight through (the overlay module converts / re-keys its
-     *   open layers itself); `set not null` is withheld from the overlay and the staged
-     *   live rows' NULLs are backfilled via ordinary overlay writes instead.
+     *   / `set default` forward through the overlay's `alterSchema` (the overlay module
+     *   converts / re-keys its open layers itself), with one preparation step: a `set collate`
+     *   that re-keys the PRIMARY KEY first discards any deletion markers the new key collapses
+     *   onto another staged row ({@link dropCollapsedPkRekeyMarkers}). `set not null` is
+     *   withheld from the overlay and the staged live rows' NULLs are backfilled via ordinary
+     *   overlay writes instead.
      * - ADD / DROP / RENAME CONSTRAINT: {@link forwardAddConstraintToOverlay} /
      *   {@link forwardConstraintNameChangeToOverlay}. A UNIQUE lands as a
      *   tombstone-narrowed unique index; CHECK forwards verbatim; FOREIGN KEY does not
@@ -1139,6 +1168,7 @@ export class IsolationModule {
         if (change.type === 'alterPrimaryKey' && ownEntry?.[1].hasChanges) {
             throw new QuereusError(`Cannot alter the primary key of '${schemaName}.${tableName}' while this transaction has uncommitted changes staged for it; commit or roll back first.`, StatusCode.UNSUPPORTED);
         }
+        this.assertColumnNameNotTombstone(schemaName, tableName, change);
         // Build the addColumn backfill context up front (undefined for other change types).
         // Derived purely from `change` + the session nullability option — no post-alter
         // schema needed — so it is valid here, before the underlying is mutated, and the
@@ -1153,22 +1183,76 @@ export class IsolationModule {
         // retype with overlays to convert). Read from the same PRE-alter overlay schema as above;
         // `inferType` cannot throw, so deriving it before the underlying mutation is safe.
         const setDataTypeCtx = this.deriveSetDataTypeConvert(change, toMigrate);
+        // Build the primary-key re-key context (undefined unless this is a collation change on a
+        // PK member with overlays to carry). Read from the same PRE-alter overlay schema as above.
+        const pkRekeyCtx = this.derivePkRekey(change, db, tableName, toMigrate);
         // Tier 2: validate the ISSUER's own overlay BEFORE mutating the shared underlying.
         // Any throw here (CONSTRAINT backfill, MISMATCH conversion, or INTERNAL tombstone guard)
         // propagates while underlying + catalog + every overlay are still untouched — the companion
         // ticket's atomic-abort guarantee, preserved unchanged for the issuer.
         if (ownEntry) {
-            await this.validateOverlayMigration(ownEntry[1], addColumnCtx, setNotNullCtx, setDataTypeCtx);
+            await this.validateOverlayMigration(ownEntry[1], addColumnCtx, setNotNullCtx, setDataTypeCtx, pkRekeyCtx);
         }
         const underlyingState = this.getUnderlyingState(schemaName, tableName);
         // Hand the underlying the issuer's effective rows so its own row-content checks
         // (`add constraint … unique`, `alter column … set collate`) see the transaction's
         // pending rows and skip the ones it has deleted. An outer wrapper's source wins if
         // one was supplied.
-        const rowSource = rows ?? (underlyingState
+        let rowSource = rows ?? (underlyingState
             ? this.issuerEffectiveRows(db, schemaName, tableName, underlyingState.underlyingTable)
             : undefined);
-        const updated = await this.underlying.alterTable(db, schemaName, tableName, change, rowSource);
+        // Tier 2, PRIMARY-KEY re-key arm: the overlay's own `alterSchema` is itself fallible —
+        // its module re-validates the re-key against the overlay's layer chain (e.g. the memory
+        // module's representability BUSY when a savepoint could restore rows that collide under
+        // the new collation) — and it runs in the migrate step, AFTER the underlying has
+        // irreversibly re-keyed. So dry-run it here, against the exact state the migrate step
+        // will see: drop the collapsed markers first (saving them), pre-flight the overlay's
+        // `alterSchema` in validate-only mode, and on ANY later refusal — the pre-flight's or
+        // the underlying's — reinsert the markers and rethrow with everything net-untouched.
+        //
+        // The drops happen BEFORE `underlying.alterTable`, and a dropped marker un-shadows the
+        // committed row it deletes in `issuerEffectiveRows`'s merged stream — so the effective
+        // rows are SNAPSHOTTED first, and the underlying judges the pre-drop view.
+        let droppedMarkerRows = [];
+        if (pkRekeyCtx && ownEntry && ownEntry[1].hasChanges && ownEntry[1].overlayTable.alterSchema) {
+            const ownOverlayState = ownEntry[1];
+            const plan = await this.planPkRekeyMarkerDrops(ownOverlayState, pkRekeyCtx);
+            if (plan.length > 0) {
+                if (rowSource) {
+                    // NOTE: materializes the issuer's whole effective row set for this one ALTER.
+                    // The underlying's own re-key probe already holds every row it sees (see
+                    // MemoryTableManager.makePrimaryKeyProbe's NOTE), so this does not change the
+                    // peak's order of magnitude.
+                    const snapshot = [];
+                    for await (const row of rowSource())
+                        snapshot.push(row);
+                    rowSource = () => (async function* () { yield* snapshot; })();
+                }
+                droppedMarkerRows = plan;
+                await this.applyPkRekeyMarkerDrops(ownOverlayState, pkRekeyCtx, plan);
+            }
+            try {
+                // NOTE: unenforced contract — an overlay module that ignores `validateOnly` (see
+                // VirtualTable.alterSchema / IsolationModuleConfig.overlay) applies the re-key
+                // here for real, and a later refusal would then reinsert the markers into an
+                // already-re-keyed overlay. If host-injected overlay modules ever become common,
+                // gate this call on a module capability rather than on the documented contract.
+                await ownOverlayState.overlayTable.alterSchema(change, true);
+            }
+            catch (e) {
+                await this.reinsertPkRekeyMarkers(ownOverlayState, droppedMarkerRows);
+                throw e;
+            }
+        }
+        let updated;
+        try {
+            updated = await this.underlying.alterTable(db, schemaName, tableName, change, rowSource);
+        }
+        catch (e) {
+            if (ownEntry)
+                await this.reinsertPkRekeyMarkers(ownEntry[1], droppedMarkerRows);
+            throw e;
+        }
         // The cached underlying VirtualTable's `tableSchema` is a construction-time
         // snapshot (e.g. MemoryTable.tableSchema); module-level alterTable rotates the
         // underlying manager's schema but not this instance's field. Refresh it so a
@@ -1191,7 +1275,7 @@ export class IsolationModule {
                     break;
                 case 'alterColumn': {
                     const alterColumnChange = change;
-                    await this.applyInPlaceOverlayChange(isIssuer, state, schemaName, tableName, ddlDescription, () => this.forwardAlterColumnToOverlay(state, alterColumnChange, setNotNullCtx));
+                    await this.applyInPlaceOverlayChange(isIssuer, state, schemaName, tableName, ddlDescription, () => this.forwardAlterColumnToOverlay(state, alterColumnChange, setNotNullCtx, pkRekeyCtx));
                     break;
                 }
                 case 'addConstraint': {
@@ -1232,7 +1316,7 @@ export class IsolationModule {
         // overlay, so one bad foreign overlay poisons only itself; healthy peers still migrate.
         for (const [key, oldState] of foreign) {
             try {
-                await this.validateOverlayMigration(oldState, addColumnCtx, setNotNullCtx, setDataTypeCtx);
+                await this.validateOverlayMigration(oldState, addColumnCtx, setNotNullCtx, setDataTypeCtx, pkRekeyCtx);
             }
             catch (e) {
                 if (e instanceof QuereusError && (e.code === StatusCode.CONSTRAINT || e.code === StatusCode.MISMATCH)) {
@@ -1245,15 +1329,42 @@ export class IsolationModule {
         }
         return updated;
     }
+    /**
+     * Rejects an ADD / RENAME COLUMN that would give a base column the same name as the
+     * overlay's private tombstone flag, BEFORE `underlying.alterTable` runs.
+     *
+     * Every overlay carries one extra column named {@link tombstoneColumn}, so a base column
+     * of that name collides with it. Forwarding the change to an open overlay makes the
+     * overlay module raise a duplicate-name `ERROR`, which is not a data condition and so
+     * rethrows out of {@link applyInPlaceOverlayChange} — after the underlying has already
+     * (irreversibly) applied. The catalog then keeps the pre-alter column set while the base
+     * holds the new one, and the next write to the table lands values against the wrong
+     * columns. Rejecting up front keeps the atomic-abort guarantee this method documents.
+     *
+     * NOTE: the check is unconditional — not gated on an overlay existing — so the answer
+     * does not depend on whether some connection happens to have a transaction open. That
+     * also stops a table from acquiring the colliding name while idle and hitting
+     * {@link createOverlaySchema}'s (silent) duplicate at the next BEGIN. A CREATE TABLE
+     * declaring the name still gets that duplicate — tracked separately.
+     */
+    assertColumnNameNotTombstone(schemaName, tableName, change) {
+        const newName = change.type === 'addColumn' ? change.columnDef.name
+            : change.type === 'renameColumn' ? change.newName
+                : undefined;
+        if (newName?.toLowerCase() !== this.tombstoneColumn.toLowerCase())
+            return;
+        throw new QuereusError(`Cannot name a column of '${schemaName}.${tableName}' '${newName}': the transaction-isolation layer reserves that name for its per-connection overlay's deletion marker.`, StatusCode.UNSUPPORTED);
+    }
     /**
      * Builds the poison message stamped onto a foreign overlay whose backfill could not
      * satisfy a cross-connection ALTER (see {@link alterTable} tier 3). Names the
      * schema.table and the offending column so the owning connection's eventual
      * read/write/commit error is self-explanatory. Poison arises on the addColumn NOT NULL
      * path (a new NOT-NULL column with no usable default), the `set not null` tightening
-     * path (a staged NULL with no usable default), and the `set data type` path (a staged value
-     * that cannot be converted to the new type); other change types never reach here but
-     * are handled defensively.
+     * path (a staged NULL with no usable default), the `set data type` path (a staged value
+     * that cannot be converted to the new type), and the primary-key `set collate` path (two
+     * staged live rows colliding under the new collation); other change types never reach
+     * here but are handled defensively.
      */
     buildAlterPoisonMessage(schemaName, tableName, change) {
         if (change.type === 'addColumn') {
@@ -1263,6 +1374,9 @@ export class IsolationModule {
             if (change.setDataType !== undefined) {
                 return `ALTER on '${schemaName}.${tableName}' changed the data type of column '${change.columnName}' to ${change.setDataType}, which this connection's uncommitted row cannot be converted to; roll back this transaction.`;
             }
+            if (change.setCollation !== undefined) {
+                return `ALTER on '${schemaName}.${tableName}' changed the collation of primary-key column '${change.columnName}', under which this connection's uncommitted rows collide; roll back this transaction.`;
+            }
             return `ALTER on '${schemaName}.${tableName}' tightened column '${change.columnName}' to NOT NULL, which this connection's uncommitted row violates; roll back this transaction.`;
         }
         if (change.type === 'alterPrimaryKey') {
@@ -1511,6 +1625,76 @@ export class IsolationModule {
             },
         };
     }
+    /**
+     * Precomputes the per-ALTER constants a primary-key re-keying `alter column … set collate`
+     * overlay migration needs: the PK column positions and a key serializer built over the
+     * POST-change collation — the PRE-alter overlay schema with the named column's collation
+     * replaced. The overlay's PK definition mirrors the base's (the tombstone flag is appended
+     * after the data columns), so the synthesized schema keys exactly as the re-keyed base will.
+     *
+     * Returns undefined unless the named column is a PRIMARY KEY member and the normalized
+     * collation actually differs from the column's current one — mirroring the underlying's
+     * metadata-only gate (`MemoryTableManager.alterColumn`'s `nameMatches`), so overlay and
+     * committed rows re-key together or not at all. `validateCollationForType` can throw for an
+     * unknown collation, but the engine already validated the name before dispatching the ALTER,
+     * and a direct module caller gets the same pre-mutation rejection the underlying would give.
+     */
+    derivePkRekey(change, db, tableName, toMigrate) {
+        if (change.type !== 'alterColumn' || change.setCollation === undefined)
+            return undefined;
+        if (toMigrate.length === 0)
+            return undefined;
+        const overlaySchema = toMigrate[0][1].overlayTable.tableSchema;
+        if (!overlaySchema)
+            return undefined;
+        const colIndex = overlaySchema.columnIndexMap.get(change.columnName.toLowerCase());
+        if (colIndex === undefined)
+            return undefined;
+        if (!overlaySchema.primaryKeyDefinition.some(def => def.index === colIndex))
+            return undefined;
+        const oldCol = overlaySchema.columns[colIndex];
+        if (!oldCol)
+            return undefined;
+        const normalized = validateCollationForType(change.setCollation, oldCol.logicalType, change.columnName, (n) => db.isCollationRegistered(n));
+        if (normalized === (oldCol.collation || 'BINARY'))
+            return undefined; // metadata-only — the underlying re-keys nothing
+        const postChangeSchema = {
+            ...overlaySchema,
+            columns: overlaySchema.columns.map((c, i) => (i === colIndex ? { ...c, collation: normalized } : c)),
+        };
+        return {
+            pkIndices: overlaySchema.primaryKeyDefinition.map(def => def.index),
+            keyOf: makePkKeySerializer(db, postChangeSchema),
+            tableName,
+        };
+    }
+    /**
+     * Buckets an overlay's staged rows by their PRIMARY KEY as the pending `set collate` will
+     * re-key it: one {@link PkRekeyGroup} per post-change key. Shared by the validation dry run
+     * (which refuses a group holding two live rows) and the migrate-step marker drop (which
+     * discards collapsed markers), so the two passes cannot disagree about what collapses.
+     *
+     * NOTE: materializes one serialized key + one PK tuple per staged row (plus the marker rows
+     * themselves), once per pass of one ALTER. If an overlay with very many staged rows ever
+     * makes this ALTER slow, group lazily or reuse the merge `effectiveRowsFor` already builds.
+     */
+    async collectPkRekeyGroups(overlay, tombstoneIdx, ctx) {
+        const groups = new Map();
+        for await (const row of overlay.query(makeFullScanFilterInfo())) {
+            const pk = ctx.pkIndices.map(i => row[i]);
+            const key = ctx.keyOf(pk);
+            let group = groups.get(key);
+            if (!group) {
+                group = { livePks: [], markerRows: [] };
+                groups.set(key, group);
+            }
+            if (row[tombstoneIdx] === 1)
+                group.markerRows.push(row);
+            else
+                group.livePks.push(pk);
+        }
+        return groups;
+    }
     /**
      * Dry-runs an overlay's ALTER migration-fallible work without mutating anything,
      * so the caller can run it for every affected overlay BEFORE the irreversible
@@ -1539,21 +1723,27 @@ export class IsolationModule {
      * and it must run before the migration so a failure becomes poison rather than a half-migrated
      * overlay.
      *
-     * Non-addColumn / non-tightening / non-retype changes (`addColumnCtx === undefined`, no
-     * reject-mode `setNotNullCtx`, no `setDataTypeCtx`) only run the tombstone guard; their row
-     * translation appends/removes nothing fallible on data grounds.
+     * For a primary-key re-keying `set collate` (`pkRekeyCtx`), the staged rows are grouped by
+     * their POST-change key: a deletion marker sharing a key with one live row is that row's
+     * before-image (the migrate step discards it — {@link dropCollapsedPkRekeyMarkers}), and
+     * markers alone on one key are one deletion — neither is a duplicate. Two LIVE rows on one
+     * key ARE, so that throws CONSTRAINT here, mirroring the underlying's own collision message.
+     * For the ISSUER this is belt-and-braces (the underlying's pre-mutation pass judges the same
+     * staged rows via {@link issuerEffectiveRows}, though this check fires first); for a FOREIGN
+     * overlay it is the only pass that sees those rows, and its throw becomes poison.
+     *
+     * Non-addColumn / non-tightening / non-retype / non-re-key changes (`addColumnCtx ===
+     * undefined`, no reject-mode `setNotNullCtx`, no `setDataTypeCtx`, no `pkRekeyCtx`) only run
+     * the tombstone guard; their row translation appends/removes nothing fallible on data grounds.
      */
-    async validateOverlayMigration(oldState, addColumnCtx, setNotNullCtx, setDataTypeCtx) {
+    async validateOverlayMigration(oldState, addColumnCtx, setNotNullCtx, setDataTypeCtx, pkRekeyCtx) {
         const oldOverlay = oldState.overlayTable;
         const oldOverlaySchema = oldOverlay.tableSchema;
         // Mirror the migrate-loop guard exactly: a clean overlay or one without a queryable
         // schema stages nothing and runs none of the fallible checks.
         if (!(oldState.hasChanges && oldOverlaySchema && oldOverlay.query))
             return;
-        const oldTombstoneIdx = oldOverlaySchema.columnIndexMap.get(this.tombstoneColumn.toLowerCase());
-        if (oldTombstoneIdx === undefined) {
-            throw new QuereusError(`Tombstone column '${this.tombstoneColumn}' missing from overlay schema`, StatusCode.INTERNAL);
-        }
+        const oldTombstoneIdx = this.requireTombstoneIndex(oldOverlaySchema);
         if (addColumnCtx) {
             for await (const oldRow of oldOverlay.query(makeFullScanFilterInfo())) {
                 // Discard the result — this is validation only. A NOT NULL violation throws here.
@@ -1579,6 +1769,19 @@ export class IsolationModule {
                 if (value !== null)
                     setDataTypeCtx.convert(value); // discard — validation only
             }
+            return;
+        }
+        // SET COLLATE re-keying the PRIMARY KEY: two staged LIVE rows landing on one post-change
+        // key are a real duplicate — refuse before the underlying re-keys (atomic for the issuer,
+        // poison for a foreign overlay via the caller). Marker/live and marker/marker groups
+        // collapse cleanly and are handled by the migrate step, not rejected here.
+        if (pkRekeyCtx) {
+            for (const group of (await this.collectPkRekeyGroups(oldOverlay, oldTombstoneIdx, pkRekeyCtx)).values()) {
+                if (group.livePks.length < 2)
+                    continue;
+                const keyDesc = group.livePks[1].map(formatKeyValue).join(', ');
+                throw new QuereusError(`UNIQUE constraint failed: ${pkRekeyCtx.tableName} primary key collides under new collation (key: ${keyDesc})`, StatusCode.CONSTRAINT);
+            }
         }
     }
     /**
@@ -1650,6 +1853,10 @@ export class IsolationModule {
      *   column lands ahead of the tombstone flag and the flag stays LAST — the layout
      *   every read/write path of this package assumes. A bare forward would append after
      *   the flag and every value written to the new column would be dropped on read.
+     *   A caller that named its own position keeps it: the overlay's data columns mirror
+     *   the base's one-for-one below the flag, so the base's index is the overlay's too.
+     *   (SQL always appends, so today the caller never names one — but silently ignoring
+     *   it would diverge the two layouts.)
      * - The column definition is made NULLABLE. The overlay's ADD re-runs the overlay
      *   module's own NOT NULL validation against a different row population than the
      *   base's: tombstone rows carry placeholder NULL in every non-PK column, so a
@@ -1675,7 +1882,7 @@ export class IsolationModule {
         return {
             ...change,
             columnDef: { ...change.columnDef, constraints },
-            insertAtIndex: tombstoneIdx,
+            insertAtIndex: change.insertAtIndex ?? tombstoneIdx,
             backfillEvaluator: (overlayRow) => this.computeAddColumnValue(addColumnCtx, overlayRow, tombstoneIdx),
         };
     }
@@ -1686,20 +1893,40 @@ export class IsolationModule {
      * `set data type` / `set collate` / `set default` forward through the overlay's own
      * `alterSchema`: the overlay module converts / re-keys its open transaction layers with
      * their savepoint snapshots intact, and NULLs pass a retype untouched, so tombstone rows —
-     * placeholder NULLs at every non-key data column — ride through unharmed.
-     *
-     * `set not null` must NOT forward: the overlay's copy of the column stays nullable.
-     * Tombstone rows carry NULL at every non-PK data column, so the overlay module's own
-     * tightening would backfill a deletion marker's placeholder NULL from the DEFAULT
-     * (corrupting a row that is not a row) or reject it outright when no usable DEFAULT
-     * exists. The base's NOT NULL is enforced where it belongs — by the underlying over the
-     * issuer's effective rows, and per overlay by {@link validateOverlayMigration} — and the
-     * staged LIVE rows' NULLs are filled here via {@link backfillStagedNotNull}. The runtime
-     * admits exactly one attribute per ALTER COLUMN, so a `setNotNull` change carries nothing
-     * else to forward. A missing overlay `alterSchema` is a no-op, as in
+     * placeholder NULLs at every non-key data column — ride through unharmed. One preparation
+     * step precedes a PRIMARY-KEY re-keying `set collate` (`pkRekeyCtx`): deletion markers the
+     * re-key collapses onto another staged row are discarded first
+     * ({@link dropCollapsedPkRekeyMarkers}) — the overlay's PK deliberately covers markers, so
+     * without the drop its own `alterSchema` would reject the collapsed pair as a PK collision.
+     *
+     * Neither NOT NULL direction forwards, and the overlay's `notNull` flag is left where it
+     * stood when the overlay was created:
+     *
+     * - **Tightening (`set not null`).** Forwarding would have the overlay module tighten a
+     *   column whose tombstone rows carry NULL at every non-PK data column, so it would either
+     *   backfill a deletion marker's placeholder NULL from the DEFAULT (corrupting a row that
+     *   is not a row) or reject it outright when no usable DEFAULT exists. The base's NOT NULL
+     *   is enforced where it belongs — by the underlying over the issuer's effective rows, and
+     *   per overlay by {@link validateOverlayMigration} — and the staged LIVE rows' NULLs are
+     *   filled here via {@link backfillStagedNotNull}.
+     * - **Relaxing (`drop not null`).** Nothing to do: the overlay never enforced the flag, and
+     *   the rows it stages are written with `preCoerced: true` through the overlay module's own
+     *   write path, which does not consult column nullability at all.
+     *
+     * NOTE: both directions therefore leave the overlay's `notNull` flag drifted from the base
+     * for the rest of the transaction. Inert today — nothing on the overlay path reads it. The
+     * one consumer that would, `MemoryTableModule.getBestAccessPlan` (which prunes `IS NULL` on
+     * a NOT NULL column to an empty result), is unreachable because the isolation layer builds
+     * the overlay's `FilterInfo` itself (see `filter-info.ts`) instead of planning against the
+     * overlay module. If overlay scans ever route through the overlay module's planner hook, or
+     * its write path starts enforcing nullability, this has to become a real forward that
+     * excludes tombstones.
+     *
+     * The runtime admits exactly one attribute per ALTER COLUMN, so a `setNotNull` change
+     * carries nothing else to forward. A missing overlay `alterSchema` is a no-op, as in
      * {@link forwardColumnShapeToOverlay}.
      */
-    async forwardAlterColumnToOverlay(overlayState, change, setNotNullCtx) {
+    async forwardAlterColumnToOverlay(overlayState, change, setNotNullCtx, pkRekeyCtx) {
         if (change.setNotNull !== undefined) {
             if (change.setNotNull === true && setNotNullCtx?.hasDefault) {
                 await this.backfillStagedNotNull(overlayState, setNotNullCtx);
@@ -1708,8 +1935,108 @@ export class IsolationModule {
         }
         if (!overlayState.overlayTable.alterSchema)
             return;
+        if (pkRekeyCtx)
+            await this.dropCollapsedPkRekeyMarkers(overlayState, pkRekeyCtx);
         await overlayState.overlayTable.alterSchema(change);
     }
+    /**
+     * Discards the deletion markers a pending PRIMARY-KEY re-key collapses onto another staged
+     * row, through the overlay's ordinary write path so its layer chain and savepoint snapshots
+     * stay intact. Two shapes collapse (see {@link PkRekeyContext}): a marker sharing its
+     * post-change key with a live row is that row's before-image — the live row's flush upsert
+     * is the complete net effect, so the marker is dropped; markers alone together on one key
+     * are one deletion, so all but the first are dropped. A group holding two live rows was
+     * already refused by {@link validateOverlayMigration} before the underlying mutated.
+     *
+     * Runs BEFORE the re-key is forwarded to the overlay, and deletes by each marker's OLD
+     * primary key — the overlay is still keyed under the old collation at that moment.
+     *
+     * NOTE: like {@link backfillStagedNotNull}'s rewrites, the drops land in the overlay's
+     * CURRENT savepoint frame. A later `rollback to savepoint` taken BEFORE this ALTER restores
+     * the marker while the table stays re-keyed — the same divergence class that method's NOTE
+     * documents (`bug-rolled-back-rows-violate-surviving-ddl`), not a new hole opened here.
+     */
+    async dropCollapsedPkRekeyMarkers(overlayState, ctx) {
+        const plan = await this.planPkRekeyMarkerDrops(overlayState, ctx);
+        await this.applyPkRekeyMarkerDrops(overlayState, ctx, plan);
+    }
+    /**
+     * The whole overlay rows of every deletion marker {@link dropCollapsedPkRekeyMarkers} will
+     * discard, computed without mutating anything. Re-planning after the drops finds nothing
+     * (each group then holds at most one marker and no marker beside a live row), so the
+     * issuer path — which drops in tier 2 for the pre-flight — can safely run the shared
+     * migrate-step drop again as a no-op.
+     */
+    async planPkRekeyMarkerDrops(overlayState, ctx) {
+        const overlay = overlayState.overlayTable;
+        const overlaySchema = overlay.tableSchema;
+        if (!overlaySchema || !overlay.query)
+            return [];
+        const tombstoneIdx = this.requireTombstoneIndex(overlaySchema);
+        const toDrop = [];
+        for (const group of (await this.collectPkRekeyGroups(overlay, tombstoneIdx, ctx)).values()) {
+            // NOTE: the marker-only arm (`slice(1)`) is defensive — no marker-only collapse can
+            // currently SURVIVE the ALTER. Two markers share a post-change key only if the rows
+            // they shadow did too, and such a pair is refused one level down: by the underlying's
+            // re-key (a committed pair) or by the overlay's own (a staged-then-deleted pair). It
+            // still runs, and is still undone, on the way to those refusals.
+            toDrop.push(...(group.livePks.length > 0 ? group.markerRows : group.markerRows.slice(1)));
+        }
+        return toDrop;
+    }
+    /** Deletes each planned marker by its OLD primary key through the overlay's ordinary write path. */
+    async applyPkRekeyMarkerDrops(overlayState, ctx, toDrop) {
+        const overlay = overlayState.overlayTable;
+        const overlaySchema = overlay.tableSchema;
+        if (!overlaySchema || toDrop.length === 0)
+            return;
+        for (const markerRow of toDrop) {
+            const pk = ctx.pkIndices.map(i => markerRow[i]);
+            const result = await overlay.update({
+                operation: 'delete',
+                values: undefined,
+                oldKeyValues: pk,
+            });
+            // A bare delete of a staged marker has no data condition to hit, so a non-ok result
+            // is a layer-invariant violation — raise INTERNAL (which applyInPlaceOverlayChange
+            // rethrows loud for issuer and foreign alike), never poison.
+            if (isConstraintViolation(result)) {
+                throw new QuereusError(`Dropping a collapsed deletion marker (pk=[${pk.join(', ')}]) from overlay '${overlaySchema.name}' hit a ${result.constraint} constraint: ${result.message ?? 'no message'}`, StatusCode.INTERNAL);
+            }
+        }
+    }
+    /**
+     * Undoes {@link applyPkRekeyMarkerDrops} after a refusal that aborts the whole ALTER —
+     * the overlay pre-flight's or the underlying's (see the tier-2 re-key block in
+     * {@link alterTable}). Reinserts each marker row verbatim: a marker minted for a committed
+     * row carries NULL at every non-key data column, but one made by deleting a row this
+     * transaction had already staged keeps that row's values (the convert-to-tombstone branch
+     * of `IsolatedTable.update`), so a reconstructed-from-PK row would not restore what was
+     * dropped. The reinserts land in the same savepoint frame the drops did, so the pair nets
+     * to nothing.
+     */
+    async reinsertPkRekeyMarkers(overlayState, droppedRows) {
+        if (droppedRows.length === 0)
+            return;
+        const overlay = overlayState.overlayTable;
+        const overlaySchema = overlay.tableSchema;
+        if (!overlaySchema)
+            return;
+        for (const markerRow of droppedRows) {
+            const result = await overlay.update({ operation: 'insert', values: [...markerRow] });
+            if (isConstraintViolation(result)) {
+                throw new QuereusError(`Restoring a deletion marker (row=[${markerRow.join(', ')}]) to overlay '${overlaySchema.name}' after a refused ALTER hit a ${result.constraint} constraint: ${result.message ?? 'no message'}`, StatusCode.INTERNAL);
+            }
+        }
+    }
+    /** The overlay's tombstone-flag column index, or INTERNAL if the schema lost it. */
+    requireTombstoneIndex(overlaySchema) {
+        const tombstoneIdx = overlaySchema.columnIndexMap.get(this.tombstoneColumn.toLowerCase());
+        if (tombstoneIdx === undefined) {
+            throw new QuereusError(`Tombstone column '${this.tombstoneColumn}' missing from overlay schema`, StatusCode.INTERNAL);
+        }
+        return tombstoneIdx;
+    }
     /**
      * Backfills a `set not null` tightening into an overlay's staged LIVE rows: every
      * non-tombstone row holding NULL at the tightened column is rewritten with the column's
@@ -1896,6 +2223,14 @@ export class IsolationModule {
             state.poison = { message: this.buildAlterPoisonMessage(schemaName, tableName, change) };
             return;
         }
+        // NOTE: this is the only remaining path that swaps an overlay table rather than adopting
+        // the change in place, so a registered `IsolatedConnection` keeps the OLD overlay's
+        // connection (captured at construction, isolated-connection.ts) and forwards its
+        // savepoint/rollback calls to a released table. Benign today: the swapped overlay is
+        // clean, so nothing staged is lost, and the fresh overlay registers its own connection
+        // with the `Database` on its first write and receives the broadcasts directly. If this
+        // swap ever becomes reachable with staged rows, or per-connection state moves onto
+        // `IsolatedConnection`, retarget the connection here.
         const fresh = await this.overlayModule.create(state.db, this.createOverlaySchema(updatedSchema));
         this.connectionOverlays.set(key, { overlayTable: fresh, hasChanges: false, db: state.db });
         await this.releaseOverlayTable(state);
@@ -1921,6 +2256,16 @@ export class IsolationModule {
      *
      * `IsolatedTable.mergedSecondaryIndexQuery` wants exactly the live overlay rows out of
      * these indexes, so narrowing them is what it already expects.
+     *
+     * NOTE: a base column already named {@link tombstoneColumn} lands here as a SECOND column
+     * of that name, and `columnIndexMap` resolves the name to the appended flag. Harmless
+     * today — every consumer reaches the flag by index, and reads, merged scans, a unique
+     * secondary index over that column and a staged deletion all behave exactly as they do
+     * for a non-colliding name. If anything ever resolves the overlay's columns BY NAME (a
+     * hand-built overlay predicate, a module that re-derives its own index map from the
+     * column list), give the overlay a collision-free flag name here instead. ALTER into the
+     * name is rejected outright — see {@link assertColumnNameNotTombstone} for why that one
+     * cannot be tolerated.
      */
     createOverlaySchema(baseSchema) {
         const tombstoneColumn = {