@quereus/quereus 4.13.1 → 4.14.0

package/dist/src/runtime/emit/materialized-view-helpers.js

@@ -41,9 +41,9 @@ export function materializedViewNotASetError(schemaName, viewName) {
  * the schema declares NOT NULL *and* that is a physical-PK member. This is the exact
  * reachable contradiction: the view's `order by <col>` seeded the column into the
  * backing's **physical** primary key ({@link computeBackingPrimaryKey}), which keeps it
- * declared NOT NULL even after the source column loosened to nullable — a physical-PK
- * column cannot drop NOT NULL (`MemoryTableManager.alterColumn` refuses it, and the
- * refresh reshape masks the doomed loosen — see {@link isPhysicalPkColumn}). Once the
+ * declared NOT NULL even after the source column loosened to nullable — the refresh reshape
+ * masks the loosening of a physical-PK column rather than emitting it (see
+ * {@link isPhysicalPkColumn}). Once the
  * source column becomes nullable and the recomputed body yields a NULL row, storing it
  * would leave the backing schema declaring NOT NULL while holding a NULL — a silent
  * contradiction. Both maintenance vectors raise this instead, naming the column, the cause,
@@ -73,11 +73,20 @@ export function nullInNotNullSeededPkError(schemaName, viewName, columnName) {
  * `keysOf` (all-columns fallback when none — such an MV is incremental-ineligible
  * until Phase 2). Re-planning here is cheap relative to materialization and keeps
  * the create/refresh emitters free of optimizer plumbing.
+ *
+ * `backing` names the module that hosts (or would host) this shape's backing table,
+ * plus (for a comparison) the table the result will be compared against. Supplying it
+ * runs the derived shape through that module's own create-time normalization
+ * ({@link normalizeBackingShape}) so every downstream comparison sees the shape the
+ * module would actually create — see that function for why, and for what `against`
+ * resolves. Omit only where no backing identity exists; every consumer that compares
+ * against a live backing must pass it.
  */
-export function deriveBackingShape(db, schemaName, bodySql, explicitColumns) {
+export function deriveBackingShape(db, schemaName, bodySql, explicitColumns, backing) {
     // Suppress the read-side rewrite: we are computing the MV body to derive/populate
     // its OWN backing, so it must not be rewritten to read that backing.
-    return db.schemaManager.withSuppressedMaterializedViewRewrite(() => deriveBackingShapeUnguarded(db, schemaName, bodySql, explicitColumns));
+    const shape = db.schemaManager.withSuppressedMaterializedViewRewrite(() => deriveBackingShapeUnguarded(db, schemaName, bodySql, explicitColumns));
+    return normalizeBackingShape(db, schemaName, shape, backing?.moduleName, backing?.moduleArgs, backing?.against);
 }
 function deriveBackingShapeUnguarded(db, schemaName, bodySql, explicitColumns) {
     // Home-schema path: the body's unqualified source names resolve next to the
@@ -278,6 +287,146 @@ tags) {
         tags: tags && Object.keys(tags).length > 0 ? tags : undefined,
     };
 }
+/**
+ * Applies the backing module's own create-time schema normalization
+ * ({@link VirtualTableModule.normalizeCreateSchema}) to a derived {@link BackingShape},
+ * so a shape derived from the body is expressed in the SAME terms as a table the module
+ * actually created.
+ *
+ * Why: a module may own per-column physical attributes the body's output type cannot
+ * predict. The store module keys an *implicit*-collation text primary-key column under
+ * its table-level key collation K (default NOCASE — `reconcilePkCollations`), so a
+ * persisted backing reads `COLLATE NOCASE` while the re-derived shape says BINARY. Every
+ * shape comparison ({@link backingShapeMatches}, {@link describeBackingShapeMismatch},
+ * {@link classifyBackingReshape}) would then report a difference that does not exist,
+ * refilling the backing on every reopen. Normalizing here — at the SOURCE, before any
+ * comparison — keeps both sides post-normalization rather than teaching each comparison
+ * site to special-case a module's rewrite.
+ *
+ * Identity for a module without the hook (memory, and every third-party module), and for
+ * an unresolvable module name — the real `no virtual table module named …` /
+ * capability diagnostics stay with {@link buildBackingTableSchema}, which is deliberately
+ * NOT reused here: it carries the `getBackingHost` gate (whose error must keep firing at
+ * its own site) and stamps primary-key flags onto `shape.columns`, neither of which
+ * belongs on a pure "what would this become?" probe.
+ *
+ * **`against` — two-valued compatibility for an implicit attribute.** The hook only ever
+ * rewrites attributes the shape left IMPLICIT (the store touches a text primary-key column
+ * only when it declares no `collate`). An implicit attribute is genuinely ambiguous once a
+ * table exists, because explicitness does not survive a catalog round-trip
+ * (`ColumnSchema.collationExplicit` is not persisted; a declared `collate binary` reloads
+ * indistinguishable from no clause at all). So an implicit shape column is compatible with
+ * a live column carrying EITHER the shape's own declared value or the module's normalized
+ * one — both are physically consistent with the body. When the caller names the table this
+ * shape will be compared against, the rewrite is therefore kept only where the RAW shape
+ * does not already agree with that table; a table-form maintained table whose text key was
+ * declared `collate binary` keeps matching its own declaration, while a materialized-view
+ * backing the module keyed under K matches that. Omit `against` on a create path (nothing
+ * to be compatible with yet) and the module's answer is taken verbatim.
+ *
+ * A collation the module would NOT produce and the body did NOT declare (a backing
+ * persisted under RTRIM, say) matches neither reading and still reports a mismatch. The
+ * agreement test looks only at the attributes the hook actually rewrote — see
+ * {@link rawReadingIsTheLiveOne}.
+ *
+ * Only `columns` can change; `primaryKey`, `ordering`, `sourceTables`, `coarsenedKey`,
+ * and `allProvedKeys` pass through untouched.
+ */
+function normalizeBackingShape(db, schemaName, shape, moduleName, moduleArgs, against) {
+    const resolvedModuleName = normalizeBackingModuleName(moduleName);
+    const module = db.schemaManager.getModule(resolvedModuleName)?.module;
+    if (!module?.normalizeCreateSchema)
+        return shape;
+    // Copy the columns into the probe: the contract says the hook is pure, but a
+    // sloppy implementation mutating a column in place would otherwise corrupt the
+    // caller's shape. The probe uses the PHYSICAL primary key
+    // ({@link computeBackingPrimaryKey} — ordering-seeded), not `shape.primaryKey`,
+    // because that is the key the module will actually be handed at create; a text
+    // ORDERING column is reconciled there too and would keep diverging otherwise.
+    const probeColumns = shape.columns.map(c => ({ ...c }));
+    const probe = {
+        name: `<materialized-view backing shape probe>`,
+        schemaName,
+        columns: probeColumns,
+        columnIndexMap: buildColumnIndexMap(probeColumns),
+        primaryKeyDefinition: computeBackingPrimaryKey(shape).map(pk => ({
+            index: pk.index,
+            desc: pk.desc,
+            collation: probeColumns[pk.index]?.collation,
+        })),
+        checkConstraints: [],
+        vtabModule: module,
+        vtabModuleName: resolvedModuleName,
+        vtabArgs: moduleArgs ? { ...moduleArgs } : {},
+        isView: false,
+        estimatedRows: 0,
+    };
+    const normalized = module.normalizeCreateSchema(probe);
+    if (normalized.columns === probe.columns)
+        return shape; // hook declined to rewrite
+    // A reshape is a contract violation, not a shape difference: the hook may only
+    // adjust per-column attributes it owns physically. Fail loud rather than let a
+    // misbehaving module silently redefine the backing's column list.
+    if (normalized.columns.length !== shape.columns.length) {
+        throw new QuereusError(`module '${resolvedModuleName}' normalizeCreateSchema changed the column count `
+            + `(${shape.columns.length} → ${normalized.columns.length}); it may only adjust per-column attributes`, StatusCode.INTERNAL);
+    }
+    for (let i = 0; i < shape.columns.length; i++) {
+        if (normalized.columns[i].name !== shape.columns[i].name) {
+            throw new QuereusError(`module '${resolvedModuleName}' normalizeCreateSchema renamed or reordered column ${i} `
+                + `('${shape.columns[i].name}' → '${normalized.columns[i].name}'); it may only adjust per-column attributes`, StatusCode.INTERNAL);
+        }
+    }
+    // Two-valued compatibility (see the docblock): where the RAW shape column already
+    // agrees with the table this shape is about to be compared against, that reading is
+    // the live one and the module's alternative is not applied.
+    // NOTE: the two readings exist only because `ColumnSchema.collationExplicit` does not
+    // survive a catalog round-trip. If explicitness is ever persisted, the live table can
+    // answer "did the author declare this?" directly and this collapses to a single
+    // reading — take the module's answer iff the live column is implicit.
+    // NOTE: `against` is read POSITIONALLY, which is what the rename cascade needs (names
+    // have shifted there, the attributes have not). It is only ever consulted for a column
+    // the hook rewrote, so a pure append/drop reshape still resolves correctly — but if a
+    // reshape ever MOVES a rewritten column to a position the live table fills with a
+    // different column, this asks the wrong column and the alignment-based
+    // {@link classifyBackingReshape} would see a collation the module did not produce. Give
+    // callers a way to say "align by name" if a reshape is ever seen misclassified this way.
+    const columns = normalized.columns.map((col, i) => {
+        const raw = shape.columns[i];
+        if (col === raw)
+            return raw;
+        const live = against?.columns[i];
+        if (live && rawReadingIsTheLiveOne(raw, col, live))
+            return raw;
+        return col;
+    });
+    if (columns.every((c, i) => c === shape.columns[i]))
+        return shape;
+    return { ...shape, columns };
+}
+/**
+ * The two-valued compatibility test of {@link normalizeBackingShape}: does the RAW
+ * (pre-normalization) reading of a column agree with the live table on every attribute
+ * the hook actually rewrote?
+ *
+ * Only the REWRITTEN attributes are compared. An attribute the hook left alone carries no
+ * second reading, so a difference in it is a genuine shape change for the comparison sites
+ * to report — it says nothing about which reading of the rewritten attribute is live.
+ * Testing all three attributes unconditionally instead would let an unrelated difference
+ * decide the collation: a NOT NULL loosening on a physical-PK column — which
+ * {@link describeBackingShapeMismatch} deliberately masks — would flip a declared
+ * `collate binary` key over to the module's K and turn a data-only refresh into a
+ * spurious "primary-key column 0 collation BINARY → NOCASE" inexpressible reshape.
+ */
+function rawReadingIsTheLiveOne(raw, normalized, live) {
+    if (!backingTypeMatches(raw, normalized) && !backingTypeMatches(raw, live))
+        return false;
+    if (!backingNotNullMatches(raw, normalized) && !backingNotNullMatches(raw, live))
+        return false;
+    if (!backingCollationMatches(raw, normalized) && !backingCollationMatches(raw, live))
+        return false;
+    return true;
+}
 /** Runs the body to completion and returns its rows (raw `Row` arrays). Uses the
  *  no-transaction-management primitive — the caller is already inside DDL execution. */
 export async function collectBodyRows(db, schemaName, bodySql) {
@@ -396,7 +545,7 @@ function warnKeyCoarsening(schemaName, viewName, info) {
  */
 export async function materializeView(db, def, preDerivedShape) {
     const sm = db.schemaManager;
-    const shape = preDerivedShape ?? deriveBackingShape(db, def.schemaName, def.bodySql, def.columns);
+    const shape = preDerivedShape ?? deriveBackingShape(db, def.schemaName, def.bodySql, def.columns, { moduleName: def.backingModuleName, moduleArgs: def.backingModuleArgs });
     // Lives here — not in deriveBackingShape — because the refresh path reaches a
     // legitimate mismatch after a source ALTER (see the assert's docstring).
     assertDeclaredColumnArity(def, shape);
@@ -913,7 +1062,7 @@ discardBackingOnFailure = false) {
     // renamed positionally to it and the name check skipped; otherwise natural output
     // names with the strict declared-shape check (the body must already be aliased to
     // the declared names — the attach verb / implicit-create posture).
-    const shape = deriveBackingShape(db, schemaName, bodySql, positionalRename ? recordedColumns : undefined);
+    const shape = deriveBackingShape(db, schemaName, bodySql, positionalRename ? recordedColumns : undefined, { moduleName: table.vtabModuleName, moduleArgs: table.vtabArgs, against: table });
     // Explicit rename-list arity guard. `deriveBackingShape` sizes the shape to the
     // BODY's arity (a surplus rename name is dropped, a missing one padded), so a
     // list whose length disagrees with the body would otherwise record an
@@ -1272,7 +1421,7 @@ export async function createMaintainedTable(db, stmt) {
     const declared = sm.buildDeclaredTableSchema(stmt);
     const recordedColumns = explicit ? declared.columns.map(c => c.name) : undefined;
     const bodySql = astToString(stmt.maintained.select);
-    const shape = deriveBackingShape(db, schemaName, bodySql, explicit ? recordedColumns : undefined);
+    const shape = deriveBackingShape(db, schemaName, bodySql, explicit ? recordedColumns : undefined, { moduleName: declared.vtabModuleName, moduleArgs: declared.vtabArgs, against: declared });
     const mismatch = describeAttachShapeMismatch(declared, shape, explicit);
     if (mismatch) {
         throw new QuereusError(`cannot create maintained table '${schemaName}.${name}': ${mismatch}`, StatusCode.ERROR, undefined, stmt.table.loc?.start.line, stmt.table.loc?.start.column);
@@ -1358,8 +1507,8 @@ export async function rebuildBacking(db, mv, validationColumns) {
     // a backing column declared NOT NULL that is also a physical-PK member cannot legally
     // hold the recomputed NULL a nullable source now produces. The ordering-seeded physical
     // PK (computeBackingPrimaryKey) pins such a column NOT NULL even after its source column
-    // dropped NOT NULL — the memory manager refuses to loosen a PK column and the reshape
-    // masks the doomed loosen — so at this point the only still-NOT-NULL columns that can
+    // dropped NOT NULL — the reshape masks the loosening of a physical-PK column rather than
+    // emitting it — so at this point the only still-NOT-NULL columns that can
     // carry source-nullable data are the physical-PK ones. Storing the NULL would make the
     // backing schema declare NOT NULL while holding a NULL; reject it before either swap.
     // NOTE: narrow loud guard until `debt-mv-ordering-seed-to-materialized-index` replaces
@@ -1462,9 +1611,17 @@ function backingCollationMatches(a, b) {
 /** Whether `columnNameLower` (already lowercased) names a column of `table`'s
  *  *physical* primary key ({@link TableSchema.primaryKeyDefinition} — which for an
  *  MV backing includes any ordering-seeded columns, {@link computeBackingPrimaryKey}).
- *  A physical-PK column is NOT NULL by definition: `MemoryTableManager.alterColumn`
- *  refuses to DROP NOT NULL on one, so the refresh reshape must never try to loosen
- *  its NOT NULL — see the two callers below. */
+ *  An MV backing keeps its physical-PK columns NOT NULL, so the refresh reshape must
+ *  never try to loosen one — see the two callers below.
+ *
+ *  NOTE: that is now a POLICY of the MV backing, not an engine constraint. A primary key
+ *  column may be nullable in general (docs/schema.md § Primary-key nullability) and
+ *  `alter column … drop not null` on one is accepted by both backends; the backing keeps
+ *  its seeded key columns NOT NULL because the ordering seed treats them as a total order
+ *  and `assertNoNullInNotNullSeededPk` is the loud guard for the contradiction. If
+ *  ordering-seeded backings ever need to carry NULLs in the seeded column, this mask and
+ *  that guard are the pair to revisit together — `debt-mv-ordering-seed-to-materialized-index`
+ *  removes the need for both. */
 function isPhysicalPkColumn(table, columnNameLower) {
     return table.primaryKeyDefinition.some(def => table.columns[def.index]?.name.toLowerCase() === columnNameLower);
 }
@@ -1487,8 +1644,8 @@ function describeBackingShapeMismatch(current, shape) {
         }
         if (!backingNotNullMatches(a, b)) {
             // A physical-PK column stays NOT NULL in the backing regardless of the
-            // re-derived logical nullability (a PK column cannot hold NULL — the memory
-            // manager refuses to DROP NOT NULL on it), so a NOT-NULL→nullable *loosening*
+            // re-derived logical nullability (an MV-backing policy — see
+            // {@link isPhysicalPkColumn}), so a NOT-NULL→nullable *loosening*
             // of a PK column is NOT a shape difference. Masking it lets refresh keep the
             // data-only rebuild path instead of emitting a doomed `loosenNotNull` op. Tight
             // on purpose: only a loosening (`current` NOT NULL, derived nullable) of a
@@ -1836,14 +1993,14 @@ export function tryRecompileMaterializedViewLive(db, mv, oldObject, newObject) {
     try {
         const d = mv.derivation;
         const bodySql = astToString(d.selectAst);
-        const shape = deriveBackingShape(db, mv.schemaName, bodySql, d.columns);
+        const schema = db.schemaManager.getSchemaOrFail(mv.schemaName);
+        const live = schema.getTable(mv.name);
+        const backing = isMaintainedTable(live) ? live : mv;
+        const shape = deriveBackingShape(db, mv.schemaName, bodySql, d.columns, { moduleName: mv.vtabModuleName, moduleArgs: mv.vtabArgs, against: backing });
         if (!sameSourceTables(d.sourceTables, shape.sourceTables)) {
             log('Marking materialized view %s.%s stale instead of recompiling: re-planned source tables (%s) disagree with the recorded set (%s) — REFRESH re-derives', mv.schemaName, mv.name, shape.sourceTables.join(', '), d.sourceTables.join(', '));
             return false;
         }
-        const schema = db.schemaManager.getSchemaOrFail(mv.schemaName);
-        const live = schema.getTable(mv.name);
-        const backing = isMaintainedTable(live) ? live : mv;
         const mismatch = describeBackingShapeMismatch(backing, shape);
         if (mismatch) {
             // Relaxed superkey gate: columns match structurally AND the existing backing
@@ -1964,10 +2121,10 @@ function classifyBackingReshape(current, shape) {
         if (!backingNotNullMatches(from, to)) {
             if (to.notNull === true)
                 postReconcileOps.push({ kind: 'tightenNotNull', name });
-            // A physical-PK column cannot hold NULL, so the backing keeps it NOT NULL and
-            // the memory manager refuses to DROP NOT NULL on it: skip the doomed
-            // `loosenNotNull` op for a physical-PK column of `current` (matched by its
-            // pre-rename name). Reached only when reshape is entered for SOME OTHER genuine
+            // An MV backing keeps its physical-PK columns NOT NULL (a backing policy — see
+            // {@link isPhysicalPkColumn}), so skip the `loosenNotNull` op for a physical-PK
+            // column of `current` (matched by its pre-rename name) rather than emitting a
+            // loosening the backing must not take. Reached only when reshape is entered for SOME OTHER genuine
             // shape change coexisting with a PK-column loosening in the same refresh;
             // touch point #1 masks the loosening-only case before reshape. See the NOTE at
             // describeBackingShapeMismatch / computeBackingPrimaryKey (~line 236).
@@ -2713,7 +2870,7 @@ export async function restoreUnaffectedMaterializedViews(db, preStale) {
             // Throws when the body no longer plans against the renamed catalog
             // (e.g. a chained MV referencing a renamed-away output name) → catch
             // below leaves it stale.
-            const shape = deriveBackingShape(db, mv.schemaName, bodySql, d.columns);
+            const shape = deriveBackingShape(db, mv.schemaName, bodySql, d.columns, { moduleName: mv.vtabModuleName, moduleArgs: mv.vtabArgs, against: schema.getTable(mv.name) });
             // The retry of a failure-marked MV must not revive an inconsistent record: a
             // rewrite that threw between the in-place AST mutation and the derived-field
             // re-key leaves the OLD derivation (un-re-keyed `sourceTables`) holding the
@@ -2769,7 +2926,7 @@ function sameSourceTables(a, b) {
  * rewrite (a mid-statement failure).
  */
 async function renameShiftedBackingColumns(db, schema, mv, bodySql, preDerivedShape) {
-    const shape = preDerivedShape ?? deriveBackingShape(db, mv.schemaName, bodySql, mv.derivation.columns);
+    const shape = preDerivedShape ?? deriveBackingShape(db, mv.schemaName, bodySql, mv.derivation.columns, { moduleName: mv.vtabModuleName, moduleArgs: mv.vtabArgs, against: schema.getTable(mv.name) });
     const backing = schema.getTable(mv.name);
     if (!backing) {
         throw new QuereusError(`Internal error: maintained table '${mv.name}' not found during backing-column rename`, StatusCode.INTERNAL);