@quereus/quereus 4.17.0 → 4.18.0

package/dist/src/planner/rules/join/rule-semi-join-pushdown.js

@@ -0,0 +1,214 @@
+/**
+ * Rule: Semi-Join Pushdown — reassociate a semi join below an inner/cross join
+ *
+ * A semi join is an existential *filter* on its left input: it emits the left
+ * rows for which the key source holds a match, and publishes the left input's
+ * attributes verbatim. When its condition reads columns from only ONE side of an
+ * inner/cross join underneath it, the filter can be applied to that side first:
+ *
+ *   Join(semi, Join(inner|cross, L, R), K, cond)   cond reads L (and K) only
+ *     →  Join(inner|cross, Join(semi, L, K, cond), R)
+ *
+ *   Join(semi, Join(inner|cross, L, R), K, cond)   cond reads R (and K) only
+ *     →  Join(inner|cross, L, Join(semi, R, K, cond))
+ *
+ * This is the same argument `rules/predicate/rule-join-predicate-pushdown.ts`
+ * makes for scalar conjuncts — see its header's null-extension table. Neither
+ * side of an `inner` / `cross` join is ever null-extended, so every output row
+ * carries that side's genuine input values, and filtering the side before the
+ * join removes exactly the output rows filtering after it would have.
+ *
+ * Soundness, stated for the semi case: `semi(X, K, p) = { x ∈ X : ∃k ∈ K. p(x, k) }`.
+ * When `p` reads only the `L` part of `x`, the test is per-`L`-row and
+ * deterministic, so it commutes with the join — whether the join drops `L` rows
+ * or fans them out, filtering before removes precisely the output rows filtering
+ * after would have.
+ *
+ * Output attributes are unchanged in identity AND order. A semi join publishes
+ * its left input's attributes verbatim, so both the original and the rewritten
+ * shape yield `L.attrs ++ R.attrs`; every `ColumnReferenceNode` above the join
+ * resolves to the same attribute id it did before.
+ *
+ * WHY: `rules/access/rule-key-set-seek.ts` — which turns a semi join into
+ * "materialize the key set once, then seek the target index per key" — only
+ * fires when the filtered side peels down to a bare access leaf. A compound
+ * query (`… from entry e join txn t on t.id = e.txn_id where e.txn_id in
+ * (select …)`) puts a join there instead, so that rule declines and `entry` is
+ * read end-to-end. After this reassociation the filtered side is a bare leaf
+ * again and the existing rule fires unchanged — no new peel through `JoinNode`
+ * is needed (walking a join in the peel would be unsound; this rule is the
+ * sound way to reach the same plan).
+ *
+ * Deeper nesting works without extra machinery: the rewrite's new semi join is a
+ * child of the returned node, so the top-down descent visits it and
+ * `Join(semi, Join(inner, Join(inner, A, B), C), K)` pushes down twice.
+ *
+ * Admission gates (all must hold):
+ * - Anchor is a semi `JoinNode` with a condition and no `exists … as` flags.
+ * - `node.left` is an `inner` or `cross` `JoinNode`.
+ * - The key source is uncorrelated, deterministic and write-free — the same
+ *   admission test `rule-key-set-seek`'s `admitJoin` applies, and for the same
+ *   reason: re-rooting the semi join must not change how the source is drained.
+ * - The condition is functional (deterministic + read-only): the reassociation
+ *   changes how many rows it is evaluated against.
+ * - Neither inner-join branch is correlated. A LATERAL right side reads the left
+ *   side, and re-rooting either branch under a semi join must not disturb that.
+ * - Neither inner-join branch carries a write.
+ * - Every attribute id the condition needs (`collectPredicateAttributeIds`,
+ *   which also accounts for correlated references inside a sub-query operand) is
+ *   either a key-source attribute or an attribute of EXACTLY ONE inner-join
+ *   branch. An id in neither branch (an outer reference, an `exists … as` flag)
+ *   declines; ids spanning both branches decline; ids touching neither branch
+ *   decline because there is nothing to gain.
+ *
+ * `inner.condition` / `inner.usingColumns` / `inner.existence` carry over
+ * verbatim to the rebuilt inner join, and `node.condition` / `node.usingColumns`
+ * to the pushed semi join — both rebuilds go through `withChildren`, which
+ * threads those fields. Existence flags on an INNER join are unreachable today
+ * (the parser only accepts `exists … as` on `left join`), and the attribute-id
+ * gate above already declines any condition that reads a flag, so no redundant
+ * guard is added for them.
+ *
+ * NOT IN SCOPE — deliberately:
+ * - NOTE: `anti` joins are declined by design, not by accident.
+ *   `anti(L ⋈ R, K) = anti(L, K) ⋈ R` when the condition reads only `L`,
+ *   exactly as above (the test is still per-`L`-row and deterministic), but
+ *   `rule-key-set-seek` admits `semi` only, so there is nothing downstream to
+ *   gain. Revisit if that rule ever admits `anti`.
+ * - NOTE: a LEFT join on the probe side with the semi condition reading the
+ *   PRESERVED (left) side is also sound — `σ(L ⟕ R) = σ(L) ⟕ R` when `σ` reads
+ *   only `L` — and is left out only to keep the first pass to the shape in the
+ *   repro. Revisit if a compound outer-join shape shows up wanting the seek.
+ *   The MIRROR case is NOT sound and must stay declined permanently: a
+ *   condition reading the NULL-EXTENDED side (the right side of a `left join`)
+ *   does not commute, because filtering `R` before the join re-pads the left row
+ *   with NULLs instead of dropping it.
+ *   `backlog/feat-outer-join-to-inner-on-null-rejecting-filter`, if it lands,
+ *   converts some outer joins to inner ones and this rule then covers them.
+ * - NOTE: the anchor's left input must be the `JoinNode` ITSELF — no wrapper is
+ *   peeled. `select … from (select … from a join b …) x where x.k in (select …)`
+ *   and a residual `Filter` that survived predicate pushdown (one spanning both
+ *   branches, e.g. `… and e.amount > t.id`) both leave a node in between and
+ *   decline. Revisit if a wrapped compound shape shows up wanting the seek; the
+ *   peel would have to re-wrap whatever it walked through, which is why the
+ *   first pass does not attempt it.
+ *
+ * NOTE: pushing leaves a semi join as a LEAF of the enclosing inner spine, and
+ * `rule-quickpick-enumeration` abandons any join graph containing a non-inner
+ * join — so a 3+-relation join under an `IN (SELECT …)` filter now gets no join
+ * enumeration where it used to. Seek beats enumeration on the shapes this rule
+ * targets; revisit (by teaching QuickPick to admit a non-inner join as one opaque
+ * leaf) if such a query shows up with a bad join order. See the matching `NOTE:`
+ * in `rule-quickpick-enumeration.ts`.
+ */
+import { createLogger } from '../../../common/logger.js';
+import { JoinNode } from '../../nodes/join-node.js';
+import { PlanNodeCharacteristics } from '../../framework/characteristics.js';
+import { collectPredicateAttributeIds } from '../../analysis/predicate-dependencies.js';
+import { isCorrelatedSubquery } from '../../cache/correlation-detector.js';
+const log = createLogger('optimizer:rule:semi-join-pushdown');
+const attributeIds = (node) => new Set(node.getAttributes().map(a => a.id));
+/**
+ * The single inner-join branch `cond` reads, or null when it spans both, escapes
+ * both (an outer reference or an existence flag), or reads neither.
+ */
+function conditionSide(cond, keyIds, leftIds, rightIds) {
+    let needsLeft = false;
+    let needsRight = false;
+    for (const id of collectPredicateAttributeIds(cond)) {
+        if (keyIds.has(id))
+            continue;
+        if (leftIds.has(id)) {
+            needsLeft = true;
+            continue;
+        }
+        if (rightIds.has(id)) {
+            needsRight = true;
+            continue;
+        }
+        // An id belonging to neither branch nor the key source: an outer
+        // reference, or an `exists … as` flag the inner join appends. Pushing
+        // below either branch would strand it.
+        return null;
+    }
+    // Equal means either "spans both branches" (unsound to push to one) or
+    // "reads neither" (nothing to gain — a condition over constants alone).
+    if (needsLeft === needsRight)
+        return null;
+    return needsLeft ? 'left' : 'right';
+}
+export function ruleSemiJoinPushdown(node, _context) {
+    if (!(node instanceof JoinNode))
+        return null;
+    if (node.joinType !== 'semi')
+        return null;
+    if (!node.condition)
+        return null;
+    // An `exists … as` flag on the anchor would be published above it, but the
+    // pushed semi join publishes only its own left input's attributes.
+    if (node.hasExistenceColumns)
+        return null;
+    const inner = node.left;
+    if (!(inner instanceof JoinNode))
+        return null;
+    if (inner.joinType !== 'inner' && inner.joinType !== 'cross')
+        return null;
+    // NOTE: unconditional — no cost gate, matching `rule-join-predicate-pushdown`'s
+    // treatment of scalar conjuncts. At Structural time the row estimates are not
+    // usable anyway (most table-backed inputs report `Infinity`; see the NOTE in
+    // `rule-join-greedy-commute`). The one shape where pushdown does extra work is a
+    // strongly *filtering* inner join, where the semi join probes |L| rows instead of
+    // the smaller |L ⋈ R| — offset by the join then receiving a smaller left input.
+    // Revisit condition: if a filtering-join shape ever shows up regressed in
+    // `yarn bench:gate`, gate this on estimated rows.
+    const keySource = node.right;
+    // The key source is drained once by the semi join wherever it sits; a
+    // correlated / non-deterministic / write-bearing source must keep its
+    // per-execution semantics. Same admission test as `rule-key-set-seek`.
+    if (isCorrelatedSubquery(keySource)) {
+        log('decline: key source is correlated');
+        return null;
+    }
+    if (!PlanNodeCharacteristics.isDeterministic(keySource)) {
+        log('decline: key source is non-deterministic');
+        return null;
+    }
+    if (PlanNodeCharacteristics.subtreeHasSideEffects(keySource)) {
+        log('decline: key source has side effects');
+        return null;
+    }
+    // Both branches are re-rooted (one under the semi join, one under the rebuilt
+    // inner join) — refuse to touch a subtree carrying a write.
+    if (PlanNodeCharacteristics.subtreeHasSideEffects(inner.left)
+        || PlanNodeCharacteristics.subtreeHasSideEffects(inner.right)) {
+        log('decline: an inner-join branch has side effects');
+        return null;
+    }
+    // A LATERAL right side reads the left side; re-rooting either branch must not
+    // disturb the row context it reads from.
+    if (isCorrelatedSubquery(inner.left) || isCorrelatedSubquery(inner.right)) {
+        log('decline: an inner-join branch is correlated');
+        return null;
+    }
+    // The condition is evaluated against a different number of rows after the
+    // move, so a non-deterministic or write-bearing one must stay put.
+    if (!PlanNodeCharacteristics.isFunctional(node.condition)) {
+        log('decline: semi-join condition is not functional');
+        return null;
+    }
+    const side = conditionSide(node.condition, attributeIds(keySource), attributeIds(inner.left), attributeIds(inner.right));
+    if (side === null)
+        return null;
+    const target = side === 'left' ? inner.left : inner.right;
+    // `withChildren` (not `new JoinNode(...)`) so `usingColumns` / `existence`
+    // thread through both rebuilds.
+    const pushedSemi = node.withChildren([target, keySource, node.condition]);
+    const innerChildren = side === 'left'
+        ? [pushedSemi, inner.right]
+        : [inner.left, pushedSemi];
+    if (inner.condition)
+        innerChildren.push(inner.condition);
+    log('Pushed semi join below %s join onto its %s branch', inner.joinType, side);
+    return inner.withChildren(innerChildren);
+}
+//# sourceMappingURL=rule-semi-join-pushdown.js.map

package/dist/src/planner/rules/join/rule-semi-join-pushdown.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"rule-semi-join-pushdown.js","sourceRoot":"","sources":["../../../../../src/planner/rules/join/rule-semi-join-pushdown.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsGG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AAGzD,OAAO,EAAE,QAAQ,EAAE,MAAM,0BAA0B,CAAC;AACpD,OAAO,EAAE,uBAAuB,EAAE,MAAM,oCAAoC,CAAC;AAC7E,OAAO,EAAE,4BAA4B,EAAE,MAAM,0CAA0C,CAAC;AACxF,OAAO,EAAE,oBAAoB,EAAE,MAAM,qCAAqC,CAAC;AAE3E,MAAM,GAAG,GAAG,YAAY,CAAC,mCAAmC,CAAC,CAAC;AAK9D,MAAM,YAAY,GAAG,CAAC,IAAwB,EAAe,EAAE,CAC9D,IAAI,GAAG,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AAE9C;;;GAGG;AACH,SAAS,aAAa,CACrB,IAAoB,EACpB,MAA2B,EAC3B,OAA4B,EAC5B,QAA6B;IAE7B,IAAI,SAAS,GAAG,KAAK,CAAC;IACtB,IAAI,UAAU,GAAG,KAAK,CAAC;IACvB,KAAK,MAAM,EAAE,IAAI,4BAA4B,CAAC,IAAI,CAAC,EAAE,CAAC;QACrD,IAAI,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;YAAE,SAAS;QAC7B,IAAI,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;YAAC,SAAS,GAAG,IAAI,CAAC;YAAC,SAAS;QAAC,CAAC;QACpD,IAAI,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;YAAC,UAAU,GAAG,IAAI,CAAC;YAAC,SAAS;QAAC,CAAC;QACtD,iEAAiE;QACjE,sEAAsE;QACtE,uCAAuC;QACvC,OAAO,IAAI,CAAC;IACb,CAAC;IACD,uEAAuE;IACvE,wEAAwE;IACxE,IAAI,SAAS,KAAK,UAAU;QAAE,OAAO,IAAI,CAAC;IAC1C,OAAO,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC;AACrC,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,IAAc,EAAE,QAAoB;IACxE,IAAI,CAAC,CAAC,IAAI,YAAY,QAAQ,CAAC;QAAE,OAAO,IAAI,CAAC;IAC7C,IAAI,IAAI,CAAC,QAAQ,KAAK,MAAM;QAAE,OAAO,IAAI,CAAC;IAC1C,IAAI,CAAC,IAAI,CAAC,SAAS;QAAE,OAAO,IAAI,CAAC;IACjC,2EAA2E;IAC3E,mEAAmE;IACnE,IAAI,IAAI,CAAC,mBAAmB;QAAE,OAAO,IAAI,CAAC;IAE1C,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC;IACxB,IAAI,CAAC,CAAC,KAAK,YAAY,QAAQ,CAAC;QAAE,OAAO,IAAI,CAAC;IAC9C,IAAI,KAAK,CAAC,QAAQ,KAAK,OAAO,IAAI,KAAK,CAAC,QAAQ,KAAK,OAAO;QAAE,OAAO,IAAI,CAAC;IAE1E,gFAAgF;IAChF,8EAA8E;IAC9E,6EAA6E;IAC7E,iFAAiF;IACjF,kFAAkF;IAClF,gFAAgF;IAChF,0EAA0E;IAC1E,kDAAkD;IAElD,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC;IAC7B,sEAAsE;IACtE,sEAAsE;IACtE,uEAAuE;IACvE,IAAI,oBAAoB,CAAC,SAAS,CAAC,EAAE,CAAC;QACrC,GAAG,CAAC,mCAAmC,CAAC,CAAC;QACzC,OAAO,IAAI,CAAC;IACb,CAAC;IACD,IAAI,CAAC,uBAAuB,CAAC,eAAe,CAAC,SAAS,CAAC,EAAE,CAAC;QACzD,GAAG,CAAC,0CAA0C,CAAC,CAAC;QAChD,OAAO,IAAI,CAAC;IACb,CAAC;IACD,IAAI,uBAAuB,CAAC,qBAAqB,CAAC,SAAS,CAAC,EAAE,CAAC;QAC9D,GAAG,CAAC,sCAAsC,CAAC,CAAC;QAC5C,OAAO,IAAI,CAAC;IACb,CAAC;IACD,8EAA8E;IAC9E,4DAA4D;IAC5D,IAAI,uBAAuB,CAAC,qBAAqB,CAAC,KAAK,CAAC,IAAI,CAAC;WACzD,uBAAuB,CAAC,qBAAqB,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;QAChE,GAAG,CAAC,gDAAgD,CAAC,CAAC;QACtD,OAAO,IAAI,CAAC;IACb,CAAC;IACD,8EAA8E;IAC9E,yCAAyC;IACzC,IAAI,oBAAoB,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,oBAAoB,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;QAC3E,GAAG,CAAC,6CAA6C,CAAC,CAAC;QACnD,OAAO,IAAI,CAAC;IACb,CAAC;IACD,0EAA0E;IAC1E,mEAAmE;IACnE,IAAI,CAAC,uBAAuB,CAAC,YAAY,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;QAC3D,GAAG,CAAC,gDAAgD,CAAC,CAAC;QACtD,OAAO,IAAI,CAAC;IACb,CAAC;IAED,MAAM,IAAI,GAAG,aAAa,CACzB,IAAI,CAAC,SAAS,EACd,YAAY,CAAC,SAAS,CAAC,EACvB,YAAY,CAAC,KAAK,CAAC,IAAI,CAAC,EACxB,YAAY,CAAC,KAAK,CAAC,KAAK,CAAC,CACzB,CAAC;IACF,IAAI,IAAI,KAAK,IAAI;QAAE,OAAO,IAAI,CAAC;IAE/B,MAAM,MAAM,GAAG,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC;IAC1D,2EAA2E;IAC3E,gCAAgC;IAChC,MAAM,UAAU,GAAG,IAAI,CAAC,YAAY,CAAC,CAAC,MAAM,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,CAAuB,CAAC;IAChG,MAAM,aAAa,GAAe,IAAI,KAAK,MAAM;QAChD,CAAC,CAAC,CAAC,UAAU,EAAE,KAAK,CAAC,KAAK,CAAC;QAC3B,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;IAC5B,IAAI,KAAK,CAAC,SAAS;QAAE,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;IAEzD,GAAG,CAAC,mDAAmD,EAAE,KAAK,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;IAC/E,OAAO,KAAK,CAAC,YAAY,CAAC,aAAa,CAAC,CAAC;AAC1C,CAAC"}

package/dist/src/planner/util/column-refs.d.ts

@@ -0,0 +1,17 @@
+/**
+ * Column-reference walks over scalar plan subtrees.
+ *
+ * Lives here rather than in `analysis/constraint-extractor.ts` so that
+ * `planner/nodes/` can import it as a *value*: the runtime cycle
+ * `constraint-extractor → nodes/reference → …` is real, and this module
+ * deliberately depends on nothing but the node-type enum.
+ */
+import type { ScalarPlanNode } from '../nodes/plan-node.js';
+/**
+ * Collect the attributeIds of every ColumnReference in a scalar subtree.
+ * Walking into children (rather than only unwrapping a top-level Cast) reaches
+ * references nested inside arithmetic, function calls, casts, etc. — e.g.
+ * `outer.id + 1`, `coalesce(outer.id, 0)`, `cast(outer.id + 1 as integer)`.
+ */
+export declare function collectColumnRefAttributeIds(node: ScalarPlanNode): number[];
+//# sourceMappingURL=column-refs.d.ts.map

package/dist/src/planner/util/column-refs.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"column-refs.d.ts","sourceRoot":"","sources":["../../../../src/planner/util/column-refs.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAGH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAG5D;;;;;GAKG;AACH,wBAAgB,4BAA4B,CAAC,IAAI,EAAE,cAAc,GAAG,MAAM,EAAE,CAa3E"}

package/dist/src/planner/util/column-refs.js

@@ -0,0 +1,30 @@
+/**
+ * Column-reference walks over scalar plan subtrees.
+ *
+ * Lives here rather than in `analysis/constraint-extractor.ts` so that
+ * `planner/nodes/` can import it as a *value*: the runtime cycle
+ * `constraint-extractor → nodes/reference → …` is real, and this module
+ * deliberately depends on nothing but the node-type enum.
+ */
+import { PlanNodeType } from '../nodes/plan-node-type.js';
+/**
+ * Collect the attributeIds of every ColumnReference in a scalar subtree.
+ * Walking into children (rather than only unwrapping a top-level Cast) reaches
+ * references nested inside arithmetic, function calls, casts, etc. — e.g.
+ * `outer.id + 1`, `coalesce(outer.id, 0)`, `cast(outer.id + 1 as integer)`.
+ */
+export function collectColumnRefAttributeIds(node) {
+    const ids = [];
+    const stack = [node];
+    while (stack.length) {
+        const n = stack.pop();
+        if (n.nodeType === PlanNodeType.ColumnReference) {
+            ids.push(n.attributeId);
+        }
+        for (const c of n.getChildren()) {
+            stack.push(c);
+        }
+    }
+    return ids;
+}
+//# sourceMappingURL=column-refs.js.map

package/dist/src/planner/util/column-refs.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"column-refs.js","sourceRoot":"","sources":["../../../../src/planner/util/column-refs.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAI1D;;;;;GAKG;AACH,MAAM,UAAU,4BAA4B,CAAC,IAAoB;IAChE,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,MAAM,KAAK,GAAqB,CAAC,IAAI,CAAC,CAAC;IACvC,OAAO,KAAK,CAAC,MAAM,EAAE,CAAC;QACrB,MAAM,CAAC,GAAG,KAAK,CAAC,GAAG,EAAG,CAAC;QACvB,IAAI,CAAC,CAAC,QAAQ,KAAK,YAAY,CAAC,eAAe,EAAE,CAAC;YACjD,GAAG,CAAC,IAAI,CAAE,CAAoC,CAAC,WAAW,CAAC,CAAC;QAC7D,CAAC;QACD,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC;YACjC,KAAK,CAAC,IAAI,CAAC,CAA8B,CAAC,CAAC;QAC5C,CAAC;IACF,CAAC;IACD,OAAO,GAAG,CAAC;AACZ,CAAC"}

package/dist/src/runtime/emit/alter-table.d.ts

@@ -6,10 +6,30 @@ export declare function emitAlterTable(plan: AlterTableNode, ctx: EmissionContex
 /**
  * Build the shadow-table CREATE TABLE DDL used by the non-memory rebuild path.
  *
- * Nullability is emitted explicitly for every column, matching the "no-db"
- * stance of `generateTableDDL` in ddl-generator.ts: safe under any session's
- * `default_column_nullability` setting. DEFAULT and COLLATE are preserved so
- * the shadow table faithfully mirrors the original schema.
+ * Rendered through the canonical emitter (`generateTableDDL`) over a copy of the
+ * REAL `TableSchema` with only the name and key substituted, so the shadow
+ * carries everything the table declares — constraints, foreign keys, tags,
+ * defaults, collations, generated columns, the key's ON CONFLICT action — not a
+ * hand-picked subset. An omission in emitted DDL is never neutral: a missing
+ * PRIMARY KEY clause re-parses as a different key (all-columns), and a missing
+ * constraint re-parses as no constraint.
+ *
+ * Called WITHOUT a `db` argument: that form fully qualifies the name, always
+ * emits USING, and annotates nullability on every column, so the text re-parses
+ * to the same shape under any session's `default_column_nullability` /
+ * `default_vtab_module` settings. An empty `newPkDef` emits `PRIMARY KEY ()` —
+ * the singleton key — rather than omitting the clause (which would re-parse as
+ * the opposite thing: keyed by every column).
+ *
+ * The copied schema's per-column `primaryKey` flags are stale (they still mirror
+ * the OLD key) — harmless: the emitter reads only `primaryKeyDefinition`, and the
+ * shadow's real schema is built by re-parsing the emitted text.
+ *
+ * Deliberately WHOLE-TABLE only (no column subset parameter): stored constraints
+ * address columns by index, so a subset copy would need every constraint and
+ * index remapped or dropped. `ALTER TABLE … DROP COLUMN` goes through
+ * `module.alterTable`, never through this rebuild; if a column-dropping rebuild
+ * is ever wanted it must come back with deliberate constraint handling.
  */
-export declare function buildShadowTableDdl(tableSchema: TableSchema, shadowName: string, survivingColumns: string[], newPkDef: PrimaryKeyColumnDefinition[]): string;
+export declare function buildShadowTableDdl(tableSchema: TableSchema, shadowName: string, newPkDef: PrimaryKeyColumnDefinition[]): string;
 //# sourceMappingURL=alter-table.d.ts.map

package/dist/src/runtime/emit/alter-table.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"alter-table.d.ts","sourceRoot":"","sources":["../../../../src/runtime/emit/alter-table.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAqC,MAAM,yCAAyC,CAAC;AACjH,OAAO,KAAK,EAAE,WAAW,EAAkB,MAAM,aAAa,CAAC;AAE/D,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AAM9D,OAAO,KAAK,EAAE,WAAW,EAAE,0BAA0B,EAAE,MAAM,uBAAuB,CAAC;AAkDrF,wBAAgB,cAAc,CAAC,IAAI,EAAE,cAAc,EAAE,GAAG,EAAE,eAAe,GAAG,WAAW,CAkJtF;AA2xDD;;;;;;;GAOG;AACH,wBAAgB,mBAAmB,CAClC,WAAW,EAAE,WAAW,EACxB,UAAU,EAAE,MAAM,EAClB,gBAAgB,EAAE,MAAM,EAAE,EAC1B,QAAQ,EAAE,0BAA0B,EAAE,GACpC,MAAM,CAuCR"}
+{"version":3,"file":"alter-table.d.ts","sourceRoot":"","sources":["../../../../src/runtime/emit/alter-table.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAqC,MAAM,yCAAyC,CAAC;AACjH,OAAO,KAAK,EAAE,WAAW,EAAkB,MAAM,aAAa,CAAC;AAE/D,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AAM9D,OAAO,KAAK,EAAE,WAAW,EAAE,0BAA0B,EAAe,MAAM,uBAAuB,CAAC;AAmDlG,wBAAgB,cAAc,CAAC,IAAI,EAAE,cAAc,EAAE,GAAG,EAAE,eAAe,GAAG,WAAW,CAkJtF;AAyyDD;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,wBAAgB,mBAAmB,CAClC,WAAW,EAAE,WAAW,EACxB,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,0BAA0B,EAAE,GACpC,MAAM,CAMR"}

package/dist/src/runtime/emit/alter-table.js

@@ -10,7 +10,8 @@ import { quoteIdentifier, expressionToString, astToString } from '../../emit/ast
 import { renameTableInAst, renameColumnInAst, renameColumnInCheckExpression, renameColumnInColumnExpressions, renameTableInCheckConstraints, renameTableInIndexPredicates, renameTableInColumnExpressions, objectRefKey } from '../../schema/rename-rewriter.js';
 import { snapshotObjectRefResolvers, tableRenameTargetsFor } from '../../schema/object-ref-resolver.js';
 import { assertCatalogObjectPersistable, assertRenameDependentsPersistable } from '../../schema/catalog-persistability.js';
-import { assertUniqueConstraintIndexNameFree, assertUniqueConstraintNotDuplicated, assertUniqueConstraintBackingNamesDistinct, uniqueConstraintColumnSetKey } from '../../schema/catalog.js';
+import { assertUniqueConstraintIndexNameFree, assertUniqueConstraintNotDuplicated, assertUniqueConstraintBackingNamesDistinct, uniqueConstraintColumnSetKey, isImplicitCoveringIndex } from '../../schema/catalog.js';
+import { generateTableDDL, generateIndexDDL } from '../../schema/ddl-generator.js';
 import { isTruthy } from '../../util/comparison.js';
 import { assertDdlTransactionPolicy, isExplicitTransactionOpen } from './ddl-transaction-policy.js';
 import { buildColumnSourceResolver } from '../../schema/column-source-resolver.js';
@@ -1553,6 +1554,18 @@ async function runAlterPrimaryKey(rctx, tableSchema, schema, columns, sql) {
             + `this one — without 'renameTable' the module would keep the rows under the shadow name and `
             + `the rebuilt table could not be opened.`, StatusCode.UNSUPPORTED);
     }
+    // Second capability check: the rebuild re-creates the table's user indexes with
+    // CREATE INDEX statements (the shadow's CREATE TABLE carries every constraint, but
+    // indexes have their own statements), and CREATE INDEX routes through
+    // `module.createIndex`. Refuse up front — failing after the original table is
+    // dropped would strand a half-rebuilt table, and silently dropping the indexes is
+    // data loss of a different kind (a UNIQUE index stops enforcing).
+    if (!module.createIndex && rebuildUserIndexes(tableSchema).length > 0) {
+        throw new QuereusError(`Module '${tableSchema.vtabModuleName ?? '<unknown>'}' does not support ALTER PRIMARY KEY on table `
+            + `'${tableSchema.name}': it cannot re-key in place, and the engine's fallback rebuild must `
+            + `re-create the table's indexes on the rebuilt table — without 'createIndex' they would be `
+            + `silently lost. Drop the indexes first, or use a module that can re-key or create indexes.`, StatusCode.UNSUPPORTED);
+    }
     // NOTE: refused on every `DdlTransactionality` tier, including a module declaring
     // 'transactional'. None does today; if one appears its DROP + RENAME would roll back
     // together with the row copy, making this refusal over-broad — exempt that tier then, the
@@ -1564,7 +1577,7 @@ async function runAlterPrimaryKey(rctx, tableSchema, schema, columns, sql) {
             + `copy does not — a rollback would leave an empty table and destroy rows committed before this `
             + `transaction began. COMMIT or ROLLBACK first, then re-issue in autocommit mode.`, StatusCode.ERROR);
     }
-    await rebuildTableWithNewShape(rctx, tableSchema, schema, tableSchema.columns.map(c => c.name), newPkDef);
+    await rebuildTableWithNewShape(rctx, tableSchema, schema, newPkDef);
     // A DEFENSIVE NO-OP on this path today, not the working re-key the native arm above does.
     // Nothing can be in the batch for this table by now: the rebuild's own events are
     // suppressed (see `rebuildViaShadowTable`), the ALTER statement writes no rows of its own,
@@ -1572,8 +1585,8 @@ async function runAlterPrimaryKey(rctx, tableSchema, schema, columns, sql) {
     // Kept anyway — it costs one walk of an empty batch, and it is the correct call the moment
     // that transaction guard is loosened; deleting it would quietly make this arm's
     // as-of-delivery `key` guarantee depend on the guard staying exactly as it is. Every column
-    // survives the rebuild in order (`survivingColumns` above is the full list), so the indices
-    // would still line up with the event images.
+    // survives the rebuild in order (the rebuild is whole-table by construction — see
+    // `buildShadowTableDdl`), so the indices would still line up with the event images.
     rctx.db._getEventEmitter().rekeyBatchedDataEvents(tableSchema.schemaName, tableSchema.name, oldPkIndices, newPkIndices);
     // The rebuild's own four statements are silent (`rebuildViaShadowTable` runs them inside
     // `withPublicEventsSuppressed`, which also swallows the inner RENAME TO arm's event). This
@@ -1596,17 +1609,17 @@ async function runAlterPrimaryKey(rctx, tableSchema, schema, columns, sql) {
     return null;
 }
 /**
- * Rebuilds a table with a new column projection and/or primary key, via the
- * shadow-table SQL approach with DROP+RENAME. This is the fallback for a module
- * whose `alterTable` throws `UNSUPPORTED` for `alterPrimaryKey` (or omits the
- * hook); the built-in memory and store modules both re-key in place and never
- * reach it. See `rebuildViaShadowTable` for the two preconditions
- * `runAlterPrimaryKey` checks before calling this.
+ * Rebuilds a table with a new primary key, via the shadow-table SQL approach
+ * with DROP+RENAME. This is the fallback for a module whose `alterTable` throws
+ * `UNSUPPORTED` for `alterPrimaryKey` (or omits the hook); the built-in memory
+ * and store modules both re-key in place and never reach it. See
+ * `rebuildViaShadowTable` for the preconditions `runAlterPrimaryKey` checks
+ * before calling this.
  */
-async function rebuildTableWithNewShape(rctx, tableSchema, schema, survivingColumns, newPkDef) {
+async function rebuildTableWithNewShape(rctx, tableSchema, schema, newPkDef) {
     const tableName = tableSchema.name;
     const schemaName = tableSchema.schemaName;
-    await rebuildViaShadowTable(rctx, tableSchema, schema, survivingColumns, newPkDef);
+    await rebuildViaShadowTable(rctx, tableSchema, newPkDef);
     const finalSchema = schema.getTable(tableName);
     if (finalSchema) {
         rctx.db.schemaManager.getChangeNotifier().notifyChange({
@@ -1621,49 +1634,54 @@ async function rebuildTableWithNewShape(rctx, tableSchema, schema, survivingColu
 /**
  * Build the shadow-table CREATE TABLE DDL used by the non-memory rebuild path.
  *
- * Nullability is emitted explicitly for every column, matching the "no-db"
- * stance of `generateTableDDL` in ddl-generator.ts: safe under any session's
- * `default_column_nullability` setting. DEFAULT and COLLATE are preserved so
- * the shadow table faithfully mirrors the original schema.
+ * Rendered through the canonical emitter (`generateTableDDL`) over a copy of the
+ * REAL `TableSchema` with only the name and key substituted, so the shadow
+ * carries everything the table declares — constraints, foreign keys, tags,
+ * defaults, collations, generated columns, the key's ON CONFLICT action — not a
+ * hand-picked subset. An omission in emitted DDL is never neutral: a missing
+ * PRIMARY KEY clause re-parses as a different key (all-columns), and a missing
+ * constraint re-parses as no constraint.
+ *
+ * Called WITHOUT a `db` argument: that form fully qualifies the name, always
+ * emits USING, and annotates nullability on every column, so the text re-parses
+ * to the same shape under any session's `default_column_nullability` /
+ * `default_vtab_module` settings. An empty `newPkDef` emits `PRIMARY KEY ()` —
+ * the singleton key — rather than omitting the clause (which would re-parse as
+ * the opposite thing: keyed by every column).
+ *
+ * The copied schema's per-column `primaryKey` flags are stale (they still mirror
+ * the OLD key) — harmless: the emitter reads only `primaryKeyDefinition`, and the
+ * shadow's real schema is built by re-parsing the emitted text.
+ *
+ * Deliberately WHOLE-TABLE only (no column subset parameter): stored constraints
+ * address columns by index, so a subset copy would need every constraint and
+ * index remapped or dropped. `ALTER TABLE … DROP COLUMN` goes through
+ * `module.alterTable`, never through this rebuild; if a column-dropping rebuild
+ * is ever wanted it must come back with deliberate constraint handling.
  */
-export function buildShadowTableDdl(tableSchema, shadowName, survivingColumns, newPkDef) {
-    const colDefs = [];
-    for (const colName of survivingColumns) {
-        const idx = tableSchema.columnIndexMap.get(colName.toLowerCase());
-        if (idx === undefined)
-            continue;
-        const col = tableSchema.columns[idx];
-        let def = quoteIdentifier(col.name) + ' ' + col.logicalType.name;
-        def += col.notNull ? ' not null' : ' null';
-        if (col.collation && col.collation !== 'BINARY')
-            def += ` collate ${col.collation}`;
-        if (col.defaultValue !== null && col.defaultValue !== undefined) {
-            def += ` default ${expressionToString(col.defaultValue)}`;
-        }
-        colDefs.push(def);
-    }
-    const pkColNames = [];
-    for (const pk of newPkDef) {
-        const colName = tableSchema.columns[pk.index].name;
-        let entry = quoteIdentifier(colName);
-        if (pk.desc)
-            entry += ' desc';
-        pkColNames.push(entry);
-    }
-    let createDdl = `create table ${qualifyTableName(tableSchema.schemaName, shadowName)} (${colDefs.join(', ')}`;
-    createDdl += pkColNames.length > 0
-        ? `, primary key (${pkColNames.join(', ')}))`
-        : `)`;
-    if (tableSchema.vtabModuleName) {
-        createDdl += ` using ${tableSchema.vtabModuleName}`;
-        if (tableSchema.vtabArgs && Object.keys(tableSchema.vtabArgs).length > 0) {
-            const args = Object.entries(tableSchema.vtabArgs)
-                .map(([key, value]) => `${key} = ${JSON.stringify(value)}`)
-                .join(', ');
-            createDdl += ` (${args})`;
-        }
-    }
-    return createDdl;
+export function buildShadowTableDdl(tableSchema, shadowName, newPkDef) {
+    return generateTableDDL({
+        ...tableSchema,
+        name: shadowName,
+        primaryKeyDefinition: newPkDef,
+    });
+}
+/**
+ * The table's user indexes — the ones the rebuild must re-create with explicit
+ * CREATE INDEX statements. Implicit covering structures (the auto-built backing
+ * of a declared UNIQUE constraint) are excluded: the shadow's CREATE TABLE
+ * carries the constraint itself, which re-materializes its backing structure.
+ * UNIQUE constraints synthesized FROM an index (`derivedFromIndex`) are the
+ * opposite case — `generateTableDDL` skips them, and the `CREATE UNIQUE INDEX`
+ * emitted for their index re-synthesizes them (see `appendIndexToTableSchema`).
+ *
+ * NOTE: an *exposed* implicit index's user tags (`exposedIndexTags` in store
+ * mode, materialized `IndexSchema.tags` in memory mode) are not carried by
+ * either channel through a rebuild; if a rebuild-path backend ever exposes
+ * implicit indexes with tags, they need explicit re-application here.
+ */
+function rebuildUserIndexes(tableSchema) {
+    return (tableSchema.indexes ?? []).filter(idx => !isImplicitCoveringIndex(tableSchema, idx.name));
 }
 /**
  * SET MAINTAINED [(cols)] AS <body> — attach a derivation to a plain table, or
@@ -1728,13 +1746,16 @@ async function runDropMaintained(rctx, tableSchema, schema) {
  * Generic rebuild via shadow table SQL, for a module without a native
  * `alterPrimaryKey`.
  *
- * Two preconditions the caller (`runAlterPrimaryKey`) enforces, because this rebuild has no
+ * Preconditions the caller (`runAlterPrimaryKey`) enforces, because this rebuild has no
  * correct outcome without them:
  *
  *  - **The module must implement `renameTable`.** The last step renames the shadow table over
  *    the original; a module that files its rows under the table's name and never hears about
  *    the rename keeps them under the shadow name while the catalog says otherwise, and the
  *    rebuilt table cannot be connected at all.
+ *  - **The module must implement `createIndex` when the table has user indexes.** The rebuild
+ *    finishes by re-creating them on the rebuilt table; without the hook they would be
+ *    silently lost.
  *  - **No explicit transaction may be open.** The two halves have different transactional
  *    lifetimes: the schema half (DROP + RENAME) escapes ROLLBACK on every
  *    `DdlTransactionality` tier a built-in module reaches, while the row copy is staged in the
@@ -1744,17 +1765,23 @@ async function runDropMaintained(rctx, tableSchema, schema) {
  *    offers; making the data half survive would commit part of the user's transaction behind
  *    their back.
  *
- * The four statements are engine scaffolding, not statements the application issued, so the
+ * The statements are engine scaffolding, not statements the application issued, so the
  * whole rebuild runs with the PUBLIC event channels suppressed — see the scope inside.
  */
-async function rebuildViaShadowTable(rctx, tableSchema, _schema, survivingColumns, newPkDef) {
+async function rebuildViaShadowTable(rctx, tableSchema, newPkDef) {
     const tableName = tableSchema.name;
     const schemaName = tableSchema.schemaName;
     const shadowName = `${tableName}__rekey_${Date.now()}`;
     const qualifiedShadow = qualifyTableName(schemaName, shadowName);
     const qualifiedTable = qualifyTableName(schemaName, tableName);
-    const createDdl = buildShadowTableDdl(tableSchema, shadowName, survivingColumns, newPkDef);
-    const projection = survivingColumns.map(c => quoteIdentifier(c)).join(', ');
+    const createDdl = buildShadowTableDdl(tableSchema, shadowName, newPkDef);
+    // Whole-table copy (see buildShadowTableDdl for why no subset is representable).
+    // Generated columns are excluded from the projection: the shadow declares the same
+    // GENERATED ALWAYS AS clauses, so it computes them itself and rejects direct writes.
+    const projection = tableSchema.columns
+        .filter(c => !c.generated)
+        .map(c => quoteIdentifier(c.name))
+        .join(', ');
     // The rebuild's statements are ordinary SQL, so without this scope they would raise
     // ordinary notifications describing the scaffolding rather than what the user asked for:
     // the copy announces every existing row as a fresh `insert` (relabelled onto the real
@@ -1777,15 +1804,70 @@ async function rebuildViaShadowTable(rctx, tableSchema, _schema, survivingColumn
     await rctx.db._getEventEmitter().withPublicEventsSuppressed(async () => {
         try {
             await rctx.db._execWithinTransaction(createDdl);
-            await rctx.db._execWithinTransaction(`insert into ${qualifiedShadow} (${projection}) select ${projection} from ${qualifiedTable}`);
-            await rctx.db._execWithinTransaction(`drop table ${qualifiedTable}`);
+            // `or abort` is load-bearing, not decoration. The shadow now declares the table's
+            // own conflict actions (the whole point of rendering it through the canonical
+            // writer), so a row-set that COLLIDES under the new key — the case the in-place
+            // re-key path refuses with CONSTRAINT — would otherwise be resolved by the
+            // declared action instead of reported: `primary key (a) on conflict replace`
+            // re-keyed onto a non-unique column silently keeps the last row and reports
+            // success. The copy is engine scaffolding, not a user INSERT, so it takes the
+            // statement-level clause that outranks every declared default
+            // (see row-constraints.ts: statement OR clause > per-constraint default > ABORT)
+            // and surfaces the collision the way every other re-key path does.
+            await rctx.db._execWithinTransaction(`insert or abort into ${qualifiedShadow} (${projection}) select ${projection} from ${qualifiedTable}`);
+            // The DROP is scaffolding: the shadow's rows ARE the table's rows, moments from
+            // being renamed back over it. Two FK situations would otherwise refuse it:
+            //  - a SELF-referencing FK — the shadow's copy references the ORIGINAL by name
+            //    (correct for the copy: the per-row EXISTS checks run against the intact
+            //    original), so the drop guard sees the shadow as a referencing child;
+            //  - another table's FK into this one with rows present — the referencing rows
+            //    stay satisfied because the rename restores the name with identical rows,
+            //    matching the in-place re-key path (which never breaks such children).
+            // Suppress the parent-side RESTRICT guard for exactly this statement; restored
+            // in the finally so a failing drop cannot latch it.
+            //
+            // NOTE: two OTHER drop-refusal shapes are not handled here, and both are
+            // reachable rather than hypothetical (verified against a stub module with no
+            // `alterTable`) — see backlog `bug-rebuild-drop-refused-by-user-facing-guards`.
+            // `emitDropTable` also runs `assertNoAssertionDependsOn` and
+            // `assertNoExpressionDependsOn`, so an assertion over this table — or ANOTHER
+            // table's CHECK / DEFAULT / generated body whose subquery names it — refuses the
+            // internal DROP, and the whole ALTER fails with `cannot drop table '<name>'`,
+            // naming a drop the user never asked for. The rebuild unwinds cleanly (the shadow
+            // is dropped in the catch below, the table is untouched), so this costs a
+            // confusing refusal, not data. The fix is a scaffolding-drop scope those guards
+            // honor too, not a wider FK flag.
+            const priorSuppress = rctx.db._setFkRestrictSuppressed(true);
+            try {
+                await rctx.db._execWithinTransaction(`drop table ${qualifiedTable}`);
+            }
+            finally {
+                rctx.db._setFkRestrictSuppressed(priorSuppress);
+            }
+            // The rename's propagate pass rewrites the shadow's dangling references to the
+            // dropped original (e.g. a self-referencing FK) onto the restored name.
             await rctx.db._execWithinTransaction(`alter table ${qualifiedShadow} rename to ${quoteIdentifier(tableName)}`);
+            // Re-create the table's user indexes on the rebuilt table (see
+            // rebuildUserIndexes for what is included and why). AFTER the rename so the
+            // index names — unique per schema — never coexist with the original's, and
+            // rendered from the ORIGINAL schema, whose name/schema/column order the rebuilt
+            // table shares. `runAlterPrimaryKey` refused up front if the module cannot
+            // createIndex, so a failure here is exceptional; it propagates rather than
+            // silently dropping the index.
+            for (const idx of rebuildUserIndexes(tableSchema)) {
+                await rctx.db._execWithinTransaction(generateIndexDDL(idx, tableSchema));
+            }
         }
         catch (e) {
             try {
                 await rctx.db._execWithinTransaction(`drop table if exists ${qualifiedShadow}`);
             }
-            catch { /* ignore */ }
+            catch (cleanupError) {
+                // Never masks the original failure (rethrown just below), but a shadow table the
+                // engine could not clean up outlives the statement under a machine-generated name
+                // — say so rather than dropping it on the floor.
+                warnLog('failed to drop leftover shadow table %s after a failed rebuild: %s', qualifiedShadow, cleanupError instanceof Error ? cleanupError.message : String(cleanupError));
+            }
             throw e;
         }
     });