@quereus/quereus 0.2.0 → 0.4.0

package/src/planner/rules/join/rule-quickpick-enumeration.ts

@@ -0,0 +1,267 @@
+import { createLogger } from '../../../common/logger.js';
+import type { PlanNode, RelationalPlanNode, ScalarPlanNode } from '../../nodes/plan-node.js';
+import { isRelationalNode } from '../../nodes/plan-node.js';
+import type { OptContext } from '../../framework/context.js';
+import { JoinNode } from '../../nodes/join-node.js';
+import { normalizePredicate } from '../../analysis/predicate-normalizer.js';
+import { BinaryOpNode } from '../../nodes/scalar.js';
+import { ColumnReferenceNode } from '../../nodes/reference.js';
+
+const log = createLogger('optimizer:rule:quickpick');
+
+interface JoinPredicate {
+  leftIndex: number;
+  rightIndex: number;
+  condition: ScalarPlanNode; // equi-predicate (or AND of equis) connecting the pair
+}
+
+interface JoinGraph {
+  relations: RelationalPlanNode[];
+  predicates: JoinPredicate[];
+}
+
+function extractJoinGraph(node: PlanNode): JoinGraph | null {
+  // Flatten INNER/CROSS join subtree into leaves (relations) and equi-predicates between them
+  const relations: RelationalPlanNode[] = [];
+  const attrIdToRel = new Map<number, number>();
+  const pairToConds = new Map<string, ScalarPlanNode[]>();
+
+  function addRelation(rel: RelationalPlanNode): number {
+    const idx = relations.length;
+    relations.push(rel);
+    for (const attr of rel.getAttributes()) {
+      attrIdToRel.set(attr.id, idx);
+    }
+    return idx;
+  }
+
+  function addPairCond(aIdx: number, bIdx: number, cond: ScalarPlanNode): void {
+    const [x, y] = aIdx < bIdx ? [aIdx, bIdx] : [bIdx, aIdx];
+    const key = `${x}:${y}`;
+    if (!pairToConds.has(key)) pairToConds.set(key, []);
+    pairToConds.get(key)!.push(cond);
+  }
+
+  function visit(n: PlanNode): void {
+    if (n instanceof JoinNode) {
+      // Only enumerate INNER/CROSS joins; bail if OUTER join encountered
+      if (n.getJoinType() !== 'inner' && n.getJoinType() !== 'cross') {
+        relations.length = 0; // mark failure
+        return;
+      }
+      visit(n.getLeftSource());
+      visit(n.getRightSource());
+
+      // Analyze equi-join predicates
+      const cond = n.getJoinCondition();
+      if (cond) {
+        const norm = normalizePredicate(cond);
+        const stack: ScalarPlanNode[] = [norm];
+        while (stack.length) {
+          const expr = stack.pop()!;
+          if (expr instanceof BinaryOpNode && expr.expression.operator === 'AND') {
+            stack.push(expr.left, expr.right);
+          } else if (expr instanceof BinaryOpNode && expr.expression.operator === '=') {
+            if (expr.left instanceof ColumnReferenceNode && expr.right instanceof ColumnReferenceNode) {
+              const lRel = attrIdToRel.get((expr.left as ColumnReferenceNode).attributeId);
+              const rRel = attrIdToRel.get((expr.right as ColumnReferenceNode).attributeId);
+              if (lRel !== undefined && rRel !== undefined && lRel !== rRel) {
+                addPairCond(lRel, rRel, expr);
+              }
+            }
+          }
+        }
+      }
+      return;
+    }
+    if (isRelationalNode(n)) {
+      addRelation(n);
+      return;
+    }
+    // Traverse generic children otherwise
+    for (const c of n.getChildren()) visit(c);
+  }
+
+  visit(node);
+  if (relations.length === 0) return null; // failure or non-join
+
+  // Build predicates array (AND-combine multiple per pair later during plan build)
+  const predicates: JoinPredicate[] = [];
+  for (const [key, conds] of pairToConds) {
+    const [aStr, bStr] = key.split(':');
+    const a = parseInt(aStr, 10);
+    const b = parseInt(bStr, 10);
+    // Leave multiple conditions to be AND-combined when consumed
+    const combined = conds.length === 1 ? conds[0] : conds.reduce((acc, cur) =>
+      new BinaryOpNode(relations[a].scope, { type: 'binary', operator: 'AND' } as any, acc, cur)
+    );
+    predicates.push({ leftIndex: a, rightIndex: b, condition: combined });
+  }
+  return { relations, predicates };
+}
+
+function estimatePlanCost(plan: RelationalPlanNode): number {
+  // Use total cost, but apply penalties for cross products (no predicates between recent components)
+  // and reward key-covered joins via lower estimatedRows coming from computePhysical.
+  // Total cost already accumulates subtree costs; keep it as primary signal.
+  return plan.getTotalCost();
+}
+
+function buildLeftDeepPlan(order: number[], graph: JoinGraph): RelationalPlanNode {
+  let current: RelationalPlanNode | null = null;
+  const chosen = new Set<number>();
+  for (let i = 0; i < order.length; i++) {
+    const idx = order[i];
+    if (current === null) {
+      current = graph.relations[idx];
+      chosen.add(idx);
+      continue;
+    }
+    const next = graph.relations[idx];
+    // Gather all predicates that connect next to the chosen set
+    const connectors: ScalarPlanNode[] = [];
+    for (const p of graph.predicates) {
+      const connects = (chosen.has(p.leftIndex) && p.rightIndex === idx) || (chosen.has(p.rightIndex) && p.leftIndex === idx);
+      if (connects) connectors.push(p.condition);
+    }
+    const cond = connectors.length === 0 ? undefined : connectors.reduce((acc, cur) =>
+      acc ? new BinaryOpNode(current!.scope, { type: 'binary', operator: 'AND' } as any, acc, cur) : cur
+    , undefined as unknown as ScalarPlanNode | undefined);
+    current = new JoinNode(current.scope, current, next, 'inner', cond);
+    chosen.add(idx);
+  }
+  return current!;
+}
+
+// Greedy bushy: repeatedly merge the pair of components with minimal estimated join cost
+function buildBushyPlan(graph: JoinGraph): RelationalPlanNode {
+  type Component = { members: Set<number>; plan: RelationalPlanNode };
+  const components: Component[] = graph.relations.map((r, i) => ({ members: new Set([i]), plan: r }));
+
+  function predicatesBetween(a: Component, b: Component): ScalarPlanNode | undefined {
+    const conns: ScalarPlanNode[] = [];
+    for (const p of graph.predicates) {
+      const lInA = a.members.has(p.leftIndex);
+      const rInA = a.members.has(p.rightIndex);
+      const lInB = b.members.has(p.leftIndex);
+      const rInB = b.members.has(p.rightIndex);
+      const crosses = (lInA && rInB) || (rInA && lInB);
+      if (crosses) conns.push(p.condition);
+    }
+    if (conns.length === 0) return undefined;
+    return conns.reduce((acc, cur) => acc ? new BinaryOpNode(graph.relations[0].scope, { type: 'binary', operator: 'AND' } as any, acc, cur) : cur, undefined as unknown as ScalarPlanNode | undefined);
+  }
+
+  while (components.length > 1) {
+    let bestI = -1, bestJ = -1;
+    let bestCost = Number.POSITIVE_INFINITY;
+    let bestPlan: RelationalPlanNode | null = null;
+
+    for (let i = 0; i < components.length; i++) {
+      for (let j = i + 1; j < components.length; j++) {
+        const a = components[i];
+        const b = components[j];
+        const cond = predicatesBetween(a, b);
+        const joined = new JoinNode(a.plan.scope, a.plan, b.plan, 'inner', cond);
+        const cost = estimatePlanCost(joined);
+        if (cost < bestCost) {
+          bestCost = cost;
+          bestI = i;
+          bestJ = j;
+          bestPlan = joined;
+        }
+      }
+    }
+
+    // Merge best pair
+    const a = components[bestI];
+    const b = components[bestJ];
+    const merged: Component = { members: new Set([...a.members, ...b.members]), plan: bestPlan! };
+    // Remove higher index first
+    components.splice(bestJ, 1);
+    components.splice(bestI, 1);
+    components.push(merged);
+  }
+
+  return components[0].plan;
+}
+
+export function ruleQuickPickJoinEnumeration(node: PlanNode, context: OptContext): PlanNode | null {
+  const qk = context.tuning.quickpick;
+  if (!qk?.enabled) return null;
+  if (!(node instanceof JoinNode)) return null;
+
+  const graph = extractJoinGraph(node);
+  if (!graph) return null;
+  if (graph.relations.length < 3) return null; // Only helpful for 3+ relations
+
+  const baselineCost = estimatePlanCost(node as unknown as RelationalPlanNode);
+  if (baselineCost < (qk.minTriggerCost ?? 0)) return null;
+
+  const maxTours = qk.maxTours ?? 100;
+  let bestPlan: RelationalPlanNode | null = null;
+  let bestCost = Number.POSITIVE_INFINITY;
+
+  // Try multiple strategies: left-deep greedy from small bases and bushy greedy
+  const sizes = graph.relations.map(r => r.estimatedRows ?? 1e9);
+  const baseOrder = [...graph.relations.keys()].sort((a, b) => (sizes[a] - sizes[b]));
+
+  const start = Date.now();
+  let tours = 0;
+  while (tours < maxTours && (Date.now() - start) <= (qk.timeLimitMs ?? 100)) {
+    // Greedy NN tour: start with random among top-2 smallest
+    const startIdx = baseOrder[Math.min(tours % 2, baseOrder.length - 1)];
+    const remaining = new Set<number>(graph.relations.keys());
+    remaining.delete(startIdx);
+    const order: number[] = [startIdx];
+
+    while (remaining.size > 0) {
+      let bestNext: number | null = null;
+      let bestIncCost = Number.POSITIVE_INFINITY;
+
+      for (const cand of remaining) {
+        // Prefer connected joins (has predicate to chosen set); penalize cross-products
+        const connected = graph.predicates.some(p =>
+          (order.some(o => o === p.leftIndex) && p.rightIndex === cand) ||
+          (order.some(o => o === p.rightIndex) && p.leftIndex === cand)
+        );
+        const plan = buildLeftDeepPlan([...order, cand], graph);
+        let cost = estimatePlanCost(plan);
+        if (!connected) cost *= 10; // strong penalty for cross product
+        if (cost < bestIncCost) {
+          bestIncCost = cost;
+          bestNext = cand;
+        }
+      }
+      order.push(bestNext!);
+      remaining.delete(bestNext!);
+    }
+
+    const plan = buildLeftDeepPlan(order, graph);
+    const cost = estimatePlanCost(plan);
+    if (cost < bestCost) {
+      bestCost = cost;
+      bestPlan = plan;
+    }
+    tours++;
+  }
+
+  // Bushy attempt (one per invocation)
+  const bushy = buildBushyPlan(graph);
+  const bushyCost = estimatePlanCost(bushy);
+  if (bushyCost < bestCost) {
+    bestCost = bushyCost;
+    bestPlan = bushy;
+  }
+
+  if (!bestPlan) return null;
+  // Always record diagnostics for visibility
+  context.diagnostics.quickpick = { tours, bestCost };
+  if (bestCost < baselineCost * 0.9) {
+    log('QuickPick replaced join plan (%.2f -> %.2f)', baselineCost, bestCost);
+    return bestPlan as unknown as PlanNode;
+  }
+  return null;
+}
+
+

package/src/planner/rules/predicate/rule-predicate-pushdown.ts

@@ -0,0 +1,144 @@
+/**
+ * Rule: Predicate Pushdown (characteristic-driven)
+ *
+ * Goal: Move Filter predicates downward across safe commuting nodes (Sort, Distinct, eligible Project)
+ * and into the Retrieve pipeline boundary so modules can execute or exploit them.
+ *
+ * Safe moves implemented now:
+ * - Across Sort: always safe (ordering unaffected by selection)
+ * - Across Distinct: safe for selection predicates (commute)
+ * - Across Project: only if predicate references attribute IDs available below the Project source
+ *   (we verify attribute-id coverage), and we keep predicate unchanged (IDs preserved by design)
+ * - Into Retrieve: wrap Retrieve.source with a Filter
+ *
+ * Non-moves (for now):
+ * - Across Limit/Offset (changes semantics)
+ * - Across Aggregate/Window/Join (requires deeper analysis)
+ */
+
+import { createLogger } from '../../../common/logger.js';
+import type { PlanNode, RelationalPlanNode } from '../../nodes/plan-node.js';
+import { isRelationalNode } from '../../nodes/plan-node.js';
+import type { OptContext } from '../../framework/context.js';
+import { PlanNodeType } from '../../nodes/plan-node-type.js';
+import { FilterNode } from '../../nodes/filter.js';
+import { SortNode } from '../../nodes/sort.js';
+import { DistinctNode } from '../../nodes/distinct-node.js';
+import { ProjectNode } from '../../nodes/project-node.js';
+import { RetrieveNode } from '../../nodes/retrieve-node.js';
+import { CapabilityDetectors } from '../../framework/characteristics.js';
+import type { ScalarPlanNode } from '../../nodes/plan-node.js';
+import { normalizePredicate } from '../../analysis/predicate-normalizer.js';
+import { collectBindingsInExpr } from '../../analysis/binding-collector.js';
+import { extractConstraints, createTableInfoFromNode } from '../../analysis/constraint-extractor.js';
+
+const log = createLogger('optimizer:rule:predicate-pushdown');
+
+export function rulePredicatePushdown(node: PlanNode, _context: OptContext): PlanNode | null {
+	// Only act on Filter nodes
+	if (node.nodeType !== PlanNodeType.Filter) return null;
+
+	const filter = node as FilterNode;
+  const normalized = normalizePredicate(filter.predicate);
+
+	// If no relational child, nothing to do
+	if (!isRelationalNode(filter.source)) return null;
+
+	const pushed = tryPushDown(filter.source, normalized, filter.scope);
+	if (!pushed) return null;
+
+	return pushed;
+}
+
+function tryPushDown(child: RelationalPlanNode, predicate: ScalarPlanNode, scope: any): PlanNode | null {
+  // Reach a Retrieve boundary: insert only the supported portion inside pipeline
+	if (child instanceof RetrieveNode) {
+    log('Pushing predicate into Retrieve pipeline (supported-only)');
+    const tableInfo = createTableInfoFromNode(child.tableRef, `${child.tableRef.tableSchema.name}`);
+    const extraction = extractConstraints(predicate, [tableInfo]);
+    const supported = extraction.supportedPredicateByTable?.get(tableInfo.relationKey);
+
+    if (!supported) {
+      log('No supported portion for this retrieve; not pushing');
+      return null;
+    }
+
+    const newInner = new FilterNode(child.source.scope, child.source, supported);
+    const newBindings = [
+      ...(child.bindings ?? []),
+      ...collectBindingsInExpr(supported, child.tableRef)
+    ];
+    const updatedRetrieve = child.withPipeline(newInner, child.moduleCtx, newBindings);
+
+    // If the supported portion equals the whole filter, remove original filter; else keep residual above
+    if (!extraction.residualPredicate) {
+      return updatedRetrieve;
+    }
+    return new FilterNode(scope, updatedRetrieve as unknown as RelationalPlanNode, extraction.residualPredicate);
+	}
+
+	// Across Sort
+	if (child instanceof SortNode) {
+		log('Pushing predicate below Sort');
+		const under = child.source;
+		const newUnder = new FilterNode(under.scope, under, predicate);
+		return new SortNode(child.scope, newUnder, child.sortKeys);
+	}
+
+	// Across Distinct
+	if (child instanceof DistinctNode) {
+		log('Pushing predicate below Distinct');
+		const under = child.source;
+		const newUnder = new FilterNode(under.scope, under, predicate);
+		return new DistinctNode(child.scope, newUnder);
+	}
+
+	// Across eligible Project
+	if (child instanceof ProjectNode) {
+		if (canPushAcrossProject(child, predicate)) {
+			log('Pushing predicate below Project (eligible)');
+			const under = child.source;
+			const newUnder = new FilterNode(under.scope, under, predicate);
+			// Rebuild Project with same projections over the filtered source
+			return new ProjectNode(child.scope, newUnder, child.projections, undefined, undefined, child.preserveInputColumns);
+		}
+		return null;
+	}
+
+	// Default: do not push across other nodes
+	return null;
+}
+
+function canPushAcrossProject(project: ProjectNode, predicate: ScalarPlanNode): boolean {
+	// If project preserves input columns and all predicate-attested attributes exist below, it's safe.
+	const sourceAttrIds = new Set(project.source.getAttributes().map(a => a.id));
+	const referenced = collectReferencedAttributeIds(predicate);
+	for (const id of referenced) {
+		if (!sourceAttrIds.has(id)) return false;
+	}
+	return true;
+}
+
+function collectReferencedAttributeIds(expr: ScalarPlanNode): Set<number> {
+	const ids = new Set<number>();
+	walkExpr(expr, node => {
+		if (CapabilityDetectors.isColumnReference(node)) {
+			ids.add((node as any).attributeId as number);
+		}
+	});
+	return ids;
+}
+
+function walkExpr(expr: ScalarPlanNode, fn: (n: ScalarPlanNode) => void): void {
+	fn(expr);
+	for (const c of expr.getChildren()) {
+		// Only scalar children
+		if (!isRelationalNode(c)) {
+			// eslint-disable-next-line @typescript-eslint/no-explicit-any
+			walkExpr(c as any as ScalarPlanNode, fn);
+		}
+	}
+}
+
+
+