@quereus/quereus 0.1.0

package/src/planner/nodes/window-node.ts

@@ -0,0 +1,193 @@
+import { PlanNodeType } from './plan-node-type.js';
+import { PlanNode, type Attribute, type RelationalPlanNode, type UnaryRelationalNode, type ScalarPlanNode } from './plan-node.js';
+import type { WindowFunctionCallNode } from './window-function.js';
+import type { RelationType } from '../../common/datatype.js';
+import type { Scope } from '../scopes/scope.js';
+import { Cached } from '../../util/cached.js';
+import type * as AST from '../../parser/ast.js';
+
+export interface WindowSpec {
+	partitionBy: AST.Expression[];
+	orderBy: AST.OrderByClause[];
+	frame?: AST.WindowFrame;
+}
+
+/**
+ * Represents a window operation that computes window functions over partitions of rows.
+ * This node groups window functions that share the same window specification for efficiency.
+ */
+export class WindowNode extends PlanNode implements UnaryRelationalNode {
+	override readonly nodeType = PlanNodeType.Window;
+
+	private outputTypeCache: Cached<RelationType>;
+	private attributesCache: Cached<Attribute[]>;
+
+	constructor(
+		scope: Scope,
+		public readonly source: RelationalPlanNode,
+		public readonly windowSpec: WindowSpec,
+		public readonly functions: WindowFunctionCallNode[],
+		public readonly partitionExpressions: ScalarPlanNode[],
+		public readonly orderByExpressions: ScalarPlanNode[],
+		public readonly functionArguments: (ScalarPlanNode | null)[],
+		estimatedCostOverride?: number,
+		/** Optional predefined attributes for preserving IDs during optimization */
+		public readonly predefinedAttributes?: Attribute[]
+	) {
+		super(scope, estimatedCostOverride);
+
+		this.outputTypeCache = new Cached(() => {
+			const sourceType = this.source.getType();
+
+			// Add window function columns to the source columns
+			const windowColumns = this.functions.map(func => ({
+				name: func.alias || func.functionName.toLowerCase(),
+				type: func.getType(),
+				generated: true
+			}));
+
+			return {
+				typeClass: 'relation',
+				isReadOnly: sourceType.isReadOnly,
+				isSet: sourceType.isSet, // Window functions preserve set/bag semantics
+				columns: [...sourceType.columns, ...windowColumns],
+				keys: sourceType.keys, // Window functions don't change key structure
+				rowConstraints: sourceType.rowConstraints,
+			} as RelationType;
+		});
+
+		this.attributesCache = new Cached(() => {
+			// If predefined attributes are provided, use them (for optimization)
+			if (this.predefinedAttributes) {
+				return this.predefinedAttributes.slice(); // Return a copy
+			}
+
+			// Preserve source attributes and add window function attributes
+			const sourceAttrs = this.source.getAttributes();
+			const windowAttrs = this.functions.map((func) => ({
+				id: PlanNode.nextAttrId(),
+				name: func.alias || func.functionName.toLowerCase(),
+				type: func.getType(),
+				sourceRelation: `${this.nodeType}:${this.id}`
+			}));
+
+			return [...sourceAttrs, ...windowAttrs];
+		});
+	}
+
+	getType(): RelationType {
+		return this.outputTypeCache.value;
+	}
+
+	getAttributes(): Attribute[] {
+		return this.attributesCache.value;
+	}
+
+	getChildren(): readonly PlanNode[] {
+		// Return all scalar expressions: partition expressions, order by expressions, and function arguments
+		const children: PlanNode[] = [];
+		children.push(...this.partitionExpressions);
+		children.push(...this.orderByExpressions);
+		children.push(...this.functionArguments.filter(arg => arg !== null) as ScalarPlanNode[]);
+		return children;
+	}
+
+	withChildren(newChildren: readonly PlanNode[]): PlanNode {
+		const expectedLength = this.partitionExpressions.length +
+			this.orderByExpressions.length +
+			this.functionArguments.filter(arg => arg !== null).length;
+
+		if (newChildren.length !== expectedLength) {
+			throw new Error(`WindowNode expects ${expectedLength} children, got ${newChildren.length}`);
+		}
+
+		// Type check
+		for (const child of newChildren) {
+			if (!('expression' in child)) {
+				throw new Error('WindowNode: all children must be ScalarPlanNodes');
+			}
+		}
+
+		// Split children back into their respective arrays
+		let childIndex = 0;
+		const newPartitionExpressions = newChildren.slice(childIndex, childIndex + this.partitionExpressions.length) as ScalarPlanNode[];
+		childIndex += this.partitionExpressions.length;
+
+		const newOrderByExpressions = newChildren.slice(childIndex, childIndex + this.orderByExpressions.length) as ScalarPlanNode[];
+		childIndex += this.orderByExpressions.length;
+
+		const newNonNullFunctionArgs = newChildren.slice(childIndex) as ScalarPlanNode[];
+
+		// Rebuild function arguments array preserving null positions
+		const newFunctionArguments: (ScalarPlanNode | null)[] = [];
+		let nonNullIndex = 0;
+		for (const arg of this.functionArguments) {
+			if (arg === null) {
+				newFunctionArguments.push(null);
+			} else {
+				newFunctionArguments.push(newNonNullFunctionArgs[nonNullIndex++]);
+			}
+		}
+
+		// Check if anything changed
+		const partitionChanged = newPartitionExpressions.some((expr, i) => expr !== this.partitionExpressions[i]);
+		const orderByChanged = newOrderByExpressions.some((expr, i) => expr !== this.orderByExpressions[i]);
+		const functionArgsChanged = newFunctionArguments.some((arg, i) => arg !== this.functionArguments[i]);
+
+		if (!partitionChanged && !orderByChanged && !functionArgsChanged) {
+			return this;
+		}
+
+		// **CRITICAL**: Preserve original attribute IDs to maintain column reference stability
+		const originalAttributes = this.getAttributes();
+
+		// Create new instance with preserved attributes
+		return new WindowNode(
+			this.scope,
+			this.source, // Source doesn't change via withChildren
+			this.windowSpec,
+			this.functions, // Functions don't change via withChildren
+			newPartitionExpressions,
+			newOrderByExpressions,
+			newFunctionArguments,
+			undefined, // estimatedCostOverride
+			originalAttributes // Preserve original attribute IDs
+		);
+	}
+
+	getRelations(): readonly [RelationalPlanNode] {
+		return [this.source];
+	}
+
+	get estimatedRows(): number | undefined {
+		return this.source.estimatedRows; // Window functions don't change row count
+	}
+
+	override toString(): string {
+		const partitionClause = this.windowSpec.partitionBy.length > 0
+			? `PARTITION BY ${this.windowSpec.partitionBy.map(_e => '...').join(', ')}`
+			: '';
+		const orderClause = this.windowSpec.orderBy.length > 0
+			? `ORDER BY ${this.windowSpec.orderBy.map(_o => '...').join(', ')}`
+			: '';
+		const clauses = [partitionClause, orderClause].filter(c => c).join(' ');
+		const funcNames = this.functions.map(f => f.functionName).join(', ');
+
+		return `WINDOW ${funcNames} OVER (${clauses})`;
+	}
+
+	override getLogicalAttributes(): Record<string, unknown> {
+		return {
+			windowSpec: {
+				partitionBy: this.windowSpec.partitionBy.length,
+				orderBy: this.windowSpec.orderBy.length,
+				frame: this.windowSpec.frame ? 'custom' : 'default'
+			},
+			functions: this.functions.map(f => ({
+				name: f.functionName,
+				alias: f.alias,
+				distinct: f.isDistinct
+			}))
+		};
+	}
+}

package/src/planner/optimizer-tuning.ts

@@ -0,0 +1,87 @@
+/**
+ * Optimizer tuning parameters - centralized configuration for magic numbers
+ */
+export interface OptimizerTuning {
+	/** Row estimation defaults */
+	readonly defaultRowEstimate: number;
+
+	/** Maximum optimization depth to prevent infinite recursion */
+	readonly maxOptimizationDepth: number;
+
+	/** Join optimization */
+	readonly join: {
+		/** Minimum left side rows to consider caching right side */
+		readonly minLeftRowsForCaching: number;
+		/** Maximum right side rows to cache */
+		readonly maxRightRowsForCaching: number;
+		/** Cache threshold multiplier (rightSize * multiplier) */
+		readonly cacheThresholdMultiplier: number;
+		/** Maximum cache threshold */
+		readonly maxCacheThreshold: number;
+	};
+
+	/** CTE optimization */
+	readonly cte: {
+		/** Maximum CTE size to consider for caching */
+		readonly maxSizeForCaching: number;
+		/** Cache threshold multiplier for CTEs */
+		readonly cacheThresholdMultiplier: number;
+		/** Maximum cache threshold for CTEs */
+		readonly maxCacheThreshold: number;
+	};
+
+	/** Recursive CTE configuration */
+	readonly recursiveCte: {
+		/** Maximum iterations before recursive CTE is terminated (0 = unlimited) */
+		readonly maxIterations: number;
+		/** Default cache threshold for CTE self-references */
+		readonly defaultCacheThreshold: number;
+	};
+
+	/** Materialization advisory configuration */
+	readonly cache: {
+		/** Row threshold for switching from memory to spill strategy */
+		readonly spillThreshold: number;
+		/** Maximum memory buffer size for spill caches */
+		readonly maxSpillBuffer: number;
+		/** Whether spill caching is enabled */
+		readonly spillEnabled: boolean;
+	};
+
+	/** Development and debugging options */
+	readonly debug: {
+		/** Whether to validate physical plans before emission */
+		readonly validatePlan: boolean;
+	};
+}
+
+/**
+ * Default optimizer tuning parameters
+ */
+export const DEFAULT_TUNING: OptimizerTuning = {
+	defaultRowEstimate: 1000,
+	maxOptimizationDepth: 50,
+	join: {
+		minLeftRowsForCaching: 1,
+		maxRightRowsForCaching: 50000,
+		cacheThresholdMultiplier: 2,
+		maxCacheThreshold: 10000
+	},
+	cte: {
+		maxSizeForCaching: 50000,
+		cacheThresholdMultiplier: 2,
+		maxCacheThreshold: 20000
+	},
+	recursiveCte: {
+		maxIterations: 10000,
+		defaultCacheThreshold: 10000
+	},
+	cache: {
+		spillThreshold: 100000,
+		maxSpillBuffer: 10000,
+		spillEnabled: true
+	},
+	debug: {
+		validatePlan: false // Default to disabled in production
+	}
+};

package/src/planner/optimizer.ts

@@ -0,0 +1,263 @@
+import { createLogger } from '../common/logger.js';
+import { PlanNode } from './nodes/plan-node.js';
+import { PlanNodeType } from './nodes/plan-node-type.js';
+import { OptimizerTuning, DEFAULT_TUNING } from './optimizer-tuning.js';
+
+// Re-export for convenience
+export { DEFAULT_TUNING };
+
+import { applyRules, clearVisitedRules, registerRules, createRule } from './framework/registry.js';
+import { tracePhaseStart, tracePhaseEnd, traceNodeStart, traceNodeEnd } from './framework/trace.js';
+import { defaultStatsProvider, type StatsProvider } from './stats/index.js';
+import { createOptContext, type OptContext } from './framework/context.js';
+// Phase 2 rules
+import { ruleMaterializationAdvisory } from './rules/cache/rule-materialization-advisory.js';
+// Phase 1.5 rules
+import { ruleSelectAccessPath } from './rules/access/rule-select-access-path.js';
+// Core optimization rules
+import { ruleAggregateStreaming } from './rules/aggregate/rule-aggregate-streaming.js';
+// Constraint rules removed - now handled in builders for correctness
+import { ruleCteOptimization } from './rules/cache/rule-cte-optimization.js';
+import { ruleMutatingSubqueryCache } from './rules/cache/rule-mutating-subquery-cache.js';
+import { ruleMarkPhysical } from './rules/physical/rule-mark-physical.js';
+// Phase 3 rules
+import { validatePhysicalTree } from './validation/plan-validator.js';
+import { Database } from '../core/database.js';
+import { performConstantFolding } from './analysis/const-pass.js';
+import { createRuntimeExpressionEvaluator } from './analysis/const-evaluator.js';
+
+const log = createLogger('optimizer');
+
+/**
+ * The query optimizer transforms logical plan trees into physical plan trees
+ */
+export class Optimizer {
+	private readonly stats: StatsProvider;
+
+	constructor(
+		public readonly tuning: OptimizerTuning = DEFAULT_TUNING,
+		stats?: StatsProvider
+	) {
+		this.stats = stats ?? defaultStatsProvider;
+
+		// Ensure global framework rules are registered once
+		Optimizer.ensureGlobalRulesRegistered();
+	}
+
+	private static globalRulesRegistered = false;
+
+	/**
+	 * Ensure global framework rules are registered only once
+	 */
+	private static ensureGlobalRulesRegistered(): void {
+		if (Optimizer.globalRulesRegistered) {
+			return;
+		}
+		Optimizer.globalRulesRegistered = true;
+
+		const toRegister = [];
+
+		// Single-pass constant folding is now done before rules, so no individual rules needed
+
+		// Core optimization rules (converted from old system)
+		toRegister.push(createRule(
+			'aggregate-streaming',
+			PlanNodeType.Aggregate,
+			'impl',
+			ruleAggregateStreaming,
+			40 // High priority - fundamental logical→physical transformation
+		));
+
+		// Mutating subquery cache injection - critical for correctness
+		toRegister.push(createRule(
+			'mutating-subquery-cache',
+			PlanNodeType.Join,
+			'rewrite',
+			ruleMutatingSubqueryCache,
+			20 // Very high priority - correctness fix to prevent multiple execution
+		));
+
+		toRegister.push(createRule(
+			'cte-optimization',
+			PlanNodeType.CTE,
+			'impl',
+			ruleCteOptimization,
+			70 // Lower priority - caching optimization
+		));
+
+		// toRegister.push(createRule(
+		// 	'project-optimization',
+		// 	PlanNodeType.Project,
+		// 	'impl',
+		// 	ruleProjectOptimization,
+		// 	50 // Medium priority - basic optimization
+		// ));
+
+		// toRegister.push(createRule(
+		// 	'filter-optimization',
+		// 	PlanNodeType.Filter,
+		// 	'impl',
+		// 	ruleFilterOptimization,
+		// 	50 // Medium priority - basic optimization
+		// ));
+
+		// Phase 1.5 rules
+		toRegister.push(createRule(
+			'select-access-path',
+			PlanNodeType.TableScan,
+			'impl',
+			ruleSelectAccessPath,
+			30 // High priority - fundamental access path selection
+		));
+
+		// Phase 2 rules
+		toRegister.push(createRule(
+			'materialization-advisory',
+			PlanNodeType.Block, // Apply to root-level nodes
+			'rewrite',
+			ruleMaterializationAdvisory,
+			90 // Low priority - run last for global analysis
+		));
+
+		// Fallback rule - must run last with lowest priority
+		// We need to register this for multiple node types that might need fallback handling
+		const fallbackNodeTypes = [
+			// Relational nodes
+			PlanNodeType.TableScan, PlanNodeType.Values, PlanNodeType.TableFunctionCall,
+			PlanNodeType.Join, PlanNodeType.NestedLoopJoin, PlanNodeType.SingleRow,
+			PlanNodeType.SetOperation, PlanNodeType.Distinct, PlanNodeType.LimitOffset,
+			PlanNodeType.Window, PlanNodeType.Block, PlanNodeType.CTEReference,
+			PlanNodeType.Cache, PlanNodeType.Sequencing, PlanNodeType.UpdateExecutor,
+			// Scalar nodes
+			PlanNodeType.Literal, PlanNodeType.ColumnReference, PlanNodeType.ParameterReference,
+			PlanNodeType.BinaryOp, PlanNodeType.UnaryOp, PlanNodeType.CaseExpr,
+			PlanNodeType.Cast, PlanNodeType.Collate, PlanNodeType.ScalarFunctionCall,
+			PlanNodeType.Between, PlanNodeType.IsNull, PlanNodeType.IsNotNull, PlanNodeType.Like,
+			PlanNodeType.ScalarSubquery, PlanNodeType.WindowFunctionCall, PlanNodeType.ArrayIndex
+		];
+
+		for (const nodeType of fallbackNodeTypes) {
+			toRegister.push(createRule(
+				`mark-physical-${nodeType.toLowerCase()}`,
+				nodeType,
+				'impl',
+				ruleMarkPhysical,
+				100 // Lowest priority - absolute fallback
+			));
+		}
+
+		// Register all rules at once
+		registerRules(toRegister);
+	}
+
+	/**
+	 * Optimize a plan tree by applying transformation rules
+	 */
+	optimize(plan: PlanNode, db: Database): PlanNode {
+		log('Starting optimization of plan', plan.nodeType);
+
+		// Clear rule tracking from previous runs
+		clearVisitedRules();
+
+		// Create optimization context
+		const context = createOptContext(this, this.stats, this.tuning, db);
+
+		tracePhaseStart('optimization');
+		try {
+			// Phase 0: Single-pass constant folding before other rules
+			log('Running constant folding pre-pass');
+			const constantFoldedPlan = this.performConstantFolding(plan, context);
+
+			// Continue with rule-based optimization
+			const result = this.optimizeNode(constantFoldedPlan, context);
+
+			// Phase 3: Validate the physical plan before returning
+			if (this.tuning.debug.validatePlan) {
+				log('Running plan validation');
+				try {
+					validatePhysicalTree(result);
+					log('Plan validation passed');
+				} catch (error) {
+					log('Plan validation failed: %s', error);
+					throw error;
+				}
+			}
+
+			return result;
+		} finally {
+			tracePhaseEnd('optimization');
+		}
+	}
+
+	/**
+	 * Perform single-pass constant folding over the entire plan tree
+	 */
+	private performConstantFolding(plan: PlanNode, context: OptContext): PlanNode {
+		try {
+			// Create runtime expression evaluator
+			const evaluator = createRuntimeExpressionEvaluator(context.db);
+
+			// Perform single-pass constant folding
+			const result = performConstantFolding(plan, evaluator);
+
+			log('Constant folding completed');
+			return result;
+		} catch (error) {
+			log('Constant folding failed: %s', error);
+			// Return original plan on failure
+			return plan;
+		}
+	}
+
+	optimizeNode(node: PlanNode, context: OptContext): PlanNode {
+		traceNodeStart(node);
+
+		// If already physical, just recurse on children
+		if (node.physical) {
+			const result = this.optimizeChildren(node, context);
+			traceNodeEnd(node, result);
+			return result;
+		}
+
+		// First optimize all children
+		const optimizedNode = this.optimizeChildren(node, context);
+
+		// Apply rules
+		const rulesApplied = applyRules(optimizedNode, context);
+
+		if (rulesApplied !== optimizedNode) {
+			// Rules transformed the node
+			log(`Rules applied to ${optimizedNode.nodeType}, transformed to ${rulesApplied.nodeType}`);
+			traceNodeEnd(node, rulesApplied);
+			return rulesApplied;
+		}
+
+		// No rule applied - assume node is physical
+		traceNodeEnd(node, optimizedNode);
+		return optimizedNode;
+	}
+
+	private optimizeChildren(node: PlanNode, context: OptContext): PlanNode {
+		// Generic tree walk using withChildren
+		const originalChildren = node.getChildren();
+		const optimizedChildren = originalChildren.map(child => this.optimizeNode(child, context));
+
+		// Check if any children changed
+		const childrenChanged = optimizedChildren.some((child, i) => child !== originalChildren[i]);
+
+		if (!childrenChanged) {
+			return node; // No changes
+		}
+
+		// Use withChildren to create new node with optimized children
+		// withChildren is a required contract - any errors should propagate
+		return node.withChildren(optimizedChildren);
+	}
+
+	/**
+	 * Get the statistics provider
+	 */
+	getStats(): StatsProvider {
+		return this.stats;
+	}
+}