@quereus/quereus 0.1.0 → 0.2.0

package/src/planner/cache/materialization-advisory.ts

@@ -4,11 +4,12 @@
  */
 
 import { createLogger } from '../../common/logger.js';
-import type { PlanNode, RelationalPlanNode } from '../nodes/plan-node.js';
+import { isRelationalNode, type PlanNode, type RelationalPlanNode } from '../nodes/plan-node.js';
 import { CacheNode, type CacheStrategy } from '../nodes/cache-node.js';
 import { PlanNodeType } from '../nodes/plan-node-type.js';
 import type { OptimizerTuning } from '../optimizer-tuning.js';
 import { ReferenceGraphBuilder, type RefStats } from './reference-graph.js';
+import { isCorrelatedSubquery } from './correlation-detector.js';
 
 const log = createLogger('optimizer:cache:materialization');
 
@@ -37,34 +38,38 @@ export class MaterializationAdvisory {
 	}
 
 	/**
-	 * Analyze a plan tree and return caching recommendations
+	 * Analyze a plan tree and inject caching where beneficial
+	 * Returns the transformed tree or the original if no caching was added
 	 */
-	analyzeTree(root: PlanNode): Map<PlanNode, CacheRecommendation> {
+	analyzeAndTransform(root: PlanNode): PlanNode {
+		// Build reference graph
 		const refGraph = this.referenceBuilder.buildReferenceGraph(root);
+
+		// Build recommendations
 		const recommendations = new Map<PlanNode, CacheRecommendation>();
 
 		for (const [node, stats] of refGraph) {
 			// Only consider relational nodes for caching
-			if (!this.isRelationalNode(node)) {
+			if (!isRelationalNode(node)) {
 				continue;
 			}
 
 			const recommendation = this.adviseCaching(node, stats);
-			recommendations.set(node, recommendation);
-
 			if (recommendation.shouldCache) {
+				recommendations.set(node, recommendation);
 				log('Recommending cache for %s: %s', node.nodeType, recommendation.reason);
 			}
 		}
 
-		return recommendations;
-	}
+		if (recommendations.size === 0) {
+			log('No caching opportunities identified');
+			return root;
+		}
 
-	/**
-	 * Apply caching recommendations to a plan tree
-	 */
-	applyCaching(root: PlanNode, recommendations: Map<PlanNode, CacheRecommendation>): PlanNode {
-		return this.transformNode(root, recommendations);
+		log('Found %d caching opportunities', recommendations.size);
+
+		// Transform the tree by wrapping recommended nodes with CacheNode
+		return this.transformTree(root, recommendations);
 	}
 
 	/**
@@ -91,7 +96,18 @@ export class MaterializationAdvisory {
 			};
 		}
 
-		// Rule 3: Single-parent nodes that don't appear in loops typically don't benefit from caching
+		// Rule 3: Correlated subqueries should not be cached
+		// Check if this node is part of a subquery context and if it's correlated
+		if (isRelationalNode(node) && this.isCorrelatedNode(node)) {
+			return {
+				shouldCache: false,
+				strategy: 'memory',
+				threshold: 0,
+				reason: 'Correlated subquery - must re-execute for each outer row'
+			};
+		}
+
+		// Rule 4: Single-parent nodes that don't appear in loops typically don't benefit from caching
 		if (stats.parentCount <= 1 && !stats.appearsInLoop) {
 			return {
 				shouldCache: false,
@@ -101,7 +117,7 @@ export class MaterializationAdvisory {
 			};
 		}
 
-		// Rule 4: Multi-parent nodes benefit from caching
+		// Rule 5: Multi-parent nodes benefit from caching
 		if (stats.parentCount > 1) {
 			const strategy = this.selectStrategy(stats.estimatedRows);
 			const threshold = this.calculateThreshold(stats.estimatedRows, strategy);
@@ -114,7 +130,7 @@ export class MaterializationAdvisory {
 			};
 		}
 
-		// Rule 5: Nodes in loop contexts benefit from caching even with single parent
+		// Rule 6: Nodes in loop contexts benefit from caching even with single parent
 		if (stats.appearsInLoop) {
 			// Check if the estimated size is reasonable for caching
 			if (stats.estimatedRows > this.tuning.join.maxRightRowsForCaching) {
@@ -146,7 +162,7 @@ export class MaterializationAdvisory {
 		};
 	}
 
-		/**
+	/**
 	 * Select appropriate cache strategy based on estimated size
 	 */
 	private selectStrategy(estimatedRows: number): CacheStrategy {
@@ -174,22 +190,27 @@ export class MaterializationAdvisory {
 	}
 
 	/**
-	 * Check if a node is relational (can be cached)
+	 * Check if a node is part of a correlated subquery
 	 */
-	private isRelationalNode(node: PlanNode): boolean {
-		return 'getAttributes' in node && typeof (node as any).getAttributes === 'function';
+	private isCorrelatedNode(node: PlanNode): boolean {
+		// Check if this is a relational node that could be correlated
+		if (isRelationalNode(node)) {
+			return isCorrelatedSubquery(node as RelationalPlanNode);
+		}
+		return false;
 	}
 
 	/**
-	 * Transform a node tree by applying cache recommendations
+	 * Transform a tree by wrapping recommended nodes with CacheNode
+	 * Uses a bottom-up approach to ensure proper transformation
 	 */
-	private transformNode(node: PlanNode, recommendations: Map<PlanNode, CacheRecommendation>): PlanNode {
-		// First, recursively transform children
+	private transformTree(node: PlanNode, recommendations: Map<PlanNode, CacheRecommendation>): PlanNode {
+		// First, transform all children recursively
 		const transformedNode = this.transformChildren(node, recommendations);
 
-		// Check if this node should be cached
+		// Then check if this node itself should be cached
 		const recommendation = recommendations.get(node);
-		if (recommendation?.shouldCache && this.isRelationalNode(transformedNode)) {
+		if (recommendation?.shouldCache && isRelationalNode(transformedNode)) {
 			log('Injecting %s cache for %s (threshold: %d)',
 				recommendation.strategy, transformedNode.nodeType, recommendation.threshold);
 
@@ -206,39 +227,39 @@ export class MaterializationAdvisory {
 
 	/**
 	 * Transform children of a node
+	 * This handles both scalar and relational children using a simpler approach
 	 */
 	private transformChildren(node: PlanNode, recommendations: Map<PlanNode, CacheRecommendation>): PlanNode {
-		// Handle different node types that have children
+		// For nodes that we know how to handle, transform their children
+		// For others, return the node as-is (the optimizer will handle it)
 
-		// For nodes with relational children, we need to transform them
-		if ('getRelations' in node) {
-			const relations = (node as any).getRelations();
-			const transformedRelations = relations.map((rel: PlanNode) =>
-				this.transformNode(rel, recommendations)
-			);
+		// First, try to transform scalar children using withChildren
+		const scalarChildren = node.getChildren();
+		const transformedScalarChildren = scalarChildren.map(child =>
+			this.transformTree(child, recommendations)
+		);
+
+		const scalarChanged = transformedScalarChildren.some((child, idx) =>
+			child !== scalarChildren[idx]
+		);
 
-			// Check if any relations were transformed
-			if (transformedRelations.some((rel: PlanNode, idx: number) => rel !== relations[idx])) {
-				// Need to create a new node with transformed relations
-				return this.recreateNodeWithNewRelations(node, transformedRelations);
+		if (scalarChanged) {
+			// Let withChildren handle the transformation
+			// This will maintain proper attribute IDs and node structure
+			try {
+				return node.withChildren(transformedScalarChildren);
+			} catch (e) {
+				// If withChildren fails, log and return original
+				log('Warning: withChildren failed for %s: %s', node.nodeType, e);
+				return node;
 			}
 		}
 
-		// If no transformations needed, return original node
-		return node;
-	}
-
-	/**
-	 * Recreate a node with new relational children
-	 * This is a simplified version - in practice, would need comprehensive node cloning
-	 */
-	private recreateNodeWithNewRelations(node: PlanNode, _newRelations: RelationalPlanNode[]): PlanNode {
-		// TODO: This is a simplified implementation
-		// In practice, would need to handle all node types properly
+		// If no scalar children changed, check if this node has relational children
+		// that might need caching. For now, we'll return the node as-is and let
+		// individual optimization rules handle relational transformations.
+		// This is safer than trying to recreate complex nodes.
 
-		// For now, just return the original node
-		// This would need to be expanded to handle all node types
-		log('Node recreation not yet implemented for %s', node.nodeType);
 		return node;
 	}
 }

package/src/planner/cache/reference-graph.ts

@@ -1,12 +1,16 @@
 /**
  * Reference graph builder for materialization advisory
  * Analyzes plan tree to identify nodes that would benefit from caching
+ *
+ * Note: This builder works with logical plan nodes and their properties.
+ * It does not make assumptions about execution strategies (e.g., whether
+ * a join will use nested loops vs hash join). Loop detection and execution
+ * multipliers should be determined during physical optimization when
+ * concrete execution strategies are chosen.
  */
 
 import { createLogger } from '../../common/logger.js';
-import type { PlanNode } from '../nodes/plan-node.js';
-import { PlanNodeType } from '../nodes/plan-node-type.js';
-import { JoinNode } from '../nodes/join-node.js';
+import { isRelationalNode, type PlanNode } from '../nodes/plan-node.js';
 import type { OptimizerTuning } from '../optimizer-tuning.js';
 
 const log = createLogger('optimizer:cache:reference-graph');
@@ -23,6 +27,22 @@ export interface RefStats {
 	estimatedRows: number;
 	/** Whether this node is deterministic (same inputs produce same outputs) */
 	deterministic: boolean;
+	/** Parent nodes that reference this node (for debugging) */
+	parents: Set<PlanNode>;
+	/** Estimated execution multiplier due to loop contexts */
+	loopMultiplier: number;
+}
+
+/**
+ * Node traversal context
+ */
+interface TraversalContext {
+	/** Current parent node */
+	parent: PlanNode | null;
+	/** Whether we're in a loop context */
+	inLoop: boolean;
+	/** Estimated loop iteration count */
+	loopIterations: number;
 }
 
 /**
@@ -30,7 +50,6 @@ export interface RefStats {
  */
 export class ReferenceGraphBuilder {
 	private refMap = new Map<PlanNode, RefStats>();
-	private visited = new Set<PlanNode>();
 
 	constructor(private tuning: OptimizerTuning) {}
 
@@ -38,90 +57,105 @@ export class ReferenceGraphBuilder {
 	 * Build reference statistics for all nodes in the plan tree
 	 */
 	buildReferenceGraph(root: PlanNode): Map<PlanNode, RefStats> {
+		if (!root) {
+			log('Warning: buildReferenceGraph called with null root');
+			return new Map();
+		}
+
 		this.refMap.clear();
-		this.visited.clear();
 
-		// First pass: count parent references
-		this.countReferences(root, false);
+		// Build the reference graph with proper parent tracking
+		const context: TraversalContext = {
+			parent: null,
+			inLoop: false,
+			loopIterations: 1
+		};
 
-		// Second pass: identify loop contexts
-		this.identifyLoopContexts(root, false);
+		this.buildReferences(root, context);
 
 		log('Built reference graph with %d nodes', this.refMap.size);
 		return new Map(this.refMap);
 	}
 
 	/**
-	 * First pass: count how many parents reference each node
+	 * Build reference statistics recursively
 	 */
-	private countReferences(node: PlanNode, inLoop: boolean): void {
-		if (this.visited.has(node)) {
-			// Node seen again - increment parent count
-			const stats = this.refMap.get(node);
-			if (stats) {
-				stats.parentCount++;
-			}
+	private buildReferences(node: PlanNode | null | undefined, context: TraversalContext): void {
+		if (!node) {
 			return;
 		}
 
-		this.visited.add(node);
-
-		// Initialize stats for this node
-		const stats: RefStats = {
-			parentCount: 1, // First time seeing this node
-			appearsInLoop: inLoop,
-			estimatedRows: this.getEstimatedRows(node),
-			deterministic: this.isDeterministic(node)
-		};
-
-		this.refMap.set(node, stats);
-
-		// Recurse to children
-		this.visitChildren(node, (child) => {
-			this.countReferences(child, inLoop);
-		});
-	}
+		// Get or create stats for this node
+		let stats = this.refMap.get(node);
+		if (!stats) {
+			// First time seeing this node
+			stats = {
+				parentCount: 0,
+				appearsInLoop: context.inLoop,
+				estimatedRows: this.getEstimatedRows(node),
+				deterministic: this.isDeterministic(node),
+				parents: new Set<PlanNode>(),
+				loopMultiplier: context.loopIterations
+			};
+			this.refMap.set(node, stats);
+		}
 
-	/**
-	 * Second pass: identify nodes that appear in loop contexts
-	 */
-	private identifyLoopContexts(node: PlanNode, inLoop: boolean): void {
-		const stats = this.refMap.get(node);
-		if (!stats) return;
+		// Update stats based on current traversal
+		if (context.parent && !stats.parents.has(context.parent)) {
+			stats.parents.add(context.parent);
+			stats.parentCount++;
+		}
 
-		// Update loop status if we're now in a loop context
-		if (inLoop && !stats.appearsInLoop) {
+		// Update loop context
+		if (context.inLoop) {
 			stats.appearsInLoop = true;
-			log('Node %s marked as appearing in loop', node.nodeType);
+			stats.loopMultiplier = Math.max(stats.loopMultiplier, context.loopIterations);
 		}
 
-		// Determine if children are in loop context
-		if (node instanceof JoinNode && node.joinType === 'inner') {
-			// Right side of nested loop join is in loop context
-			const [left, right] = node.getRelations();
-			this.identifyLoopContexts(left, inLoop);
-			this.identifyLoopContexts(right, true); // Right side is in loop
-		} else {
-			// Recurse to all children with current loop status
-			this.visitChildren(node, (child) => {
-				this.identifyLoopContexts(child, inLoop);
-			});
-		}
+		// Create child context - for now, we propagate the parent context
+		// In the future, if nodes expose execution strategy hints, we could use those
+		const childContext: TraversalContext = {
+			parent: node,
+			inLoop: context.inLoop,
+			loopIterations: context.loopIterations
+		};
+
+		// Visit all children uniformly
+		this.visitAllChildren(node, childContext);
 	}
 
 	/**
 	 * Visit all children of a node
 	 */
-	private visitChildren(node: PlanNode, visitor: (child: PlanNode) => void): void {
-		// Visit scalar expression children
-		for (const child of node.getChildren()) {
-			visitor(child);
+	private visitAllChildren(node: PlanNode, childContext: TraversalContext): void {
+		// 1. Scalar children (expressions)
+		try {
+			const children = node.getChildren();
+			for (const child of children) {
+				if (child) {
+					this.buildReferences(child, childContext);
+				}
+			}
+		} catch (e) {
+			log('Warning: Failed to get children for node %s: %s', node.nodeType, e);
 		}
 
-		// Visit relational children
-		if ('getRelations' in node) {
-			for (const relation of (node as any).getRelations()) {
-				visitor(relation);
+		// 2. Relational children
+		// Note: getRelations() returns a subset of getChildren() for nodes that have relational children
+		// We need to be careful not to double-count, but since we're using a Set for parents,
+		// and checking if we've already added a parent, this should be fine
+		if (isRelationalNode(node)) {
+			try {
+				const relations = node.getRelations();
+				for (const relation of relations) {
+					if (relation) {
+						// For now, treat all relational children the same
+						// In the future, nodes could provide hints about execution patterns
+						this.buildReferences(relation, childContext);
+					}
+				}
+			} catch (e) {
+				log('Warning: Failed to get relations for node %s: %s', node.nodeType, e);
 			}
 		}
 	}
@@ -129,44 +163,34 @@ export class ReferenceGraphBuilder {
 	/**
 	 * Get estimated row count for a node
 	 */
-	private getEstimatedRows(node: PlanNode): number {
-		if ('estimatedRows' in node && typeof node.estimatedRows === 'number') {
-			return node.estimatedRows;
+	private getEstimatedRows(node: PlanNode | null | undefined): number {
+		if (!node) {
+			return this.tuning.defaultRowEstimate;
 		}
-		if (node.physical?.estimatedRows) {
+
+		// Use physical properties if available
+		if (node.physical?.estimatedRows !== undefined) {
 			return node.physical.estimatedRows;
 		}
+
+		// Fall back to node-specific estimates (for relational nodes)
+		if (isRelationalNode(node) && node.estimatedRows !== undefined) {
+			return node.estimatedRows;
+		}
+
+		// Default estimate
 		return this.tuning.defaultRowEstimate;
 	}
 
 	/**
 	 * Determine if a node is deterministic
 	 */
-	private isDeterministic(node: PlanNode): boolean {
-		// Check physical properties first
-		if (node.physical?.deterministic !== undefined) {
-			return node.physical.deterministic;
+	private isDeterministic(node: PlanNode | null | undefined): boolean {
+		if (!node) {
+			return true;
 		}
 
-		// Node-type specific deterministic analysis
-		switch (node.nodeType) {
-			case PlanNodeType.TableScan:
-			case PlanNodeType.Values:
-			case PlanNodeType.Project:
-			case PlanNodeType.Filter:
-			case PlanNodeType.Sort:
-			case PlanNodeType.Aggregate:
-			case PlanNodeType.StreamAggregate:
-				return true;
-
-			case PlanNodeType.TableFunctionCall:
-				// Would need to check if the table function is deterministic
-				// For now, assume non-deterministic to be safe
-				return false;
-
-			default:
-				// Conservative default - assume deterministic unless proven otherwise
-				return true;
-		}
+		// Use physical properties to determine determinism
+		return node.physical?.deterministic ?? true;
 	}
 }

package/src/planner/debug.ts

@@ -314,3 +314,166 @@ export function generateTraceReport(
   lines.push('=== END TRACE ===');
   return lines.join('\n');
 }
+
+/**
+ * Options for plan formatting
+ */
+export interface PlanDisplayOptions {
+	/** Show concise plan by default (true) or full details (false) */
+	concise?: boolean;
+	/** Node IDs to expand with full details (only applies when concise=true) */
+	expandNodes?: string[];
+	/** Maximum depth to display (default: no limit) */
+	maxDepth?: number;
+	/** Show physical properties if available */
+	showPhysical?: boolean;
+}
+
+/**
+ * Creates a concise, tree-like representation of the plan
+ */
+export function formatPlanTree(rootNode: PlanNode, options: PlanDisplayOptions = {}): string {
+	const { concise = true, expandNodes = [], maxDepth, showPhysical = true } = options;
+	const lines: string[] = [];
+	const nodesSeen = new Set<PlanNode>();
+
+	function formatNode(node: PlanNode, depth: number, isLast: boolean, prefix: string): void {
+		if (maxDepth !== undefined && depth > maxDepth) {
+			return;
+		}
+
+		// Avoid infinite recursion for circular references
+		if (nodesSeen.has(node)) {
+			lines.push(`${prefix}├─ [CIRCULAR: ${node.nodeType}#${node.id}]`);
+			return;
+		}
+		nodesSeen.add(node);
+
+		// Determine if this node should be expanded
+		const shouldExpand = !concise || expandNodes.includes(node.id);
+
+		// Node header with connection lines
+		const connector = isLast ? '└─ ' : '├─ ';
+		const nodeType = node.nodeType;
+		const nodeId = `#${node.id}`;
+		const description = node.toString();
+
+		// Build the header line
+		let headerLine = `${prefix}${connector}${nodeType}${nodeId}`;
+		if (description && description !== nodeType) {
+			headerLine += `: ${description}`;
+		}
+
+		// Add cost information if available
+		const cost = node.estimatedCost;
+		const totalCost = node.getTotalCost();
+		if (cost > 0 || totalCost > 0) {
+			headerLine += ` [cost: ${cost}, total: ${totalCost}]`;
+		}
+
+		// Add physical properties if requested and available
+		if (showPhysical && (node as any).physical) {
+			const physical = (node as any).physical;
+			const physicalInfo = [];
+			if (physical.estimatedRows !== undefined) {
+				physicalInfo.push(`rows: ${physical.estimatedRows}`);
+			}
+			if (physical.ordering && physical.ordering.length > 0) {
+				physicalInfo.push(`ordered: ${physical.ordering.map((o: any) => `${o.attributeId}:${o.direction}`).join(',')}`);
+			}
+			if (physical.readonly !== undefined) {
+				physicalInfo.push(`readonly: ${physical.readonly}`);
+			}
+			if (physicalInfo.length > 0) {
+				headerLine += ` {${physicalInfo.join(', ')}}`;
+			}
+		}
+
+		lines.push(headerLine);
+
+		// Add expanded details if requested
+		if (shouldExpand) {
+			const logical = node.getLogicalAttributes();
+			if (logical && Object.keys(logical).length > 0) {
+				const logicalLines = JSON.stringify(logical, null, 2).split('\n');
+				const extendedPrefix = prefix + (isLast ? '    ' : '│   ');
+				lines.push(`${extendedPrefix}┌─ Logical Attributes:`);
+				for (let i = 0; i < logicalLines.length; i++) {
+					const line = logicalLines[i];
+					const isLastLogicalLine = i === logicalLines.length - 1;
+					const logicalConnector = isLastLogicalLine ? '└─ ' : '│  ';
+					lines.push(`${extendedPrefix}${logicalConnector}${line}`);
+				}
+			}
+		}
+
+		// Process children
+		const children = node.getChildren();
+		const relations = node.getRelations();
+		const allChildren = [...children, ...relations];
+
+		// Filter out duplicates (in case a child is both a child and relation)
+		const uniqueChildren = Array.from(new Set(allChildren));
+
+		for (let i = 0; i < uniqueChildren.length; i++) {
+			const child = uniqueChildren[i];
+			const isLastChild = i === uniqueChildren.length - 1;
+			const childPrefix = prefix + (isLast ? '    ' : '│   ');
+			formatNode(child, depth + 1, isLastChild, childPrefix);
+		}
+
+		nodesSeen.delete(node);
+	}
+
+	lines.push('Query Plan:');
+	formatNode(rootNode, 0, true, '');
+
+	// Add help text
+	if (concise && expandNodes.length === 0) {
+		lines.push('');
+		lines.push('Tip: Use --expand-nodes node1,node2,... to see detailed properties for specific nodes');
+	}
+
+	return lines.join('\n');
+}
+
+/**
+ * Generates a compact plan summary showing just the execution path
+ */
+export function formatPlanSummary(rootNode: PlanNode): string {
+	const path: string[] = [];
+	const visited = new Set<PlanNode>();
+
+	function collectPath(node: PlanNode): void {
+		if (visited.has(node)) return;
+		visited.add(node);
+
+		const description = node.toString();
+		const nodeInfo = description && description !== node.nodeType
+			? `${node.nodeType}(${description})`
+			: node.nodeType;
+
+		path.push(nodeInfo);
+
+		// Follow the main execution path (first child for most nodes)
+		const children = node.getChildren();
+		if (children.length > 0) {
+			collectPath(children[0]);
+		}
+	}
+
+	collectPath(rootNode);
+	return `Execution Path: ${path.join(' → ')}`;
+}
+
+/**
+ * Enhanced plan serialization with formatting options
+ */
+export function serializePlanTreeWithOptions(rootNode: PlanNode, options: PlanDisplayOptions = {}): string {
+	if (options.concise !== false) {
+		return formatPlanTree(rootNode, options);
+	} else {
+		// Use the existing detailed serialization
+		return serializePlanTree(rootNode);
+	}
+}