@quereus/quereus 0.7.3 → 0.7.4

package/src/planner/cache/materialization-advisory.ts

@@ -1,265 +0,0 @@
-/**
- * Materialization advisory framework
- * Decides when and how to inject caching based on reference graph analysis
- */
-
-import { createLogger } from '../../common/logger.js';
-import { isRelationalNode, type PlanNode, type RelationalPlanNode } from '../nodes/plan-node.js';
-import { CacheNode, type CacheStrategy } from '../nodes/cache-node.js';
-import { PlanNodeType } from '../nodes/plan-node-type.js';
-import type { OptimizerTuning } from '../optimizer-tuning.js';
-import { ReferenceGraphBuilder, type RefStats } from './reference-graph.js';
-import { isCorrelatedSubquery } from './correlation-detector.js';
-
-const log = createLogger('optimizer:cache:materialization');
-
-/**
- * Cache recommendation for a specific node
- */
-export interface CacheRecommendation {
-	/** Whether to inject caching for this node */
-	shouldCache: boolean;
-	/** Recommended cache strategy */
-	strategy: CacheStrategy;
-	/** Recommended cache threshold */
-	threshold: number;
-	/** Reason for the recommendation (for debugging) */
-	reason: string;
-}
-
-/**
- * Materialization advisory that analyzes plan trees and recommends caching
- */
-export class MaterializationAdvisory {
-	private referenceBuilder: ReferenceGraphBuilder;
-
-	constructor(private tuning: OptimizerTuning) {
-		this.referenceBuilder = new ReferenceGraphBuilder(tuning);
-	}
-
-	/**
-	 * Analyze a plan tree and inject caching where beneficial
-	 * Returns the transformed tree or the original if no caching was added
-	 */
-	analyzeAndTransform(root: PlanNode): PlanNode {
-		// Build reference graph
-		const refGraph = this.referenceBuilder.buildReferenceGraph(root);
-
-		// Build recommendations
-		const recommendations = new Map<PlanNode, CacheRecommendation>();
-
-		for (const [node, stats] of refGraph) {
-			// Only consider relational nodes for caching
-			if (!isRelationalNode(node)) {
-				continue;
-			}
-
-			const recommendation = this.adviseCaching(node, stats);
-			if (recommendation.shouldCache) {
-				recommendations.set(node, recommendation);
-				log('Recommending cache for %s: %s', node.nodeType, recommendation.reason);
-			}
-		}
-
-		if (recommendations.size === 0) {
-			log('No caching opportunities identified');
-			return root;
-		}
-
-		log('Found %d caching opportunities', recommendations.size);
-
-		// Transform the tree by wrapping recommended nodes with CacheNode
-		return this.transformTree(root, recommendations);
-	}
-
-	/**
-	 * Core advisory algorithm
-	 */
-	private adviseCaching(node: PlanNode, stats: RefStats): CacheRecommendation {
-		// Rule 1: Non-deterministic nodes should not be cached
-		if (!stats.deterministic) {
-			return {
-				shouldCache: false,
-				strategy: 'memory',
-				threshold: 0,
-				reason: 'Non-deterministic node'
-			};
-		}
-
-		// Rule 2: Nodes that are already cached don't need additional caching
-		if (node.nodeType === PlanNodeType.Cache) {
-			return {
-				shouldCache: false,
-				strategy: 'memory',
-				threshold: 0,
-				reason: 'Already cached'
-			};
-		}
-
-		// Rule 3: Correlated subqueries should not be cached
-		// Check if this node is part of a subquery context and if it's correlated
-		if (isRelationalNode(node) && this.isCorrelatedNode(node)) {
-			return {
-				shouldCache: false,
-				strategy: 'memory',
-				threshold: 0,
-				reason: 'Correlated subquery - must re-execute for each outer row'
-			};
-		}
-
-		// Rule 4: Single-parent nodes that don't appear in loops typically don't benefit from caching
-		if (stats.parentCount <= 1 && !stats.appearsInLoop) {
-			return {
-				shouldCache: false,
-				strategy: 'memory',
-				threshold: 0,
-				reason: 'Single parent, not in loop'
-			};
-		}
-
-		// Rule 5: Multi-parent nodes benefit from caching
-		if (stats.parentCount > 1) {
-			const strategy = this.selectStrategy(stats.estimatedRows);
-			const threshold = this.calculateThreshold(stats.estimatedRows, strategy);
-
-			return {
-				shouldCache: true,
-				strategy,
-				threshold,
-				reason: `Multiple parents (${stats.parentCount})`
-			};
-		}
-
-		// Rule 6: Nodes in loop contexts benefit from caching even with single parent
-		if (stats.appearsInLoop) {
-			// Check if the estimated size is reasonable for caching
-			if (stats.estimatedRows > this.tuning.join.maxRightRowsForCaching) {
-				return {
-					shouldCache: false,
-					strategy: 'memory',
-					threshold: 0,
-					reason: `In loop but too large (${stats.estimatedRows} rows)`
-				};
-			}
-
-			const strategy = this.selectStrategy(stats.estimatedRows);
-			const threshold = this.calculateThreshold(stats.estimatedRows, strategy);
-
-			return {
-				shouldCache: true,
-				strategy,
-				threshold,
-				reason: 'Appears in loop context'
-			};
-		}
-
-		// Default: no caching
-		return {
-			shouldCache: false,
-			strategy: 'memory',
-			threshold: 0,
-			reason: 'No caching criteria met'
-		};
-	}
-
-	/**
-	 * Select appropriate cache strategy based on estimated size
-	 */
-	private selectStrategy(estimatedRows: number): CacheStrategy {
-		// Use tuning configuration for strategy selection
-		if (this.tuning.cache.spillEnabled && estimatedRows > this.tuning.cache.spillThreshold) {
-			return 'spill';
-		}
-
-		return 'memory';
-	}
-
-	/**
-	 * Calculate appropriate cache threshold
-	 */
-	private calculateThreshold(estimatedRows: number, strategy: CacheStrategy): number {
-		const multiplier = this.tuning.join.cacheThresholdMultiplier;
-		const maxThreshold = strategy === 'spill' ?
-			this.tuning.join.maxCacheThreshold * 2 : // Allow larger thresholds for spill
-			this.tuning.join.maxCacheThreshold;
-
-		return Math.min(
-			Math.max(estimatedRows * multiplier, 1000), // Minimum threshold
-			maxThreshold
-		);
-	}
-
-	/**
-	 * Check if a node is part of a correlated subquery
-	 */
-	private isCorrelatedNode(node: PlanNode): boolean {
-		// Check if this is a relational node that could be correlated
-		if (isRelationalNode(node)) {
-			return isCorrelatedSubquery(node as RelationalPlanNode);
-		}
-		return false;
-	}
-
-	/**
-	 * Transform a tree by wrapping recommended nodes with CacheNode
-	 * Uses a bottom-up approach to ensure proper transformation
-	 */
-	private transformTree(node: PlanNode, recommendations: Map<PlanNode, CacheRecommendation>): PlanNode {
-		// First, transform all children recursively
-		const transformedNode = this.transformChildren(node, recommendations);
-
-		// Then check if this node itself should be cached
-		const recommendation = recommendations.get(node);
-		if (recommendation?.shouldCache && isRelationalNode(transformedNode)) {
-			log('Injecting %s cache for %s (threshold: %d)',
-				recommendation.strategy, transformedNode.nodeType, recommendation.threshold);
-
-			return new CacheNode(
-				transformedNode.scope,
-				transformedNode as RelationalPlanNode,
-				recommendation.strategy,
-				recommendation.threshold
-			);
-		}
-
-		return transformedNode;
-	}
-
-	/**
-	 * Transform children of a node
-	 * This handles both scalar and relational children using a simpler approach
-	 */
-	private transformChildren(node: PlanNode, recommendations: Map<PlanNode, CacheRecommendation>): PlanNode {
-		// For nodes that we know how to handle, transform their children
-		// For others, return the node as-is (the optimizer will handle it)
-
-		// First, try to transform scalar children using withChildren
-		const scalarChildren = node.getChildren();
-		const transformedScalarChildren = scalarChildren.map(child =>
-			this.transformTree(child, recommendations)
-		);
-
-		const scalarChanged = transformedScalarChildren.some((child, idx) =>
-			child !== scalarChildren[idx]
-		);
-
-		if (scalarChanged) {
-			// Let withChildren handle the transformation
-			// This will maintain proper attribute IDs and node structure
-			try {
-				return node.withChildren(transformedScalarChildren);
-			} catch (e) {
-				// If withChildren fails, log and return original
-				log('Warning: withChildren failed for %s: %s', node.nodeType, e);
-				return node;
-			}
-		}
-
-		// If no scalar children changed, check if this node has relational children
-		// that might need caching. For now, we'll return the node as-is and let
-		// individual optimization rules handle relational transformations.
-		// This is safer than trying to recreate complex nodes.
-
-		return node;
-	}
-}

package/src/planner/cache/reference-graph.ts

@@ -1,196 +0,0 @@
-/**
- * Reference graph builder for materialization advisory
- * Analyzes plan tree to identify nodes that would benefit from caching
- *
- * Note: This builder works with logical plan nodes and their properties.
- * It does not make assumptions about execution strategies (e.g., whether
- * a join will use nested loops vs hash join). Loop detection and execution
- * multipliers should be determined during physical optimization when
- * concrete execution strategies are chosen.
- */
-
-import { createLogger } from '../../common/logger.js';
-import { isRelationalNode, type PlanNode } from '../nodes/plan-node.js';
-import type { OptimizerTuning } from '../optimizer-tuning.js';
-
-const log = createLogger('optimizer:cache:reference-graph');
-
-/**
- * Statistics about how a node is referenced in the plan tree
- */
-export interface RefStats {
-	/** Number of parent nodes referencing this node */
-	parentCount: number;
-	/** Whether this node appears on the inner side of a nested loop or correlated subquery */
-	appearsInLoop: boolean;
-	/** Estimated number of rows this node produces */
-	estimatedRows: number;
-	/** Whether this node is deterministic (same inputs produce same outputs) */
-	deterministic: boolean;
-	/** Parent nodes that reference this node (for debugging) */
-	parents: Set<PlanNode>;
-	/** Estimated execution multiplier due to loop contexts */
-	loopMultiplier: number;
-}
-
-/**
- * Node traversal context
- */
-interface TraversalContext {
-	/** Current parent node */
-	parent: PlanNode | null;
-	/** Whether we're in a loop context */
-	inLoop: boolean;
-	/** Estimated loop iteration count */
-	loopIterations: number;
-}
-
-/**
- * Builds a reference graph for materialization decisions
- */
-export class ReferenceGraphBuilder {
-	private refMap = new Map<PlanNode, RefStats>();
-
-	constructor(private tuning: OptimizerTuning) {}
-
-	/**
-	 * Build reference statistics for all nodes in the plan tree
-	 */
-	buildReferenceGraph(root: PlanNode): Map<PlanNode, RefStats> {
-		if (!root) {
-			log('Warning: buildReferenceGraph called with null root');
-			return new Map();
-		}
-
-		this.refMap.clear();
-
-		// Build the reference graph with proper parent tracking
-		const context: TraversalContext = {
-			parent: null,
-			inLoop: false,
-			loopIterations: 1
-		};
-
-		this.buildReferences(root, context);
-
-		log('Built reference graph with %d nodes', this.refMap.size);
-		return new Map(this.refMap);
-	}
-
-	/**
-	 * Build reference statistics recursively
-	 */
-	private buildReferences(node: PlanNode | null | undefined, context: TraversalContext): void {
-		if (!node) {
-			return;
-		}
-
-		// Get or create stats for this node
-		let stats = this.refMap.get(node);
-		if (!stats) {
-			// First time seeing this node
-			stats = {
-				parentCount: 0,
-				appearsInLoop: context.inLoop,
-				estimatedRows: this.getEstimatedRows(node),
-				deterministic: this.isDeterministic(node),
-				parents: new Set<PlanNode>(),
-				loopMultiplier: context.loopIterations
-			};
-			this.refMap.set(node, stats);
-		}
-
-		// Update stats based on current traversal
-		if (context.parent && !stats.parents.has(context.parent)) {
-			stats.parents.add(context.parent);
-			stats.parentCount++;
-		}
-
-		// Update loop context
-		if (context.inLoop) {
-			stats.appearsInLoop = true;
-			stats.loopMultiplier = Math.max(stats.loopMultiplier, context.loopIterations);
-		}
-
-		// Create child context - for now, we propagate the parent context
-		// In the future, if nodes expose execution strategy hints, we could use those
-		const childContext: TraversalContext = {
-			parent: node,
-			inLoop: context.inLoop,
-			loopIterations: context.loopIterations
-		};
-
-		// Visit all children uniformly
-		this.visitAllChildren(node, childContext);
-	}
-
-	/**
-	 * Visit all children of a node
-	 */
-	private visitAllChildren(node: PlanNode, childContext: TraversalContext): void {
-		// 1. Scalar children (expressions)
-		try {
-			const children = node.getChildren();
-			for (const child of children) {
-				if (child) {
-					this.buildReferences(child, childContext);
-				}
-			}
-		} catch (e) {
-			log('Warning: Failed to get children for node %s: %s', node.nodeType, e);
-		}
-
-		// 2. Relational children
-		// Note: getRelations() returns a subset of getChildren() for nodes that have relational children
-		// We need to be careful not to double-count, but since we're using a Set for parents,
-		// and checking if we've already added a parent, this should be fine
-		if (isRelationalNode(node)) {
-			try {
-				const relations = node.getRelations();
-				for (const relation of relations) {
-					if (relation) {
-						// For now, treat all relational children the same
-						// In the future, nodes could provide hints about execution patterns
-						this.buildReferences(relation, childContext);
-					}
-				}
-			} catch (e) {
-				log('Warning: Failed to get relations for node %s: %s', node.nodeType, e);
-			}
-		}
-	}
-
-	/**
-	 * Get estimated row count for a node
-	 */
-	private getEstimatedRows(node: PlanNode | null | undefined): number {
-		if (!node) {
-			return this.tuning.defaultRowEstimate;
-		}
-
-		// Use physical properties if available
-		if (node.physical?.estimatedRows !== undefined) {
-			return node.physical.estimatedRows;
-		}
-
-		// Fall back to node-specific estimates (for relational nodes)
-		if (isRelationalNode(node) && node.estimatedRows !== undefined) {
-			return node.estimatedRows;
-		}
-
-		// Default estimate
-		return this.tuning.defaultRowEstimate;
-	}
-
-	/**
-	 * Determine if a node is deterministic
-	 */
-	private isDeterministic(node: PlanNode | null | undefined): boolean {
-		if (!node) {
-			return true;
-		}
-
-		// Use physical properties to determine determinism
-		return node.physical?.deterministic ?? true;
-	}
-}

package/src/planner/cost/index.ts

@@ -1,169 +0,0 @@
-/**
- * Cost model helpers for query optimization
- * Provides consistent cost estimation formulas across the optimizer
- */
-
-import { quereusError } from "../../common/errors.js";
-import { StatusCode } from "../../common/types.js";
-
-/**
- * Basic cost constants (in arbitrary "virtual CPU units")
- */
-export const COST_CONSTANTS = {
-	/** Cost per row for sequential scan */
-	SEQ_SCAN_PER_ROW: 1.0,
-	/** Base cost for starting a sequential scan */
-	SEQ_SCAN_BASE: 0.1,
-
-	/** Cost per row for index seek */
-	INDEX_SEEK_PER_ROW: 0.3,
-	/** Base cost for index seek operation */
-	INDEX_SEEK_BASE: 0.5,
-
-	/** Cost per row for index scan */
-	INDEX_SCAN_PER_ROW: 0.5,
-	/** Base cost for index scan operation */
-	INDEX_SCAN_BASE: 0.3,
-
-	/** Cost per row for sorting */
-	SORT_PER_ROW: 2.0,
-	/** Log base for sort cost calculations */
-	SORT_LOG_BASE: 2,
-
-	/** Cost per row for filtering */
-	FILTER_PER_ROW: 0.2,
-
-	/** Cost per row for projection */
-	PROJECT_PER_ROW: 0.1,
-
-	/** Cost per output row for aggregation */
-	AGGREGATE_PER_GROUP: 1.5,
-	/** Cost per input row for aggregation */
-	AGGREGATE_PER_INPUT_ROW: 0.3,
-
-	/** Cost per row for nested loop join (inner side) */
-	NL_JOIN_PER_INNER_ROW: 0.1,
-	/** Cost per row for nested loop join (outer side) */
-	NL_JOIN_PER_OUTER_ROW: 1.0,
-
-	/** Cost per row for hash join build phase */
-	HASH_JOIN_BUILD_PER_ROW: 0.8,
-	/** Cost per row for hash join probe phase */
-	HASH_JOIN_PROBE_PER_ROW: 0.4,
-
-	/** Cost per row for distinct operation */
-	DISTINCT_PER_ROW: 1.2,
-
-	/** Cost per row for limit operation */
-	LIMIT_PER_ROW: 0.05,
-
-	/** Cost per row for cache access */
-	CACHE_ACCESS_PER_ROW: 0.1,
-	/** Cost per row for cache population */
-	CACHE_POPULATE_PER_ROW: 0.2,
-} as const;
-
-/**
- * Calculate cost for sequential scan
- */
-export function seqScanCost(rows: number): number {
-	return COST_CONSTANTS.SEQ_SCAN_BASE + (rows * COST_CONSTANTS.SEQ_SCAN_PER_ROW);
-}
-
-/**
- * Calculate cost for index seek (point lookup or tight range)
- */
-export function indexSeekCost(rows: number): number {
-	return COST_CONSTANTS.INDEX_SEEK_BASE + (rows * COST_CONSTANTS.INDEX_SEEK_PER_ROW);
-}
-
-/**
- * Calculate cost for index scan (range scan with ordering)
- */
-export function indexScanCost(rows: number): number {
-	return COST_CONSTANTS.INDEX_SCAN_BASE + (rows * COST_CONSTANTS.INDEX_SCAN_PER_ROW);
-}
-
-/**
- * Calculate cost for sorting operation
- * Uses O(n log n) complexity
- */
-export function sortCost(rows: number): number {
-	if (rows <= 1) return COST_CONSTANTS.SORT_PER_ROW;
-	return rows * Math.log2(rows) * COST_CONSTANTS.SORT_PER_ROW;
-}
-
-/**
- * Calculate cost for filter operation
- */
-export function filterCost(inputRows: number): number {
-	return inputRows * COST_CONSTANTS.FILTER_PER_ROW;
-}
-
-/**
- * Calculate cost for projection operation
- */
-export function projectCost(rows: number, projectionCount: number = 1): number {
-	return rows * projectionCount * COST_CONSTANTS.PROJECT_PER_ROW;
-}
-
-/**
- * Calculate cost for aggregation operation
- */
-export function aggregateCost(inputRows: number, outputRows: number): number {
-	return (inputRows * COST_CONSTANTS.AGGREGATE_PER_INPUT_ROW) +
-		   (outputRows * COST_CONSTANTS.AGGREGATE_PER_GROUP);
-}
-
-/**
- * Calculate cost for nested loop join
- */
-export function nestedLoopJoinCost(outerRows: number, innerRows: number): number {
-	return (outerRows * COST_CONSTANTS.NL_JOIN_PER_OUTER_ROW) +
-		   (outerRows * innerRows * COST_CONSTANTS.NL_JOIN_PER_INNER_ROW);
-}
-
-/**
- * Calculate cost for hash join
- */
-export function hashJoinCost(buildRows: number, probeRows: number): number {
-	return (buildRows * COST_CONSTANTS.HASH_JOIN_BUILD_PER_ROW) +
-		   (probeRows * COST_CONSTANTS.HASH_JOIN_PROBE_PER_ROW);
-}
-
-/**
- * Calculate cost for distinct operation
- */
-export function distinctCost(rows: number): number {
-	// Distinct typically involves sorting or hashing
-	return rows * COST_CONSTANTS.DISTINCT_PER_ROW;
-}
-
-/**
- * Calculate cost for limit operation
- */
-export function limitCost(inputRows: number, limitValue: number): number {
-	const processedRows = Math.min(inputRows, limitValue);
-	return processedRows * COST_CONSTANTS.LIMIT_PER_ROW;
-}
-
-/**
- * Calculate cost for cache operations
- */
-export function cacheCost(rows: number, accessCount: number = 1): number {
-	const populateCost = rows * COST_CONSTANTS.CACHE_POPULATE_PER_ROW;
-	const accessCost = rows * accessCount * COST_CONSTANTS.CACHE_ACCESS_PER_ROW;
-	return populateCost + accessCost;
-}
-
-/**
- * Helper to choose the minimum cost option
- */
-export function chooseCheapest<T>(options: Array<{ cost: number; option: T }>): T {
-	if (options.length === 0) {
-		quereusError('No options provided to chooseCheapest', StatusCode.INTERNAL);
-	}
-	return options.reduce((min, current) =>
-		current.cost < min.cost ? current : min
-	).option;
-}

package/src/planner/debug/logger-utils.ts

@@ -1,68 +0,0 @@
-/**
- * Debug namespace utilities for the Titan optimizer
- * Provides standardized logger creation and debug conventions
- */
-
-import { createLogger } from '../../common/logger.js';
-
-/**
- * Create a logger for optimizer rules
- * @param ruleName The name of the optimization rule
- * @returns A debug logger with the standard optimizer:rule namespace
- */
-export function ruleLog(ruleName: string) {
-	return createLogger(`optimizer:rule:${ruleName}`);
-}
-
-/**
- * Create a logger for optimizer validation
- * @param component The validation component (e.g., 'plan', 'attributes', 'ordering')
- * @returns A debug logger with the standard optimizer:validate namespace
- */
-export function validateLog(component: string = '') {
-	const namespace = component ? `optimizer:validate:${component}` : 'optimizer:validate';
-	return createLogger(namespace);
-}
-
-/**
- * Create a logger for runtime statistics
- * @param component The stats component (e.g., 'metrics', 'summary', 'cache')
- * @returns A debug logger with the standard runtime:stats namespace
- */
-export function statsLog(component: string = '') {
-	const namespace = component ? `runtime:stats:${component}` : 'runtime:stats';
-	return createLogger(namespace);
-}
-
-/**
- * Create a logger for constant folding operations
- * @param component The folding component (e.g., 'evaluation', 'binary-ops', 'functions')
- * @returns A debug logger with the standard optimizer:folding namespace
- */
-export function foldingLog(component: string = '') {
-	const namespace = component ? `optimizer:folding:${component}` : 'optimizer:folding';
-	return createLogger(namespace);
-}
-
-/**
- * Create a logger for plan emission
- * @param emitterName The name of the emitter
- * @returns A debug logger with the standard runtime:emit namespace
- */
-export function emitLog(emitterName: string) {
-	return createLogger(`runtime:emit:${emitterName}`);
-}
-
-
-
-
-
-/**
- * Examples of debug namespace usage
- *
- * Enable all optimizer rules: DEBUG=optimizer:rule:* yarn test
- * Enable plan validation: DEBUG=optimizer:validate yarn test
- * Enable runtime stats: DEBUG=runtime:stats yarn test
- * Enable specific rule: DEBUG=optimizer:rule:aggregate-streaming yarn test
- * Enable everything: DEBUG=* yarn test
- */