@quereus/quereus 0.1.0

package/src/planner/debug.ts

@@ -0,0 +1,316 @@
+import type { PlanNode } from './nodes/plan-node.js';
+import { safeJsonStringify } from '../util/serialization.js';
+import { astToString } from '../util/ast-stringify.js';
+import type { Instruction, InstructionTracer } from '../runtime/types.js';
+import type * as AST from '../parser/ast.js';
+import { quereusError } from '../common/errors.js';
+import { StatusCode } from '../common/types.js';
+
+/**
+ * Detailed information about a PlanNode for debugging purposes.
+ */
+export interface PlanNodeDebugInfo {
+  id: string;
+  nodeType: string;
+  type: any; // The result of getType()
+  estimatedCost: number;
+  estimatedRows?: number;
+  totalCost: number;
+  children: PlanNodeDebugInfo[];
+  relations: PlanNodeDebugInfo[];
+  properties: Record<string, any>; // Node-specific properties
+}
+
+/**
+ * Information about an instruction in the execution program.
+ */
+export interface InstructionDebugInfo {
+  index: number;
+  note?: string;
+  paramCount: number;
+  paramIndices: number[];
+  destination: number | null;
+  subPrograms?: SubProgramDebugInfo[];
+}
+
+/**
+ * Information about a sub-program for debugging purposes.
+ */
+export interface SubProgramDebugInfo {
+  programIndex: number;
+  instructionCount: number;
+  rootNote?: string;
+  instructions: InstructionDebugInfo[];
+}
+
+/**
+ * Checks if a value is an AST node
+ */
+function isAstNode(value: any): value is AST.AstNode {
+	return value && typeof value === 'object' && 'type' in value && typeof value.type === 'string';
+}
+
+/**
+ * Recursively processes a value, converting AST nodes to SQL strings
+ */
+function processValue(value: any): any {
+	if (value === null || value === undefined) {
+		return value;
+	}
+
+	// If it's an AST node, convert to SQL string
+	if (isAstNode(value)) {
+		try {
+			return astToString(value);
+		} catch {
+			return `[AST:${value.type}]`; // Fallback if stringify fails
+		}
+	}
+
+	// If it's an array, process each element
+	if (Array.isArray(value)) {
+		return value.map(processValue);
+	}
+
+	// If it's an object, process each property
+	if (typeof value === 'object') {
+		// Skip circular references and complex objects
+		if (value.constructor !== Object && value.constructor !== Array) {
+			return '[COMPLEX_OBJECT]';
+		}
+
+		const processed: Record<string, any> = {};
+		for (const [key, val] of Object.entries(value)) {
+			try {
+				processed[key] = processValue(val);
+			} catch {
+				processed[key] = '[UNPROCESSABLE]';
+			}
+		}
+		return processed;
+	}
+
+	// For primitives, return as-is
+	return value;
+}
+
+/**
+ * Serializes a PlanNode tree to a detailed JSON representation using the existing visit pattern.
+ */
+export function serializePlanTree(rootNode: PlanNode): string {
+	const nodeMap = new Map<PlanNode, PlanNodeDebugInfo>();
+
+	// First pass: collect all nodes using the visit pattern
+	rootNode.visit((node) => {
+		if (!nodeMap.has(node)) {
+			// Get node-specific properties by examining the node object
+			const properties: Record<string, any> = {};
+
+			// Extract interesting properties from the node (excluding functions and circular refs)
+			for (const [key, value] of Object.entries(node)) {
+				if (key === 'scope' || key === 'id' || key === 'nodeType' || key === 'estimatedCost') {
+					continue; // Skip these as they're handled separately
+				}
+
+				if (typeof value === 'function') {
+					continue; // Skip functions
+				}
+
+				if (value && typeof value === 'object' && 'nodeType' in value) {
+					// This is likely another PlanNode, skip to avoid duplication
+					continue;
+				}
+
+				try {
+					// Process the property value, converting AST nodes to SQL strings
+					properties[key] = processValue(value);
+				} catch {
+					properties[key] = '[UNSERIALIZABLE]';
+				}
+			}
+
+			nodeMap.set(node, {
+				id: node.id,
+				nodeType: node.nodeType,
+				type: node.getType(),
+				estimatedCost: node.estimatedCost,
+				estimatedRows: (node as any).estimatedRows,
+				totalCost: node.getTotalCost(),
+				children: [], // Will be filled in second pass
+				relations: [], // Will be filled in second pass
+				properties
+			});
+		}
+	});
+
+	// Second pass: establish relationships
+	for (const [node, info] of nodeMap) {
+		info.children = node.getChildren()
+			.map(child => nodeMap.get(child))
+			.filter(Boolean) as PlanNodeDebugInfo[];
+
+		info.relations = node.getRelations()
+			.map(relation => nodeMap.get(relation))
+			.filter(Boolean) as PlanNodeDebugInfo[];
+	}
+
+	const rootInfo = nodeMap.get(rootNode);
+	if (!rootInfo) {
+		quereusError('Root node not found in serialization map', StatusCode.INTERNAL);
+	}
+
+	return safeJsonStringify(rootInfo, 2);
+}
+
+/**
+ * Generates a human-readable program listing of instructions.
+ */
+export function generateInstructionProgram(
+  instructions: readonly Instruction[],
+  destinations: readonly (number | null)[]
+): string {
+  const lines: string[] = [];
+  lines.push('=== INSTRUCTION PROGRAM ===');
+  lines.push('');
+
+  const subProgramMap = new Map<number, { scheduler: any; parentIndex: number }>();
+  let nextSubProgramId = 0;
+
+  for (let i = 0; i < instructions.length; i++) {
+    const instruction = instructions[i];
+    const dest = destinations[i];
+    const note = instruction.note ? ` ; ${instruction.note}` : '';
+    const destStr = dest !== null ? ` -> [${dest}]` : ' -> [RESULT]';
+
+    let subProgramInfo = '';
+    if (instruction.programs && instruction.programs.length > 0) {
+      const programIds: number[] = [];
+      for (const program of instruction.programs) {
+        const programId = nextSubProgramId++;
+        subProgramMap.set(programId, { scheduler: program, parentIndex: i });
+        programIds.push(programId);
+      }
+      subProgramInfo = ` SUB-PROGRAMS: [${programIds.join(', ')}]`;
+    }
+
+    lines.push(`[${i.toString().padStart(3)}] PARAMS: [${instruction.params.map((_, idx) =>
+      instructions.findIndex(inst => inst === instruction.params[idx])
+    ).join(', ')}]${destStr}${note}${subProgramInfo}`);
+  }
+
+  // Add sub-program listings
+  if (subProgramMap.size > 0) {
+    lines.push('');
+    lines.push('=== SUB-PROGRAMS ===');
+
+    for (const [programId, { scheduler, parentIndex }] of subProgramMap) {
+      lines.push('');
+      lines.push(`--- SUB-PROGRAM ${programId} (called by instruction ${parentIndex}) ---`);
+      const subProgram = generateInstructionProgram(scheduler.instructions, scheduler.destinations);
+      // Remove the header and footer from sub-program and indent
+      const subLines = subProgram.split('\n').slice(2, -2);
+      for (const line of subLines) {
+        if (line.trim()) {
+          lines.push(`  ${line}`);
+        }
+      }
+    }
+  }
+
+  lines.push('');
+  lines.push('=== END PROGRAM ===');
+  return lines.join('\n');
+}
+
+/**
+ * Extracts detailed information about the instruction program structure.
+ */
+export function getInstructionDebugInfo(
+  instructions: readonly Instruction[],
+  destinations: readonly (number | null)[]
+): InstructionDebugInfo[] {
+  let nextSubProgramId = 0;
+
+  return instructions.map((instruction, index) => {
+    let subPrograms: SubProgramDebugInfo[] | undefined;
+
+    if (instruction.programs && instruction.programs.length > 0) {
+      subPrograms = instruction.programs.map(scheduler => {
+        const programId = nextSubProgramId++;
+        const subInstructions = getInstructionDebugInfo(scheduler.instructions, scheduler.destinations);
+
+        return {
+          programIndex: programId,
+          instructionCount: scheduler.instructions.length,
+          rootNote: scheduler.instructions[scheduler.instructions.length - 1]?.note,
+          instructions: subInstructions
+        };
+      });
+    }
+
+    return {
+      index,
+      note: instruction.note,
+      paramCount: instruction.params.length,
+      paramIndices: instruction.params.map(param =>
+        instructions.findIndex(inst => inst === param)
+      ),
+      destination: destinations[index],
+      subPrograms
+    };
+  });
+}
+
+/**
+ * Generates a comprehensive trace report that includes sub-program execution details.
+ */
+export function generateTraceReport(
+  tracer: InstructionTracer
+): string {
+  const events = tracer.getTraceEvents?.() || [];
+  const subPrograms = tracer.getSubPrograms?.() || new Map();
+
+  const lines: string[] = [];
+  lines.push('=== EXECUTION TRACE ===');
+  lines.push('');
+
+  for (const event of events) {
+    const timestamp = new Date(event.timestamp).toISOString();
+    const typeStr = event.type.toUpperCase().padEnd(6);
+    const note = event.note ? ` (${event.note})` : '';
+
+    lines.push(`[${event.instructionIndex.toString().padStart(3)}] ${typeStr} ${timestamp}${note}`);
+
+    if (event.type === 'input' && event.subPrograms) {
+      for (const subProgram of event.subPrograms) {
+        lines.push(`     └─ SUB-PROGRAM ${subProgram.programIndex}: ${subProgram.instructionCount} instructions${subProgram.rootNote ? ` (${subProgram.rootNote})` : ''}`);
+      }
+    }
+
+    if (event.type === 'error') {
+      lines.push(`     ERROR: ${event.error}`);
+    }
+  }
+
+  if (subPrograms.size > 0) {
+    lines.push('');
+    lines.push('=== SUB-PROGRAM DETAILS ===');
+
+    for (const [programId, { scheduler, parentInstructionIndex }] of subPrograms) {
+      lines.push('');
+      lines.push(`--- SUB-PROGRAM ${programId} (parent instruction: ${parentInstructionIndex}) ---`);
+      const programListing = generateInstructionProgram(scheduler.instructions, scheduler.destinations);
+      // Remove header/footer and indent
+      const programLines = programListing.split('\n').slice(2, -2);
+      for (const line of programLines) {
+        if (line.trim()) {
+          lines.push(`  ${line}`);
+        }
+      }
+    }
+  }
+
+  lines.push('');
+  lines.push('=== END TRACE ===');
+  return lines.join('\n');
+}

package/src/planner/framework/README.md

@@ -0,0 +1,132 @@
+# Titan Optimizer Framework (Phase 1)
+
+This directory contains the core framework components for the Titan optimizer Phase 1 implementation.
+
+## Components
+
+### Registry (`registry.ts`)
+- **RuleHandle**: Structured rule registration with ID, node type, phase, and priority
+- **Loop Detection**: Prevents infinite rule application using visited rule tracking
+- **Priority Ordering**: Rules execute in priority order (lower numbers first)
+- **Comprehensive Tracing**: Detailed logging and trace hooks for debugging
+
+### Trace Framework (`trace.ts`)
+- **TraceHook Interface**: Extensible hooks for rule and node processing events
+- **DebugTraceHook**: Logs all optimizer activity to debug channels
+- **PerformanceTraceHook**: Measures and logs rule execution times
+- **CompositeTraceHook**: Combines multiple trace hooks
+- **Environment Integration**: Automatically enables tracing based on DEBUG environment
+
+### Context (`context.ts`)
+- **OptContext**: Unified interface combining optimizer, stats provider, and tuning
+- **Depth Tracking**: Prevents infinite recursion in optimization rules
+- **Phase Management**: Supports 'rewrite' (logical→logical) and 'impl' (logical→physical) phases
+- **Context Data**: Key-value store for rule communication and state
+
+### Physical Utilities (`physical-utils.ts`)
+- **Property Inference**: Utilities for combining and propagating physical properties
+- **Ordering Operations**: Functions for merging and inferring result orderings
+- **Unique Key Handling**: Logic for combining unique keys across joins and projections
+- **Cost Estimation**: Helpers for consistent cost model application
+
+## Architecture Integration
+
+### Rule Registration
+```typescript
+import { registerRule, createRule } from '../framework/registry.js';
+
+registerRule(createRule(
+  'Aggregate→StreamAggregate',
+  PlanNodeType.Aggregate,
+  'impl',
+  ruleAggregateStreaming,
+  10 // priority
+));
+```
+
+### Rule Implementation
+```typescript
+import type { RuleFn } from '../framework/registry.js';
+
+const ruleAggregateStreaming: RuleFn = (node, optimizer) => {
+  if (!(node instanceof AggregateNode)) return null;
+  
+  const context = optimizer.getContext();
+  const stats = optimizer.getStats();
+  
+  // Rule logic here...
+  return transformedNode;
+};
+```
+
+### Tracing Integration
+Tracing is automatically enabled based on environment variables:
+- `DEBUG=quereus:optimizer*` - enables debug tracing
+- `QUEREUS_OPTIMIZER_PERF=true` - enables performance tracing
+
+### Statistics Provider
+```typescript
+// Use built-in providers
+import { defaultStatsProvider, vtabStatsProvider } from '../stats/index.js';
+
+// Or create custom provider
+const customStats = createStatsProvider(
+  new Map([['users', 50000]]), // table row counts
+  new Map([['users:BinaryOp', 0.1]]) // predicate selectivity
+);
+```
+
+## Phase 1 Status
+
+✅ **Implemented Components:**
+- Rule registration and management framework
+- Comprehensive trace framework with multiple hook types
+- Optimizer context with statistics provider integration
+- Physical property utilities
+- Golden plan test harness
+- Environment-based configuration
+
+✅ **Integration Complete:**
+- Optimizer updated to use new framework
+- Emitter metadata support added
+- Trace hooks integrated into rule application
+- Statistics provider abstraction ready
+
+🔄 **Next Steps (Phase 2):**
+- Advanced optimization rules implementation
+- Seek/range scan access path selection  
+- Materialization advisory framework
+- Plan validation pass
+
+## Usage Examples
+
+### Enabling Tracing
+```bash
+# Enable all optimizer tracing
+DEBUG=quereus:optimizer* yarn test
+
+# Enable performance measurement
+QUEREUS_OPTIMIZER_PERF=true DEBUG=quereus:optimizer* yarn test
+
+# Trace specific rule categories
+DEBUG=quereus:optimizer:rule:* yarn test
+```
+
+### Running Golden Plan Tests
+```bash
+# Run all golden plan tests
+yarn test:plans
+
+# Update golden files when plans change
+UPDATE_PLANS=true yarn test:plans
+```
+
+### Custom Statistics Provider
+```typescript
+const optimizer = new Optimizer(
+  DEFAULT_TUNING,
+  new VTabStatsProvider() // Uses VTab module statistics when available
+);
+```
+
+This framework provides the foundation for sophisticated query optimization while maintaining clear separation of concerns and comprehensive debugging capabilities. 

package/src/planner/framework/context.ts

@@ -0,0 +1,188 @@
+/**
+ * Optimizer context that wraps Optimizer with StatsProvider and other utilities
+ * Provides unified interface for optimization rules
+ */
+
+import type { Optimizer } from '../optimizer.js';
+import type { StatsProvider } from '../stats/index.js';
+import type { OptimizerTuning } from '../optimizer-tuning.js';
+import { createLogger } from '../../common/logger.js';
+import { StatusCode } from '../../common/types.js';
+import { quereusError } from '../../common/errors.js';
+import { Database } from '../../core/database.js';
+
+const log = createLogger('optimizer:framework:context');
+
+/**
+ * Context object passed to optimization rules
+ * Contains all the utilities and data sources rules need
+ */
+export interface OptContext {
+	/** The optimizer instance */
+	readonly optimizer: Optimizer;
+
+	/** Statistics provider for cardinality and selectivity estimates */
+	readonly stats: StatsProvider;
+
+	/** Optimizer tuning parameters */
+	readonly tuning: OptimizerTuning;
+
+	/** Current optimization phase */
+	readonly phase: 'rewrite' | 'impl';
+
+	/** Rule application depth (for detecting infinite recursion) */
+	readonly depth: number;
+
+	/** Additional context data that rules can use */
+	readonly context: Map<string, any>;
+
+	/** Database instance */
+	readonly db: Database;
+}
+
+/**
+ * Implementation of optimization context
+ */
+export class OptimizationContext implements OptContext {
+	readonly context = new Map<string, any>();
+
+	constructor(
+		public readonly optimizer: Optimizer,
+		public readonly stats: StatsProvider,
+		public readonly tuning: OptimizerTuning,
+		public readonly phase: 'rewrite' | 'impl' = 'rewrite',
+		public readonly db: Database,
+		public readonly depth: number = 0,
+	) {
+		log('Created optimization context (phase: %s, depth: %d)', phase, depth);
+	}
+
+	/**
+	 * Create a new context for a different phase
+	 */
+	withPhase(phase: 'rewrite' | 'impl'): OptimizationContext {
+		return new OptimizationContext(
+			this.optimizer,
+			this.stats,
+			this.tuning,
+			phase,
+			this.db,
+			this.depth,
+		);
+	}
+
+	/**
+	 * Create a new context with incremented depth
+	 */
+	withIncrementedDepth(): OptimizationContext {
+		if (this.depth >= this.tuning.maxOptimizationDepth) {
+			quereusError(`Maximum optimization depth exceeded: ${this.depth}`, StatusCode.ERROR);
+		}
+
+		return new OptimizationContext(
+			this.optimizer,
+			this.stats,
+			this.tuning,
+			this.phase,
+			this.db,
+			this.depth + 1,
+		);
+	}
+
+	/**
+	 * Create a new context with additional context data
+	 */
+	withContext(key: string, value: any): OptimizationContext {
+		const newContext = new OptimizationContext(
+			this.optimizer,
+			this.stats,
+			this.tuning,
+			this.phase,
+			this.db,
+			this.depth,
+		);
+
+		// Copy existing context
+		for (const [k, v] of this.context) {
+			newContext.context.set(k, v);
+		}
+
+		// Add new context
+		newContext.context.set(key, value);
+
+		return newContext;
+	}
+
+	/**
+	 * Get context value
+	 */
+	getContext<T>(key: string): T | undefined {
+		return this.context.get(key) as T | undefined;
+	}
+
+	/**
+	 * Check if context has a key
+	 */
+	hasContext(key: string): boolean {
+		return this.context.has(key);
+	}
+
+	/**
+	 * Set context value (mutates current context)
+	 */
+	setContext(key: string, value: any): void {
+		this.context.set(key, value);
+	}
+
+	/**
+	 * Remove context value (mutates current context)
+	 */
+	deleteContext(key: string): boolean {
+		return this.context.delete(key);
+	}
+
+	/**
+	 * Clear all context data (mutates current context)
+	 */
+	clearContext(): void {
+		this.context.clear();
+	}
+
+	/**
+	 * Get a snapshot of all context data
+	 */
+	getContextSnapshot(): Record<string, any> {
+		const snapshot: Record<string, any> = {};
+		for (const [key, value] of this.context) {
+			snapshot[key] = value;
+		}
+		return snapshot;
+	}
+}
+
+/**
+ * Factory function to create optimization context
+ */
+export function createOptContext(
+	optimizer: Optimizer,
+	stats: StatsProvider,
+	tuning: OptimizerTuning,
+	db: Database,
+	phase: 'rewrite' | 'impl' = 'rewrite',
+): OptContext {
+	return new OptimizationContext(optimizer, stats, tuning, phase, db);
+}
+
+/**
+ * Type guard to check if an object is an OptContext
+ */
+export function isOptContext(obj: any): obj is OptContext {
+	return obj &&
+		typeof obj === 'object' &&
+		'optimizer' in obj &&
+		'stats' in obj &&
+		'tuning' in obj &&
+		'phase' in obj &&
+		'depth' in obj &&
+		'context' in obj;
+}