@quereus/quereus 0.1.0

package/src/planner/rules/cache/rule-materialization-advisory.ts

@@ -0,0 +1,66 @@
+/**
+ * Rule: Materialization Advisory
+ *
+ * Transforms: Any relational node → CacheNode (when beneficial)
+ * Conditions: Node would benefit from caching based on reference analysis
+ * Benefits: Reduces redundant computation for repeated scans and loop contexts
+ */
+
+import { createLogger } from '../../../common/logger.js';
+import type { PlanNode } from '../../nodes/plan-node.js';
+import type { OptContext } from '../../framework/context.js';
+import { MaterializationAdvisory } from '../../cache/materialization-advisory.js';
+
+const log = createLogger('optimizer:rule:materialization-advisory');
+
+export function ruleMaterializationAdvisory(node: PlanNode, context: OptContext): PlanNode | null {
+	// This is a global rule that analyzes the entire tree
+	// It should only be applied to root nodes to avoid duplicate analysis
+
+	// Simple heuristic: only apply to certain root-level node types
+	// to avoid analyzing the same tree multiple times
+	const rootNodeTypes = new Set([
+		'Select', 'Insert', 'Update', 'Delete', 'Block', 'CTE'
+	]);
+
+	if (!rootNodeTypes.has(node.nodeType)) {
+		return null;
+	}
+
+	log('Applying materialization advisory to tree rooted at %s', node.nodeType);
+
+	try {
+		// Create advisory with current tuning parameters
+		const advisory = new MaterializationAdvisory(context.tuning);
+
+		// Analyze the tree for caching opportunities
+		const recommendations = advisory.analyzeTree(node);
+
+		// Count how many cache recommendations were made
+		const cacheCount = Array.from(recommendations.values())
+			.filter(rec => rec.shouldCache).length;
+
+		if (cacheCount === 0) {
+			log('No caching opportunities identified');
+			return null;
+		}
+
+		log('Found %d caching opportunities', cacheCount);
+
+		// Apply the caching recommendations
+		const cachedTree = advisory.applyCaching(node, recommendations);
+
+		// If the tree was modified, return the new tree
+		if (cachedTree !== node) {
+			log('Applied materialization advisory, tree modified');
+			return cachedTree;
+		}
+
+		return null;
+
+	} catch (error) {
+		log('Error in materialization advisory: %s', error);
+		// Don't fail optimization - just skip caching
+		return null;
+	}
+}

package/src/planner/rules/cache/rule-mutating-subquery-cache.ts

@@ -0,0 +1,110 @@
+/**
+ * Rule: Mutating Subquery Cache Injection
+ *
+ * Transforms: JoinNode with mutating right side → JoinNode with cached right side
+ * Conditions: Right side contains operations with readonly=false physical property
+ * Benefits: Prevents mutating subqueries from being executed multiple times in nested loop joins
+ */
+
+import { createLogger } from '../../../common/logger.js';
+import { PlanNode } from '../../nodes/plan-node.js';
+import type { OptContext } from '../../framework/context.js';
+import { JoinNode } from '../../nodes/join-node.js';
+import { CacheNode } from '../../nodes/cache-node.js';
+import { PlanNodeType } from '../../nodes/plan-node-type.js';
+
+const log = createLogger('optimizer:rule:mutating-subquery-cache');
+
+export function ruleMutatingSubqueryCache(node: PlanNode, context: OptContext): PlanNode | null {
+	// Guard: only apply to JoinNode
+	if (!(node instanceof JoinNode)) {
+		return null;
+	}
+
+	log('Checking JoinNode for operations with side effects on right side');
+
+	// Check if the right side contains operations with side effects
+	const rightSide = node.right;
+	const hasSideEffects = containsOperationsWithSideEffects(rightSide);
+
+	if (!hasSideEffects) {
+		log('Right side does not contain operations with side effects, skipping');
+		return null;
+	}
+
+	// Check if right side is already cached
+	if (rightSide.nodeType === PlanNodeType.Cache) {
+		log('Right side is already cached, skipping');
+		return null;
+	}
+
+	log('Detected operations with side effects on right side of join, injecting cache');
+
+	// Calculate appropriate cache threshold
+	const estimatedRows = rightSide.estimatedRows ?? context.tuning.defaultRowEstimate;
+	const threshold = Math.min(
+		Math.max(estimatedRows * context.tuning.join.cacheThresholdMultiplier, 1000),
+		context.tuning.join.maxCacheThreshold
+	);
+
+	// Wrap the right side with a cache node
+	const cachedRightSide = new CacheNode(
+		rightSide.scope,
+		rightSide,
+		'memory',
+		threshold
+	);
+
+	// Create new join node with cached right side
+	const result = new JoinNode(
+		node.scope,
+		node.left, // Left side unchanged
+		cachedRightSide, // Right side now cached
+		node.joinType,
+		node.condition,
+		node.usingColumns
+	);
+
+	log('Successfully injected cache for operations with side effects (threshold: %d)', threshold);
+	return result;
+}
+
+/**
+ * Recursively check if a plan tree contains operations with side effects
+ */
+function containsOperationsWithSideEffects(node: PlanNode): boolean {
+	// Check if this node has side effects using physical properties
+	if (hasNodeSideEffects(node)) {
+		return true;
+	}
+
+	// Recursively check children
+	for (const child of node.getChildren()) {
+		if (containsOperationsWithSideEffects(child)) {
+			return true;
+		}
+	}
+
+	// Check relational children if available
+	if ('getRelations' in node) {
+		for (const relation of (node as any).getRelations()) {
+			if (containsOperationsWithSideEffects(relation)) {
+				return true;
+			}
+		}
+	}
+
+	return false;
+}
+
+/**
+ * Check if a node has side effects using physical properties
+ */
+function hasNodeSideEffects(node: PlanNode): boolean {
+	// By the time rules run, all children should have physical properties set by the framework
+	if (!node.physical) {
+		throw new Error(`Internal error: Node ${node.nodeType}:${node.id} missing physical properties during rule application`);
+	}
+
+	return PlanNode.hasSideEffects(node.physical);
+}

package/src/planner/rules/physical/rule-mark-physical.ts

@@ -0,0 +1,37 @@
+/**
+ * Rule: Mark Physical (Fallback)
+ *
+ * Transforms: Any logical node → Same node (marked as physical)
+ * Conditions: When no other rule has made the node physical
+ * Benefits: Ensures all nodes become physical even if no specific transformation is needed
+ */
+
+import { createLogger } from '../../../common/logger.js';
+import type { PlanNode } from '../../nodes/plan-node.js';
+import type { OptContext } from '../../framework/context.js';
+
+const log = createLogger('optimizer:rule:mark-physical');
+
+export function ruleMarkPhysical(node: PlanNode, _context: OptContext): PlanNode | null {
+	// This is a fallback rule - only apply if the node is not already physical
+	if (node.physical) {
+		return null; // Already physical
+	}
+
+	// Some node types cannot be made physical directly (like AggregateNode)
+	// These should have been handled by specific transformation rules
+	const cannotBePhysical = new Set([
+		'Aggregate', // Should be transformed to StreamAggregate/HashAggregate
+	]);
+
+	if (cannotBePhysical.has(node.nodeType)) {
+		log('Cannot mark %s as physical - requires transformation', node.nodeType);
+		return null; // Let the optimizer handle this as an error
+	}
+
+	log('Marking %s as physical (fallback rule)', node.nodeType);
+
+	// Return the node - framework will set physical properties via markPhysical()
+	// Don't hardcode properties, let the framework compute them from children
+	return node;
+}

package/src/planner/scopes/aliased.ts

@@ -0,0 +1,50 @@
+import type { PlanNode } from "../nodes/plan-node.js";
+import type { Scope } from "./scope.js";
+import { Ambiguous } from "./scope.js";
+import * as AST from "../../parser/ast.js";
+import { BaseScope } from "./base.js";
+
+/**
+ * A Scope that aliases a parent scope.
+ *
+ * @param parent The parent scope, assumed to already be populated with symbols..
+ */
+export class AliasedScope extends BaseScope {
+	private readonly _parentName: string;
+	private readonly _alias: string;
+
+	constructor(
+		public readonly parent: Scope,
+		parentName: string,
+		alias: string
+	) {
+		super();
+		this._parentName = parentName.toLowerCase();
+		this._alias = alias.toLowerCase();
+	}
+
+	resolveSymbol(symbolKey: string, expression: AST.Expression): PlanNode | typeof Ambiguous | undefined {
+		const parts = symbolKey.split('.');
+
+		// Handle schema-qualified symbols like "main.j.value"
+		if (parts.length === 3 && parts[1].toLowerCase() === this._alias) {
+			// Replace alias with parent name: "main.j.value" -> "main..value" -> "main.value" (if parent name is empty)
+			if (this._parentName === '') {
+				// For table-valued functions, remove the alias entirely: "main.j.value" -> "value"
+				return this.parent.resolveSymbol(parts[2], expression);
+			} else {
+				parts[1] = this._parentName;
+				return this.parent.resolveSymbol(parts.join('.'), expression);
+			}
+		}
+		// Handle qualified symbols like "j.value"
+		else if (parts.length === 2 && parts[0].toLowerCase() === this._alias) {
+			// For both cases (table-valued functions and regular tables),
+			// just use the column name since the parent scope knows what table it represents
+			return this.parent.resolveSymbol(parts[1], expression);
+		}
+
+		// For all other symbols (including unqualified columns), delegate to parent
+		return this.parent.resolveSymbol(symbolKey, expression);
+	}
+}

package/src/planner/scopes/base.ts

@@ -0,0 +1,27 @@
+import type { PlanNode } from '../nodes/plan-node.js';
+import * as AST from '../../parser/ast.js';
+import { type Scope, Ambiguous } from './scope.js';
+
+/**
+ * Scope that tracks references.
+ */
+export abstract class BaseScope implements Scope {
+	/** References that have been resolved through this scope. */
+	private _references: PlanNode[] = [];
+
+	abstract resolveSymbol(symbolKey: string, expression: AST.Expression): PlanNode | typeof Ambiguous | undefined;
+
+	/**
+	 * Returns all references that have been resolved through this scope.
+	 * This includes references from both this scope and any parent scopes.
+	 *
+	 * @returns An array of all resolved references.
+	 */
+	getReferences(): readonly PlanNode[] {
+		return this._references;
+	}
+
+	addReference(reference: PlanNode): void {
+		this._references.push(reference);
+	}
+}

package/src/planner/scopes/empty.ts

@@ -0,0 +1,22 @@
+import { QuereusError } from '../../common/errors.js';
+import { StatusCode } from '../../common/types.js';
+import * as AST from '../../parser/ast.js';
+import type { PlanNode } from '../nodes/plan-node.js';
+import { type Scope, Ambiguous } from './scope.js';
+
+/** Scope that contains no symbols.  */
+export class EmptyScope implements Scope {
+	resolveSymbol(_symbolKey: string, _expression: AST.Expression): PlanNode | typeof Ambiguous | undefined {
+		return undefined;
+	}
+
+	getReferences(): readonly PlanNode[] {
+		return [];
+	}
+
+	addReference(_reference: PlanNode): void {
+		throw new QuereusError('EmptyScope does not support adding references.', StatusCode.MISUSE);
+	}
+
+	static readonly instance = new EmptyScope();
+}

package/src/planner/scopes/global.ts

@@ -0,0 +1,73 @@
+import { BaseScope } from "./base.js";
+import type { SchemaManager } from "../../schema/manager.js";
+import type { PlanNode } from "../nodes/plan-node.js";
+import * as AST from "../../parser/ast.js";
+import { FunctionReferenceNode, TableReferenceNode } from "../nodes/reference.js";
+import { Ambiguous } from "./scope.js";
+import type { ScalarType } from "../../common/datatype.js";
+import { SqlDataType } from "../../common/types.js";
+import { isScalarFunctionSchema } from "../../schema/function.js";
+
+export class GlobalScope extends BaseScope {
+	constructor(public readonly manager: SchemaManager) {
+		super();
+	}
+
+	resolveSymbol(symbolKey: string, _expression: AST.Expression): PlanNode | typeof Ambiguous | undefined {
+		if (symbolKey.includes('/')) {// Function: [schema.]name/nArgs
+			const [name, nArgsStr] = symbolKey.split('/');
+			const nArgs = parseInt(nArgsStr);
+			const func = this.manager.findFunction(name, nArgs);
+			if (!func) {
+				return undefined;
+			}
+
+			// Get the proper scalar type from the function schema
+			const scalarType: ScalarType = isScalarFunctionSchema(func)
+				? func.returnType
+				: { typeClass: 'scalar', affinity: SqlDataType.NUMERIC, nullable: true, isReadOnly: true };
+
+			return new FunctionReferenceNode(this, func, scalarType);
+		}
+		// Table: [schema.]table
+		const [first, second] = symbolKey.split('.');
+		const schema = second ? first : undefined;
+		const table = second ? second : first;
+		const tableSchema = this.manager.findTable(table, schema);
+		if (!tableSchema) {
+			return undefined;
+		}
+		// Note: GlobalScope can't resolve vtab modules without a planning context
+		// This path is mainly used for constraint checking where we don't need full resolution
+		const vtabModule = this.manager.getModule(tableSchema.vtabModuleName);
+		if (!vtabModule) {
+			return undefined;
+		}
+		return new TableReferenceNode(this, tableSchema, vtabModule.module, vtabModule.auxData);
+	}
+
+	findUnqualifiedName(name: string): PlanNode | typeof Ambiguous | undefined {
+		// Functions have priority over tables.
+		// Check for zero-argument functions first
+		const func = this.manager.findFunction(name, 0);
+		if (func) {
+			// Get the proper scalar type from the function schema
+			const scalarType: ScalarType = isScalarFunctionSchema(func)
+				? func.returnType
+				: { typeClass: 'scalar', affinity: SqlDataType.NUMERIC, nullable: true, isReadOnly: true };
+
+			return new FunctionReferenceNode(this, func, scalarType);
+		}
+		// Table: [schema.]table
+		const table = this.manager.findTable(name);
+		if (table) {
+			// TODO: Create a proper ColumnScope to allow column references
+			const vtabModule = this.manager.getModule(table.vtabModuleName);
+			if (!vtabModule) {
+				return undefined;
+			}
+			return new TableReferenceNode(this, table, vtabModule.module, vtabModule.auxData);
+		}
+		return undefined;
+	}
+}

package/src/planner/scopes/multi.ts

@@ -0,0 +1,41 @@
+import { Ambiguous } from "./scope.js";
+import * as AST from "../../parser/ast.js";
+import { QuereusError } from "../../common/errors.js";
+import { StatusCode } from "../../common/types.js";
+import type { PlanNode } from "../nodes/plan-node.js";
+import type { Scope } from "./scope.js";
+import { BaseScope } from "./base.js";
+
+/**
+ * A Scope that contains multiple other scopes.
+ * Symbols are resolved in order, and the first match is used.
+ * If a symbol is found in multiple scopes, an error is thrown.
+ */
+export class MultiScope extends BaseScope {
+	constructor(
+		public readonly scopes: Scope[]
+	) {
+		super();
+	}
+
+	registerSymbol(_symbolKey: string, _getReference: (expression: AST.Expression, currentScope: Scope) => PlanNode): void {
+		throw new QuereusError('MultiScope does not support registering symbols.', StatusCode.ERROR);
+	}
+
+	resolveSymbol(symbolKey: string, expression: AST.Expression): PlanNode | typeof Ambiguous | undefined {
+		// In SQL, inner scopes shadow outer scopes for unqualified names
+		// Return the first match, don't treat multiple matches as ambiguous
+		for (const scope of this.scopes) {
+			const result = scope.resolveSymbol(symbolKey, expression);
+			if (result === Ambiguous) {
+				return Ambiguous;
+			}
+			if (result) {
+				this.addReference(result);
+				return result;
+			}
+		}
+		return undefined;
+	}
+}
+

package/src/planner/scopes/param.ts

@@ -0,0 +1,96 @@
+import type * as AST from '../../parser/ast.js';
+import { ParameterReferenceNode } from '../nodes/reference.js'; // Corrected import
+import { BaseScope } from './base.js';
+import { Ambiguous, type Scope } from './scope.js';
+import type { ScalarType } from '../../common/datatype.js';
+import { SqlDataType } from '../../common/types.js';
+import type { PlanNode } from '../nodes/plan-node.js';
+
+// Default type for parameters when not otherwise specified.
+const DEFAULT_PARAMETER_TYPE: ScalarType = {
+	typeClass: 'scalar',
+	affinity: SqlDataType.TEXT,
+	nullable: true,
+};
+
+/**
+ * A scope that resolves query parameters (e.g., :name, :1, ?).
+ * It makes these parameters available via an accessor.
+ */
+export class ParameterScope extends BaseScope {
+	private _nextAnonymousIndex: number = 1;
+	private readonly _parameters: Map<string | number, ParameterReferenceNode> = new Map();
+	private readonly _parameterTypeHints: ReadonlyMap<string | number, ScalarType>;
+
+	constructor(
+		public readonly parentScope: Scope,
+		parameterTypeHints?: ReadonlyMap<string | number, ScalarType>
+	) {
+		super();
+		this._parameterTypeHints = parameterTypeHints || new Map();
+	}
+
+	resolveSymbol(symbolKey: string, expression: AST.Expression): PlanNode | typeof Ambiguous | undefined {
+		let identifier: string | number;
+		let parameterNode: ParameterReferenceNode | undefined;
+
+		// The expression should be an AST.ParameterExpr when symbolKey indicates a parameter
+		const parameterExpression = expression as AST.ParameterExpr;
+		let resolvedType = DEFAULT_PARAMETER_TYPE;
+
+		if (symbolKey === '?') {
+			// Use the current _nextAnonymousIndex as the potential identifier for this '?'
+			const currentAnonymousId = this._nextAnonymousIndex;
+
+			// Check if this specific anonymous parameter (by its future index) has a type hint
+			if (this._parameterTypeHints.has(currentAnonymousId)) {
+				resolvedType = this._parameterTypeHints.get(currentAnonymousId)!;
+			}
+			// Note: We don't check _parameters here for '?' because each '?' AST node should resolve,
+			// potentially creating a new ParameterReferenceNode if it's a new '?' instance in the query,
+			// even if it gets the same numeric index as a previous one *if* they were different AST nodes.
+			// The _parameters map is more for caching resolved nodes per unique AST node or name.
+			// For '?', the ParameterReferenceNode constructor expects the numeric index.
+			// We use currentAnonymousId as the identifier and increment after creation.
+			identifier = currentAnonymousId;
+			parameterNode = new ParameterReferenceNode(this, parameterExpression, identifier, resolvedType);
+			this._parameters.set(identifier, parameterNode); // Cache it by its assigned numeric ID
+			this._nextAnonymousIndex++; // Increment for the *next* '?'
+		} else if (symbolKey.startsWith(':')) {
+			const nameOrIndex = symbolKey.substring(1);
+			const numIndex = parseInt(nameOrIndex, 10);
+			identifier = isNaN(numIndex) ? nameOrIndex : numIndex;
+
+			if (this._parameters.has(identifier)) {
+				parameterNode = this._parameters.get(identifier)!;
+				// If already exists, its type was set at creation. Type hints are for new nodes.
+			} else {
+				if (this._parameterTypeHints.has(identifier)) {
+					resolvedType = this._parameterTypeHints.get(identifier)!;
+				}
+				parameterNode = new ParameterReferenceNode(this, parameterExpression, identifier, resolvedType);
+				this._parameters.set(identifier, parameterNode);
+			}
+		} else {
+			// Not a parameter symbol, delegate to parent scope
+			return this.parentScope.resolveSymbol(symbolKey, expression);
+		}
+
+		this.addReference(parameterNode!);
+		return parameterNode;
+	}
+
+	/**
+	 * Returns all parameters resolved by this scope.
+	 */
+	getParameters(): ReadonlyMap<string | number, ParameterReferenceNode> {
+		return this._parameters;
+	}
+
+	/**
+	 * Gets the next available anonymous parameter index (1-based) for assigning to new ' ? ' params.
+	 */
+	// getNextAnonymousIndex(): number { // This method might be misleading as index is auto-assigned.
+	// 	return this._nextAnonymousIndex;
+	// }
+}

package/src/planner/scopes/registered.ts

@@ -0,0 +1,78 @@
+import { StatusCode } from '../../common/types.js';
+import { QuereusError } from '../../common/errors.js';
+import type { PlanNode } from '../nodes/plan-node.js';
+import * as AST from '../../parser/ast.js';
+import { type ReferenceCallback, type Scope, Ambiguous } from './scope.js';
+
+/**
+ * The Scope object provides context for symbol resolution during query planning.
+ * It encapsulates the logic for looking up columns, parameters, functions, and CTEs
+ * based on the current position in the PlanNode tree.
+ */
+export class RegisteredScope implements Scope {
+	/** References that have been resolved through this scope. */
+	private _references: PlanNode[] = [];
+
+	/** Symbols that have been registered in this scope. */
+	private registeredSymbols: Map<string, ReferenceCallback> = new Map();
+
+	constructor(
+		/** The parent scope, if any. The root scope of a query has no parent. */
+		public readonly parent?: Scope,
+	) { }
+
+	/**
+	 * Registers a symbol (like a table alias, CTE name, or parameter) with a factory function
+	 * that can produce a ReferenceNode for it when encountered in an expression.
+	 *
+	 * @param symbolKey The unique string key for this symbol in the current scope.
+	 *  For tables/aliases: lower-case name.
+	 *  For qualified schema.table: "schema.table" (lower-case).
+	 *  For parameters: ":name" or ":index".
+	 * @param getReference A factory function that takes the matching symbol and the current Scope,
+	 *  and returns an appropriate ReferenceNode.
+	 */
+	registerSymbol(symbolKey: string, getReference: ReferenceCallback): void {
+		const lowerSymbolKey = symbolKey.toLowerCase();
+		if (this.registeredSymbols.has(lowerSymbolKey)) {
+			throw new QuereusError(`Symbol '${lowerSymbolKey}' already exists in the same scope.`, StatusCode.ERROR);
+		}
+		this.registeredSymbols.set(lowerSymbolKey, getReference);
+	}
+
+	resolveSymbol(symbolKey: string, expression: AST.Expression): PlanNode | typeof Ambiguous | undefined {
+		const reference = this.registeredSymbols.get(symbolKey.toLowerCase());
+		if (reference) {
+			const node = reference(expression, this);
+			this.addReference(node);
+			return node;
+		}
+		if (this.parent) {
+			return this.parent.resolveSymbol(symbolKey, expression);
+		}
+		return undefined;
+	}
+
+	/**
+	 * Returns all references that have been resolved through this scope.
+	 * This includes references from both this scope and any parent scopes.
+	 *
+	 * @returns An array of all resolved references.
+	 */
+	getReferences(): readonly PlanNode[] {
+		return this._references;
+	}
+
+	addReference(reference: PlanNode): void {
+		this._references.push(reference);
+	}
+
+	/**
+	 * Returns all symbols that have been registered in this scope.
+	 *
+	 * @returns An array of all [symbolKey, ReferenceCallback].
+	 */
+	getSymbols(): readonly [string, ReferenceCallback][] {
+		return Array.from(this.registeredSymbols.entries());
+	}
+}

package/src/planner/scopes/scope.ts

@@ -0,0 +1,28 @@
+import type { PlanNode } from '../nodes/plan-node.js';
+import * as AST from '../../parser/ast.js';
+
+// Create a javascript symbol that represents an ambiguous symbol.
+export const Ambiguous = Symbol();
+
+export type ReferenceCallback = (expression: AST.Expression, currentScope: Scope) => PlanNode;
+
+/**
+ * The Scope object provides context for symbol resolution during query planning.
+ * It encapsulates the logic for looking up columns, parameters, functions, and CTEs
+ * based on the current position in the PlanNode tree.
+ */
+export interface Scope {
+	resolveSymbol(symbolKey: string, expression: AST.Expression): PlanNode | typeof Ambiguous | undefined;
+
+	/**
+	 * Returns all references that have been resolved through this scope.
+	 * This includes references from both this scope and any parent scopes.
+	 *
+	 * @returns An array of all resolved references.
+	 */
+	getReferences(): readonly PlanNode[];
+
+	addReference(reference: PlanNode): void;
+}
+
+