@quereus/quereus 0.1.0

package/src/planner/validation/plan-validator.ts

@@ -0,0 +1,340 @@
+/**
+ * Plan validator pass for the Titan optimizer
+ * Validates that a plan tree meets all invariants before emission
+ */
+
+import { PlanNode, type RelationalPlanNode } from '../nodes/plan-node.js';
+import { PlanNodeType } from '../nodes/plan-node-type.js';
+import { QuereusError } from '../../common/errors.js';
+import { StatusCode } from '../../common/types.js';
+import { validateLog } from '../debug/logger-utils.js';
+
+const log = validateLog();
+
+/**
+ * Validation options
+ */
+export interface ValidationOptions {
+	/** Validate that all physical properties are set (default: true) */
+	requirePhysical?: boolean;
+	/** Validate attribute consistency (default: true) */
+	validateAttributes?: boolean;
+	/** Validate ordering properties (default: true) */
+	validateOrdering?: boolean;
+}
+
+/**
+ * Validate a physical plan tree before emission
+ * @param root The root node of the plan tree
+ * @param options Validation options
+ * @throws QuereusError if validation fails
+ */
+export function validatePhysicalTree(root: PlanNode, options: ValidationOptions = {}): void {
+	const opts = {
+		requirePhysical: true,
+		validateAttributes: true,
+		validateOrdering: true,
+		...options
+	};
+
+	log('Starting plan validation for tree rooted at %s', root.nodeType);
+
+	const context = new ValidationContext(opts);
+	validateNode(root, context, []);
+
+	log('Plan validation completed successfully');
+}
+
+/**
+ * Validation context for tracking state during traversal
+ */
+class ValidationContext {
+	/** All attribute IDs seen so far */
+	private attributeIds = new Set<number>();
+
+	/** Map of attribute ID to node path for debugging */
+	private attributeLocations = new Map<number, string>();
+
+	constructor(public readonly options: ValidationOptions) {}
+
+	/**
+	 * Register an attribute ID and check for duplicates
+	 */
+	registerAttribute(attrId: number, nodePath: string): void {
+		if (this.attributeIds.has(attrId)) {
+			const existingLocation = this.attributeLocations.get(attrId);
+			throw new QuereusError(
+				`Duplicate attribute ID ${attrId} found at ${nodePath} (previously seen at ${existingLocation})`,
+				StatusCode.INTERNAL
+			);
+		}
+
+		this.attributeIds.add(attrId);
+		this.attributeLocations.set(attrId, nodePath);
+	}
+
+	/**
+	 * Check if an attribute ID exists
+	 */
+	hasAttribute(attrId: number): boolean {
+		return this.attributeIds.has(attrId);
+	}
+}
+
+/**
+ * Validate a single node and recursively validate its children
+ */
+function validateNode(node: PlanNode, context: ValidationContext, path: string[]): void {
+	const nodePath = path.concat(node.nodeType).join(' > ');
+
+	try {
+		log('Validating node %s at path: %s', node.nodeType, nodePath);
+
+		// 1. Validate physical properties are present
+		if (context.options.requirePhysical) {
+			validatePhysicalProperties(node, nodePath);
+		}
+
+		// 2. Validate that this is a physical (not logical-only) node type
+		validatePhysicalNodeType(node, nodePath);
+
+		// 3. Validate relational node specific properties
+		if (isRelationalNode(node)) {
+			validateRelationalNode(node as RelationalPlanNode, context, nodePath);
+		}
+
+		// 4. Validate column references point to valid attributes
+		if (node.nodeType === PlanNodeType.ColumnReference) {
+			validateColumnReference(node as any, context, nodePath);
+		}
+
+		// 5. Recursively validate children
+		for (const child of node.getChildren()) {
+			validateNode(child, context, path.concat(node.nodeType));
+		}
+
+	} catch (error) {
+		if (error instanceof QuereusError) {
+			// Add context to the error
+			error.message = `Validation failed at ${nodePath}: ${error.message}`;
+		}
+		throw error;
+	}
+}
+
+/**
+ * Validate that physical properties are present
+ */
+function validatePhysicalProperties(node: PlanNode, nodePath: string): void {
+	if (!node.physical) {
+		throw new QuereusError(
+			`Node ${node.nodeType} at ${nodePath} lacks physical properties`,
+			StatusCode.INTERNAL
+		);
+	}
+
+	// Basic sanity checks on physical properties
+	const physical = node.physical;
+
+	if (typeof physical.deterministic !== 'boolean') {
+		throw new QuereusError(
+			`Node ${node.nodeType} has invalid deterministic flag: ${physical.deterministic}`,
+			StatusCode.INTERNAL
+		);
+	}
+
+	if (typeof physical.readonly !== 'boolean') {
+		throw new QuereusError(
+			`Node ${node.nodeType} has invalid readonly flag: ${physical.readonly}`,
+			StatusCode.INTERNAL
+		);
+	}
+
+	if (physical.idempotent !== undefined && typeof physical.idempotent !== 'boolean') {
+		throw new QuereusError(
+			`Node ${node.nodeType} has invalid idempotent flag: ${physical.idempotent}`,
+			StatusCode.INTERNAL
+		);
+	}
+
+	if (physical.estimatedRows !== undefined && physical.estimatedRows < 0) {
+		throw new QuereusError(
+			`Node ${node.nodeType} has negative estimated rows: ${physical.estimatedRows}`,
+			StatusCode.INTERNAL
+		);
+	}
+
+	// Validate side effect consistency for DML nodes
+	if (PlanNode.hasSideEffects(physical)) {
+		// Nodes with side effects should not be constant
+		if (physical.constant === true) {
+			throw new QuereusError(
+				`Node ${node.nodeType} has side effects but is marked as constant`,
+				StatusCode.INTERNAL
+			);
+		}
+	}
+}
+
+/**
+ * Validate that the node type is physical (not logical-only)
+ */
+function validatePhysicalNodeType(node: PlanNode, nodePath: string): void {
+	// Node types that should NOT appear in a fully optimized physical tree
+	const logicalOnlyTypes = new Set([
+		PlanNodeType.Aggregate, // Should be StreamAggregate or HashAggregate
+		// Add other logical-only types here as needed
+	]);
+
+	if (logicalOnlyTypes.has(node.nodeType)) {
+		throw new QuereusError(
+			`Logical-only node type ${node.nodeType} found in physical tree at ${nodePath}`,
+			StatusCode.INTERNAL
+		);
+	}
+}
+
+/**
+ * Validate relational node specific properties
+ */
+function validateRelationalNode(node: RelationalPlanNode, context: ValidationContext, nodePath: string): void {
+	if (!context.options.validateAttributes) {
+		return;
+	}
+
+	// Get attributes for this node
+	const attributes = node.getAttributes();
+
+	// Validate that we have attributes if this is a real relational node
+	// (Some nodes like DDL operations might not have attributes)
+	const needsAttributes = !isDDLNode(node.nodeType);
+	if (needsAttributes && attributes.length === 0) {
+		log('Warning: Relational node %s has no attributes at %s', node.nodeType, nodePath);
+	}
+
+	// Register all attribute IDs and check for duplicates within this node
+	for (const attr of attributes) {
+		if (typeof attr.id !== 'number') {
+			throw new QuereusError(
+				`Invalid attribute ID ${attr.id} (must be number) at ${nodePath}`,
+				StatusCode.INTERNAL
+			);
+		}
+
+		context.registerAttribute(attr.id, nodePath);
+
+		// Validate attribute properties
+		if (!attr.name || typeof attr.name !== 'string') {
+			throw new QuereusError(
+				`Attribute ${attr.id} has invalid name "${attr.name}" at ${nodePath}`,
+				StatusCode.INTERNAL
+			);
+		}
+
+		if (!attr.sourceRelation || typeof attr.sourceRelation !== 'string') {
+			throw new QuereusError(
+				`Attribute ${attr.id} has invalid source relation "${attr.sourceRelation}" at ${nodePath}`,
+				StatusCode.INTERNAL
+			);
+		}
+	}
+
+	// Validate ordering properties if present
+	if (context.options.validateOrdering && node.physical?.ordering) {
+		validateOrdering(node.physical.ordering, attributes.length, nodePath);
+	}
+}
+
+/**
+ * Validate column references point to valid attributes
+ */
+function validateColumnReference(node: any, context: ValidationContext, nodePath: string): void {
+	if (!context.options.validateAttributes) {
+		return;
+	}
+
+	const attributeId = node.attributeId;
+	if (typeof attributeId !== 'number') {
+		throw new QuereusError(
+			`ColumnReference has invalid attribute ID ${attributeId} at ${nodePath}`,
+			StatusCode.INTERNAL
+		);
+	}
+
+	if (!context.hasAttribute(attributeId)) {
+		throw new QuereusError(
+			`ColumnReference refers to unknown attribute ID ${attributeId} at ${nodePath}`,
+			StatusCode.INTERNAL
+		);
+	}
+}
+
+/**
+ * Validate ordering specification
+ */
+function validateOrdering(ordering: any[], columnCount: number, nodePath: string): void {
+	for (let i = 0; i < ordering.length; i++) {
+		const orderSpec = ordering[i];
+
+		if (typeof orderSpec.column !== 'number') {
+			throw new QuereusError(
+				`Invalid ordering column index ${orderSpec.column} at ${nodePath}[${i}]`,
+				StatusCode.INTERNAL
+			);
+		}
+
+		if (orderSpec.column < 0 || orderSpec.column >= columnCount) {
+			throw new QuereusError(
+				`Ordering column index ${orderSpec.column} out of range (0-${columnCount-1}) at ${nodePath}[${i}]`,
+				StatusCode.INTERNAL
+			);
+		}
+
+		if (typeof orderSpec.desc !== 'boolean') {
+			throw new QuereusError(
+				`Invalid ordering desc flag ${orderSpec.desc} at ${nodePath}[${i}]`,
+				StatusCode.INTERNAL
+			);
+		}
+	}
+}
+
+/**
+ * Check if a node is a relational node
+ */
+function isRelationalNode(node: PlanNode): boolean {
+	return 'getAttributes' in node && typeof (node as any).getAttributes === 'function';
+}
+
+/**
+ * Check if a node type is a DDL node that doesn't produce attributes
+ */
+function isDDLNode(nodeType: PlanNodeType): boolean {
+	const ddlTypes = new Set([
+		PlanNodeType.CreateTable,
+		PlanNodeType.DropTable,
+		PlanNodeType.CreateIndex,
+		PlanNodeType.CreateView,
+		PlanNodeType.DropView,
+		PlanNodeType.Transaction,
+		PlanNodeType.Pragma,
+		PlanNodeType.AddConstraint,
+	]);
+
+	return ddlTypes.has(nodeType);
+}
+
+/**
+ * Quick validation function for development/testing
+ * Validates plan with default options and logs results
+ */
+export function quickValidate(root: PlanNode): boolean {
+	try {
+		validatePhysicalTree(root);
+		log('✓ Plan validation passed');
+		return true;
+	} catch (error) {
+		log('✗ Plan validation failed: %s', error);
+		return false;
+	}
+}

package/src/runtime/async-util.ts

@@ -0,0 +1,282 @@
+/**
+ * Async stream utilities for processing async iterables
+ * Used by CacheNode emitter, NestedLoopJoin inner side, and other streaming operations
+ */
+
+import type { MaybePromise, Row } from '../common/types.js';
+import { createLogger } from '../common/logger.js';
+
+const log = createLogger('runtime:async-util');
+
+/**
+ * Transform rows using a mapping function
+ */
+export async function* mapRows<T extends Row, R>(
+	src: AsyncIterable<T>,
+	fn: (row: T) => R
+): AsyncIterable<R> {
+	for await (const row of src) {
+		yield fn(row);
+	}
+}
+
+/**
+ * Filter rows using a predicate function
+ */
+export async function* filterRows<T>(
+	src: AsyncIterable<T>,
+	pred: (row: T) => MaybePromise<boolean>
+): AsyncIterable<T> {
+	for await (const row of src) {
+		const include = await pred(row);
+		if (include) {
+			yield row;
+		}
+	}
+}
+
+/**
+ * Duplicate an async iterable into two independent streams
+ * This materializes chunks internally as needed
+ */
+export function tee<T>(src: AsyncIterable<T>): [AsyncIterable<T>, AsyncIterable<T>] {
+	const buffer: T[] = [];
+	let srcIterator: AsyncIterator<T> | null = null;
+	let srcDone = false;
+	let consumer1Index = 0;
+	let consumer2Index = 0;
+
+	async function fillBuffer(targetIndex: number): Promise<void> {
+		if (srcDone || buffer.length > targetIndex) {
+			return;
+		}
+
+		if (!srcIterator) {
+			srcIterator = src[Symbol.asyncIterator]();
+		}
+
+		while (buffer.length <= targetIndex && !srcDone) {
+			const result = await srcIterator.next();
+			if (result.done) {
+				srcDone = true;
+			} else {
+				buffer.push(result.value);
+			}
+		}
+	}
+
+	const stream1: AsyncIterable<T> = {
+		async *[Symbol.asyncIterator]() {
+			while (true) {
+				await fillBuffer(consumer1Index);
+
+				if (consumer1Index >= buffer.length) {
+					if (srcDone) break;
+					continue;
+				}
+
+				yield buffer[consumer1Index];
+				consumer1Index++;
+
+				// Clean up buffer when both consumers have passed this point
+				const minIndex = Math.min(consumer1Index, consumer2Index);
+				if (minIndex > 100) { // Keep some buffer for efficiency
+					buffer.splice(0, minIndex - 100);
+					consumer1Index -= (minIndex - 100);
+					consumer2Index -= (minIndex - 100);
+				}
+			}
+		}
+	};
+
+	const stream2: AsyncIterable<T> = {
+		async *[Symbol.asyncIterator]() {
+			while (true) {
+				await fillBuffer(consumer2Index);
+
+				if (consumer2Index >= buffer.length) {
+					if (srcDone) break;
+					continue;
+				}
+
+				yield buffer[consumer2Index];
+				consumer2Index++;
+
+				// Clean up buffer when both consumers have passed this point
+				const minIndex = Math.min(consumer1Index, consumer2Index);
+				if (minIndex > 100) { // Keep some buffer for efficiency
+					buffer.splice(0, minIndex - 100);
+					consumer1Index -= (minIndex - 100);
+					consumer2Index -= (minIndex - 100);
+				}
+			}
+		}
+	};
+
+	return [stream1, stream2];
+}
+
+/**
+ * Add buffering to an async iterable with back-pressure
+ */
+export async function* buffered<T>(
+	src: AsyncIterable<T>,
+	maxBuffer: number
+): AsyncIterable<T> {
+	const buffer: T[] = [];
+	const srcIterator = src[Symbol.asyncIterator]();
+	let srcDone = false;
+	let fillPromise: Promise<void> | null = null;
+
+	// Fill buffer in background
+	async function fillBuffer(): Promise<void> {
+		while (buffer.length < maxBuffer && !srcDone) {
+			const result = await srcIterator.next();
+			if (result.done) {
+				srcDone = true;
+			} else {
+				buffer.push(result.value);
+			}
+		}
+	}
+
+	// Start initial fill
+	fillPromise = fillBuffer();
+
+	while (true) {
+		// Wait for buffer to have items or source to be done
+		await fillPromise;
+
+		if (buffer.length === 0 && srcDone) {
+			break;
+		}
+
+		if (buffer.length > 0) {
+			const item = buffer.shift()!;
+			yield item;
+
+			// Start refilling buffer if below threshold
+			if (buffer.length < maxBuffer / 2 && !srcDone) {
+				fillPromise = fillBuffer();
+			}
+		}
+	}
+}
+
+/**
+ * Take only the first N items from an async iterable
+ */
+export async function* take<T>(src: AsyncIterable<T>, count: number): AsyncIterable<T> {
+	let taken = 0;
+	for await (const item of src) {
+		if (taken >= count) break;
+		yield item;
+		taken++;
+	}
+}
+
+/**
+ * Skip the first N items from an async iterable
+ */
+export async function* skip<T>(src: AsyncIterable<T>, count: number): AsyncIterable<T> {
+	let skipped = 0;
+	for await (const item of src) {
+		if (skipped < count) {
+			skipped++;
+			continue;
+		}
+		yield item;
+	}
+}
+
+/**
+ * Count the number of items in an async iterable (consumes the iterable)
+ */
+export async function count<T>(src: AsyncIterable<T>): Promise<number> {
+	let total = 0;
+	for await (const _ of src) {
+		total++;
+	}
+	return total;
+}
+
+/**
+ * Collect all items from an async iterable into an array
+ * Use with caution on large iterables
+ */
+export async function collect<T>(src: AsyncIterable<T>): Promise<T[]> {
+	const result: T[] = [];
+	for await (const item of src) {
+		result.push(item);
+	}
+	return result;
+}
+
+/**
+ * Merge multiple async iterables into a single stream
+ * Items are yielded as soon as they become available from any source
+ */
+export async function* merge<T>(...sources: AsyncIterable<T>[]): AsyncIterable<T> {
+	const iterators = sources.map(src => src[Symbol.asyncIterator]());
+	const pending = new Map<number, Promise<IteratorResult<T>>>();
+
+	// Start initial reads
+	for (let i = 0; i < iterators.length; i++) {
+		pending.set(i, iterators[i].next());
+	}
+
+	while (pending.size > 0) {
+		// Wait for the first iterator to produce a result
+		const entries = Array.from(pending.entries());
+		const promises = entries.map(([index, promise]) =>
+			promise.then(result => ({ index, result }))
+		);
+
+		const { index, result } = await Promise.race(promises);
+		pending.delete(index);
+
+		if (!result.done) {
+			yield result.value;
+			// Start next read from this iterator
+			pending.set(index, iterators[index].next());
+		}
+		// If iterator is done, it's removed from pending and won't be read again
+	}
+}
+
+/**
+ * Convert an array to an async iterable
+ */
+export async function* fromArray<T>(items: readonly T[]): AsyncIterable<T> {
+	for (const item of items) {
+		yield item;
+	}
+}
+
+/**
+ * Add logging to an async iterable for debugging
+ */
+export async function* traced<T>(
+	src: AsyncIterable<T>,
+	name: string,
+	itemLogger?: (item: T, index: number) => string
+): AsyncIterable<T> {
+	log('Starting trace for %s', name);
+	let index = 0;
+
+	try {
+		for await (const item of src) {
+			if (itemLogger) {
+				log('%s[%d]: %s', name, index, itemLogger(item, index));
+			} else {
+				log('%s[%d]: %O', name, index, item);
+			}
+			yield item;
+			index++;
+		}
+		log('Completed trace for %s (%d items)', name, index);
+	} catch (error) {
+		log('Error in trace for %s after %d items: %s', name, index, error);
+		throw error;
+	}
+}