@quereus/quereus 0.7.2 → 0.7.4

package/src/runtime/cache/shared-cache.ts

@@ -1,169 +0,0 @@
-/**
- * Shared cache utilities based on the existing working cache emitter
- * Provides reusable caching patterns for CacheNode, NLJ inner caching, and CTE materialization
- */
-
-import type { Row } from '../../common/types.js';
-import { createLogger } from '../../common/logger.js';
-
-const log = createLogger('runtime:cache:shared');
-
-/**
- * Cache strategy for handling overflow
- */
-export type CacheStrategy = 'memory' | 'spill';
-
-/**
- * Configuration for shared cache behavior
- */
-export interface SharedCacheConfig {
-	/** Maximum rows before abandoning cache */
-	threshold: number;
-	/** Strategy when threshold exceeded */
-	strategy: CacheStrategy;
-	/** Optional identifier for logging */
-	name?: string;
-}
-
-/**
- * Cache state for a shared cache instance
- * Based on the proven closure pattern from the existing cache emitter
- */
-export interface CacheState {
-	/** Cached result rows (undefined if not cached or abandoned) */
-	cachedResult?: Row[];
-	/** Whether caching was abandoned due to threshold */
-	cacheAbandoned: boolean;
-	/** Number of times cache has been consumed */
-	consumeCount: number;
-}
-
-/**
- * Create a cache state object
- */
-export function createCacheState(): CacheState {
-	return {
-		cachedResult: undefined,
-		cacheAbandoned: false,
-		consumeCount: 0
-	};
-}
-
-/**
- * Shared cache implementation using the proven streaming-first pattern
- * This is extracted from the existing working cache emitter
- */
-export async function* streamWithCache(
-	sourceIterable: AsyncIterable<Row>,
-	config: SharedCacheConfig,
-	state: CacheState
-): AsyncIterable<Row> {
-	const { threshold, name = 'cache' } = config;
-
-	// If we already have cached data, return it
-	if (state.cachedResult) {
-		state.consumeCount++;
-		log('Serving %s from cache (access #%d, %d rows)',
-			name, state.consumeCount, state.cachedResult.length);
-		yield* state.cachedResult;
-		return;
-	}
-
-	// If we previously abandoned caching, just stream through
-	if (state.cacheAbandoned) {
-		log('%s cache abandoned due to previous threshold exceed, streaming directly', name);
-		yield* sourceIterable;
-		return;
-	}
-
-	// First time - pipeline results while building cache
-	log('Building %s cache with threshold %d while pipelining', name, threshold);
-	let cache: Row[] | undefined = [];
-
-	// Pipeline while caching (proven pattern from existing emitter)
-	for await (const row of sourceIterable) {
-		// Always yield immediately (streaming-first)
-		yield row;
-
-		// Try to cache if we haven't exceeded threshold
-		if (cache) {
-			if (cache.length < threshold) {
-				// Cache the row (deep copy to avoid reference issues)
-				cache.push([...row] as Row);
-			} else {
-				// Hit threshold - dump cache and abandon caching
-				log('%s cache threshold %d exceeded at row %d, dumping cache and continuing to pipeline',
-					name, threshold, cache.length);
-				cache = undefined;
-			}
-		}
-	}
-
-	// If we finished without exceeding threshold, cache is ready
-	if (cache) {
-		log('%s cache built successfully with %d rows', name, cache.length);
-		state.cachedResult = cache;
-	} else {
-		state.cacheAbandoned = true;
-	}
-}
-
-/**
- * Factory function to create a reusable cache function
- * Returns a function that can be called multiple times with different sources
- */
-export function createCacheFunction(config: SharedCacheConfig) {
-	const state = createCacheState();
-
-	return async function* cachedIterable(source: AsyncIterable<Row>): AsyncIterable<Row> {
-		yield* streamWithCache(source, config, state);
-	};
-}
-
-/**
- * Helper to wrap an async iterable with caching using the proven pattern
- * This is the main utility that other components should use
- */
-export function withSharedCache(
-	source: AsyncIterable<Row>,
-	config: SharedCacheConfig
-): {
-	iterable: AsyncIterable<Row>;
-	state: CacheState;
-} {
-	const state = createCacheState();
-
-	const iterable = {
-		async *[Symbol.asyncIterator](): AsyncIterator<Row> {
-			yield* streamWithCache(source, config, state);
-		}
-	};
-
-	return { iterable, state };
-}
-
-/**
- * Get cache effectiveness metrics
- */
-export function getCacheMetrics(state: CacheState): {
-	isCached: boolean;
-	isAbandoned: boolean;
-	consumeCount: number;
-	cachedRows?: number;
-} {
-	return {
-		isCached: !!state.cachedResult,
-		isAbandoned: state.cacheAbandoned,
-		consumeCount: state.consumeCount,
-		cachedRows: state.cachedResult?.length
-	};
-}
-
-/**
- * Clear cache state (useful for testing or manual cache invalidation)
- */
-export function clearCache(state: CacheState): void {
-	state.cachedResult = undefined;
-	state.cacheAbandoned = false;
-	state.consumeCount = 0;
-}

package/src/runtime/context-helpers.ts

@@ -1,191 +0,0 @@
-import type { RuntimeContext } from './types.js';
-import type { RowDescriptor, RowGetter } from '../planner/nodes/plan-node.js';
-import type { SqlValue, Row } from '../common/types.js';
-import { QuereusError } from '../common/errors.js';
-import { StatusCode } from '../common/types.js';
-import { createLogger } from '../common/logger.js';
-
-const ctxLog = createLogger('runtime:context');
-const ctxLookupLog = createLogger('runtime:context:lookup');
-
-/**
- * A mutable slot for efficient row context management in streaming operations.
- * Avoids per-row Map mutations while maintaining context safety.
- */
-export interface RowSlot {
-	/** Replace the current row (cheap field write) */
-	set(row: Row): void;
-	/** Tear down (removes descriptor from context) */
-	close(): void;
-}
-
-/**
- * Create a row slot for efficient streaming context management.
- * The slot installs a context entry once and updates it by reference.
- * Perfect for scan/join/window operations that process many rows.
- */
-export function createRowSlot(
-	rctx: RuntimeContext,
-	descriptor: RowDescriptor
-): RowSlot {
-	// Internal boxed reference - one allocation per slot
-	const ref = { current: undefined as Row | undefined };
-
-	// Install only once
-	rctx.context.set(descriptor, () => ref.current!);
-
-	const attrs = Object.keys(descriptor).filter(k => descriptor[parseInt(k)] !== undefined);
-	ctxLog('CREATE slot with attrs=[%s]', attrs.join(','));
-
-	return {
-		set(row: Row) {
-			ref.current = row;
-		},
-		close() {
-			rctx.context.delete(descriptor);
-			ctxLog('CLOSE slot with attrs=[%s]', attrs.join(','));
-		}
-	};
-}
-
-/**
- * Resolve an attribute ID to its column value in the current context.
- * Searches from newest (innermost) to oldest (outermost) scope.
- */
-export function resolveAttribute(rctx: RuntimeContext, attributeId: number, columnName?: string): SqlValue {
-	// Iterate newest → oldest so the most recently pushed scope wins
-	const contextsReversed = Array.from(rctx.context.entries()).reverse();
-
-	ctxLookupLog('LOOKUP column %s (attr#%d) in %d contexts', columnName || '?', attributeId, contextsReversed.length);
-
-	for (const [descriptor, rowGetter] of contextsReversed) {
-		const columnIndex = descriptor[attributeId];
-		if (columnIndex !== undefined) {
-			const row = rowGetter();
-			if (Array.isArray(row) && columnIndex < row.length) {
-				ctxLookupLog('FOUND column %s at index %d in row', columnName || '?', columnIndex);
-				return row[columnIndex];
-			}
-		}
-	}
-
-	// Log available attributes for debugging
-	if (ctxLookupLog.enabled) {
-		ctxLookupLog('Available contexts:');
-		for (const [descriptor, _] of contextsReversed) {
-			const attrs = Object.keys(descriptor).filter(k => descriptor[parseInt(k)] !== undefined);
-			ctxLookupLog('  - Descriptor with attrs=[%s]', attrs.join(','));
-		}
-
-		// Log current plan execution stack if available
-		if (rctx.planStack && rctx.planStack.length > 0) {
-			const currentNode = rctx.planStack[rctx.planStack.length - 1];
-			ctxLookupLog('LOOKUP FAILED in node %s id=%d', currentNode.nodeType, currentNode.id);
-			ctxLookupLog('Execution stack: %s',
-				rctx.planStack.map(n => `${n.nodeType}#${n.id}`).join(' → '));
-		}
-	}
-
-	throw new QuereusError(
-		`No row context found for column ${columnName || `attr#${attributeId}`}. The column reference must be evaluated within the context of its source relation.`,
-		StatusCode.ERROR
-	);
-}
-
-/**
- * Look up a specific column by descriptor and index.
- * Useful when you already know which descriptor contains the column.
- */
-export function lookupColumn(rctx: RuntimeContext, descriptor: RowDescriptor, columnIndex: number): SqlValue | undefined {
-	const rowGetter = rctx.context.get(descriptor);
-	if (!rowGetter) {
-		ctxLookupLog('LOOKUP by index %d - no context found', columnIndex);
-		return undefined;
-	}
-
-	const row = rowGetter();
-	if (Array.isArray(row) && columnIndex < row.length) {
-		ctxLookupLog('LOOKUP by index %d - found value', columnIndex);
-		return row[columnIndex];
-	}
-	ctxLookupLog('LOOKUP by index %d - index out of bounds', columnIndex);
-	return undefined;
-}
-
-/**
- * Execute a function with a row context, ensuring proper cleanup.
- * This is the recommended pattern for all row-processing emitters.
- */
-export async function withAsyncRowContext<T>(
-	rctx: RuntimeContext,
-	descriptor: RowDescriptor,
-	rowGetter: RowGetter,
-	fn: () => T | Promise<T>
-): Promise<T> {
-	const attrs = Object.keys(descriptor).filter(k => descriptor[parseInt(k)] !== undefined);
-	ctxLog('PUSH async context with attrs=[%s]', attrs.join(','));
-
-	rctx.context.set(descriptor, rowGetter);
-	try {
-		return await fn();
-	} finally {
-		rctx.context.delete(descriptor);
-		ctxLog('POP async context with attrs=[%s]', attrs.join(','));
-	}
-}
-
-/**
- * Execute a function with a row context, ensuring proper cleanup.
- * This is the recommended pattern for all row-processing emitters.
- */
-export function withRowContext<T>(
-	rctx: RuntimeContext,
-	descriptor: RowDescriptor,
-	rowGetter: RowGetter,
-	fn: () => T
-): T {
-	const attrs = Object.keys(descriptor).filter(k => descriptor[parseInt(k)] !== undefined);
-	ctxLog('PUSH context with attrs=[%s]', attrs.join(','));
-
-	rctx.context.set(descriptor, rowGetter);
-	try {
-		return fn();
-	} finally {
-		rctx.context.delete(descriptor);
-		ctxLog('POP context with attrs=[%s]', attrs.join(','));
-	}
-}
-
-/**
- * Execute a generator function with row context management.
- * Useful for streaming operations that process multiple rows.
- */
-export async function* withRowContextGenerator<T>(
-	rctx: RuntimeContext,
-	descriptor: RowDescriptor,
-	rows: AsyncIterable<Row>,
-	fn: (row: Row) => AsyncIterable<T>
-): AsyncIterable<T> {
-	const attrs = Object.keys(descriptor).filter(k => descriptor[parseInt(k)] !== undefined);
-	let rowCount = 0;
-
-	for await (const row of rows) {
-		if (rowCount === 0) {
-			ctxLog('PUSH generator context with attrs=[%s]', attrs.join(','));
-		}
-		rctx.context.set(descriptor, () => row);
-		try {
-			yield* fn(row);
-			rowCount++;
-		} finally {
-			rctx.context.delete(descriptor);
-			if (ctxLog.enabled && rowCount % 1000 === 0) {
-				ctxLog('Generator context processed %d rows', rowCount);
-			}
-		}
-	}
-
-	if (rowCount > 0) {
-		ctxLog('POP generator context after %d rows', rowCount);
-	}
-}

package/src/runtime/deferred-constraint-queue.ts

@@ -1,196 +0,0 @@
-import type { Row, SqlValue, OutputValue } from '../common/types.js';
-import type { RowDescriptor } from '../planner/nodes/plan-node.js';
-import type { RuntimeContext } from './types.js';
-import type { Database } from '../core/database.js';
-import { QuereusError } from '../common/errors.js';
-import { StatusCode } from '../common/types.js';
-import { createRowSlot } from './context-helpers.js';
-import type { VirtualTableConnection } from '../vtab/connection.js';
-
-export interface DeferredConstraintRow {
-	row: Row;
-	descriptor: RowDescriptor;
-	evaluator: (ctx: RuntimeContext) => OutputValue;
-	constraintName: string;
-	connectionId?: string;
-	contextRow?: Row; // Mutation context values
-	contextDescriptor?: RowDescriptor; // Mutation context row descriptor
-}
-
-type DeferredConstraintBuckets = Map<string, Map<string, DeferredConstraintRow[]>>;
-
-export class DeferredConstraintQueue {
-	private readonly entries: DeferredConstraintBuckets = new Map();
-	private layers: DeferredConstraintBuckets[] = [];
-
-	constructor(private readonly db: Database) { }
-
-	enqueue(baseTable: string, constraintName: string, row: Row, descriptor: RowDescriptor, evaluator: (ctx: RuntimeContext) => OutputValue, connectionId?: string, contextRow?: Row, contextDescriptor?: RowDescriptor): void {
-		const store = this.getActiveStore();
-		const tableKey = baseTable.toLowerCase();
-		if (!store.has(tableKey)) store.set(tableKey, new Map());
-		const constraints = store.get(tableKey)!;
-		if (!constraints.has(constraintName)) constraints.set(constraintName, []);
-		constraints.get(constraintName)!.push({
-			row: row.slice() as Row,
-			descriptor,
-			evaluator,
-			constraintName,
-			connectionId,
-			contextRow: contextRow ? contextRow.slice() as Row : undefined,
-			contextDescriptor
-		});
-	}
-
-	beginLayer(): void {
-		this.layers.push(new Map());
-	}
-
-	rollbackLayer(): void {
-		this.layers.pop();
-	}
-
-	releaseLayer(): void {
-		const top = this.layers.pop();
-		if (!top) return;
-		const target = this.getActiveStore();
-		this.merge(target, top);
-	}
-
-	clear(): void {
-		this.entries.clear();
-		this.layers = [];
-	}
-
-	async runDeferredRows(): Promise<void> {
-		if (!this.hasPending()) return;
-		const activeConnections = this.db.getAllConnections();
-		const runtimeCtx: RuntimeContext = {
-			db: this.db,
-			stmt: undefined,
-			params: {},
-			context: new Map(),
-			tableContexts: new Map(),
-			tracer: this.db.getInstructionTracer(),
-			enableMetrics: this.db.options.getBooleanOption('runtime_stats'),
-		};
-		const store = this.cloneAll();
-		for (const [table, constraints] of store) {
-			for (const [_, rows] of constraints) {
-				for (const entry of rows) {
-					const connection = this.findConnection(activeConnections, table, entry.connectionId);
-					runtimeCtx.activeConnection = connection;
-					await this.evaluateEntry(runtimeCtx, entry);
-				}
-			}
-		}
-		this.clear();
-	}
-
-	private async evaluateEntry(runtimeCtx: RuntimeContext, entry: DeferredConstraintRow): Promise<void> {
-		// Compose context row with the flat row if context is present
-		const evaluationRow = entry.contextRow ? [...entry.contextRow, ...entry.row] : entry.row;
-		const evaluationDescriptor = entry.contextRow && entry.contextDescriptor
-			? this.composeCombinedDescriptor(entry.contextDescriptor, entry.descriptor)
-			: entry.descriptor;
-
-
-		const slot = createRowSlot(runtimeCtx, evaluationDescriptor);
-		try {
-			slot.set(evaluationRow);
-			const value = await entry.evaluator(runtimeCtx) as SqlValue;
-			if (value === false || value === 0) {
-				throw new QuereusError(`CHECK constraint failed: ${entry.constraintName}`, StatusCode.CONSTRAINT);
-			}
-		} finally {
-			slot.close();
-		}
-	}
-
-	private composeCombinedDescriptor(contextDescriptor: RowDescriptor, flatRowDescriptor: RowDescriptor): RowDescriptor {
-		const combined: RowDescriptor = [];
-		const contextLength = Object.keys(contextDescriptor).filter(k => contextDescriptor[parseInt(k)] !== undefined).length;
-
-		// Copy context descriptor as-is (indices 0..contextLength-1)
-		for (const attrIdStr in contextDescriptor) {
-			const attrId = parseInt(attrIdStr);
-			if (contextDescriptor[attrId] !== undefined) {
-				combined[attrId] = contextDescriptor[attrId];
-			}
-		}
-
-		// Copy flat descriptor with offset indices (indices contextLength..end)
-		for (const attrIdStr in flatRowDescriptor) {
-			const attrId = parseInt(attrIdStr);
-			if (flatRowDescriptor[attrId] !== undefined) {
-				combined[attrId] = flatRowDescriptor[attrId] + contextLength;
-			}
-		}
-
-		return combined;
-	}
-
-	private getActiveStore(): DeferredConstraintBuckets {
-		return this.layers.length > 0 ? this.layers[this.layers.length - 1] : this.entries;
-	}
-
-	private hasPending(): boolean {
-		if (this.entries.size > 0) return true;
-		return this.layers.some(layer => layer.size > 0);
-	}
-
-	private cloneAll(): DeferredConstraintBuckets {
-		const clone: DeferredConstraintBuckets = new Map();
-		const append = (source: DeferredConstraintBuckets) => {
-			for (const [table, constraints] of source) {
-				const lowerTable = table.toLowerCase();
-				if (!clone.has(lowerTable)) clone.set(lowerTable, new Map());
-				const targetConstraints = clone.get(lowerTable)!;
-				for (const [constraintName, rows] of constraints) {
-					if (!targetConstraints.has(constraintName)) targetConstraints.set(constraintName, []);
-					targetConstraints.get(constraintName)!.push(...rows.map(entry => ({
-						row: entry.row.slice() as Row,
-						descriptor: entry.descriptor,
-						evaluator: entry.evaluator,
-						constraintName: entry.constraintName,
-						connectionId: entry.connectionId,
-						contextRow: entry.contextRow ? entry.contextRow.slice() as Row : undefined,
-						contextDescriptor: entry.contextDescriptor,
-					})));
-				}
-			}
-		};
-
-		append(this.entries);
-		for (const layer of this.layers) append(layer);
-		return clone;
-	}
-
-	private merge(target: DeferredConstraintBuckets, source: DeferredConstraintBuckets): void {
-		for (const [table, constraints] of source) {
-			const lowerTable = table.toLowerCase();
-			if (!target.has(lowerTable)) target.set(lowerTable, new Map());
-			const destConstraints = target.get(lowerTable)!;
-			for (const [constraintName, rows] of constraints) {
-				if (!destConstraints.has(constraintName)) destConstraints.set(constraintName, []);
-				destConstraints.get(constraintName)!.push(...rows);
-			}
-		}
-	}
-
-	private findConnection(connections: VirtualTableConnection[], tableKey: string, preferredId?: string): VirtualTableConnection | undefined {
-		if (preferredId) {
-			const direct = connections.find(conn => conn.connectionId === preferredId);
-			if (direct) {
-				return direct;
-			}
-		}
-		const normalized = tableKey.toLowerCase();
-		const simple = normalized.includes('.') ? normalized.substring(normalized.lastIndexOf('.') + 1) : normalized;
-		return connections.find(conn => {
-			const connName = conn.tableName.toLowerCase();
-			return connName === normalized || connName === simple;
-		});
-	}
-}
-