@quereus/quereus 0.1.0

package/src/runtime/types.ts

@@ -0,0 +1,191 @@
+import type { RuntimeValue, SqlParameters, OutputValue, Row } from "../common/types.js";
+import type { Database } from "../core/database.js";
+import type { Statement } from "../core/statement.js";
+import type { RowDescriptor, RowGetter, TableDescriptor, TableGetter } from "../planner/nodes/plan-node.js";
+import type { Scheduler } from "./scheduler.js";
+import type { EmissionContext } from "./emission-context.js";
+import type { VirtualTableConnection } from "../vtab/connection.js";
+
+export type RuntimeContext = {
+	db: Database;
+	stmt: Statement | null; // Can be null for transient exec statements
+	params: SqlParameters; // User-provided values for the current execution
+	/** Row contexts by row descriptor */
+	context: Map<RowDescriptor, RowGetter>;
+	/** Table contexts by table name, used for recursive CTEs or other temporary table situations */
+	tableContexts: Map<TableDescriptor, TableGetter>;
+	/** Debug tracer for instruction execution, if enabled */
+	tracer?: InstructionTracer;
+	/** Active connection for the current transaction context */
+	activeConnection?: VirtualTableConnection;
+	/** Whether to collect runtime execution metrics */
+	enableMetrics: boolean;
+};
+
+export type InstructionRun = (ctx: RuntimeContext, ...args: any[]) => OutputValue;
+
+export type Instruction = {
+	params: Instruction[];
+	run: InstructionRun;
+	/** Optional human-readable note about what this instruction does */
+	note?: string;
+	/** Optional sub-programs used to execute this instruction - this is here for tracing purposes */
+	programs?: Scheduler[];
+	/** Optional emission context for schema validation */
+	emissionContext?: EmissionContext;
+	/** Optional runtime statistics collected during execution */
+	runtimeStats?: InstructionRuntimeStats;
+};
+
+/**
+ * Runtime statistics for instruction execution
+ */
+export interface InstructionRuntimeStats {
+	/** Number of input values/rows processed */
+	in: number;
+	/** Number of output values/rows produced */
+	out: number;
+	/** Total execution time in nanoseconds */
+	elapsedNs: bigint;
+	/** Number of times this instruction was executed */
+	executions: number;
+}
+
+/** * Trace event for instruction execution. */
+export interface InstructionTraceEvent {
+	instructionIndex: number;
+	note?: string;
+	type: 'input' | 'output' | 'row' | 'error';
+	timestamp: number;
+	args?: RuntimeValue[];
+	result?: OutputValue;
+	error?: string;
+	/** Information about sub-programs if this instruction has any */
+	subPrograms?: SubProgramInfo[];
+	/** Row index within the async iterable (for 'row' type events) */
+	rowIndex?: number;
+	/** Row data (for 'row' type events) */
+	row?: Row;
+}
+
+/** Information about a sub-program for tracing purposes */
+export interface SubProgramInfo {
+	programIndex: number;
+	instructionCount: number;
+	rootNote?: string;
+}
+
+/** * Interface for tracing instruction execution. */
+export interface InstructionTracer {
+	/** Called before an instruction executes */
+	traceInput(instructionIndex: number, instruction: Instruction, args: RuntimeValue[]): void;
+	/** Called after an instruction executes */
+	traceOutput(instructionIndex: number, instruction: Instruction, result: OutputValue): void;
+	/** Called when an instruction throws an error */
+	traceError(instructionIndex: number, instruction: Instruction, error: Error): void;
+	/** Called for each row emitted by an async iterable instruction */
+	traceRow(instructionIndex: number, instruction: Instruction, rowIndex: number, row: Row): void;
+	/** Gets collected trace events (if supported by the tracer) */
+	getTraceEvents?(): InstructionTraceEvent[];
+	/** Gets information about all sub-programs encountered during tracing */
+	getSubPrograms?(): Map<number, { scheduler: Scheduler; parentInstructionIndex: number }>;
+}
+
+/** * Tracer that collects execution events for later analysis. */
+export class CollectingInstructionTracer implements InstructionTracer {
+	private events: InstructionTraceEvent[] = [];
+	private subPrograms = new Map<number, { scheduler: Scheduler; parentInstructionIndex: number }>();
+	private nextSubProgramId = 0;
+
+	traceInput(instructionIndex: number, instruction: Instruction, args: RuntimeValue[]): void {
+		const subPrograms = this.collectSubProgramInfo(instructionIndex, instruction);
+
+		this.events.push({
+			instructionIndex,
+			note: instruction.note,
+			type: 'input',
+			timestamp: Date.now(),
+			args: this.cloneArgs(args),
+			subPrograms
+		});
+	}
+
+	traceOutput(instructionIndex: number, instruction: Instruction, result: OutputValue): void {
+		this.events.push({
+			instructionIndex,
+			note: instruction.note,
+			type: 'output',
+			timestamp: Date.now(),
+			result: this.cloneResult(result)
+		});
+	}
+
+	traceError(instructionIndex: number, instruction: Instruction, error: Error): void {
+		this.events.push({
+			instructionIndex,
+			note: instruction.note,
+			type: 'error',
+			timestamp: Date.now(),
+			error: error.message
+		});
+	}
+
+	traceRow(instructionIndex: number, instruction: Instruction, rowIndex: number, row: Row): void {
+		this.events.push({
+			instructionIndex,
+			note: instruction.note,
+			type: 'row',
+			timestamp: Date.now(),
+			rowIndex,
+			row
+		});
+	}
+
+	getTraceEvents(): InstructionTraceEvent[] {
+		return [...this.events];
+	}
+
+	getSubPrograms(): Map<number, { scheduler: Scheduler; parentInstructionIndex: number }> {
+		return new Map(this.subPrograms);
+	}
+
+	clear(): void {
+		this.events = [];
+		this.subPrograms.clear();
+		this.nextSubProgramId = 0;
+	}
+
+	private collectSubProgramInfo(instructionIndex: number, instruction: Instruction): SubProgramInfo[] | undefined {
+		if (!instruction.programs || instruction.programs.length === 0) {
+			return undefined;
+		}
+
+		return instruction.programs.map(scheduler => {
+			const programId = this.nextSubProgramId++;
+			this.subPrograms.set(programId, { scheduler, parentInstructionIndex: instructionIndex });
+
+			return {
+				programIndex: programId,
+				instructionCount: scheduler.instructions.length,
+				rootNote: scheduler.instructions[scheduler.instructions.length - 1]?.note
+			};
+		});
+	}
+
+	private cloneArgs(args: RuntimeValue[]): RuntimeValue[] {
+		return args.map(arg => this.cloneValue(arg));
+	}
+
+	private cloneResult(result: OutputValue): OutputValue {
+		return this.cloneValue(result);
+	}
+
+	private cloneValue(value: any): any {
+		if (value === null || value === undefined) return value;
+		if (typeof value === 'function') return '[Function]';
+		if (typeof value === 'object' && Symbol.asyncIterator in value) return '[AsyncIterable]';
+		if (Array.isArray(value)) return value.map(v => this.cloneValue(v));
+		if (typeof value === 'object') return '[Object]';
+		return value;
+	}
+}

package/src/runtime/utils.ts

@@ -0,0 +1,136 @@
+import { QuereusError } from '../common/errors.js';
+import { StatusCode } from '../common/types.js';
+import type { TableSchema } from '../schema/table.js';
+import type { VirtualTable } from '../vtab/table.js';
+import type { RuntimeContext } from './types.js';
+import type { VirtualTableConnection } from '../vtab/connection.js';
+import { createLogger } from '../common/logger.js';
+import type { MemoryVirtualTableConnection } from '../vtab/memory/connection.js';
+import type { MemoryTable } from '../vtab/memory/table.js';
+import type { RowDescriptor } from '../planner/nodes/plan-node.js';
+
+const log = createLogger('runtime:utils');
+const errorLog = log.extend('error');
+export const ctxLog = createLogger('runtime:context');
+
+export function isAsyncIterable<T>(value: unknown): value is AsyncIterable<T> {
+	return typeof value === 'object' && value !== null && Symbol.asyncIterator in value;
+}
+
+export async function asyncIterableToArray<T>(iterable: AsyncIterable<T>): Promise<T[]> {
+	const result: T[] = [];
+	for await (const item of iterable) {
+		result.push(item);
+	}
+	return result;
+}
+
+/**
+ * Helper to get or create a VirtualTable connection for a given table.
+ * This ensures transaction consistency by reusing connections within the same context.
+ */
+export async function getVTableConnection(ctx: RuntimeContext, tableSchema: TableSchema): Promise<VirtualTableConnection> {
+	const tableName = tableSchema.name; // Use just the table name, not fully qualified
+
+	// Check if we already have an active connection for this table
+	const existingConnections = ctx.db.getConnectionsForTable(tableName);
+	if (existingConnections.length > 0) {
+		log(`Reusing existing connection for table ${tableName}`);
+		return existingConnections[0];
+	}
+
+	// Create a new VirtualTable instance
+	const vtab = await getVTable(ctx, tableSchema);
+
+	// Try to create a connection if the table supports it
+	let connection: VirtualTableConnection;
+	if (vtab.createConnection) {
+		connection = await vtab.createConnection();
+		log(`Created new connection ${connection.connectionId} for table ${tableName}`);
+	} else if (vtab.getConnection) {
+		const existingConn = vtab.getConnection();
+		if (existingConn) {
+			connection = existingConn;
+			log(`Using existing internal connection ${connection.connectionId} for table ${tableName}`);
+		} else {
+			throw new QuereusError(`Table '${tableName}' does not support connections`, StatusCode.INTERNAL);
+		}
+	} else {
+		throw new QuereusError(`Table '${tableName}' does not support connections`, StatusCode.INTERNAL);
+	}
+
+	// Register the connection with the database
+	await ctx.db.registerConnection(connection);
+
+	// Set as the active connection in the runtime context if none is set
+	if (!ctx.activeConnection) {
+		ctx.activeConnection = connection;
+	}
+
+	return connection;
+}
+
+/**
+ * Helper to get the VirtualTable instance for a given TableReferenceNode.
+ * This is the legacy method that creates ephemeral instances.
+ * When reusing connections, this will also inject the existing connection into the VirtualTable.
+ */
+export async function getVTable(ctx: RuntimeContext, tableSchema: TableSchema): Promise<VirtualTable> {
+	// All tables are virtual, so vtabModuleName should always be present.
+	if (!tableSchema.vtabModuleName) {
+		throw new QuereusError(`Table schema for '${tableSchema.name}' is missing vtabModuleName.`, StatusCode.INTERNAL);
+	}
+	const moduleInfo = ctx.db._getVtabModule(tableSchema.vtabModuleName);
+	if (!moduleInfo) {
+		throw new QuereusError(`Virtual table module '${tableSchema.vtabModuleName}' not found for table '${tableSchema.name}'`, StatusCode.ERROR);
+	}
+	const module = moduleInfo.module;
+	if (typeof module.xConnect !== 'function') {
+		throw new QuereusError(`Virtual table module '${tableSchema.vtabModuleName}' does not implement xConnect`, StatusCode.MISUSE);
+	}
+	const vtabArgs = tableSchema.vtabArgs || {};
+	const vtabInstance = module.xConnect(ctx.db, moduleInfo.auxData, tableSchema.vtabModuleName, tableSchema.schemaName, tableSchema.name, vtabArgs);
+
+	// If we have an active connection for this table, inject it into the VirtualTable
+	const tableName = tableSchema.name;
+	const existingConnections = ctx.db.getConnectionsForTable(tableName);
+	if (existingConnections.length > 0 && tableSchema.vtabModuleName === 'memory') {
+		const memoryConnection = existingConnections[0] as MemoryVirtualTableConnection;
+		const memoryTable = vtabInstance as MemoryTable;
+		if (memoryConnection.getMemoryConnection && memoryTable.setConnection) {
+			memoryTable.setConnection(memoryConnection.getMemoryConnection());
+			log(`Injected existing connection into VirtualTable for table ${tableName}`);
+		}
+	}
+
+	return vtabInstance;
+}
+
+/**
+ * Helper to properly disconnect and unregister a VirtualTable instance.
+ */
+export async function disconnectVTable(ctx: RuntimeContext, vtab: VirtualTable): Promise<void> {
+	// Disconnect the VirtualTable instance
+	if (typeof vtab.xDisconnect === 'function') {
+		await vtab.xDisconnect().catch((e: any) => {
+			errorLog(`Error during xDisconnect for table '${vtab.tableName}': ${e}`);
+		});
+	}
+}
+
+/**
+ * Helper function to log context push operations
+ */
+export function logContextPush(descriptor: RowDescriptor, note: string, attributes?: any[]) {
+	const attrs = Object.keys(descriptor).filter(k => descriptor[parseInt(k)] !== undefined);
+	const attrNames = attributes ? attributes.map(attr => `${attr.name}(#${attr.id})`).join(',') : 'unknown';
+	ctxLog('PUSH context %s: attrs=[%s] names=[%s]', note, attrs.join(','), attrNames);
+}
+
+/**
+ * Helper function to log context pop operations
+ */
+export function logContextPop(descriptor: RowDescriptor, note: string) {
+	const attrs = Object.keys(descriptor).filter(k => descriptor[parseInt(k)] !== undefined);
+	ctxLog('POP context %s: attrs=[%s]', note, attrs.join(','));
+}

package/src/schema/change-events.ts

@@ -0,0 +1,80 @@
+import { createLogger } from '../common/logger.js';
+
+const log = createLogger('schema:change-events');
+
+/**
+ * Represents a schema change event.
+ */
+export interface SchemaChangeEvent {
+	type: 'table_added' | 'table_removed' | 'table_modified' |
+	      'function_added' | 'function_removed' | 'function_modified' |
+	      'module_added' | 'module_removed' | 'collation_added' | 'collation_removed';
+	schemaName?: string;
+	objectName: string;
+	oldObject?: any;
+	newObject?: any;
+}
+
+/**
+ * Function that handles schema change events.
+ */
+export type SchemaChangeListener = (event: SchemaChangeEvent) => void;
+
+/**
+ * Manages schema change listeners and notifications.
+ */
+export class SchemaChangeNotifier {
+	private listeners = new Set<SchemaChangeListener>();
+
+	/**
+	 * Adds a schema change listener.
+	 * @returns A function to unsubscribe the listener.
+	 */
+	addListener(listener: SchemaChangeListener): () => void {
+		this.listeners.add(listener);
+		log('Added schema change listener, total listeners: %d', this.listeners.size);
+		return () => this.removeListener(listener);
+	}
+
+	/**
+	 * Removes a schema change listener.
+	 */
+	removeListener(listener: SchemaChangeListener): void {
+		const removed = this.listeners.delete(listener);
+		if (removed) {
+			log('Removed schema change listener, total listeners: %d', this.listeners.size);
+		}
+	}
+
+	/**
+	 * Notifies all listeners of a schema change event.
+	 */
+	notifyChange(event: SchemaChangeEvent): void {
+		log('Notifying %d listeners of schema change: %s %s',
+			this.listeners.size, event.type, event.objectName);
+
+		for (const listener of this.listeners) {
+			try {
+				listener(event);
+			} catch (error) {
+				log('Error in schema change listener: %s', error);
+			}
+		}
+	}
+
+	/**
+	 * Gets the number of active listeners.
+	 */
+	getListenerCount(): number {
+		return this.listeners.size;
+	}
+
+	/**
+	 * Clears all listeners.
+	 */
+	clearListeners(): void {
+		const count = this.listeners.size;
+		this.listeners.clear();
+		log('Cleared all %d schema change listeners', count);
+	}
+}

package/src/schema/column.ts

@@ -0,0 +1,45 @@
+import { SqlDataType } from '../common/types.js';
+import type { Expression } from '../parser/ast.js';
+
+/**
+ * Represents the schema definition of a single column in a table.
+ */
+export interface ColumnSchema {
+	/** Column name */
+	name: string;
+	/** Data type affinity (TEXT, INTEGER, REAL, BLOB, NUMERIC) */
+	affinity: SqlDataType;
+	/** Whether the column has a NOT NULL constraint */
+	notNull: boolean;
+	/** Whether the column is part of the primary key */
+	primaryKey: boolean;
+	/** Order within the primary key (1-based) or 0 if not PK */
+	pkOrder: number;
+	/** Default value expression */
+	defaultValue: Expression | null;
+	/** Declared collation sequence name (e.g., "BINARY", "NOCASE", "RTRIM") */
+	collation: string;
+	/** Is the column generated? */
+	generated: boolean;
+}
+
+/**
+ * Creates a default ColumnSchema with basic properties
+ * Following Third Manifesto principles, columns default to NOT NULL unless explicitly specified otherwise
+ *
+ * @param name The name for the column
+ * @param defaultNotNull Whether columns should be NOT NULL by default (defaults to true for Third Manifesto compliance)
+ * @returns A new column schema with default values
+ */
+export function createDefaultColumnSchema(name: string, defaultNotNull: boolean = true): ColumnSchema {
+	return {
+		name: name,
+		affinity: SqlDataType.TEXT,
+		notNull: defaultNotNull, // Third Manifesto: default to NOT NULL
+		primaryKey: false,
+		pkOrder: 0,
+		defaultValue: null,
+		collation: 'BINARY', // SQLite's default
+		generated: false,
+	};
+}

package/src/schema/function.ts

@@ -0,0 +1,139 @@
+import type { MaybePromise, Row, SqlValue } from '../common/types.js';
+import { FunctionFlags } from '../common/constants.js';
+import { SqlDataType } from '../common/types.js';
+import type { Database } from '../core/database.js';
+import type { BaseType, ScalarType, RelationType } from '../common/datatype.js';
+
+/**
+ * Type for a scalar function implementation.
+ */
+export type ScalarFunc = (...args: SqlValue[]) => MaybePromise<SqlValue>;
+
+/**
+ * Type for a table-valued function implementation.
+ */
+export type TableValuedFunc = (...args: SqlValue[]) => MaybePromise<AsyncIterable<Row>>;
+
+/**
+ * Type for a database-aware table-valued function implementation.
+ * Takes a database instance and SQL values, returns an async iterable of rows.
+ */
+export type IntegratedTableValuedFunc = (db: Database, ...args: SqlValue[]) => MaybePromise<AsyncIterable<Row>>;
+
+/**
+ * Type for aggregate step function.
+ */
+export type AggregateReducer<T = any> = (accumulator: T, ...args: SqlValue[]) => T;
+
+/**
+ * Type for aggregate finalizer function.
+ */
+export type AggregateFinalizer<T = any> = (accumulator: T) => SqlValue;
+
+/**
+ * Base interface for all function schemas with common properties.
+ */
+interface BaseFunctionSchema {
+	/** Function name (lowercase for consistent lookup) */
+	name: string;
+	/** Number of arguments (-1 for variable) */
+	numArgs: number;
+	/** Combination of FunctionFlags */
+	flags: FunctionFlags;
+	/** User data pointer passed during registration */
+	userData?: unknown;
+	/** Return type information */
+	returnType: BaseType;
+}
+
+/**
+ * Schema for scalar functions that return a single value.
+ */
+export interface ScalarFunctionSchema extends BaseFunctionSchema {
+	returnType: ScalarType;
+	/** Direct scalar function implementation */
+	implementation: ScalarFunc;
+}
+
+/**
+ * Schema for table-valued functions that return rows.
+ */
+export interface TableValuedFunctionSchema extends BaseFunctionSchema {
+	returnType: RelationType;
+	/** Table-valued function implementation */
+	implementation: TableValuedFunc | IntegratedTableValuedFunc;
+	/** Whether this TVF requires database access as first parameter */
+	isIntegrated?: boolean;
+}
+
+/**
+ * Schema for aggregate functions.
+ */
+export interface AggregateFunctionSchema extends BaseFunctionSchema {
+	returnType: ScalarType;
+	/** Aggregate step function */
+	stepFunction: AggregateReducer;
+	/** Aggregate finalizer function */
+	finalizeFunction: AggregateFinalizer;
+	/** Initial accumulator value for aggregates */
+	initialValue?: any;
+}
+
+/**
+ * Schema for window functions (for future use).
+ */
+export interface WindowFunctionSchema extends BaseFunctionSchema {
+	returnType: ScalarType;
+	/** Window function implementation */
+	implementation: (...args: any[]) => any;
+}
+
+/**
+ * Union type representing all possible function schemas.
+ */
+export type FunctionSchema =
+	| ScalarFunctionSchema
+	| TableValuedFunctionSchema
+	| AggregateFunctionSchema
+	| WindowFunctionSchema;
+
+/**
+ * Type guards for function schema types.
+ */
+export function isScalarFunctionSchema(schema: FunctionSchema): schema is ScalarFunctionSchema {
+	return schema.returnType.typeClass === 'scalar' && 'implementation' in schema && typeof schema.implementation === 'function';
+}
+
+export function isTableValuedFunctionSchema(schema: FunctionSchema): schema is TableValuedFunctionSchema {
+	return schema.returnType.typeClass === 'relation';
+}
+
+export function isAggregateFunctionSchema(schema: FunctionSchema): schema is AggregateFunctionSchema {
+	return 'stepFunction' in schema && 'finalizeFunction' in schema;
+}
+
+export function isWindowFunctionSchema(schema: FunctionSchema): schema is WindowFunctionSchema {
+	return 'implementation' in schema && schema.returnType.typeClass === 'scalar' && !isScalarFunctionSchema(schema) && !isAggregateFunctionSchema(schema);
+}
+
+/**
+ * Creates a consistent key for storing/looking up functions
+ *
+ * @param name Function name
+ * @param numArgs Number of arguments (-1 for variable argument count)
+ * @returns A string key in the format "name/numArgs"
+ */
+export function getFunctionKey(name: string, numArgs: number): string {
+	return `${name.toLowerCase()}/${numArgs}`;
+}
+
+// Legacy compatibility - deprecated interfaces and column info
+/**
+ * @deprecated Use RelationType.columns instead
+ * Column information for table-valued functions.
+ */
+export interface TVFColumnInfo {
+	name: string;
+	type: SqlDataType;
+	nullable?: boolean;
+}