@quereus/quereus 0.1.0

package/src/vtab/memory/table.ts

@@ -0,0 +1,253 @@
+import { VirtualTable } from '../table.js';
+import type { VirtualTableModule, SchemaChangeInfo } from '../module.js';
+import type { Database } from '../../core/database.js';
+import type { Row } from '../../common/types.js';
+import { type IndexSchema, type TableSchema } from '../../schema/table.js';
+import { MemoryTableManager } from './layer/manager.js';
+import type { MemoryTableConnection } from './layer/connection.js';
+import { QuereusError } from '../../common/errors.js';
+import { StatusCode } from '../../common/types.js';
+import type { FilterInfo } from '../filter-info.js';
+import { buildScanPlanFromFilterInfo } from './layer/scan-plan.js';
+import type { ColumnDef as ASTColumnDef } from '../../parser/ast.js'; // Assuming this will be updated for renameColumn
+import { createMemoryTableLoggers } from './utils/logging.js';
+import { safeJsonStringify } from '../../util/serialization.js';
+import type { VirtualTableConnection } from '../connection.js';
+import { MemoryVirtualTableConnection } from './connection.js';
+
+const logger = createMemoryTableLoggers('table');
+
+/**
+ * Represents a connection-specific instance of an in-memory table using the layer-based MVCC model.
+ * This class acts as a thin wrapper around the shared MemoryTableManager,
+ * holding the connection state.
+ */
+export class MemoryTable extends VirtualTable {
+	/** @internal The shared manager handling layers, schema, and global state */
+	public readonly manager: MemoryTableManager;
+	/** @internal Connection state specific to this table instance (lazily initialized) */
+	private connection: MemoryTableConnection | null = null;
+
+	/**
+	 * @internal - Use MemoryTableModule.xConnect or xCreate
+	 * Creates a connection-specific instance linked to a manager.
+	 */
+	constructor(
+		db: Database,
+		module: VirtualTableModule<any, any>,
+		manager: MemoryTableManager // Pass the shared manager instance
+	) {
+		// Use manager's schema and name for the base class constructor
+		super(db, module, manager.schemaName, manager.tableName);
+		this.manager = manager;
+		// Set the tableSchema directly from the manager's current canonical schema
+		// This ensures the VirtualTable base class has the correct schema reference.
+		this.tableSchema = manager.tableSchema;
+	}
+
+	/** Returns the canonical schema from the manager */
+	getSchema(): TableSchema | undefined {
+		// Always return the potentially updated schema from the manager
+		return this.manager.tableSchema;
+	}
+
+	/** Checks read-only status via the manager */
+	isReadOnly(): boolean {
+		// Access readOnly via a public method on the manager
+		return this.manager.isReadOnly;
+	}
+
+	/** Ensures the connection to the manager is established */
+	private async ensureConnection(): Promise<MemoryTableConnection> {
+		if (!this.connection) {
+			// Check if there's already an active connection for this table in the database
+			const existingConnections = this.db.getConnectionsForTable(this.tableName);
+			if (existingConnections.length > 0 && existingConnections[0] instanceof MemoryVirtualTableConnection) {
+				const memoryVirtualConnection = existingConnections[0] as MemoryVirtualTableConnection;
+				this.connection = memoryVirtualConnection.getMemoryConnection();
+				logger.debugLog(`ensureConnection: Reused existing connection ${this.connection.connectionId} for table ${this.tableName}`);
+			} else {
+				// Establish connection state with the manager upon first use
+				this.connection = this.manager.connect();
+
+				// Create a VirtualTableConnection wrapper and register it with the database
+				const vtabConnection = new MemoryVirtualTableConnection(this.tableName, this.connection);
+				await this.db.registerConnection(vtabConnection);
+
+				logger.debugLog(`ensureConnection: Created and registered new connection ${this.connection.connectionId} for table ${this.tableName}`);
+			}
+		}
+		return this.connection;
+	}
+
+	/** Sets an existing connection for this table instance (for transaction reuse) */
+	setConnection(memoryConnection: MemoryTableConnection): void {
+		logger.debugLog(`Setting connection ${memoryConnection.connectionId} for table ${this.tableName}`);
+		this.connection = memoryConnection;
+	}
+
+	/** Creates a new VirtualTableConnection for transaction support */
+	createConnection(): VirtualTableConnection {
+		const memoryConnection = this.manager.connect();
+		return new MemoryVirtualTableConnection(this.tableName, memoryConnection);
+	}
+
+	/** Gets the current connection if this table maintains one internally */
+	getConnection(): VirtualTableConnection | undefined {
+		if (!this.connection) {
+			return undefined;
+		}
+		return new MemoryVirtualTableConnection(this.tableName, this.connection);
+	}
+
+	// New xQuery method for direct async iteration
+	async* xQuery(filterInfo: FilterInfo): AsyncIterable<Row> {
+		const conn = await this.ensureConnection();
+		logger.debugLog(`xQuery using connection ${conn.connectionId} (pending: ${conn.pendingTransactionLayer?.getLayerId()}, read: ${conn.readLayer.getLayerId()})`);
+		const currentSchema = this.manager.tableSchema;
+		if (!currentSchema) {
+			logger.error('xQuery', this.tableName, 'Table schema is undefined');
+			return;
+		}
+		const plan = buildScanPlanFromFilterInfo(filterInfo, currentSchema);
+		logger.debugLog(`xQuery invoked for ${this.tableName} with plan: ${safeJsonStringify(plan)}`);
+
+		const startLayer = conn.pendingTransactionLayer ?? conn.readLayer;
+		logger.debugLog(`xQuery reading from layer ${startLayer.getLayerId()}`);
+
+		// Delegate scanning to the manager, which handles layer recursion
+		yield* this.manager.scanLayer(startLayer, plan);
+	}
+
+	// Note: xBestIndex is handled by the MemoryTableModule, not the table instance.
+
+	/** Performs mutation through the connection's transaction layer */
+	async xUpdate(
+		operation: 'insert' | 'update' | 'delete',
+		values: Row | undefined,
+		oldKeyValues?: Row
+	): Promise<Row | undefined> {
+		const conn = await this.ensureConnection();
+		// Delegate mutation to the manager.
+		// This assumes manager.performMutation will be updated to this signature and logic.
+		return this.manager.performMutation(conn, operation, values, oldKeyValues);
+	}
+
+	/** Begins a transaction for this connection */
+	async xBegin(): Promise<void> {
+		(await this.ensureConnection()).begin();
+	}
+
+	/** Commits this connection's transaction */
+	async xCommit(): Promise<void> {
+		// Only commit if a connection has actually been established
+		if (this.connection) {
+			await this.connection.commit();
+		}
+	}
+
+	/** Rolls back this connection's transaction */
+	async xRollback(): Promise<void> {
+		// Only rollback if a connection has actually been established
+		if (this.connection) {
+			this.connection.rollback();
+		}
+	}
+
+	/** Sync operation (currently no-op for memory table layers) */
+	async xSync(): Promise<void> {
+		// This might trigger background collapse in the manager in the future
+		// await this.manager.tryCollapseLayers(); // Optional: trigger collapse on sync?
+		return Promise.resolve();
+	}
+
+	/** Renames the underlying table via the manager */
+	async xRename(newName: string): Promise<void> {
+		logger.operation('Rename', this.tableName, { newName });
+		await this.manager.renameTable(newName);
+		// Update this instance's schema reference after rename
+		this.tableSchema = this.manager.tableSchema;
+	}
+
+	// --- Savepoint operations ---
+	async xSavepoint(savepointIndex: number): Promise<void> {
+		const conn = await this.ensureConnection();
+		conn.createSavepoint(savepointIndex);
+	}
+
+	async xRelease(savepointIndex: number): Promise<void> {
+		if (!this.connection) return; // No connection, no savepoints to release
+		this.connection.releaseSavepoint(savepointIndex);
+	}
+
+	async xRollbackTo(savepointIndex: number): Promise<void> {
+		if (!this.connection) return; // No connection, no savepoints to rollback to
+		this.connection.rollbackToSavepoint(savepointIndex);
+	}
+	// --- End Savepoint operations ---
+
+
+	/** Handles schema changes via the manager */
+	async xAlterSchema(changeInfo: SchemaChangeInfo): Promise<void> {
+		const originalManagerSchema = this.manager.tableSchema; // For potential error recovery
+		try {
+			switch (changeInfo.type) {
+				case 'addColumn':
+					await this.manager.addColumn(changeInfo.columnDef);
+					break;
+				case 'dropColumn':
+					await this.manager.dropColumn(changeInfo.columnName);
+					break;
+				case 'renameColumn':
+					if (!('newColumnDefAst' in changeInfo)) {
+						throw new QuereusError('SchemaChangeInfo for renameColumn missing newColumnDefAst', StatusCode.INTERNAL);
+					}
+					await this.manager.renameColumn(changeInfo.oldName, changeInfo.newColumnDefAst as ASTColumnDef);
+					break;
+				default: {
+					const exhaustiveCheck: never = changeInfo;
+					throw new QuereusError(`Unhandled schema change: ${(exhaustiveCheck as any)?.type}`, StatusCode.INTERNAL);
+				}
+			}
+			this.tableSchema = this.manager.tableSchema; // Refresh local schema ref
+		} catch (e) {
+			logger.error('Schema Change', this.tableName, e);
+			// Manager DDL methods should handle reverting their own BaseLayer schema updates on error.
+			// Refresh local schema ref to ensure it's consistent with manager after potential error/revert.
+			this.tableSchema = originalManagerSchema;
+			// It might be safer for manager DDL to not alter its own this.tableSchema until baseLayer op succeeds.
+			// And if baseLayer op fails, manager DDL reverts baseLayer.tableSchema.
+			// Then here, we always sync from manager: this.tableSchema = this.manager.tableSchema;
+			throw e;
+		}
+	}
+
+	/** Disconnects this connection instance from the manager */
+	async xDisconnect(): Promise<void> {
+		if (this.connection) {
+			// Manager handles cleanup and potential layer collapse trigger
+			await this.manager.disconnect(this.connection.connectionId);
+			this.connection = null; // Clear connection reference on this instance
+		}
+	}
+
+	// --- Index DDL methods delegate to the manager ---
+	async xCreateIndex(indexSchema: IndexSchema): Promise<void> {
+		logger.operation('Create Index', this.tableName, { indexName: indexSchema.name });
+		await this.manager.createIndex(indexSchema);
+		this.tableSchema = this.manager.tableSchema; // Refresh local schema ref
+	}
+
+	async xDropIndex(indexName: string): Promise<void> {
+		logger.operation('Drop Index', this.tableName, { indexName });
+		await this.manager.dropIndex(indexName);
+		// Update schema reference
+		this.tableSchema = this.manager.tableSchema;
+	}
+	// --- End Index DDL methods ---
+}
+
+// Helper function (moved from MemoryTableCursor and adapted)
+// function buildScanPlanInternal(filterInfo: FilterInfo, tableSchema: TableSchema): ScanPlan { ... MOVED ... }
+
+

package/src/vtab/memory/types.ts

@@ -0,0 +1,23 @@
+import type { SqlValue } from '../../common/types.js';
+
+/** Key type used in B-Trees (primary key or index key part) */
+export type BTreeKey = BTreeKeyForPrimary | BTreeKeyForIndex;
+
+/** Alias for BTreeKey when explicitly referring to a primary key. */
+export type BTreeKeyForPrimary = SqlValue | SqlValue[];
+
+/** Alias for BTreeKey when explicitly referring to a key of a secondary index. */
+export type BTreeKeyForIndex = SqlValue | SqlValue[];
+
+/** Represents an entry in a MemoryIndex BTree, mapping an IndexKey to an array of PrimaryKeys */
+export interface MemoryIndexEntry {
+	indexKey: BTreeKeyForIndex;
+	primaryKeys: Set<BTreeKeyForPrimary>;
+}
+
+/**
+ * Configuration options for MemoryTable creation
+ */
+export interface MemoryTableConfig {
+	readOnly?: boolean;
+}

package/src/vtab/memory/utils/logging.ts

@@ -0,0 +1,33 @@
+import { createLogger } from '../../../common/logger.js';
+import { safeJsonStringify } from '../../../util/serialization.js';
+
+/**
+ * Creates consistent logging utilities for memory vtable components
+ */
+export function createMemoryTableLoggers(namespace: string) {
+	const log = createLogger(`vtab:memory:${namespace}`);
+
+	const warnLog = log.extend('warn');
+	const errorLog = log.extend('error');
+	const debugLog = log.extend('debug');
+
+	return {
+		info: log,
+		warnLog,
+		errorLog,
+		debugLog,
+
+		operation: (operation: string, tableName: string, details?: any) => {
+			log(`[${tableName}] ${operation}${details ? `: ${safeJsonStringify(details)}` : ''}`);
+		},
+
+		warn: (operation: string, tableName: string, message: string, details?: any) => {
+			warnLog(`[${tableName}] ${operation}: ${message}${details ? ` - ${safeJsonStringify(details)}` : ''}`);
+		},
+
+		error: (operation: string, tableName: string, innerError: unknown, details?: any) => {
+			const errorMessage = innerError instanceof Error ? innerError.message : innerError;
+			errorLog(`[${tableName}] ${operation} failed: ${errorMessage}${details ? ` - ${safeJsonStringify(details)}` : ''}`);
+		},
+	};
+}

package/src/vtab/memory/utils/primary-key.ts

@@ -0,0 +1,158 @@
+import type { TableSchema, PrimaryKeyColumnDefinition } from '../../../schema/table.js';
+import type { Row, SqlValue } from '../../../common/types.js';
+import type { BTreeKeyForPrimary } from '../types.js';
+import { compareSqlValuesFast, resolveCollation, type CollationFunction } from '../../../util/comparison.js';
+import { QuereusError } from '../../../common/errors.js';
+import { StatusCode } from '../../../common/types.js';
+
+/**
+ * Result of creating primary key functions for a given schema
+ */
+export interface PrimaryKeyFunctions {
+	extractFromRow: (row: Row) => BTreeKeyForPrimary;
+	compare: (a: BTreeKeyForPrimary, b: BTreeKeyForPrimary) => number;
+}
+
+/**
+ * Creates optimized primary key extraction and comparison functions for a given table schema.
+ * This centralizes the logic that was previously duplicated across BaseLayer and TransactionLayer.
+ */
+export function createPrimaryKeyFunctions(schema: TableSchema): PrimaryKeyFunctions {
+	const pkDefinition = schema.primaryKeyDefinition
+		// Use all columns if no primary key is defined (that's different from an empty primary key)
+		// This is an important design change and documented deviation from SQLite behavior, and not something we want to change
+		?? schema.columns.map((col, index) => ({ index, collation: col.collation || 'BINARY' }));
+
+	if (pkDefinition.length === 0) {
+		return createSingletonPrimaryKeyFunctions();
+	} else if (pkDefinition.length === 1) {
+		return createSingleColumnPrimaryKeyFunctions(pkDefinition[0]);
+	} else {
+		return createCompositeColumnPrimaryKeyFunctions(pkDefinition);
+	}
+}
+
+/**
+ * Creates functions for tables with empty primary keys (zero or one rows possible)
+ */
+function createSingletonPrimaryKeyFunctions(): PrimaryKeyFunctions {
+	return {
+		extractFromRow: (): BTreeKeyForPrimary => {
+			return [];
+		},
+		compare: (): number => {
+			return 0;	// Always equal
+		}
+	};
+}
+
+/**
+ * Creates functions for single-column primary keys (optimized path)
+ */
+function createSingleColumnPrimaryKeyFunctions(
+	columnDef: PrimaryKeyColumnDefinition
+): PrimaryKeyFunctions {
+	const pkColIndex = columnDef.index;
+	const collation = columnDef.collation || 'BINARY';
+	const descMultiplier = columnDef.desc ? -1 : 1;
+
+	// Pre-resolve collation function for optimal performance
+	const collationFunc = resolveCollation(collation);
+
+	const extractFromRow = (row: Row): BTreeKeyForPrimary => {
+		if (!row || !Array.isArray(row)) {
+			throw new QuereusError(
+				`Primary key extraction requires a valid row array, got: ${typeof row}`,
+				StatusCode.INTERNAL
+			);
+		}
+		if (pkColIndex < 0 || pkColIndex >= row.length) {
+			throw new QuereusError(
+				`PK index ${pkColIndex} is out of bounds for row length ${row.length}`,
+				StatusCode.INTERNAL
+			);
+		}
+		return row[pkColIndex];
+	};
+
+	const compare = (a: BTreeKeyForPrimary, b: BTreeKeyForPrimary): number => {
+		return compareSqlValuesFast(a as SqlValue, b as SqlValue, collationFunc) * descMultiplier;
+	};
+
+	return { extractFromRow, compare };
+}
+
+/**
+ * Creates functions for composite (multi-column) primary keys
+ */
+function createCompositeColumnPrimaryKeyFunctions(
+	pkDefinition: ReadonlyArray<PrimaryKeyColumnDefinition>
+): PrimaryKeyFunctions {
+	// Pre-resolve all collation functions for optimal performance
+	const collationFuncs: CollationFunction[] = pkDefinition.map(def =>
+		resolveCollation(def.collation || 'BINARY')
+	);
+
+	const extractFromRow = (row: Row): BTreeKeyForPrimary => {
+		if (!row || !Array.isArray(row)) {
+			throw new QuereusError(
+				`Primary key extraction requires a valid row array, got: ${typeof row}`,
+				StatusCode.INTERNAL
+			);
+		}
+		return pkDefinition.map(def => {
+			if (def.index < 0 || def.index >= row.length) {
+				throw new QuereusError(
+					`PK index ${def.index} is out of bounds for row length ${row.length}`,
+					StatusCode.INTERNAL
+				);
+			}
+			return row[def.index];
+		});
+	};
+
+	const compare = (a: BTreeKeyForPrimary, b: BTreeKeyForPrimary): number => {
+		const arrA = a as SqlValue[];
+		const arrB = b as SqlValue[];
+
+		for (let i = 0; i < pkDefinition.length; i++) {
+			if (i >= arrA.length || i >= arrB.length) {
+				return arrA.length - arrB.length;
+			}
+
+			const def = pkDefinition[i];
+			const comparison = compareSqlValuesFast(arrA[i], arrB[i], collationFuncs[i]);
+
+			if (comparison !== 0) {
+				return def.desc ? -comparison : comparison;
+			}
+		}
+
+		return 0;
+	};
+
+	return { extractFromRow, compare };
+}
+
+/**
+ * Builds a primary key from key values and a primary key definition.
+ * Used for constructing keys from old key values in UPDATE/DELETE operations.
+ */
+export function buildPrimaryKeyFromValues(
+	keyValues: Row,
+	pkDefinition: ReadonlyArray<PrimaryKeyColumnDefinition>
+): BTreeKeyForPrimary {
+	if (pkDefinition.length === 0) {
+		// Empty primary key definition means singleton table - return empty array
+		return [];
+	}
+
+	if (keyValues.length !== pkDefinition.length) {
+		throw new QuereusError(
+			`Key value count mismatch. Expected ${pkDefinition.length}, got ${keyValues.length}.`,
+			StatusCode.INTERNAL
+		);
+	}
+
+	return pkDefinition.length === 1 ? keyValues[0] : keyValues;
+}

package/src/vtab/module.ts

@@ -0,0 +1,140 @@
+import type { Database } from '../core/database.js'; // Assuming Database class exists
+import type { VirtualTable } from './table.js';
+import type { IndexInfo } from './index-info.js';
+import type { ColumnDef } from '../parser/ast.js'; // <-- Add parser AST import
+import type { TableSchema, IndexSchema } from '../schema/table.js'; // Add import for TableSchema and IndexSchema
+import type { BestAccessPlanRequest, BestAccessPlanResult } from './best-access-plan.js';
+
+/**
+ * Base interface for module-specific configuration passed to xCreate/xConnect.
+ * Modules should define their own interface extending this if they need options.
+ */
+// eslint-disable-next-line @typescript-eslint/no-empty-object-type
+export interface BaseModuleConfig {}
+
+/**
+ * Interface defining the methods for a virtual table module implementation.
+ * The module primarily acts as a factory for connection-specific VirtualTable instances.
+ *
+ * @template TTable The specific type of VirtualTable managed by this module.
+ * @template TConfig The type defining module-specific configuration options.
+ */
+export interface VirtualTableModule<
+	TTable extends VirtualTable,
+	TConfig extends BaseModuleConfig = BaseModuleConfig
+> {
+
+	/**
+	 * Creates the persistent definition of a virtual table.
+	 * Called by CREATE VIRTUAL TABLE to define schema and initialize storage.
+	 *
+	 * @param db The database connection
+	 * @param tableSchema The schema definition for the table being created
+	 * @returns The new VirtualTable instance
+	 * @throws QuereusError on failure
+	 */
+	xCreate(
+		db: Database,
+		tableSchema: TableSchema,
+	): TTable;
+
+	/**
+	 * Connects to an existing virtual table definition.
+	 * Called when the schema is loaded or a connection needs to interact with the table.
+	 *
+	 * @param db The database connection
+	 * @param pAux Client data passed during module registration
+	 * @param moduleName The name the module was registered with
+	 * @param schemaName The name of the database schema
+	 * @param tableName The name of the virtual table to connect to
+	 * @param options Module-specific configuration options from the original CREATE VIRTUAL TABLE
+	 * @returns The connection-specific VirtualTable instance
+	 * @throws QuereusError on failure
+	 */
+	xConnect(
+		db: Database,
+		pAux: unknown,
+		moduleName: string,
+		schemaName: string,
+		tableName: string,
+		options: TConfig
+	): TTable;
+
+	/**
+	 * Modern, type-safe access planning interface.
+	 * Preferred over xBestIndex for new implementations.
+	 *
+	 * @param db The database connection
+	 * @param tableInfo The schema information for the table being planned
+	 * @param request Planning request with constraints and requirements
+	 * @returns Access plan result describing the chosen strategy
+	 */
+	getBestAccessPlan?(
+		db: Database,
+		tableInfo: TableSchema,
+		request: BestAccessPlanRequest
+	): BestAccessPlanResult;
+
+	/**
+	 * Determines the best query plan for a given set of constraints and orderings.
+	 * This method MUST be synchronous for performance. It modifies the passed IndexInfo object.
+	 *
+	 * @deprecated Use getBestAccessPlan instead for better type safety and extensibility
+	 * @param db The database connection
+	 * @param tableInfo The schema information for the table being planned
+	 * @param indexInfo Input constraints/orderings and output plan details
+	 * @returns StatusCode.OK on success, or an error code
+	 */
+	xBestIndex(db: Database, tableInfo: TableSchema, indexInfo: IndexInfo): number;
+
+	/**
+	 * Destroys the underlying persistent representation of the virtual table.
+	 * Called by DROP TABLE.
+	 *
+	 * @param db The database connection
+	 * @param pAux Client data passed during module registration
+	 * @param moduleName The name the module was registered with
+	 * @param schemaName The name of the database schema
+	 * @param tableName The name of the virtual table being destroyed
+	 * @throws QuereusError on failure
+	 */
+	xDestroy(
+		db: Database,
+		pAux: unknown,
+		moduleName: string,
+		schemaName: string,
+		tableName: string
+	): Promise<void>;
+
+	/**
+	 * Creates an index on a virtual table.
+	 * Called by CREATE INDEX.
+	 *
+	 * @param db The database connection
+	 * @param schemaName The name of the database schema
+	 * @param tableName The name of the virtual table
+	 * @param indexSchema The schema definition for the index being created
+	 * @throws QuereusError on failure
+	 */
+	xCreateIndex?(
+		db: Database,
+		schemaName: string,
+		tableName: string,
+		indexSchema: IndexSchema
+	): Promise<void>;
+
+	/**
+	 * Checks for shadow table name conflicts
+	 * @param name The name to check
+	 * @returns true if the name would conflict
+	 */
+	xShadowName?(name: string): boolean;
+}
+
+/**
+ * Defines the structure for schema change information passed to xAlterSchema
+ */
+export type SchemaChangeInfo =
+	| { type: 'addColumn'; columnDef: ColumnDef }
+	| { type: 'dropColumn'; columnName: string }
+	| { type: 'renameColumn'; oldName: string; newName: string; newColumnDefAst?: ColumnDef };