@quereus/quereus 0.1.0

package/src/vtab/memory/layer/base.ts

@@ -0,0 +1,273 @@
+import { BTree } from 'inheritree';
+import type { TableSchema } from '../../../schema/table.js';
+import type { BTreeKeyForPrimary, BTreeKeyForIndex, MemoryIndexEntry } from '../types.js';
+import type { Layer } from './interface.js';
+import { MemoryIndex } from '../index.js';
+import { safeJsonStringify } from '../../../util/serialization.js';
+import type { Row, SqlValue } from '../../../common/types.js';
+import { type ColumnSchema } from '../../../schema/column.js';
+import type { IndexSchema } from '../../../schema/table.js';
+import { createPrimaryKeyFunctions, type PrimaryKeyFunctions } from '../utils/primary-key.js';
+import { createMemoryTableLoggers } from '../utils/logging.js';
+
+let baseLayerCounter = 0;
+const logger = createMemoryTableLoggers('layer:base');
+
+export class BaseLayer implements Layer {
+	private readonly layerId: number;
+	public tableSchema: TableSchema;
+	private primaryKeyFunctions!: PrimaryKeyFunctions;
+	public primaryTree: BTree<BTreeKeyForPrimary, Row>;
+	public readonly secondaryIndexes: Map<string, MemoryIndex>;
+
+	constructor(schema: TableSchema) {
+		this.layerId = baseLayerCounter++;
+		this.tableSchema = schema;
+		this.initializePrimaryKeyFunctions();
+
+		// Use the same key extraction pattern as TransactionLayer for consistency
+		const btreeKeyFromValue = (value: Row): BTreeKeyForPrimary =>
+			this.primaryKeyFunctions.extractFromRow(value);
+
+		this.primaryTree = new BTree<BTreeKeyForPrimary, Row>(
+			btreeKeyFromValue,
+			this.primaryKeyFunctions.compare
+		);
+		this.secondaryIndexes = new Map();
+		this.rebuildAllSecondaryIndexes();
+	}
+
+	public updateSchema(newSchema: TableSchema): void {
+		logger.operation('Schema Update', this.tableSchema.name, {
+			from: this.tableSchema.name,
+			to: newSchema.name
+		});
+		this.tableSchema = newSchema;
+		this.initializePrimaryKeyFunctions();
+	}
+
+	private initializePrimaryKeyFunctions(): void {
+		this.primaryKeyFunctions = createPrimaryKeyFunctions(this.tableSchema);
+	}
+
+	public rebuildAllSecondaryIndexes(): void {
+		this.clearExistingSecondaryIndexes();
+
+		if (!this.hasSecondaryIndexes()) {
+			return;
+		}
+
+		const newIndexes = this.createSecondaryIndexes();
+		this.populateSecondaryIndexes(newIndexes);
+		this.replaceSecondaryIndexes(newIndexes);
+	}
+
+	private clearExistingSecondaryIndexes(): void {
+		this.secondaryIndexes.forEach(index => index.clear());
+	}
+
+	private hasSecondaryIndexes(): boolean {
+		return Boolean(this.tableSchema.indexes && this.tableSchema.indexes.length > 0);
+	}
+
+	private createSecondaryIndexes(): Map<string, MemoryIndex> {
+		const newIndexes = new Map<string, MemoryIndex>();
+
+		for (const indexSchema of this.tableSchema.indexes!) {
+			try {
+				const memoryIndex = new MemoryIndex(indexSchema, this.tableSchema.columns);
+				newIndexes.set(indexSchema.name, memoryIndex);
+			} catch (e: any) {
+				logger.error('Create Index', this.tableSchema.name, e, { indexName: indexSchema.name });
+			}
+		}
+
+		return newIndexes;
+	}
+
+	private populateSecondaryIndexes(newIndexes: Map<string, MemoryIndex>): void {
+		for (const path of this.primaryTree.ascending(this.primaryTree.first())) {
+			const row = this.primaryTree.at(path)!;
+			this.addRowToSecondaryIndexes(row, newIndexes);
+		}
+	}
+
+	private addRowToSecondaryIndexes(row: Row, indexes: Map<string, MemoryIndex>): void {
+		const primaryKey = this.primaryKeyFunctions.extractFromRow(row);
+
+		indexes.forEach(index => {
+			try {
+				const indexKey = index.keyFromRow(row);
+				index.addEntry(indexKey, primaryKey);
+			} catch (e: any) {
+				logger.error('Re-index Row', this.tableSchema.name, e, { indexName: index.name });
+			}
+		});
+	}
+
+	private replaceSecondaryIndexes(newIndexes: Map<string, MemoryIndex>): void {
+		this.secondaryIndexes.clear();
+		newIndexes.forEach((idx, name) => this.secondaryIndexes.set(name, idx));
+	}
+
+	getLayerId = (): number => this.layerId;
+	getParent = (): Layer | null => null;
+	getSchema = (): TableSchema => this.tableSchema;
+	isCommitted = (): boolean => true;
+
+	getModificationTree = (indexName: string | 'primary'): BTree<BTreeKeyForPrimary, Row> | null =>
+		indexName === 'primary' ? this.primaryTree : null;
+
+	getSecondaryIndexTree = (indexName: string): BTree<BTreeKeyForIndex, MemoryIndexEntry> | null =>
+		this.secondaryIndexes.get(indexName)?.data ?? null;
+
+	public getPkExtractorsAndComparators(schema: TableSchema): {
+		primaryKeyExtractorFromRow: (row: Row) => BTreeKeyForPrimary;
+		primaryKeyComparator: (a: BTreeKeyForPrimary, b: BTreeKeyForPrimary) => number
+	} {
+		if (schema !== this.tableSchema) {
+			logger.warn('PK Extractors', this.tableSchema.name, 'Called with different schema');
+		}
+		return {
+			primaryKeyExtractorFromRow: this.primaryKeyFunctions.extractFromRow,
+			primaryKeyComparator: this.primaryKeyFunctions.compare
+		};
+	}
+
+	applyChange(
+		primaryKey: BTreeKeyForPrimary,
+		modification: Row,
+		secondaryIndexChanges: Map<string, Array<{op: 'ADD' | 'DELETE', indexKey: BTreeKeyForIndex}>>
+	): void {
+		this.applySecondaryIndexChanges(primaryKey, secondaryIndexChanges);
+		this.applyPrimaryChange(primaryKey, modification);
+	}
+
+	private applySecondaryIndexChanges(
+		primaryKey: BTreeKeyForPrimary,
+		secondaryIndexChanges: Map<string, Array<{op: 'ADD' | 'DELETE', indexKey: BTreeKeyForIndex}>>
+	): void {
+		secondaryIndexChanges.forEach((changes, indexName) => {
+			const memoryIndex = this.secondaryIndexes.get(indexName);
+			if (!memoryIndex) {
+				logger.warn('Apply Change', this.tableSchema.name, 'Secondary index not found', {
+					indexName,
+					primaryKey: safeJsonStringify(primaryKey)
+				});
+				return;
+			}
+
+			changes.forEach(change => {
+				if (change.op === 'DELETE') {
+					memoryIndex.removeEntry(change.indexKey, primaryKey);
+				} else {
+					memoryIndex.addEntry(change.indexKey, primaryKey);
+				}
+			});
+		});
+	}
+
+	private applyPrimaryChange(primaryKey: BTreeKeyForPrimary, modification: Row): void {
+		this.primaryTree.insert(modification);
+	}
+
+	has = (key: BTreeKeyForPrimary): boolean => {
+		const value = this.primaryTree.get(key);
+		return value !== undefined;
+	};
+
+	async addColumnToBase(newColumnSchema: ColumnSchema, defaultValue: SqlValue): Promise<void> {
+		logger.operation('Add Column', this.tableSchema.name, {
+			columnName: newColumnSchema.name,
+			defaultValue
+		});
+
+		const oldPrimaryTree = this.primaryTree;
+
+		// First, reinitialize primary key functions with the updated schema (which already includes the new column)
+		this.initializePrimaryKeyFunctions();
+
+		// Create new primary tree with the updated schema and migrate data
+		this.recreatePrimaryTreeWithNewColumn(oldPrimaryTree, defaultValue);
+
+		this.rebuildAllSecondaryIndexes();
+	}
+
+	private recreatePrimaryTreeWithNewColumn(
+		oldTree: BTree<BTreeKeyForPrimary, Row>,
+		defaultValue: SqlValue
+	): void {
+		// Use the updated primary key functions for the new tree
+		const btreeKeyFromValue = (value: Row): BTreeKeyForPrimary =>
+			this.primaryKeyFunctions.extractFromRow(value);
+
+		this.primaryTree = new BTree<BTreeKeyForPrimary, Row>(
+			btreeKeyFromValue,
+			this.primaryKeyFunctions.compare
+		);
+
+		for (const path of oldTree.ascending(oldTree.first())) {
+			const oldRow = oldTree.at(path)!;
+			const newRow = [...oldRow, defaultValue];
+			this.primaryTree.insert(newRow);
+		}
+	}
+
+	async dropColumnFromBase(columnIndexInOldSchema: number): Promise<void> {
+		logger.operation('Drop Column', this.tableSchema.name, {
+			columnIndex: columnIndexInOldSchema
+		});
+
+		const oldPrimaryTree = this.primaryTree;
+		this.recreatePrimaryTreeWithoutColumn(oldPrimaryTree, columnIndexInOldSchema);
+		await this.rebuildAllSecondaryIndexes();
+	}
+
+	private recreatePrimaryTreeWithoutColumn(oldTree: BTree<BTreeKeyForPrimary, Row>, columnIndex: number): void {
+		const btreeKeyFromValue = (value: Row): BTreeKeyForPrimary =>
+			this.primaryKeyFunctions.extractFromRow(value);
+
+		this.primaryTree = new BTree<BTreeKeyForPrimary, Row>(
+			btreeKeyFromValue,
+			this.primaryKeyFunctions.compare
+		);
+
+		for (const path of oldTree.ascending(oldTree.first())) {
+			const oldRow = oldTree.at(path)!;
+			const newRow = oldRow.filter((_, idx) => idx !== columnIndex);
+			this.primaryTree.insert(newRow);
+		}
+	}
+
+	async handleColumnRename(): Promise<void> {
+		logger.operation('Handle Column Rename', this.tableSchema.name);
+		await this.rebuildAllSecondaryIndexes();
+	}
+
+	async addIndexToBase(indexSchema: IndexSchema): Promise<void> {
+		logger.operation('Add Index', this.tableSchema.name, {
+			indexName: indexSchema.name
+		});
+
+		const newMemoryIndex = new MemoryIndex(indexSchema, this.tableSchema.columns);
+		this.populateNewIndex(newMemoryIndex);
+		this.secondaryIndexes.set(indexSchema.name, newMemoryIndex);
+	}
+
+	private populateNewIndex(newIndex: MemoryIndex): void {
+		for (const path of this.primaryTree.ascending(this.primaryTree.first())) {
+			const currentRow = this.primaryTree.at(path)!;
+			const indexKey = newIndex.keyFromRow(currentRow);
+			const primaryKey = this.primaryKeyFunctions.extractFromRow(currentRow);
+			newIndex.addEntry(indexKey, primaryKey);
+		}
+	}
+
+	async dropIndexFromBase(indexName: string): Promise<void> {
+		if (this.secondaryIndexes.delete(indexName)) {
+			logger.operation('Drop Index', this.tableSchema.name, { indexName });
+		} else {
+			logger.warn('Drop Index', this.tableSchema.name, 'Index not found', { indexName });
+		}
+	}
+}

package/src/vtab/memory/layer/connection.ts

@@ -0,0 +1,203 @@
+import type { Layer } from './interface.js';
+import { TransactionLayer } from './transaction.js';
+import type { MemoryTableManager } from './manager.js';
+import { createLogger } from '../../../common/logger.js';
+import { StatusCode, type Row } from '../../../common/types.js';
+import { quereusError } from '../../../common/errors.js';
+
+let connectionCounter = 0;
+const log = createLogger('vtab:memory:layer:connection');
+const warnLog = log.extend('warn');
+const debugLog = log;
+
+/**
+ * Represents the state of a single connection to a MemoryTable
+ * within the layer-based MVCC model.
+ */
+export class MemoryTableConnection {
+	public readonly connectionId: number;
+	public readonly tableManager: MemoryTableManager;
+	public readLayer: Layer;
+	public pendingTransactionLayer: TransactionLayer | null = null;
+	public explicitTransaction: boolean = false; // Track if transaction was explicitly started
+	private savepoints: Map<number, TransactionLayer> = new Map();
+
+	constructor(manager: MemoryTableManager, initialReadLayer: Layer) {
+		this.connectionId = connectionCounter++;
+		this.tableManager = manager;
+		this.readLayer = initialReadLayer;
+	}
+
+	/** Begins a transaction by marking explicitTransaction. The pending layer is created lazily on first mutation */
+	begin(): void {
+		if (this.pendingTransactionLayer) {
+			// Already in transaction – same SQLite semantics: BEGIN is a no-op
+			this.explicitTransaction = true; // upgrade auto txn to explicit
+			return;
+		}
+
+		// Do NOT create a TransactionLayer yet.  It will be created lazily by
+		// ensureTransactionLayer() on the first data-mutation, so its parent
+		// will always be the then-current committed layer.
+		this.explicitTransaction = true;
+
+		debugLog(`Connection %d: BEGIN (lazy layer creation)`, this.connectionId);
+	}
+
+	/** Commits the current transaction */
+	async commit(): Promise<void> {
+		if (!this.pendingTransactionLayer) {
+			// Commit without an active transaction is a no-op
+			return;
+		}
+
+		await this.tableManager.commitTransaction(this);
+		// commitTransaction handles updating connection state (readLayer, pendingTransactionLayer)
+		this.clearTransactionState();
+	}
+
+	/** Rolls back the current transaction */
+	rollback(): void {
+		if (!this.pendingTransactionLayer) {
+			// Rollback without an active transaction is a no-op
+			return;
+		}
+
+		// Reset readLayer to the current committed layer
+		// Important: We need to ensure we're reading from a clean state
+		this.readLayer = this.tableManager.currentCommittedLayer;
+
+		// Simply discard the pending layer
+		this.pendingTransactionLayer = null;
+		this.clearTransactionState();
+
+		debugLog(`Connection %d: Rolled back transaction, readLayer reset to ${this.readLayer.getLayerId()}`,
+			this.connectionId);
+	}
+
+	/** Helper method to clear transaction-related state */
+	private clearTransactionState(): void {
+		this.savepoints.clear();
+		this.explicitTransaction = false;
+	}
+
+	/** Creates a savepoint with the given identifier */
+	createSavepoint(savepointIndex: number): void {
+		if (savepointIndex < 0) {
+			quereusError(`Invalid savepoint index: ${savepointIndex}. Must be non-negative.`, StatusCode.INTERNAL);
+		}
+
+		if (!this.pendingTransactionLayer) {
+			// Start an implicit transaction and create a fresh layer immediately so the savepoint has something to snapshot
+			this.pendingTransactionLayer = new TransactionLayer(this.tableManager.currentCommittedLayer);
+			// Not marking explicitTransaction; this is still auto-txn unless caller ran BEGIN earlier.
+		}
+
+		if (!this.pendingTransactionLayer) {
+			quereusError(`Failed to create transaction for savepoint ${savepointIndex}`);
+		}
+
+		// Create a snapshot of the current transaction state
+		const savepointLayer = this.createTransactionSnapshot(this.pendingTransactionLayer);
+
+		// Store the snapshot as the savepoint
+		this.savepoints.set(savepointIndex, savepointLayer);
+
+		// A SAVEPOINT implicitly puts the connection into explicit-transaction mode
+		// so that subsequent statements do NOT auto-commit and invalidate the savepoint.
+		this.explicitTransaction = true;
+	}
+
+	/**
+	 * Creates a snapshot of a transaction layer by copying its effective data to a new independent layer.
+	 * This is necessary because BTree inheritance can cause shared mutable state issues.
+	 * The snapshot becomes immutable and independent of the source layer.
+	 */
+	private createTransactionSnapshot(sourceLayer: TransactionLayer): TransactionLayer {
+		// Create a new transaction layer based on the source layer's parent
+		// This ensures the snapshot is independent of the source layer
+		const snapshotLayer = new TransactionLayer(sourceLayer.getParent());
+
+		// Copy all data from the source layer to the snapshot
+		const primaryTree = sourceLayer.getModificationTree('primary');
+		if (!primaryTree) {
+			// Empty transaction layer - just return the empty snapshot
+			snapshotLayer.markCommitted();
+			return snapshotLayer;
+		}
+
+		const firstPath = primaryTree.first();
+		if (!firstPath.on) {
+			// No data in the tree - return the empty snapshot
+			snapshotLayer.markCommitted();
+			return snapshotLayer;
+		}
+
+		// Copy all rows from the source layer
+		const { primaryKeyExtractorFromRow } = sourceLayer.getPkExtractorsAndComparators(sourceLayer.getSchema());
+		for (const path of primaryTree.ascending(firstPath)) {
+			const row = primaryTree.at(path)!;
+			try {
+				// Extract primary key and record the row in the snapshot
+				const primaryKey = primaryKeyExtractorFromRow(row as Row);
+				snapshotLayer.recordUpsert(primaryKey, row as Row, null);
+			} catch (error) {
+				warnLog(`Connection %d: Failed to copy row to savepoint snapshot: %o`, this.connectionId, error);
+			}
+		}
+
+		// Mark the snapshot as committed to make it immutable
+		snapshotLayer.markCommitted();
+
+		return snapshotLayer;
+	}
+
+	/** Releases a savepoint and any higher-numbered savepoints */
+	releaseSavepoint(savepointIndex: number): void {
+		if (!this.pendingTransactionLayer) return; // No transaction, nothing to release
+
+		if (savepointIndex < 0) {
+			warnLog(`Connection %d: Invalid savepoint index %d for release.`, this.connectionId, savepointIndex);
+			return;
+		}
+
+		// Remove this savepoint and any with higher indices
+		for (const idx of Array.from(this.savepoints.keys())) {
+			if (idx >= savepointIndex) {
+				this.savepoints.delete(idx);
+			}
+		}
+		debugLog(`Connection %d: Released savepoint %d`, this.connectionId, savepointIndex);
+	}
+
+	/** Rolls back to a savepoint while preserving the transaction */
+	rollbackToSavepoint(savepointIndex: number): void {
+		if (!this.pendingTransactionLayer) return; // No transaction, nothing to rollback to
+
+		if (savepointIndex < 0) {
+			warnLog(`Connection %d: Invalid savepoint index %d for rollback.`, this.connectionId, savepointIndex);
+			return;
+		}
+
+		const savepoint = this.savepoints.get(savepointIndex);
+		if (!savepoint) {
+			warnLog(`Connection %d: Savepoint %d not found for rollback.`, this.connectionId, savepointIndex);
+			return;
+		}
+
+		// Restore the transaction layer to the savepoint state
+		// Instead of creating a new layer, we directly set the pending layer to the savepoint
+		this.pendingTransactionLayer = savepoint;
+
+		// Remove this savepoint and any with higher indices
+		for (const idx of Array.from(this.savepoints.keys())) {
+			if (idx >= savepointIndex) {
+				this.savepoints.delete(idx);
+			}
+		}
+	}
+
+	public clearSavepoints(): void {
+		this.savepoints.clear();
+	}
+}

package/src/vtab/memory/layer/interface.ts

@@ -0,0 +1,47 @@
+import type { BTree } from 'inheritree';
+import type { TableSchema } from '../../../schema/table.js';
+import type { BTreeKeyForPrimary, BTreeKeyForIndex, MemoryIndexEntry } from '../types.js';
+import type { Row } from '../../../common/types.js';
+
+/**
+ * Represents a snapshot or a set of changes in the MemoryTable MVCC model.
+ * Layers form a chain, starting from a BaseLayer.
+ */
+export interface Layer {
+	/** Returns the layer ID (unique identifier, potentially timestamp or sequence) */
+	getLayerId(): number;
+
+	/** Returns the parent layer in the chain, or null for the BaseLayer */
+	getParent(): Layer | null;
+
+	/**
+	 * Gets the BTree containing modifications specific to this layer for a given index.
+	 * For BaseLayer, this returns the main data BTree.
+	 * For TransactionLayer, this returns the inherited BTree for that index.
+	 *
+	 * @param indexName The name of the secondary index, or 'primary' for the primary key index.
+	 * @returns The BTree containing modifications/data for the index in this layer, or null if no modifications exist for that index in this layer.
+	 */
+	getModificationTree(indexName: string | 'primary'): BTree<BTreeKeyForPrimary, Row> | null;
+
+	/**
+	 * Returns the table schema as it existed when this layer was created or relevant.
+	 * This is important for interpreting modifications during layer collapse, especially
+	 * if schema changes occurred after this layer was created.
+	 */
+	getSchema(): TableSchema;
+
+	/** Indicates if this layer represents a committed transaction state */
+	isCommitted(): boolean;
+
+	/** Helper to get the specific BTree for a secondary index's underlying data */
+	getSecondaryIndexTree?(indexName: string): BTree<BTreeKeyForIndex, MemoryIndexEntry> | null;
+
+	/**
+	 * This method provides PK extractor and comparator based on a given schema (usually its own)
+	 */
+	getPkExtractorsAndComparators(schema: TableSchema): {
+		primaryKeyExtractorFromRow: (row: Row) => BTreeKeyForPrimary;
+		primaryKeyComparator: (a: BTreeKeyForPrimary, b: BTreeKeyForPrimary) => number;
+	};
+}