@quereus/quereus 0.1.0

package/src/vtab/memory/layer/safe-iterate.ts

@@ -0,0 +1,49 @@
+import type { BTree } from 'inheritree';
+
+/**
+ * A mutation-safe iterator for BTree that automatically handles tree mutations
+ * by storing the current key and reopening the path when needed.
+ */
+export async function* safeIterate<TKey, TValue>(tree: BTree<TKey, TValue>, isAscending: boolean, startKey?: { value: TKey }): AsyncIterable<TValue> {
+	let currentKey: TKey | undefined;
+
+	// Start iteration
+	let path = startKey ? tree.find(startKey.value) : isAscending ? tree.first() : tree.last();
+	moveNearest(tree, isAscending, path);
+
+	while (path.on) {
+		const entry = tree.at(path);
+		// Store the current key for potential path recovery
+		currentKey = tree.keyFromEntry(entry!);
+		yield entry!;
+
+		// Check if path is still valid before advancing
+		if (!tree.isValid(path)) {
+			path = tree.find(currentKey);
+			// Don't move nearest; the following moveNext/movePrior will move us off of the "nearest crack" if we failed to find the exact key
+		}
+
+		// Try to advance to the next position
+		if (isAscending) {
+			tree.moveNext(path);
+		} else {
+			tree.movePrior(path);
+		}
+	}
+}
+
+/* Attempts to move the path to the nearest valid position if not found - preferring the direction of the iteration */
+function moveNearest<TKey, TValue>(tree: BTree<TKey, TValue>, isAscending: boolean, path: any): void {
+	if (!path.on) {
+		// If not found, the path will point to the closest "crack".  All we need to do is move off of it in the correct direction
+		if (isAscending) {
+			tree.moveNext(path);
+			// In case we're at the end, move prior
+			if (!path.on) tree.movePrior(path);
+		} else {
+			tree.movePrior(path);
+			// In case we're at the beginning, move next
+			if (!path.on) tree.moveNext(path);
+		}
+	}
+}

package/src/vtab/memory/layer/scan-plan.ts

@@ -0,0 +1,84 @@
+import type { IndexConstraintOp } from '../../../common/constants.js';
+import type { SqlValue } from '../../../common/types.js';
+import type { BTreeKey } from '../types.js';
+import type { FilterInfo } from '../../filter-info.js';
+import type { TableSchema } from '../../../schema/table.js';
+import { IndexConstraintOp as ActualIndexConstraintOp } from '../../../common/constants.js';
+
+/** Describes an equality constraint for a scan plan */
+export interface ScanPlanEqConstraint {
+	op: IndexConstraintOp.EQ;
+	value: BTreeKey; // Can be composite for multi-column EQ
+}
+
+/** Describes a range bound for a scan plan */
+export interface ScanPlanRangeBound {
+	op: IndexConstraintOp.GT | IndexConstraintOp.GE | IndexConstraintOp.LT | IndexConstraintOp.LE;
+	value: SqlValue; // Range bounds typically apply to the first column
+}
+
+/**
+ * Encapsulates the details needed to execute a scan across layers.
+ * Derived from IndexInfo during xBestIndex/xFilter.
+ */
+export interface ScanPlan {
+	/** Name of the index to scan ('primary' or secondary index name) */
+	indexName: string | 'primary';
+	/** Scan direction */
+	descending: boolean;
+	/** Specific key for an equality scan (used if planType is EQ) */
+	equalityKey?: BTreeKey;
+	/** Lower bound for a range scan (used if planType is RANGE_*) */
+	lowerBound?: ScanPlanRangeBound;
+	/** Upper bound for a range scan (used if planType is RANGE_*) */
+	upperBound?: ScanPlanRangeBound;
+	/** The original idxNum from xBestIndex, potentially useful for cursor logic */
+	idxNum?: number;
+	/** The original idxStr from xBestIndex, potentially useful for debugging */
+	idxStr?: string | null;
+
+	// Additional fields might be needed for complex filtering passed down
+	// e.g., remaining constraints not handled by index bounds/equality.
+	// remainingConstraints?: ReadonlyArray<{ constraint: IndexConstraint, value: SqlValue }>;
+}
+
+// Helper function (moved from MemoryTableCursor and adapted)
+export function buildScanPlanFromFilterInfo(filterInfo: FilterInfo, tableSchema: TableSchema): ScanPlan {
+	const { idxNum, idxStr, constraints, args, indexInfoOutput } = filterInfo;
+	let indexName: string | 'primary' = 'primary';
+	let descending = false;
+	let equalityKey: BTreeKey | undefined = undefined;
+	let lowerBound: ScanPlanRangeBound | undefined = undefined;
+	let upperBound: ScanPlanRangeBound | undefined = undefined;
+	const params = new Map<string, string>();
+	idxStr?.split(';').forEach(part => { const [key, value] = part.split('=', 2); if (key && value !== undefined) params.set(key, value); });
+	const idxNameMatch = params.get('idx')?.match(/^(.*?)\\((\\d+)\\)$/);
+	if (idxNameMatch) indexName = idxNameMatch[1] === '_primary_' ? 'primary' : idxNameMatch[1];
+	const planType = parseInt(params.get('plan') ?? '0', 10);
+	descending = params.get('ordCons') === 'DESC' || planType === 1 || planType === 4;
+	const argvMap = new Map<number, number>();
+	params.get('argvMap')?.match(/\\[(\\d+),(\\d+)\\]/g)?.forEach(m => { const p = m.match(/\\[(\\d+),(\\d+)\\]/); if (p) argvMap.set(parseInt(p[1]), parseInt(p[2])); });
+	const currentSchema = tableSchema;
+	const indexSchemaForPlan = indexName === 'primary' ? { name: '_primary_', columns: currentSchema.primaryKeyDefinition ?? [{ index: -1, desc: false, collation: 'BINARY' }] } : currentSchema.indexes?.find(i => i.name === indexName);
+	if (planType === 2) { // EQ Plan
+		if (indexName === 'primary' && args.length === 1 && argvMap.size === 1 && currentSchema.primaryKeyDefinition.length <=1 ) equalityKey = args[0];
+		else if (indexSchemaForPlan?.columns) {
+			const keyParts: SqlValue[] = []; let keyComplete = true;
+			for (const colSpec of indexSchemaForPlan.columns) {
+				let foundArg = false;
+				argvMap.forEach((constraintArrIdx, queryArgIdx) => { if (foundArg) return; const constraintInfo = indexInfoOutput.aConstraint[constraintArrIdx]; if (constraintInfo && constraintInfo.iColumn === colSpec.index && constraintInfo.op === ActualIndexConstraintOp.EQ) { keyParts.push(args[queryArgIdx - 1]); foundArg = true; } });
+				if (!foundArg) { for (const cInfo of constraints) { if(foundArg) break; if (cInfo.constraint.iColumn === colSpec.index && cInfo.constraint.op === ActualIndexConstraintOp.EQ && cInfo.argvIndex > 0) { keyParts.push(args[cInfo.argvIndex - 1]); foundArg = true;}}}
+				if (!foundArg) { keyComplete = false; break; }
+			}
+			if (keyComplete && keyParts.length > 0) equalityKey = keyParts.length === 1 && indexSchemaForPlan.columns.length === 1 ? keyParts[0] : keyParts;
+		}
+	} else if (planType === 3 || planType === 4) { // Range Scan
+		const firstPkColDef = indexSchemaForPlan?.columns?.[0];
+		if (firstPkColDef) {
+			const firstColSchemaIdx = firstPkColDef.index;
+			argvMap.forEach((constraintArrIdx, queryArgIdx) => { const cI = indexInfoOutput.aConstraint[constraintArrIdx]; if (cI && cI.iColumn === firstColSchemaIdx) { const v = args[queryArgIdx-1]; const op = cI.op; if (op === ActualIndexConstraintOp.GT || op === ActualIndexConstraintOp.GE) {if (!lowerBound || op > lowerBound.op) lowerBound = {value:v,op};} else if (op === ActualIndexConstraintOp.LT || op === ActualIndexConstraintOp.LE){if (!upperBound || op < upperBound.op) upperBound = {value:v,op};}}});
+			constraints.forEach(cInfo => { if (cInfo.constraint.iColumn === firstColSchemaIdx && cInfo.argvIndex > 0) { const v = args[cInfo.argvIndex-1]; const op = cInfo.constraint.op; if (op===ActualIndexConstraintOp.GT || op===ActualIndexConstraintOp.GE) {if(!lowerBound || op>lowerBound.op)lowerBound={value:v,op:op};} else if (op===ActualIndexConstraintOp.LT || op===ActualIndexConstraintOp.LE){if(!upperBound||op<upperBound.op)upperBound={value:v,op:op};}}});
+		}
+	}
+	return { indexName, descending, equalityKey, lowerBound, upperBound, idxNum, idxStr };
+}

package/src/vtab/memory/layer/transaction-cursor.ts

@@ -0,0 +1,162 @@
+import type { ScanPlan } from './scan-plan.js';
+import type { TransactionLayer } from './transaction.js';
+import type { BTreeKeyForPrimary, BTreeKeyForIndex } from '../types.js';
+import type { Row } from '../../../common/types.js';
+import { IndexConstraintOp } from '../../../common/constants.js';
+import { compareSqlValues } from '../../../util/comparison.js';
+import { QuereusError } from '../../../common/errors.js';
+import { safeIterate } from './safe-iterate.js';
+
+export async function* scanTransactionLayer(
+	layer: TransactionLayer,
+	plan: ScanPlan,
+	_parentIterable: AsyncIterable<Row>
+): AsyncIterable<Row> {
+	const tableSchema = layer.getSchema();
+
+	// General plan application check
+	const planAppliesToKey = (key: BTreeKeyForPrimary | BTreeKeyForIndex, isIndexKey: boolean): boolean => {
+		if (!plan) return true;
+
+		// For index keys, we need the comparator for that specific index
+		let comparator;
+		if (isIndexKey && plan.indexName !== 'primary') {
+			// We can get the comparator from the MemoryIndex, but this is simplified for now
+			// In practice, we'd need access to the MemoryIndex's compareKeys method
+			comparator = (a: any, b: any) => compareSqlValues(a, b); // Simplified
+		} else {
+			const { primaryKeyComparator } = layer.getPkExtractorsAndComparators(tableSchema);
+			comparator = primaryKeyComparator;
+		}
+
+		if (plan.equalityKey) {
+			return comparator(key, plan.equalityKey as any) === 0;
+		}
+
+		// For range checks on composite keys, this still simplifies to first column.
+		const keyForBoundComparison = Array.isArray(key) ? key[0] : key;
+
+		if (plan.lowerBound) {
+			const cmp = compareSqlValues(keyForBoundComparison, plan.lowerBound.value);
+			if (cmp < 0 || (cmp === 0 && plan.lowerBound.op === IndexConstraintOp.GT)) return false;
+		}
+		if (plan.upperBound) {
+			const cmp = compareSqlValues(keyForBoundComparison, plan.upperBound.value);
+			if (cmp > 0 || (cmp === 0 && plan.upperBound.op === IndexConstraintOp.LT)) return false;
+		}
+		return true;
+	};
+
+	if (plan.indexName === 'primary') {
+		// Primary key scan - much simpler with inherited BTrees
+		const primaryTree = layer.getModificationTree('primary');
+		if (!primaryTree) return;
+
+		const { primaryKeyExtractorFromRow } = layer.getPkExtractorsAndComparators(tableSchema);
+
+		// Use proper Inheritree API for iteration
+		const isAscending = !plan.descending;
+
+		if (plan.equalityKey) {
+			// For equality scans, just get the specific value
+			const value = primaryTree.get(plan.equalityKey as BTreeKeyForPrimary);
+			if (value) {
+				const row = value as Row;
+				const primaryKey = primaryKeyExtractorFromRow(row);
+				if (planAppliesToKey(primaryKey, false)) {
+					yield row;
+				}
+			}
+		} else {
+			// Determine start key for range scans
+			let startKey: { value: BTreeKeyForPrimary } | undefined;
+			if (plan.lowerBound) {
+				startKey = { value: plan.lowerBound.value as BTreeKeyForPrimary };
+			}
+
+			// Use mutation-safe iterator for full scans or range scans with range support
+			for await (const value of safeIterate(primaryTree, isAscending, startKey)) {
+				// With inheritree, deleted entries simply don't appear in iteration
+				// No need to check for deletion markers
+
+				const row = value as Row;
+				const primaryKey = primaryKeyExtractorFromRow(row);
+
+				// Apply plan filters
+				if (planAppliesToKey(primaryKey, false)) {
+					yield row;
+				} else {
+					// Early termination for ascending scans past upper bound
+					if (isAscending && plan.upperBound) {
+						const keyForComparison = Array.isArray(primaryKey) ? primaryKey[0] : primaryKey;
+						const cmp = compareSqlValues(keyForComparison, plan.upperBound.value);
+						if (cmp > 0 || (cmp === 0 && plan.upperBound.op === IndexConstraintOp.LT)) {
+							break;
+						}
+					}
+				}
+			}
+		}
+	} else {
+		// Secondary index scan - also simplified with inherited BTrees
+		const secondaryTree = layer.getSecondaryIndexTree(plan.indexName);
+		if (!secondaryTree) {
+			throw new QuereusError(`Secondary index ${plan.indexName} not found in TransactionLayer.`);
+		}
+
+		// Use proper Inheritree API for secondary index iteration
+		const isAscending = !plan.descending;
+
+		if (plan.equalityKey) {
+			// For equality scans on secondary index
+			const indexEntry = secondaryTree.get(plan.equalityKey as BTreeKeyForIndex);
+			if (indexEntry && planAppliesToKey(indexEntry.indexKey, true)) {
+				// Get the primary tree to fetch actual rows
+				const primaryTree = layer.getModificationTree('primary');
+				if (primaryTree) {
+					for (const pk of indexEntry.primaryKeys) {
+						const value = primaryTree.get(pk);
+						if (value) {
+							const row = value as Row;
+							yield row;
+						}
+					}
+				}
+			}
+		} else {
+			// Determine start key for range scans on secondary index
+			let startKey: { value: BTreeKeyForIndex } | undefined;
+			if (plan.lowerBound) {
+				startKey = { value: plan.lowerBound.value as BTreeKeyForIndex };
+			}
+
+			// Use mutation-safe iterator for full scans or range scans on secondary index with range support
+			for await (const indexEntry of safeIterate(secondaryTree, isAscending, startKey)) {
+				// Apply plan filters to the index key
+				if (planAppliesToKey(indexEntry.indexKey, true)) {
+					// Get the primary tree to fetch actual rows
+					const primaryTree = layer.getModificationTree('primary');
+					if (!primaryTree) continue;
+
+					// For each primary key in this index entry, fetch the row
+					for (const pk of indexEntry.primaryKeys) {
+						const value = primaryTree.get(pk);
+						if (value) {
+							const row = value as Row;
+							yield row;
+						}
+					}
+				} else {
+					// Early termination for ascending scans past upper bound
+					if (isAscending && plan.upperBound) {
+						const keyForComparison = Array.isArray(indexEntry.indexKey) ? indexEntry.indexKey[0] : indexEntry.indexKey;
+						const cmp = compareSqlValues(keyForComparison, plan.upperBound.value);
+						if (cmp > 0 || (cmp === 0 && plan.upperBound.op === IndexConstraintOp.LT)) {
+							break;
+						}
+					}
+				}
+			}
+		}
+	}
+}

package/src/vtab/memory/layer/transaction.ts

@@ -0,0 +1,229 @@
+import { BTree } from 'inheritree';
+import type { TableSchema } from '../../../schema/table.js';
+import { MemoryIndex } from '../index.js';
+import type { Row } from '../../../common/types.js';
+import type { BTreeKeyForPrimary, BTreeKeyForIndex, MemoryIndexEntry } from '../types.js';
+import type { Layer } from './interface.js';
+import { createLogger } from '../../../common/logger.js';
+import { createPrimaryKeyFunctions } from '../utils/primary-key.js';
+import { QuereusError } from '../../../common/errors.js';
+
+const log = createLogger('vtab:memory:layer:transaction');
+const warnLog = log.extend('warn');
+
+let transactionLayerCounter = 1000;
+
+/**
+ * Represents a set of modifications (inserts, updates, deletes) applied
+ * on top of a parent Layer using inherited BTrees with copy-on-write semantics.
+ * These layers are immutable once committed.
+ */
+export class TransactionLayer implements Layer {
+	private readonly layerId: number;
+	public readonly parentLayer: Layer;
+	private readonly tableSchemaAtCreation: TableSchema; // Schema when this layer was started
+
+	// Primary modifications BTree that inherits from parent
+	private primaryModifications: BTree<BTreeKeyForPrimary, Row>;
+
+	// Secondary index BTrees that inherit from parent's indexes
+	private secondaryIndexes: Map<string, MemoryIndex>;
+
+	private _isCommitted: boolean = false;
+
+	// Cache for BTree funcs to avoid recalculation
+	private btreeFuncsCacheForKeyExtraction: Map<string | 'primary', {
+		primaryKeyExtractorFromRow: (row: Row) => BTreeKeyForPrimary;
+		indexKeyExtractorFromRow?: (row: Row) => BTreeKeyForIndex;
+		primaryKeyComparator: (a: BTreeKeyForPrimary, b: BTreeKeyForPrimary) => number;
+		indexKeyComparator?: (a: BTreeKeyForIndex, b: BTreeKeyForIndex) => number;
+	}> = new Map();
+
+	constructor(parent: Layer) {
+		this.layerId = transactionLayerCounter++;
+		this.parentLayer = parent;
+		this.tableSchemaAtCreation = parent.getSchema(); // Schema is fixed at creation
+
+		// Initialize primary modifications BTree with parent's primary tree as base
+		const { primaryKeyExtractorFromRow, primaryKeyComparator } = this.getPkExtractorsAndComparators(this.tableSchemaAtCreation);
+		const btreeKeyFromValue = (value: Row): BTreeKeyForPrimary => {
+			const result = primaryKeyExtractorFromRow(value);
+			return result;
+		};
+
+		const parentPrimaryTree = parent.getModificationTree('primary');
+
+		this.primaryModifications = new BTree(
+			btreeKeyFromValue,
+			primaryKeyComparator,
+			parentPrimaryTree || undefined // Use parent's primary tree as base
+		);
+
+		// Initialize secondary indexes that inherit from parent's secondary indexes
+		this.secondaryIndexes = new Map();
+		this.initializeSecondaryIndexes();
+	}
+
+	private initializeSecondaryIndexes(): void {
+		const schema = this.tableSchemaAtCreation;
+		if (!schema.indexes) return;
+
+		for (const indexSchema of schema.indexes) {
+			const parentSecondaryTree = this.parentLayer.getSecondaryIndexTree?.(indexSchema.name);
+			// Create MemoryIndex with inherited BTree
+			const memoryIndex = new MemoryIndex(
+				indexSchema,
+				schema.columns,
+				parentSecondaryTree || undefined // Use parent's secondary index tree as base
+			);
+			this.secondaryIndexes.set(indexSchema.name, memoryIndex);
+		}
+	}
+
+	getLayerId(): number {
+		return this.layerId;
+	}
+
+	getParent(): Layer {
+		return this.parentLayer;
+	}
+
+	getSchema(): TableSchema {
+		// Return the schema as it was when this transaction started
+		return this.tableSchemaAtCreation;
+	}
+
+	isCommitted(): boolean {
+		return this._isCommitted;
+	}
+
+	/** Marks this layer as committed. Should only be done by MemoryTable. */
+	markCommitted(): void {
+		if (!this._isCommitted) {
+			this._isCommitted = true;
+			// With inherited BTrees, we don't need to freeze complex change tracking structures
+		}
+	}
+
+	public getPkExtractorsAndComparators(schema: TableSchema): {
+		primaryKeyExtractorFromRow: (row: Row) => BTreeKeyForPrimary;
+		primaryKeyComparator: (a: BTreeKeyForPrimary, b: BTreeKeyForPrimary) => number
+	} {
+		if (schema !== this.tableSchemaAtCreation) {
+			warnLog("TransactionLayer.getPkExtractorsAndComparators called with a schema different from its creation schema. Using creation schema.");
+		}
+
+		// Use the centralized primary key functions instead of duplicating the logic
+		// This ensures consistent handling of empty primary key definitions
+		const pkFunctions = createPrimaryKeyFunctions(this.tableSchemaAtCreation);
+		return {
+			primaryKeyExtractorFromRow: pkFunctions.extractFromRow,
+			primaryKeyComparator: pkFunctions.compare
+		};
+	}
+
+	getModificationTree(indexName: string | 'primary'): BTree<BTreeKeyForPrimary, Row> | null {
+		if (indexName === 'primary') return this.primaryModifications;
+		return null; // Secondary indexes are accessed via getSecondaryIndexTree
+	}
+
+	getSecondaryIndexTree(indexName: string): BTree<BTreeKeyForIndex, MemoryIndexEntry> | null {
+		return this.secondaryIndexes.get(indexName)?.data ?? null;
+	}
+
+	/** Records an insert or update in this transaction layer */
+	recordUpsert(primaryKey: BTreeKeyForPrimary, newRowData: Row, oldRowDataIfUpdate?: Row | null): void {
+		if (this._isCommitted) throw new QuereusError("Cannot modify a committed layer");
+
+		this.primaryModifications.upsert(newRowData);
+
+		// Update secondary indexes
+		const schema = this.getSchema();
+		if (schema.indexes) {
+			for (const indexSchema of schema.indexes) {
+				const memoryIndex = this.secondaryIndexes.get(indexSchema.name);
+				if (!memoryIndex) continue;
+
+				if (oldRowDataIfUpdate) { // UPDATE
+					const oldIndexKey = memoryIndex.keyFromRow(oldRowDataIfUpdate);
+					const newIndexKey = memoryIndex.keyFromRow(newRowData);
+
+					// If index key changed, remove old and add new
+					if (memoryIndex.compareKeys(oldIndexKey, newIndexKey) !== 0) {
+						memoryIndex.removeEntry(oldIndexKey, primaryKey);
+						memoryIndex.addEntry(newIndexKey, primaryKey);
+					} else {
+						// Index key is same, but we might need to update the entry
+						// With inherited BTrees, the existing entry will be copied on write
+						memoryIndex.addEntry(newIndexKey, primaryKey);
+					}
+				} else { // INSERT
+					const newIndexKey = memoryIndex.keyFromRow(newRowData);
+					memoryIndex.addEntry(newIndexKey, primaryKey);
+				}
+			}
+		}
+	}
+
+	/** Records a delete in this transaction layer */
+	recordDelete(primaryKey: BTreeKeyForPrimary, oldRowDataForIndexes: Row): void {
+		if (this._isCommitted) throw new QuereusError("Cannot modify a committed layer");
+
+		// Find the existing entry
+		const existingPath = this.primaryModifications.find(primaryKey);
+		if (existingPath.on) {
+			// Entry exists (locally or inherited) - use deleteAt to remove it
+			this.primaryModifications.deleteAt(existingPath);
+		}
+		// If key doesn't exist, there's nothing to delete - no deletion marker needed
+		// Inheritree's copy-on-write semantics handle this properly
+
+		// Update secondary indexes to remove entries
+		const schema = this.getSchema();
+		if (schema.indexes) {
+			for (const indexSchema of schema.indexes) {
+				const memoryIndex = this.secondaryIndexes.get(indexSchema.name);
+				if (!memoryIndex) continue;
+
+				const oldIndexKey = memoryIndex.keyFromRow(oldRowDataForIndexes);
+				memoryIndex.removeEntry(oldIndexKey, primaryKey);
+			}
+		}
+	}
+
+	public hasChanges(): boolean {
+		// Check if primary modifications BTree has any entries beyond its base
+		if (this.primaryModifications.getCount() > 0) {
+			// Note: getCount() might include inherited entries, so we need a better way
+			// to check if this layer has modifications. This depends on inheritree's API.
+			// For now, assume any count > 0 means changes (might need refinement)
+			return true;
+		}
+
+		// Check secondary indexes for changes
+		for (const memoryIndex of this.secondaryIndexes.values()) {
+			if (memoryIndex.size > 0) {
+				// Same caveat as above - this might include inherited entries
+				return true;
+			}
+		}
+
+		return false;
+	}
+
+	/**
+	 * Detaches this layer's BTrees from their base, making them self-contained.
+	 * This should be called when the layer becomes the new effective base.
+	 */
+	public clearBase(): void {
+		// Clear base for primary modifications
+		if (typeof (this.primaryModifications as any).clearBase === 'function') {
+			(this.primaryModifications as any).clearBase();
+		}
+
+		// Clear base for secondary indexes
+		for (const memoryIndex of this.secondaryIndexes.values()) {
+			memoryIndex.clearBase();
+		}
+	}
+}