@quereus/quereus 0.6.12 → 0.7.1

package/src/schema/manager.ts

@@ -1,7 +1,7 @@
 import { Schema } from './schema.js';
 import type { IntegrityAssertionSchema } from './assertion.js';
 import type { Database } from '../core/database.js';
-import type { TableSchema, RowConstraintSchema, IndexSchema } from './table.js';
+import type { TableSchema, RowConstraintSchema, IndexSchema, IndexColumnSchema } from './table.js';
 import type { FunctionSchema } from './function.js';
 import { quereusError, QuereusError } from '../common/errors.js';
 import { StatusCode, type SqlValue } from '../common/types.js';
@@ -12,6 +12,7 @@ import { buildColumnIndexMap, columnDefToSchema, findPKDefinition, opsToMask, mu
 import type { ViewSchema } from './view.js';
 import { createLogger } from '../common/logger.js';
 import type * as AST from '../parser/ast.js';
+import { Parser } from '../parser/parser.js';
 import { SchemaChangeNotifier } from './change-events.js';
 import { checkDeterministic } from '../planner/validation/determinism-validator.js';
 import { buildExpression } from '../planner/building/expression.js';
@@ -797,4 +798,237 @@ export class SchemaManager {
 
 		return completeTableSchema;
 	}
+
+	/**
+	 * Import catalog objects from DDL statements without triggering storage creation.
+	 * Used when connecting to existing storage that already contains data.
+	 *
+	 * This method:
+	 * 1. Parses each DDL statement
+	 * 2. Registers the schema objects (tables, indexes)
+	 * 3. Calls module.connect() instead of module.create()
+	 * 4. Skips schema change hooks (since these are existing objects)
+	 *
+	 * @param ddlStatements Array of DDL strings (CREATE TABLE, CREATE INDEX, etc.)
+	 * @returns Array of imported object names
+	 */
+	async importCatalog(ddlStatements: string[]): Promise<{ tables: string[]; indexes: string[] }> {
+		const imported = { tables: [] as string[], indexes: [] as string[] };
+
+		for (const ddl of ddlStatements) {
+			try {
+				const result = await this.importSingleDDL(ddl);
+				if (result.type === 'table') {
+					imported.tables.push(result.name);
+				} else if (result.type === 'index') {
+					imported.indexes.push(result.name);
+				}
+			} catch (e) {
+				const message = e instanceof Error ? e.message : String(e);
+				errorLog('Failed to import DDL: %s - Error: %s', ddl.substring(0, 100), message);
+				throw e;
+			}
+		}
+
+		log('Imported catalog: %d tables, %d indexes', imported.tables.length, imported.indexes.length);
+		return imported;
+	}
+
+	/**
+	 * Import a single DDL statement without creating storage.
+	 */
+	private async importSingleDDL(ddl: string): Promise<{ type: 'table' | 'index'; name: string }> {
+		// Parse the DDL using the parser
+		const parser = new Parser();
+		const statements = parser.parseAll(ddl);
+		if (statements.length !== 1) {
+			throw new QuereusError(`importCatalog expects exactly one statement per DDL, got ${statements.length}`, StatusCode.ERROR);
+		}
+
+		const stmt = statements[0];
+
+		if (stmt.type === 'createTable') {
+			return this.importTable(stmt as AST.CreateTableStmt);
+		} else if (stmt.type === 'createIndex') {
+			return this.importIndex(stmt as AST.CreateIndexStmt);
+		} else {
+			throw new QuereusError(`importCatalog does not support statement type: ${stmt.type}`, StatusCode.ERROR);
+		}
+	}
+
+	/**
+	 * Import a table schema without calling module.create().
+	 * Uses module.connect() to bind to existing storage.
+	 */
+	private async importTable(stmt: AST.CreateTableStmt): Promise<{ type: 'table'; name: string }> {
+		const targetSchemaName = stmt.table.schema || this.getCurrentSchemaName();
+		const tableName = stmt.table.name;
+
+		let moduleName: string;
+		let effectiveModuleArgs: Record<string, SqlValue>;
+
+		if (stmt.moduleName) {
+			moduleName = stmt.moduleName;
+			effectiveModuleArgs = Object.freeze(stmt.moduleArgs || {});
+		} else {
+			const defaultVtab = this.getDefaultVTabModule();
+			moduleName = defaultVtab.name;
+			effectiveModuleArgs = Object.freeze(defaultVtab.args || {});
+		}
+
+		const moduleInfo = this.getModule(moduleName);
+		if (!moduleInfo || !moduleInfo.module) {
+			throw new QuereusError(`No virtual table module named '${moduleName}'`, StatusCode.ERROR);
+		}
+
+		// Get default nullability setting from database options
+		const defaultNullability = this.db.options.getStringOption('default_column_nullability');
+		const defaultNotNull = defaultNullability === 'not_null';
+
+		const astColumnsToProcess = stmt.columns || [];
+		const astConstraintsToProcess = stmt.constraints;
+
+		const preliminaryColumnSchemas: ColumnSchema[] = astColumnsToProcess.map(colDef => columnDefToSchema(colDef, defaultNotNull));
+		const pkDefinition = findPKDefinition(preliminaryColumnSchemas, astConstraintsToProcess);
+
+		const finalColumnSchemas = preliminaryColumnSchemas.map((col, idx) => {
+			const isPkColumn = pkDefinition.some(pkCol => pkCol.index === idx);
+			let pkOrder = 0;
+			if (isPkColumn) {
+				pkOrder = pkDefinition.findIndex(pkC => pkC.index === idx) + 1;
+			}
+			return {
+				...col,
+				primaryKey: isPkColumn,
+				pkOrder: pkOrder,
+				notNull: isPkColumn ? true : col.notNull,
+			};
+		});
+
+		const checkConstraintsSchema: RowConstraintSchema[] = [];
+		astColumnsToProcess.forEach(colDef => {
+			colDef.constraints?.forEach(con => {
+				if (con.type === 'check' && con.expr) {
+					checkConstraintsSchema.push({
+						name: con.name ?? `_check_${colDef.name}`,
+						expr: con.expr,
+						operations: opsToMask(con.operations),
+						deferrable: con.deferrable,
+						initiallyDeferred: con.initiallyDeferred
+					});
+				}
+			});
+		});
+		(astConstraintsToProcess || []).forEach(con => {
+			if (con.type === 'check' && con.expr) {
+				checkConstraintsSchema.push({
+					name: con.name,
+					expr: con.expr,
+					operations: opsToMask(con.operations),
+					deferrable: con.deferrable,
+					initiallyDeferred: con.initiallyDeferred
+				});
+			}
+		});
+
+		// Process mutation context definitions if present
+		const mutationContextSchemas = stmt.contextDefinitions
+			? stmt.contextDefinitions.map(varDef => mutationContextVarToSchema(varDef, defaultNotNull))
+			: undefined;
+
+		const tableSchema: TableSchema = {
+			name: tableName,
+			schemaName: targetSchemaName,
+			columns: Object.freeze(finalColumnSchemas),
+			columnIndexMap: buildColumnIndexMap(finalColumnSchemas),
+			primaryKeyDefinition: pkDefinition,
+			checkConstraints: Object.freeze(checkConstraintsSchema),
+			isTemporary: !!stmt.isTemporary,
+			isView: false,
+			vtabModuleName: moduleName,
+			vtabArgs: effectiveModuleArgs,
+			vtabModule: moduleInfo.module,
+			vtabAuxData: moduleInfo.auxData,
+			estimatedRows: 0,
+			mutationContext: mutationContextSchemas ? Object.freeze(mutationContextSchemas) : undefined,
+		};
+
+		// Use connect() instead of create() - the storage already exists
+		try {
+			moduleInfo.module.connect(
+				this.db,
+				moduleInfo.auxData,
+				moduleName,
+				targetSchemaName,
+				tableName,
+				effectiveModuleArgs as BaseModuleConfig,
+				tableSchema // Pass the full schema so the module can use it
+			);
+		} catch (e: unknown) {
+			const message = e instanceof Error ? e.message : String(e);
+			throw new QuereusError(`Module '${moduleName}' connect failed during import for table '${tableName}': ${message}`, StatusCode.ERROR);
+		}
+
+		// Ensure schema exists
+		let schema = this.getSchema(targetSchemaName);
+		if (!schema) {
+			schema = new Schema(targetSchemaName);
+			this.schemas.set(targetSchemaName.toLowerCase(), schema);
+		}
+
+		// Register without notifying change listeners (this is an import, not a create)
+		schema.addTable(tableSchema);
+		log(`Imported table %s.%s using module %s`, targetSchemaName, tableName, moduleName);
+
+		return { type: 'table', name: `${targetSchemaName}.${tableName}` };
+	}
+
+	/**
+	 * Import an index schema without calling module.createIndex().
+	 */
+	private async importIndex(stmt: AST.CreateIndexStmt): Promise<{ type: 'index'; name: string }> {
+		const targetSchemaName = stmt.table.schema || this.getCurrentSchemaName();
+		const tableName = stmt.table.name;
+		const indexName = stmt.index.name;
+
+		// Find the table
+		const tableSchema = this.findTable(tableName, targetSchemaName);
+		if (!tableSchema) {
+			throw new QuereusError(`Cannot import index '${indexName}': table '${tableName}' not found`, StatusCode.ERROR);
+		}
+
+		// Build index columns schema
+		const indexColumns: IndexColumnSchema[] = stmt.columns.map(col => {
+			const colName = col.name;
+			if (!colName) {
+				throw new QuereusError(`Expression-based index columns are not supported during import`, StatusCode.ERROR);
+			}
+			const colIdx = tableSchema.columnIndexMap.get(colName.toLowerCase());
+			if (colIdx === undefined) {
+				throw new QuereusError(`Column '${colName}' not found in table '${tableName}'`, StatusCode.ERROR);
+			}
+			return {
+				index: colIdx,
+				desc: col.direction === 'desc',
+			};
+		});
+
+		const indexSchema: IndexSchema = {
+			name: indexName,
+			columns: Object.freeze(indexColumns),
+		};
+
+		// Add index to table without calling module.createIndex()
+		const updatedIndexes = [...(tableSchema.indexes || []), indexSchema];
+		const updatedTableSchema: TableSchema = {
+			...tableSchema,
+			indexes: Object.freeze(updatedIndexes),
+		};
+
+		const schema = this.getSchemaOrFail(targetSchemaName);
+		schema.addTable(updatedTableSchema);
+		log(`Imported index %s on table %s.%s`, indexName, targetSchemaName, tableName);
+
+		return { type: 'index', name: `${targetSchemaName}.${tableName}.${indexName}` };
+	}
 }

package/src/vtab/best-access-plan.ts

@@ -85,6 +85,8 @@ export interface BestAccessPlanResult {
 	rows: number | undefined;
 	/** Ordering guaranteed by this access plan */
 	providesOrdering?: readonly OrderingSpec[];
+	/** Name of the index that provides the ordering (if any) */
+	orderingIndexName?: string;
 	/** Whether this plan guarantees unique rows (helps DISTINCT optimization) */
 	isSet?: boolean;
 	/** Free-text explanation for debugging */

package/src/vtab/memory/layer/scan-plan.ts

@@ -52,12 +52,12 @@ export function buildScanPlanFromFilterInfo(filterInfo: FilterInfo, tableSchema:
 	let upperBound: ScanPlanRangeBound | undefined = undefined;
 	const params = new Map<string, string>();
 	idxStr?.split(';').forEach(part => { const [key, value] = part.split('=', 2); if (key && value !== undefined) params.set(key, value); });
-	const idxNameMatch = params.get('idx')?.match(/^(.*?)\\((\\d+)\\)$/);
+	const idxNameMatch = params.get('idx')?.match(/^(.*?)\((\d+)\)$/);
 	if (idxNameMatch) indexName = idxNameMatch[1] === '_primary_' ? 'primary' : idxNameMatch[1];
 	const planType = parseInt(params.get('plan') ?? '0', 10);
 	descending = params.get('ordCons') === 'DESC' || planType === 1 || planType === 4;
 	const argvMap = new Map<number, number>();
-	params.get('argvMap')?.match(/\\[(\\d+),(\\d+)\\]/g)?.forEach(m => { const p = m.match(/\\[(\\d+),(\\d+)\\]/); if (p) argvMap.set(parseInt(p[1]), parseInt(p[2])); });
+	params.get('argvMap')?.match(/\[(\d+),(\d+)\]/g)?.forEach(m => { const p = m.match(/\[(\d+),(\d+)\]/); if (p) argvMap.set(parseInt(p[1]), parseInt(p[2])); });
 	const currentSchema = tableSchema;
 	const indexSchemaForPlan = indexName === 'primary' ? { name: '_primary_', columns: currentSchema.primaryKeyDefinition ?? [{ index: -1, desc: false, collation: 'BINARY' }] } : currentSchema.indexes?.find(i => i.name === indexName);
 	if (planType === 2) { // EQ Plan

package/src/vtab/memory/module.ts

@@ -58,7 +58,7 @@ export class MemoryTableModule implements VirtualTableModule<MemoryTable, Memory
 	/**
 	 * Connects to an existing memory table definition
 	 */
-	connect(db: Database, pAux: unknown, moduleName: string, schemaName: string, tableName: string, _options: MemoryTableConfig): MemoryTable {
+	connect(db: Database, pAux: unknown, moduleName: string, schemaName: string, tableName: string, _options: MemoryTableConfig, _tableSchema?: TableSchema): MemoryTable {
 		const tableKey = `${schemaName}.${tableName}`.toLowerCase();
 		const existingManager = this.tables.get(tableKey);
 
@@ -272,6 +272,7 @@ export class MemoryTableModule implements VirtualTableModule<MemoryTable, Memory
 					...plan,
 					cost: adjustedCost,
 					providesOrdering: request.requiredOrdering,
+					orderingIndexName: index.name,
 					explains: `${plan.explains} with ordering from ${index.name}`
 				};
 			}

package/src/vtab/module.ts

@@ -1,160 +1,162 @@
-import type { Database } from '../core/database.js'; // Assuming Database class exists
-import type { VirtualTable } from './table.js';
-import type { IndexInfo } from './index-info.js';
-import type { ColumnDef } from '../parser/ast.js'; // <-- Add parser AST import
-import type { TableSchema, IndexSchema } from '../schema/table.js'; // Add import for TableSchema and IndexSchema
-import type { BestAccessPlanRequest, BestAccessPlanResult } from './best-access-plan.js';
-import type { PlanNode } from '../planner/nodes/plan-node.js';
-
-/**
- * Base interface for module-specific configuration passed to create/connect.
- * Modules should define their own interface extending this if they need options.
- */
-// eslint-disable-next-line @typescript-eslint/no-empty-object-type
-export interface BaseModuleConfig {}
-
-/**
- * Assessment result from a module's supports() method indicating
- * whether it can execute a plan subtree and at what cost.
- */
-export interface SupportAssessment {
-	/** Estimated cost comparable to local evaluation cost */
-	cost: number;
-	/** Optional context data persisted for the emitter */
-	ctx?: unknown;
-}
-
-/**
- * Interface defining the methods for a virtual table module implementation.
- * The module primarily acts as a factory for connection-specific VirtualTable instances.
- *
- * @template TTable The specific type of VirtualTable managed by this module.
- * @template TConfig The type defining module-specific configuration options.
- */
-export interface VirtualTableModule<
-	TTable extends VirtualTable,
-	TConfig extends BaseModuleConfig = BaseModuleConfig
-> {
-
-	/**
-	 * Creates the persistent definition of a virtual table.
-	 * Called by CREATE VIRTUAL TABLE to define schema and initialize storage.
-	 *
-	 * @param db The database connection
-	 * @param tableSchema The schema definition for the table being created
-	 * @returns The new VirtualTable instance
-	 * @throws QuereusError on failure
-	 */
-	create(
-		db: Database,
-		tableSchema: TableSchema,
-	): TTable;
-
-	/**
-	 * Connects to an existing virtual table definition.
-	 * Called when the schema is loaded or a connection needs to interact with the table.
-	 *
-	 * @param db The database connection
-	 * @param pAux Client data passed during module registration
-	 * @param moduleName The name the module was registered with
-	 * @param schemaName The name of the database schema
-	 * @param tableName The name of the virtual table to connect to
-	 * @param options Module-specific configuration options from the original CREATE VIRTUAL TABLE
-	 * @returns The connection-specific VirtualTable instance
-	 * @throws QuereusError on failure
-	 */
-	connect(
-		db: Database,
-		pAux: unknown,
-		moduleName: string,
-		schemaName: string,
-		tableName: string,
-		options: TConfig
-	): TTable;
-
-	/**
-	 * Determines if this module can execute a plan subtree starting at the given node.
-	 * Used for query push-down to virtual table modules that support arbitrary queries.
-	 *
-	 * @param node The root node of the subtree to evaluate
-	 * @returns Assessment with cost and optional context, or undefined if not supported
-	 */
-	supports?(
-		node: PlanNode
-	): SupportAssessment | undefined;
-
-	/**
-	 * Modern, type-safe access planning interface.
-	 * Preferred over xBestIndex for new implementations.
-	 *
-	 * @param db The database connection
-	 * @param tableInfo The schema information for the table being planned
-	 * @param request Planning request with constraints and requirements
-	 * @returns Access plan result describing the chosen strategy
-	 */
-	getBestAccessPlan?(
-		db: Database,
-		tableInfo: TableSchema,
-		request: BestAccessPlanRequest
-	): BestAccessPlanResult;
-
-
-
-	/**
-	 * Destroys the underlying persistent representation of the virtual table.
-	 * Called by DROP TABLE.
-	 *
-	 * @param db The database connection
-	 * @param pAux Client data passed during module registration
-	 * @param moduleName The name the module was registered with
-	 * @param schemaName The name of the database schema
-	 * @param tableName The name of the virtual table being destroyed
-	 * @throws QuereusError on failure
-	 */
-	destroy(
-		db: Database,
-		pAux: unknown,
-		moduleName: string,
-		schemaName: string,
-		tableName: string
-	): Promise<void>;
-
-	/**
-	 * Creates an index on a virtual table.
-	 * Called by CREATE INDEX.
-	 *
-	 * @param db The database connection
-	 * @param schemaName The name of the database schema
-	 * @param tableName The name of the virtual table
-	 * @param indexSchema The schema definition for the index being created
-	 * @throws QuereusError on failure
-	 */
-	createIndex?(
-		db: Database,
-		schemaName: string,
-		tableName: string,
-		indexSchema: IndexSchema
-	): Promise<void>;
-
-	/**
-	 * Checks for shadow table name conflicts
-	 * @param name The name to check
-	 * @returns true if the name would conflict
-	 */
-	shadowName?(name: string): boolean;
-}
-
-/**
- * Defines the structure for schema change information passed to xAlterSchema
- */
-export type SchemaChangeInfo =
-	| { type: 'addColumn'; columnDef: ColumnDef }
-	| { type: 'dropColumn'; columnName: string }
-	| { type: 'renameColumn'; oldName: string; newName: string; newColumnDefAst?: ColumnDef };
-
-/**
- * Type alias for the common usage pattern where specific table and config types are not known.
- * Use this for storage scenarios like the SchemaManager where modules of different types are stored together.
- * eslint-disable-next-line @typescript-eslint/no-explicit-any
- */
-export type AnyVirtualTableModule = VirtualTableModule<any, any>;
+import type { Database } from '../core/database.js'; // Assuming Database class exists
+import type { VirtualTable } from './table.js';
+import type { IndexInfo } from './index-info.js';
+import type { ColumnDef } from '../parser/ast.js'; // <-- Add parser AST import
+import type { TableSchema, IndexSchema } from '../schema/table.js'; // Add import for TableSchema and IndexSchema
+import type { BestAccessPlanRequest, BestAccessPlanResult } from './best-access-plan.js';
+import type { PlanNode } from '../planner/nodes/plan-node.js';
+
+/**
+ * Base interface for module-specific configuration passed to create/connect.
+ * Modules should define their own interface extending this if they need options.
+ */
+// eslint-disable-next-line @typescript-eslint/no-empty-object-type
+export interface BaseModuleConfig {}
+
+/**
+ * Assessment result from a module's supports() method indicating
+ * whether it can execute a plan subtree and at what cost.
+ */
+export interface SupportAssessment {
+	/** Estimated cost comparable to local evaluation cost */
+	cost: number;
+	/** Optional context data persisted for the emitter */
+	ctx?: unknown;
+}
+
+/**
+ * Interface defining the methods for a virtual table module implementation.
+ * The module primarily acts as a factory for connection-specific VirtualTable instances.
+ *
+ * @template TTable The specific type of VirtualTable managed by this module.
+ * @template TConfig The type defining module-specific configuration options.
+ */
+export interface VirtualTableModule<
+	TTable extends VirtualTable,
+	TConfig extends BaseModuleConfig = BaseModuleConfig
+> {
+
+	/**
+	 * Creates the persistent definition of a virtual table.
+	 * Called by CREATE VIRTUAL TABLE to define schema and initialize storage.
+	 *
+	 * @param db The database connection
+	 * @param tableSchema The schema definition for the table being created
+	 * @returns The new VirtualTable instance
+	 * @throws QuereusError on failure
+	 */
+	create(
+		db: Database,
+		tableSchema: TableSchema,
+	): TTable;
+
+	/**
+	 * Connects to an existing virtual table definition.
+	 * Called when the schema is loaded or a connection needs to interact with the table.
+	 *
+	 * @param db The database connection
+	 * @param pAux Client data passed during module registration
+	 * @param moduleName The name the module was registered with
+	 * @param schemaName The name of the database schema
+	 * @param tableName The name of the virtual table to connect to
+	 * @param options Module-specific configuration options from the original CREATE VIRTUAL TABLE
+	 * @param tableSchema Optional table schema when connecting during import (columns, PK, etc.)
+	 * @returns The connection-specific VirtualTable instance
+	 * @throws QuereusError on failure
+	 */
+	connect(
+		db: Database,
+		pAux: unknown,
+		moduleName: string,
+		schemaName: string,
+		tableName: string,
+		options: TConfig,
+		tableSchema?: TableSchema
+	): TTable;
+
+	/**
+	 * Determines if this module can execute a plan subtree starting at the given node.
+	 * Used for query push-down to virtual table modules that support arbitrary queries.
+	 *
+	 * @param node The root node of the subtree to evaluate
+	 * @returns Assessment with cost and optional context, or undefined if not supported
+	 */
+	supports?(
+		node: PlanNode
+	): SupportAssessment | undefined;
+
+	/**
+	 * Modern, type-safe access planning interface.
+	 * Preferred over xBestIndex for new implementations.
+	 *
+	 * @param db The database connection
+	 * @param tableInfo The schema information for the table being planned
+	 * @param request Planning request with constraints and requirements
+	 * @returns Access plan result describing the chosen strategy
+	 */
+	getBestAccessPlan?(
+		db: Database,
+		tableInfo: TableSchema,
+		request: BestAccessPlanRequest
+	): BestAccessPlanResult;
+
+
+
+	/**
+	 * Destroys the underlying persistent representation of the virtual table.
+	 * Called by DROP TABLE.
+	 *
+	 * @param db The database connection
+	 * @param pAux Client data passed during module registration
+	 * @param moduleName The name the module was registered with
+	 * @param schemaName The name of the database schema
+	 * @param tableName The name of the virtual table being destroyed
+	 * @throws QuereusError on failure
+	 */
+	destroy(
+		db: Database,
+		pAux: unknown,
+		moduleName: string,
+		schemaName: string,
+		tableName: string
+	): Promise<void>;
+
+	/**
+	 * Creates an index on a virtual table.
+	 * Called by CREATE INDEX.
+	 *
+	 * @param db The database connection
+	 * @param schemaName The name of the database schema
+	 * @param tableName The name of the virtual table
+	 * @param indexSchema The schema definition for the index being created
+	 * @throws QuereusError on failure
+	 */
+	createIndex?(
+		db: Database,
+		schemaName: string,
+		tableName: string,
+		indexSchema: IndexSchema
+	): Promise<void>;
+
+	/**
+	 * Checks for shadow table name conflicts
+	 * @param name The name to check
+	 * @returns true if the name would conflict
+	 */
+	shadowName?(name: string): boolean;
+}
+
+/**
+ * Defines the structure for schema change information passed to xAlterSchema
+ */
+export type SchemaChangeInfo =
+	| { type: 'addColumn'; columnDef: ColumnDef }
+	| { type: 'dropColumn'; columnName: string }
+	| { type: 'renameColumn'; oldName: string; newName: string; newColumnDefAst?: ColumnDef };
+
+/**
+ * Type alias for the common usage pattern where specific table and config types are not known.
+ * Use this for storage scenarios like the SchemaManager where modules of different types are stored together.
+ * eslint-disable-next-line @typescript-eslint/no-explicit-any
+ */
+export type AnyVirtualTableModule = VirtualTableModule<any, any>;