@quereus/quereus 0.2.1 → 0.4.0

package/src/planner/rules/retrieve/rule-grow-retrieve.ts

@@ -0,0 +1,337 @@
+/**
+ * Rule: Grow Retrieve
+ *
+ * Structural sliding rule that maximizes the query segment each virtual table module can execute.
+ * This is a bottom-up transformation that slides RetrieveNode boundaries upward to encompass
+ * as much of the query pipeline as each module can handle.
+ *
+ * Applied When:
+ * - Node is a unary relational operation (Filter, Project, Sort, LimitOffset)
+ * - Child is a RetrieveNode
+ * - Virtual table module supports executing the expanded pipeline
+ *
+ * Benefits:
+ * - Maximizes push-down opportunities for query-based modules
+ * - Provides fallback support for index-style modules via constraint extraction
+ * - Establishes optimal module execution boundaries before cost-based optimization
+ */
+
+import { createLogger } from '../../../common/logger.js';
+import { isRelationalNode, type PlanNode, type RelationalPlanNode } from '../../nodes/plan-node.js';
+import type { OptContext } from '../../framework/context.js';
+import { RetrieveNode } from '../../nodes/retrieve-node.js';
+import { FilterNode } from '../../nodes/filter.js';
+import type { TableReferenceNode } from '../../nodes/reference.js';
+import { PlanNodeType } from '../../nodes/plan-node-type.js';
+import type { SupportAssessment } from '../../../vtab/module.js';
+import type { BestAccessPlanRequest, BestAccessPlanResult, PredicateConstraint } from '../../../vtab/best-access-plan.js';
+import { extractConstraints, createTableInfoFromNode, type TableInfo } from '../../analysis/constraint-extractor.js';
+import { normalizePredicate } from '../../analysis/predicate-normalizer.js';
+import { seqScanCost } from '../../cost/index.js';
+import { SortNode } from '../../nodes/sort.js';
+import { extractOrderingFromSortKeys } from '../../framework/physical-utils.js';
+import { LimitOffsetNode } from '../../nodes/limit-offset.js';
+import { PlanNode as _PlanNode } from '../../nodes/plan-node.js';
+import { PlanNodeType as _PlanNodeType } from '../../nodes/plan-node-type.js';
+import { LiteralNode } from '../../nodes/scalar.js';
+import { collectBindingsInPlan } from '../../analysis/binding-collector.js';
+
+const log = createLogger('optimizer:rule:grow-retrieve');
+
+/**
+ * Context data stored in RetrieveNode.moduleCtx for index-style fallback
+ */
+interface IndexStyleContext {
+	kind: 'index-style';
+	accessPlan: BestAccessPlanResult;
+	residualPredicate?: PlanNode;
+	originalConstraints: PredicateConstraint[];
+}
+
+function isIndexStyleContext(ctx: unknown): ctx is IndexStyleContext {
+	return !!ctx && typeof ctx === 'object' && (ctx as any).kind === 'index-style';
+}
+
+export function ruleGrowRetrieve(node: PlanNode, context: OptContext): PlanNode | null {
+	// This rule runs in a TOP-DOWN pass, looking for any relational operation
+	// above a RetrieveNode that can be pushed into the module's execution boundary
+
+	// Must be a relational node to be growable
+	if (!isRelationalNode(node)) {
+		return null;
+	}
+
+	// Find the RetrieveNode child (if any)
+	const retrieveChild = findRetrieveChild(node);
+	if (!retrieveChild) {
+		return null;
+	}
+
+	const tableRef = retrieveChild.tableRef;
+
+	// Guard: ensure we have required properties
+	if (!tableRef?.tableSchema) {
+		log('RetrieveNode missing tableRef or tableSchema');
+		return null;
+	}
+
+	const tableSchema = tableRef.tableSchema;
+	const vtabModule = tableRef.vtabModule;
+
+	log('Evaluating growth for %s over table %s', node.nodeType, tableSchema.name);
+
+	// If no vtabModule, can't grow
+	if (!vtabModule) {
+		log('No vtabModule available for table %s', tableSchema.name);
+		return null;
+	}
+
+	// Create candidate pipeline by sliding the operation into the retrieve boundary
+	// This replaces the RetrieveNode child with its source in the parent operation
+	const candidatePipeline = replaceRetrieveWithSource(node, retrieveChild);
+
+	// Try module's supports() method first (if available)
+	let assessment: SupportAssessment | undefined;
+
+	if (vtabModule.supports && typeof vtabModule.supports === 'function') {
+		// Query-based module: let it decide if it can handle the pipeline
+		log('Testing module.supports() for %s pipeline', node.nodeType);
+		assessment = vtabModule.supports(candidatePipeline);
+
+		if (assessment) {
+			log('Module supports expanded pipeline (cost: %d)', assessment.cost);
+		} else {
+			log('Module declined expanded pipeline');
+		}
+	}
+
+	// If module doesn't have supports() or declined, try index-style fallback
+	// but ONLY for operations we know can be translated to index constraints
+	if (!assessment && vtabModule.getBestAccessPlan && typeof vtabModule.getBestAccessPlan === 'function') {
+		if (canTranslateToIndexConstraints(node)) {
+			log('Testing index-style fallback for %s', node.nodeType);
+			assessment = fallbackIndexSupports(node, candidatePipeline, context, tableRef);
+
+			if (assessment) {
+				log('Index-style fallback supports pipeline (cost: %d)', assessment.cost);
+			} else {
+				log('Index-style fallback declined pipeline');
+			}
+		} else {
+			log('Node type %s cannot be translated to index constraints', node.nodeType);
+		}
+	}
+
+	if (!assessment) {
+		// Module cannot handle the expanded pipeline
+		return null;
+	}
+
+	// Determine how to slide depending on assessment origin
+	let newPipeline: RelationalPlanNode;
+	let newBindings = [...(retrieveChild.bindings ?? []), ...collectBindingsInPlan(node, retrieveChild.tableRef)];
+
+	if (isIndexStyleContext(assessment.ctx)) {
+		// Index-style fallback: only place supported fragments under Retrieve; keep residuals above
+		newPipeline = candidatePipeline as RelationalPlanNode;
+		if (node instanceof FilterNode) {
+			const tableInfo: TableInfo = createTableInfoFromNode(retrieveChild.tableRef, tableSchema.name);
+			const extraction = extractConstraints(normalizePredicate(node.predicate), [tableInfo]);
+			const supported = extraction.supportedPredicateByTable?.get(tableInfo.relationKey);
+			if (supported) {
+				newPipeline = new FilterNode(
+					retrieveChild.source.scope,
+					(candidatePipeline as FilterNode).source,
+					supported
+				) as unknown as RelationalPlanNode;
+				newBindings = [...(retrieveChild.bindings ?? []), ...collectBindingsInPlan(newPipeline, retrieveChild.tableRef)];
+			}
+		}
+	} else {
+		// Query-based module with supports(): move the entire node into the module boundary
+		newPipeline = candidatePipeline as RelationalPlanNode;
+	}
+
+	const grownRetrieve = new RetrieveNode(
+		node.scope,
+		newPipeline,
+		retrieveChild.tableRef,
+		assessment.ctx,
+		newBindings
+	);
+
+	log('Grew retrieve pipeline for table %s: %s → %s',
+		tableSchema.name, retrieveChild.source.nodeType, candidatePipeline.nodeType);
+
+	return grownRetrieve;
+}
+
+/**
+ * Find a RetrieveNode among the children of this node
+ */
+function findRetrieveChild(node: PlanNode): RetrieveNode | undefined {
+	const children = node.getChildren();
+	for (const child of children) {
+		if (child instanceof RetrieveNode) {
+			return child;
+		}
+	}
+	return undefined;
+}
+
+/**
+ * Replace the RetrieveNode child with its source in the parent operation
+ */
+function replaceRetrieveWithSource(parent: PlanNode, retrieveNode: RetrieveNode): PlanNode {
+	const children = parent.getChildren();
+	const newChildren = children.map(child =>
+		child === retrieveNode ? retrieveNode.source : child
+	);
+	return parent.withChildren(newChildren);
+}
+
+/**
+ * Check if this node type can be translated to index constraints
+ * This is used for the fallback when modules don't implement supports()
+ */
+function canTranslateToIndexConstraints(node: PlanNode): boolean {
+	switch (node.nodeType) {
+		case PlanNodeType.Filter:
+			// Filters can be translated to predicates
+			return true;
+		case PlanNodeType.Sort:
+			// Sort can be translated to ordering requirements
+			return true;
+		case PlanNodeType.LimitOffset:
+			// Limit can be passed to index access
+			return true;
+		default:
+			// Other operations (Project, Aggregate, etc.) can't be
+			// meaningfully translated to index constraints
+			return false;
+	}
+}
+
+/**
+ * Fallback assessment for index-style modules using getBestAccessPlan
+ * Translates various operations to index constraints
+ */
+function fallbackIndexSupports(
+	node: PlanNode,
+	candidatePipeline: PlanNode,
+	context: OptContext,
+	tableRef: TableReferenceNode
+): SupportAssessment | undefined {
+
+	const vtabModule = tableRef.vtabModule;
+	const tableSchema = tableRef.tableSchema;
+
+	// Build BestAccessPlanRequest based on node type
+	const request: BestAccessPlanRequest = {
+		columns: tableSchema.columns.map((col, index) => ({
+			index,
+			name: col.name,
+			type: col.affinity,
+			isPrimaryKey: col.primaryKey || false,
+			isUnique: col.primaryKey || false
+		})),
+		filters: [],
+		requiredOrdering: undefined,
+		limit: undefined,
+		estimatedRows: tableRef.estimatedRows ?? context.stats.tableRows(tableSchema) ?? 1000
+	};
+
+	// Extract information based on node type
+	let residualPredicate: PlanNode | undefined;
+
+	if (node instanceof FilterNode) {
+		// Extract constraints from filter predicate
+		const tableInfo: TableInfo = createTableInfoFromNode(tableRef, tableSchema.name);
+		const normalizedPredicate = normalizePredicate(node.predicate);
+		const extraction = extractConstraints(normalizedPredicate, [tableInfo]);
+
+		if (extraction.allConstraints.length === 0) {
+			log('No extractable constraints from filter predicate');
+			return undefined;
+		}
+
+		request.filters = extraction.allConstraints;
+		residualPredicate = extraction.residualPredicate;
+		log('Extracted %d constraints from Filter', extraction.allConstraints.length);
+
+	} else if (node.nodeType === PlanNodeType.Sort) {
+		// Extract ordering requirements from Sort node
+		const sort = node as unknown as SortNode;
+		const ordering = extractOrderingFromSortKeys(sort.getSortKeys(), sort.source.getAttributes());
+		if (!ordering) {
+			log('Sort node has non-trivial expressions; cannot translate to ordering spec');
+			return undefined;
+		}
+		request.requiredOrdering = ordering.map(o => ({ columnIndex: o.column, desc: o.desc }));
+		log('Extracted ordering requirement of length %d', request.requiredOrdering.length);
+
+	} else if (node.nodeType === PlanNodeType.LimitOffset) {
+		// Extract limit from LimitOffset node when constant
+		const lim = node as unknown as LimitOffsetNode;
+		if (lim.limit && lim.limit.nodeType === _PlanNodeType.Literal) {
+			const limitVal = (lim.limit as unknown as LiteralNode).expression.value;
+			if (typeof limitVal === 'number') {
+				request.limit = Math.max(0, Math.floor(limitVal));
+				log('Extracted limit value: %d', request.limit);
+			}
+		}
+		// We ignore OFFSET for now (modules can implement it internally if desired)
+		if (!request.limit) {
+			log('No usable constant LIMIT found');
+			return undefined;
+		}
+
+	} else {
+		log('Node type %s not supported by index-style fallback', node.nodeType);
+		return undefined;
+	}
+
+	log('Built access plan request: %d filters, ordering: %s, limit: %s',
+		request.filters.length,
+		request.requiredOrdering ? 'yes' : 'no',
+		request.limit ?? 'none');
+
+	// Get access plan from module
+	const accessPlan = vtabModule.getBestAccessPlan!(context.db, tableSchema, request);
+
+	// Check if the plan is beneficial
+	const handlesAnyFilter = request.filters.length > 0 &&
+		accessPlan.handledFilters.some(handled => handled);
+	const providesOrdering = request.requiredOrdering &&
+		accessPlan.providesOrdering;
+
+	// Calculate baseline cost
+	const estimatedRows = request.estimatedRows ?? 1000;
+	const seqCost = seqScanCost(estimatedRows);
+
+	// Accept the plan if it handles filters OR provides required ordering
+	if (!handlesAnyFilter && !providesOrdering) {
+		log('Access plan provides no benefit');
+		return undefined;
+	}
+
+	if (accessPlan.cost >= seqCost && !providesOrdering) {
+		log('Access plan cost (%d) not better than sequential scan (%d)', accessPlan.cost, seqCost);
+		return undefined;
+	}
+
+	log('Index-style fallback beneficial: cost %d vs %d seq scan', accessPlan.cost, seqCost);
+
+	// Store context for later use in ruleSelectAccessPath
+	const indexCtx: IndexStyleContext = {
+		kind: 'index-style',
+		accessPlan,
+		residualPredicate,
+		originalConstraints: [...request.filters] // Copy to satisfy mutable type
+	};
+
+	return {
+		cost: accessPlan.cost,
+		ctx: indexCtx
+	};
+}

package/src/planner/scopes/registered.ts

@@ -37,6 +37,14 @@ export class RegisteredScope implements Scope {
 		this.registeredSymbols.set(lowerSymbolKey, getReference);
 	}
 
+	subscribeFactory(symbolKey: string, factory: ReferenceCallback): void {
+		const lower = symbolKey.toLowerCase();
+		if (this.registeredSymbols.has(lower)) {
+			throw new QuereusError(`Symbol '${lower}' already exists in the same scope.`, StatusCode.ERROR);
+		}
+		this.registeredSymbols.set(lower, factory);
+	}
+
 	resolveSymbol(symbolKey: string, expression: AST.Expression): PlanNode | typeof Ambiguous | undefined {
 		const reference = this.registeredSymbols.get(symbolKey.toLowerCase());
 		if (reference) {

package/src/planner/stats/index.ts

@@ -133,76 +133,11 @@ export class NaiveStatsProvider implements StatsProvider {
 	}
 }
 
-/**
- * Statistics provider that delegates to virtual table modules
- * Allows VTab modules to provide their own statistics
- */
-export class VTabStatsProvider implements StatsProvider {
-	constructor(
-		private readonly fallback: StatsProvider = new NaiveStatsProvider()
-	) {
-		log('Created VTab stats provider with fallback');
-	}
-
-	tableRows(table: TableSchema): number | undefined {
-		// Try to get stats from VTab module if it supports it
-		if (table.vtabModule && typeof table.vtabModule === 'object' && 'getTableStats' in table.vtabModule) {
-			try {
-				const stats = (table.vtabModule as any).getTableStats?.(table);
-				if (stats?.rowCount !== undefined) {
-					log('Got row count from VTab module for %s: %d', table.name, stats.rowCount);
-					return stats.rowCount;
-				}
-			} catch (error) {
-				log('Error getting VTab stats for %s: %s', table.name, error);
-			}
-		}
-
-		// Fall back to default provider
-		return this.fallback.tableRows(table);
-	}
-
-	selectivity(table: TableSchema, predicate: ScalarPlanNode): number | undefined {
-		// Try VTab module selectivity
-		if (table.vtabModule && typeof table.vtabModule === 'object' && 'getSelectivity' in table.vtabModule) {
-			try {
-				const selectivity = (table.vtabModule as any).getSelectivity?.(table, predicate);
-				if (selectivity !== undefined) {
-					log('Got selectivity from VTab module for %s: %f', table.name, selectivity);
-					return selectivity;
-				}
-			} catch (error) {
-				log('Error getting VTab selectivity for %s: %s', table.name, error);
-			}
-		}
-
-		// Fall back to default provider
-		return this.fallback.selectivity(table, predicate);
-	}
-
-	joinSelectivity(leftTable: TableSchema, rightTable: TableSchema, joinCondition: ScalarPlanNode): number | undefined {
-		return this.fallback.joinSelectivity?.(leftTable, rightTable, joinCondition);
-	}
-
-	distinctValues(table: TableSchema, columnName: string): number | undefined {
-		return this.fallback.distinctValues?.(table, columnName);
-	}
-
-	indexSelectivity(table: TableSchema, indexName: string, predicate: ScalarPlanNode): number | undefined {
-		return this.fallback.indexSelectivity?.(table, indexName, predicate);
-	}
-}
-
 /**
  * Default statistics provider instance
  */
 export const defaultStatsProvider = new NaiveStatsProvider();
 
-/**
- * VTab-aware statistics provider instance
- */
-export const vtabStatsProvider = new VTabStatsProvider();
-
 /**
  * Create a custom statistics provider
  */

package/src/planner/util/key-utils.ts

@@ -0,0 +1,46 @@
+import type { ColRef } from '../../common/datatype.js';
+
+/**
+ * Project unique keys through a projection mapping.
+ * - sourceKeys: keys defined on the source relation (arrays of column refs by source column index)
+ * - projectionMap: mapping from source column index -> projected column index
+ * Returns keys that survive projection (all columns present), with indices remapped to output.
+ */
+export function projectKeys(sourceKeys: ReadonlyArray<ReadonlyArray<ColRef>>, projectionMap: ReadonlyMap<number, number>): ColRef[][] {
+	const result: ColRef[][] = [];
+	for (const key of sourceKeys) {
+		const projected: ColRef[] = [];
+		let missing = false;
+		for (const col of key) {
+			const projectedIndex = projectionMap.get(col.index);
+			if (projectedIndex === undefined) {
+				missing = true;
+				break;
+			}
+			projected.push({ index: projectedIndex, desc: col.desc });
+		}
+		if (!missing) {
+			result.push(projected);
+		}
+	}
+	return result;
+}
+
+/**
+ * Combine unique keys across a join.
+ * - For inner/cross joins: keys from left and right are preserved; right indices are shifted by left column count.
+ * - For outer joins: return [] conservatively (null padding may break uniqueness).
+ */
+export function combineJoinKeys(leftKeys: ReadonlyArray<ReadonlyArray<ColRef>>, rightKeys: ReadonlyArray<ReadonlyArray<ColRef>>, joinType: 'inner' | 'left' | 'right' | 'full' | 'cross', leftColumnCount: number): ColRef[][] {
+	if (joinType !== 'inner' && joinType !== 'cross') return [];
+	const result: ColRef[][] = [];
+	for (const key of leftKeys) {
+		result.push(key.map(c => ({ index: c.index, desc: c.desc })));
+	}
+	for (const key of rightKeys) {
+		result.push(key.map(c => ({ index: c.index + leftColumnCount, desc: c.desc })));
+	}
+	return result;
+}
+
+

package/src/planner/validation/plan-validator.ts

@@ -8,6 +8,7 @@ import { PlanNodeType } from '../nodes/plan-node-type.js';
 import { QuereusError } from '../../common/errors.js';
 import { StatusCode } from '../../common/types.js';
 import { validateLog } from '../debug/logger-utils.js';
+import type { ColumnReferenceNode } from '../nodes/reference.js';
 
 const log = validateLog();
 
@@ -105,7 +106,7 @@ function validateNode(node: PlanNode, context: ValidationContext, path: string[]
 
 		// 4. Validate column references point to valid attributes
 		if (node.nodeType === PlanNodeType.ColumnReference) {
-			validateColumnReference(node as any, context, nodePath);
+			validateColumnReference(node as ColumnReferenceNode, context, nodePath);
 		}
 
 		// 5. Recursively validate children
@@ -183,6 +184,7 @@ function validatePhysicalNodeType(node: PlanNode, nodePath: string): void {
 	// Node types that should NOT appear in a fully optimized physical tree
 	const logicalOnlyTypes = new Set([
 		PlanNodeType.Aggregate, // Should be StreamAggregate or HashAggregate
+		PlanNodeType.Retrieve,  // Must be rewritten to a physical access node
 		// Add other logical-only types here as needed
 	]);
 
@@ -248,7 +250,7 @@ function validateRelationalNode(node: RelationalPlanNode, context: ValidationCon
 /**
  * Validate column references point to valid attributes
  */
-function validateColumnReference(node: any, context: ValidationContext, nodePath: string): void {
+function validateColumnReference(node: ColumnReferenceNode, context: ValidationContext, nodePath: string): void {
 	if (!context.options.validateAttributes) {
 		return;
 	}
@@ -272,7 +274,7 @@ function validateColumnReference(node: any, context: ValidationContext, nodePath
 /**
  * Validate ordering specification
  */
-function validateOrdering(ordering: any[], columnCount: number, nodePath: string): void {
+function validateOrdering(ordering: { column: number; desc: boolean }[], columnCount: number, nodePath: string): void {
 	for (let i = 0; i < ordering.length; i++) {
 		const orderSpec = ordering[i];