@optimystic/quereus-plugin-optimystic 0.3.0

package/src/schema/schema-manager.ts

@@ -0,0 +1,217 @@
+/**
+ * SchemaManager - Manages table schemas in Optimystic trees
+ *
+ * Stores and retrieves table schema definitions from distributed Optimystic trees.
+ * Schema is stored in a dedicated tree at `tree://schema/{tableName}`.
+ */
+
+import type { Tree } from '@optimystic/db-core';
+import type { TableSchema, ColumnSchema } from '@quereus/quereus';
+import type { ITransactor } from '@optimystic/db-core';
+
+// IndexSchema type from TableSchema.indexes
+type IndexSchema = NonNullable<TableSchema['indexes']>[number];
+
+/**
+ * Serializable schema storage format
+ */
+export interface StoredTableSchema {
+	name: string;
+	schemaName: string;
+	columns: StoredColumnSchema[];
+	primaryKeyDefinition: StoredPrimaryKeyColumn[];
+	indexes: StoredIndexSchema[];
+	vtabModuleName: string;
+	vtabArgs?: Record<string, any>;
+	isTemporary?: boolean;
+	estimatedRows?: number;
+}
+
+export interface StoredColumnSchema {
+	name: string;
+	affinity: string;
+	notNull: boolean;
+	primaryKey: boolean;
+	pkOrder: number;
+	defaultValue?: any;
+	collation: string;
+	generated: boolean;
+	pkDirection?: 'asc' | 'desc';
+}
+
+export interface StoredPrimaryKeyColumn {
+	index: number;
+	desc?: boolean;
+	autoIncrement?: boolean;
+	collation?: string;
+}
+
+export interface StoredIndexSchema {
+	name: string;
+	columns: StoredIndexColumn[];
+	unique?: boolean;
+}
+
+export interface StoredIndexColumn {
+	index: number;
+	desc?: boolean;
+	collation?: string;
+}
+
+/**
+ * Manages schema storage and retrieval in Optimystic trees
+ */
+export class SchemaManager {
+	private schemaCache = new Map<string, StoredTableSchema>();
+
+	constructor(
+		private readonly getSchemaTree: (transactor?: ITransactor) => Promise<Tree<string, any>>
+	) {}
+
+	/**
+	 * Store a table schema
+	 */
+	async storeSchema(schema: TableSchema, transactor?: ITransactor): Promise<void> {
+		const stored = this.tableSchemaToStored(schema);
+		this.schemaCache.set(schema.name, stored);
+
+		const tree = await this.getSchemaTree(transactor);
+		await tree.replace([[schema.name, stored]]);
+	}
+
+	/**
+	 * Retrieve a table schema
+	 */
+	async getSchema(tableName: string, transactor?: ITransactor): Promise<StoredTableSchema | undefined> {
+		// Check cache first
+		const cached = this.schemaCache.get(tableName);
+		if (cached) {
+			return cached;
+		}
+
+		// Load from tree
+		const tree = await this.getSchemaTree(transactor);
+		const path = await tree.find(tableName);
+		if (!tree.isValid(path)) {
+			return undefined;
+		}
+
+		const entry = tree.at(path) as [string, StoredTableSchema];
+		if (entry && entry.length >= 2) {
+			const stored = entry[1];
+			this.schemaCache.set(tableName, stored);
+			return stored;
+		}
+
+		return undefined;
+	}
+
+	/**
+	 * Delete a table schema
+	 */
+	async deleteSchema(tableName: string, transactor?: ITransactor): Promise<void> {
+		this.schemaCache.delete(tableName);
+
+		const tree = await this.getSchemaTree(transactor);
+		await tree.replace([[tableName, undefined]]);
+	}
+
+	/**
+	 * List all table names
+	 */
+	async listTables(transactor?: ITransactor): Promise<string[]> {
+		const tree = await this.getSchemaTree(transactor);
+		const tables: string[] = [];
+
+		for await (const path of tree.range({ isAscending: true } as any)) {
+			if (!tree.isValid(path)) {
+				continue;
+			}
+
+			const entry = tree.at(path) as [string, any];
+			if (entry && entry.length >= 1) {
+				tables.push(entry[0]);
+			}
+		}
+
+		return tables;
+	}
+
+	/**
+	 * Clear the schema cache
+	 */
+	clearCache(): void {
+		this.schemaCache.clear();
+	}
+
+	/**
+	 * Convert TableSchema to storable format
+	 */
+	private tableSchemaToStored(schema: TableSchema): StoredTableSchema {
+		return {
+			name: schema.name,
+			schemaName: schema.schemaName,
+			columns: schema.columns.map(col => this.columnSchemaToStored(col)),
+			primaryKeyDefinition: schema.primaryKeyDefinition.map(pk => ({
+				index: pk.index,
+				desc: pk.desc,
+				autoIncrement: pk.autoIncrement,
+				collation: pk.collation,
+			})),
+			indexes: (schema.indexes || []).map(idx => this.indexSchemaToStored(idx)),
+			vtabModuleName: schema.vtabModuleName,
+			vtabArgs: schema.vtabArgs as Record<string, any>,
+			isTemporary: schema.isTemporary,
+			estimatedRows: schema.estimatedRows,
+		};
+	}
+
+	/**
+	 * Convert ColumnSchema to storable format
+	 */
+	private columnSchemaToStored(col: ColumnSchema): StoredColumnSchema {
+		return {
+			name: col.name,
+			affinity: col.logicalType.name, // Use logicalType.name for storage
+			notNull: col.notNull,
+			primaryKey: col.primaryKey,
+			pkOrder: col.pkOrder,
+			defaultValue: col.defaultValue ? this.serializeExpression(col.defaultValue) : undefined,
+			collation: col.collation,
+			generated: col.generated,
+			pkDirection: col.pkDirection,
+		};
+	}
+
+	/**
+	 * Convert IndexSchema to storable format
+	 */
+	private indexSchemaToStored(idx: IndexSchema): StoredIndexSchema {
+		return {
+			name: idx.name,
+			columns: idx.columns.map((col: { index: number; desc?: boolean; collation?: string }) => ({
+				index: col.index,
+				desc: col.desc,
+				collation: col.collation,
+			})),
+		};
+	}
+
+	/**
+	 * Serialize an expression for storage
+	 * For now, we'll store a simplified representation
+	 */
+	private serializeExpression(expr: any): any {
+		// TODO: Implement proper expression serialization
+		// For now, just store the expression as-is if it's a simple value
+		if (typeof expr === 'object' && expr !== null) {
+			if ('type' in expr && expr.type === 'literal') {
+				return { type: 'literal', value: expr.value };
+			}
+			// For complex expressions, we'll need to implement full serialization
+			return { type: 'complex', raw: JSON.stringify(expr) };
+		}
+		return expr;
+	}
+}
+

package/src/schema/statistics-collector.ts

@@ -0,0 +1,173 @@
+/**
+ * StatisticsCollector - Collects and maintains table statistics for query optimization
+ *
+ * Tracks row counts, distinct values, and provides cost estimates for query planning.
+ */
+
+import type { StoredTableSchema } from './schema-manager.js';
+
+/**
+ * Statistics for a single column
+ */
+export interface ColumnStatistics {
+	/** Approximate number of distinct values */
+	distinctCount: number;
+	/** Approximate number of NULL values */
+	nullCount: number;
+	/** Sample of values for histogram (optional) */
+	sampleValues?: unknown[];
+}
+
+/**
+ * Statistics for a table
+ */
+export interface TableStatistics {
+	/** Total number of rows (approximate) */
+	rowCount: number;
+	/** Statistics per column */
+	columnStats: Map<number, ColumnStatistics>;
+	/** Last update timestamp */
+	lastUpdated: number;
+}
+
+/**
+ * Collects and maintains statistics for query optimization
+ */
+export class StatisticsCollector {
+	private stats: TableStatistics;
+
+	constructor(private schema: StoredTableSchema) {
+		this.stats = {
+			rowCount: 0,
+			columnStats: new Map(),
+			lastUpdated: Date.now(),
+		};
+
+		// Initialize column stats
+		for (let i = 0; i < schema.columns.length; i++) {
+			this.stats.columnStats.set(i, {
+				distinctCount: 0,
+				nullCount: 0,
+			});
+		}
+	}
+
+	/**
+	 * Get current table statistics
+	 */
+	getStatistics(): TableStatistics {
+		return this.stats;
+	}
+
+	/**
+	 * Get estimated row count
+	 */
+	getRowCount(): number {
+		return this.stats.rowCount;
+	}
+
+	/**
+	 * Get estimated distinct count for a column
+	 */
+	getDistinctCount(columnIndex: number): number {
+		const colStats = this.stats.columnStats.get(columnIndex);
+		return colStats?.distinctCount || 0;
+	}
+
+	/**
+	 * Increment row count (called on INSERT)
+	 */
+	incrementRowCount(): void {
+		this.stats.rowCount++;
+		this.stats.lastUpdated = Date.now();
+	}
+
+	/**
+	 * Decrement row count (called on DELETE)
+	 */
+	decrementRowCount(): void {
+		this.stats.rowCount = Math.max(0, this.stats.rowCount - 1);
+		this.stats.lastUpdated = Date.now();
+	}
+
+	/**
+	 * Estimate selectivity of an equality constraint
+	 * Returns a value between 0 and 1 representing the fraction of rows that match
+	 */
+	estimateEqualitySelectivity(columnIndex: number): number {
+		const distinctCount = this.getDistinctCount(columnIndex);
+		if (distinctCount === 0) {
+			return 0.1; // Default estimate
+		}
+		return 1.0 / distinctCount;
+	}
+
+	/**
+	 * Estimate selectivity of a range constraint
+	 * Returns a value between 0 and 1 representing the fraction of rows that match
+	 */
+	estimateRangeSelectivity(_columnIndex: number): number {
+		// Simple heuristic: assume range matches 25% of rows
+		return 0.25;
+	}
+
+	/**
+	 * Estimate cost of a full table scan
+	 */
+	estimateTableScanCost(): number {
+		// Cost is proportional to row count
+		// Base cost of 1.0 per row
+		return Math.max(1000, this.stats.rowCount);
+	}
+
+	/**
+	 * Estimate cost of an index scan
+	 */
+	estimateIndexScanCost(selectivity: number): number {
+		// Cost includes:
+		// 1. Index lookup cost (logarithmic)
+		// 2. Row fetch cost (proportional to selected rows)
+		const indexLookupCost = Math.log2(Math.max(1, this.stats.rowCount)) * 2;
+		const rowFetchCost = this.stats.rowCount * selectivity;
+		return indexLookupCost + rowFetchCost;
+	}
+
+	/**
+	 * Estimate number of rows returned by a constraint
+	 */
+	estimateRowsForConstraint(columnIndex: number, isEquality: boolean): number {
+		if (isEquality) {
+			const selectivity = this.estimateEqualitySelectivity(columnIndex);
+			return Math.max(1, Math.floor(this.stats.rowCount * selectivity));
+		} else {
+			const selectivity = this.estimateRangeSelectivity(columnIndex);
+			return Math.max(1, Math.floor(this.stats.rowCount * selectivity));
+		}
+	}
+
+	/**
+	 * Update statistics based on actual data (called periodically)
+	 * This is a simplified version - a real implementation would sample the data
+	 */
+	async updateStatistics(sampleSize = 1000): Promise<void> {
+		// TODO: Implement actual statistics collection by sampling the table
+		// For now, we just update the timestamp
+		this.stats.lastUpdated = Date.now();
+	}
+
+	/**
+	 * Reset statistics
+	 */
+	reset(): void {
+		this.stats.rowCount = 0;
+		this.stats.columnStats.clear();
+		for (let i = 0; i < this.schema.columns.length; i++) {
+			this.stats.columnStats.set(i, {
+				distinctCount: 0,
+				nullCount: 0,
+			});
+		}
+		this.stats.lastUpdated = Date.now();
+	}
+}
+

package/src/transaction/index.ts

@@ -0,0 +1,16 @@
+/**
+ * Transaction support for Quereus-Optimystic integration.
+ *
+ * This module provides the QuereusEngine for executing SQL transactions
+ * through the Optimystic distributed transaction system.
+ */
+
+export {
+	QuereusEngine,
+	QUEREUS_ENGINE_ID,
+	createQuereusStatement,
+	createQuereusStatements
+} from './quereus-engine.js';
+
+export type { QuereusStatement } from './quereus-engine.js';
+

package/src/transaction/quereus-engine.ts

@@ -0,0 +1,174 @@
+import type { Database, SqlParameters } from '@quereus/quereus';
+import type {
+	ITransactionEngine,
+	Transaction,
+	ExecutionResult,
+	CollectionActions,
+	TransactionCoordinator
+} from '@optimystic/db-core';
+import { sha256 } from '@noble/hashes/sha256';
+
+/**
+ * Engine ID for Quereus SQL transactions.
+ * Format: "quereus@{version}" where version matches the quereus package version.
+ */
+export const QUEREUS_ENGINE_ID = 'quereus@0.5.3';
+
+/**
+ * Statement format for Quereus transactions.
+ * Each statement is a SQL string with optional parameters.
+ */
+export type QuereusStatement = {
+	/** The SQL statement to execute */
+	sql: string;
+	/** Optional parameters for the statement */
+	params?: SqlParameters;
+};
+
+/**
+ * Quereus-specific transaction engine for SQL execution.
+ *
+ * This engine:
+ * 1. Executes SQL statements through a Quereus database
+ * 2. Collects resulting actions from the virtual table module
+ * 3. Computes schema hash for validation
+ *
+ * Used for both initial execution (client creating transaction) and
+ * re-execution (validators verifying transaction).
+ */
+export class QuereusEngine implements ITransactionEngine {
+	private schemaHashCache: string | undefined;
+	private schemaVersion: number = 0;
+
+	constructor(
+		private readonly db: Database,
+		private readonly coordinator: TransactionCoordinator
+	) {}
+
+	/**
+	 * Execute a transaction's statements and produce actions.
+	 *
+	 * For initial execution: Executes SQL through Quereus, which triggers
+	 * the Optimystic virtual table module to apply actions.
+	 *
+	 * For validation: Re-executes the same SQL statements to verify
+	 * they produce the same operations.
+	 */
+	async execute(transaction: Transaction): Promise<ExecutionResult> {
+		try {
+			const allActions: CollectionActions[] = [];
+
+			for (const statementJson of transaction.statements) {
+				const statement = JSON.parse(statementJson) as QuereusStatement;
+
+				// Execute SQL through Quereus
+				// The Optimystic virtual table module will:
+				// 1. Translate SQL mutations to actions
+				// 2. Call coordinator.applyActions() with the stampId
+				await this.db.exec(statement.sql, statement.params);
+
+				// Note: Actions are collected by the coordinator's trackers,
+				// not returned directly from exec(). The coordinator tracks
+				// all actions applied during this transaction.
+			}
+
+			return {
+				success: true,
+				actions: allActions
+			};
+		} catch (error) {
+			return {
+				success: false,
+				error: `Failed to execute SQL transaction: ${error instanceof Error ? error.message : String(error)}`
+			};
+		}
+	}
+
+	/**
+	 * Get the schema hash for this engine.
+	 *
+	 * The schema hash is used for validation - all participants must have
+	 * matching schema hashes for a transaction to be valid.
+	 *
+	 * Uses caching to avoid recomputing if schema hasn't changed.
+	 */
+	async getSchemaHash(): Promise<string> {
+		// Check if we have a cached hash
+		if (this.schemaHashCache !== undefined) {
+			return this.schemaHashCache;
+		}
+
+		// Compute and cache the schema hash
+		this.schemaHashCache = await this.computeSchemaHash();
+		return this.schemaHashCache;
+	}
+
+	/**
+	 * Invalidate the schema hash cache.
+	 * Call this when the schema changes (e.g., after DDL statements).
+	 */
+	invalidateSchemaCache(): void {
+		this.schemaHashCache = undefined;
+		this.schemaVersion++;
+	}
+
+	/**
+	 * Get the current schema version number.
+	 * Increments each time the schema cache is invalidated.
+	 */
+	getSchemaVersion(): number {
+		return this.schemaVersion;
+	}
+
+	/**
+	 * Compute the schema hash from the database catalog.
+	 *
+	 * Uses the schema() table-valued function to get schema information,
+	 * then hashes the canonical representation using SHA-256.
+	 */
+	private async computeSchemaHash(): Promise<string> {
+		// Query schema information from Quereus
+		const schemaInfo: Array<{ type: string; name: string; sql: string | null }> = [];
+
+		for await (const row of this.db.eval("select type, name, sql from schema() order by type, name")) {
+			schemaInfo.push({
+				type: row.type as string,
+				name: row.name as string,
+				sql: row.sql as string | null
+			});
+		}
+
+		// Serialize to canonical JSON
+		const catalogJson = JSON.stringify(schemaInfo);
+
+		// Compute SHA-256 hash using @noble/hashes
+		const hashBytes = sha256(new TextEncoder().encode(catalogJson));
+		// Use first 16 bytes encoded as base64url for compact representation
+		const hashBase64 = bytesToBase64url(hashBytes.slice(0, 16));
+		return `schema:${hashBase64}`;
+	}
+}
+
+/**
+ * Convert bytes to base64url encoding (URL-safe, no padding).
+ */
+function bytesToBase64url(bytes: Uint8Array): string {
+	const base64 = btoa(String.fromCharCode(...bytes));
+	return base64.replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/, '');
+}
+
+/**
+ * Helper to create Quereus statement JSON for a transaction.
+ */
+export function createQuereusStatement(sql: string, params?: SqlParameters): string {
+	const statement: QuereusStatement = { sql, params };
+	return JSON.stringify(statement);
+}
+
+/**
+ * Helper to create an array of Quereus statements for a transaction.
+ */
+export function createQuereusStatements(statements: Array<{ sql: string; params?: SqlParameters }>): string[] {
+	return statements.map(s => createQuereusStatement(s.sql, s.params));
+}
+

package/src/types.ts

@@ -0,0 +1,91 @@
+import type { Libp2p } from '@libp2p/interface';
+import type { IKeyNetwork, ITransactor } from '@optimystic/db-core';
+
+/**
+ * Configuration for the optimystic virtual table
+ */
+export interface OptimysticOptions {
+  /** URI for the collection (e.g., 'tree://mydb/users') */
+  collectionUri: string;
+
+  /** Transactor type - 'network', 'test', or custom class name */
+  transactor?: 'network' | 'test' | string;
+
+  /** Key network type - 'libp2p', 'test', or custom class name */
+  keyNetwork?: 'libp2p' | 'test' | string;
+
+  /** Existing libp2p instance to use (optional) */
+  libp2p?: Libp2p;
+
+  /** Options for creating a new libp2p node */
+  libp2pOptions?: LibP2PNodeOptions;
+
+  /** Enable local snapshot cache */
+  cache?: boolean;
+
+  /** Row encoding format */
+  encoding?: 'json' | 'msgpack';
+}
+
+/**
+ * Options for creating a libp2p node
+ */
+export interface LibP2PNodeOptions {
+  /** Network port to listen on */
+  port?: number;
+
+  /** Network name for protocol prefixes */
+  networkName?: string;
+
+  /** Bootstrap nodes for peer discovery */
+  bootstrapNodes?: string[];
+}
+
+/**
+ * Internal configuration after parsing and validation
+ */
+export interface ParsedOptimysticOptions {
+  collectionUri: string;
+  transactor: 'network' | 'test' | string;
+  keyNetwork: 'libp2p' | 'test' | string;
+  libp2p?: Libp2p;
+  libp2pOptions: LibP2PNodeOptions;
+  cache: boolean;
+  encoding: 'json' | 'msgpack';
+}
+
+/**
+ * Registry for custom transactor and key network implementations
+ */
+export interface CustomImplementationRegistry {
+  transactors: Map<string, new (...args: any[]) => ITransactor>;
+  keyNetworks: Map<string, new (...args: any[]) => IKeyNetwork>;
+}
+
+/**
+ * Column definition for the virtual table
+ */
+export interface ColumnDefinition {
+  name: string;
+  type: 'TEXT' | 'INTEGER' | 'REAL' | 'BLOB' | 'NULL';
+  isPrimaryKey: boolean;
+  isNotNull: boolean;
+}
+
+/**
+ * Row data as stored in the tree
+ * Format: [primaryKey, encodedRow]
+ * - primaryKey: string representation of the primary key (composite keys are joined with \x00)
+ * - encodedRow: JSON-encoded row data
+ */
+export type RowData = [string, string];
+
+/**
+ * Transaction state for managing Optimystic transactions
+ */
+export interface TransactionState {
+  transactor: ITransactor;
+  isActive: boolean;
+  collections: Map<string, any>; // Tree collections used in this transaction
+  transactionId: string; // Unique identifier for this transaction (stable within transaction, cycles between)
+}

package/src/util/generate-transaction-id.ts

@@ -0,0 +1,41 @@
+import { randomBytes } from '@libp2p/crypto';
+import { toString as uint8ArrayToString } from 'uint8arrays/to-string';
+import { sha256 } from '@noble/hashes/sha256';
+
+/**
+ * Generates a unique transaction ID that includes a hash of the peer ID
+ * to reduce collision probability across distributed nodes.
+ * 
+ * Format: {peerIdHash}-{randomBytes}
+ * - peerIdHash: First 16 bytes of SHA-256 hash of peer ID (base64url)
+ * - randomBytes: 16 random bytes (base64url)
+ * - Total: 32 bytes, base64url encoded
+ * 
+ * @param peerId Optional peer ID to include in the hash. If not provided, only random bytes are used.
+ * @returns A unique transaction ID string
+ */
+export function generateTransactionId(peerId?: string): string {
+	const randomPart = randomBytes(16);
+	
+	if (peerId) {
+		// Hash the peer ID and take first 16 bytes
+		const peerIdBytes = new TextEncoder().encode(peerId);
+		const peerIdHash = sha256(peerIdBytes);
+		const peerIdHashPart = peerIdHash.slice(0, 16);
+		
+		// Combine peer ID hash and random bytes
+		const combined = new Uint8Array(32);
+		combined.set(peerIdHashPart, 0);
+		combined.set(randomPart, 16);
+		
+		return uint8ArrayToString(combined, 'base64url');
+	}
+	
+	// Fallback: just use random bytes (padded to 32 bytes for consistency)
+	const fallback = new Uint8Array(32);
+	fallback.set(randomPart, 0);
+	fallback.set(randomBytes(16), 16);
+	
+	return uint8ArrayToString(fallback, 'base64url');
+}
+