@optimystic/db-core 0.5.2 → 0.7.0

package/src/transaction/coordinator.ts

@@ -1,11 +1,14 @@
 import type { ITransactor, BlockId, CollectionId, Transforms, PendRequest, CommitRequest, ActionId, IBlock, BlockOperations } from "../index.js";
-import type { Transaction, ExecutionResult, ITransactionEngine, CollectionActions } from "./transaction.js";
+import type { Transaction, ExecutionResult, ITransactionEngine, CollectionActions, ReadDependency } from "./transaction.js";
 import type { PeerId } from "../network/types.js";
 import type { Collection } from "../collection/collection.js";
 import { TransactionContext } from "./context.js";
 import { ActionsEngine } from "./actions-engine.js";
-import { createActionsStatements, createTransactionStamp, createTransactionId } from "./transaction.js";
+import { createActionsStatements, createTransactionStamp, createTransactionId, isTransactionExpired } from "./transaction.js";
 import { Log, blockIdsForTransforms, transformsFromTransform, hashString } from "../index.js";
+import { createLogger } from "../logger.js";
+
+const log = createLogger('trx:coordinator');
 
 /**
  * Represents an operation on a block within a collection.
@@ -26,6 +29,14 @@ type Operation =
  * - Commit transactions by running consensus phases (GATHER, PEND, COMMIT)
  */
 export class TransactionCoordinator {
+	/** Per-stampId tracking: snapshot before first apply + accumulated actions for replay */
+	private stampData = new Map<string, {
+		order: number;
+		preSnapshot: Map<CollectionId, Transforms>;
+		actionBatches: CollectionActions[][];
+	}>();
+	private nextStampOrder = 0;
+
 	constructor(
 		private readonly transactor: ITransactor,
 		private readonly collections: Map<CollectionId, Collection<any>>
@@ -44,15 +55,37 @@ export class TransactionCoordinator {
 	async applyActions(
 		actions: CollectionActions[],
 		stampId: string
+	): Promise<void> {
+		// On first call for this stampId, snapshot all collections for potential rollback
+		if (!this.stampData.has(stampId)) {
+			const snapshot = new Map<CollectionId, Transforms>();
+			for (const [id, col] of this.collections) {
+				snapshot.set(id, structuredClone(col.tracker.transforms));
+			}
+			this.stampData.set(stampId, {
+				order: this.nextStampOrder++,
+				preSnapshot: snapshot,
+				actionBatches: []
+			});
+		}
+		this.stampData.get(stampId)!.actionBatches.push(actions);
+
+		await this.applyActionsRaw(actions, stampId);
+	}
+
+	/**
+	 * Apply actions without tracking (used internally and for replay during rollback).
+	 */
+	private async applyActionsRaw(
+		actions: CollectionActions[],
+		stampId: string
 	): Promise<void> {
 		for (const { collectionId, actions: collectionActions } of actions) {
-			// Get collection
 			const collection = this.collections.get(collectionId);
 			if (!collection) {
 				throw new Error(`Collection not found: ${collectionId}`);
 			}
 
-			// Apply each action (tagged with stampId)
 			for (const action of collectionActions) {
 				const taggedAction = { ...(action as any), transaction: stampId };
 				await collection.act(taggedAction);
@@ -70,6 +103,10 @@ export class TransactionCoordinator {
 	 * @param transaction - The transaction to commit
 	 */
 	async commit(transaction: Transaction): Promise<void> {
+		if (isTransactionExpired(transaction.stamp)) {
+			throw new Error(`Transaction expired at ${transaction.stamp.expiration}`);
+		}
+
 		// Collect transforms and determine critical blocks for each affected collection
 		const collectionData = Array.from(this.collections.entries())
 			.map(([collectionId, collection]) => ({
@@ -122,7 +159,7 @@ export class TransactionCoordinator {
 			)
 		]);
 
-		const operationsHash = this.hashOperations(allOperations);
+		const operationsHash = await this.hashOperations(allOperations);
 
 		// Execute consensus phases (GATHER, PEND, COMMIT)
 		const coordResult = await this.coordinateTransaction(
@@ -135,24 +172,75 @@ export class TransactionCoordinator {
 		if (!coordResult.success) {
 			throw new Error(`Transaction commit failed: ${coordResult.error}`);
 		}
+
+		// Reset trackers and update actionContext after successful commit
+		for (const { collection } of collectionData) {
+			const newRev = (collection['source'].actionContext?.rev ?? 0) + 1;
+			collection['source'].actionContext = {
+				committed: [
+					...(collection['source'].actionContext?.committed ?? []),
+					{ actionId: transaction.id, rev: newRev }
+				],
+				rev: newRev,
+			};
+			collection.tracker.reset();
+		}
+
+		// Clean up stamp tracking data
+		this.stampData.delete(transaction.stamp.id);
 	}
 
 	/**
-	 * Rollback a transaction (undo applied actions).
+	 * Rollback a transaction (undo only the given stampId's applied actions).
 	 *
-	 * This is called by TransactionSession.rollback() to undo all actions
-	 * that were applied via applyActions().
+	 * Restores tracker state to the snapshot taken before the stampId's first
+	 * applyActions call, then replays any later stamps' actions to preserve
+	 * other sessions' transforms.
 	 *
-	 * @param _stampId - The transaction stamp ID to rollback (currently unused - we clear all trackers)
+	 * @param stampId - The transaction stamp ID to rollback
 	 */
-	async rollback(_stampId: string): Promise<void> {
-		// Clear trackers for all collections
-		// This discards all pending changes that were applied via applyActions()
-		// TODO: In the future, we may want to track which collections were affected by
-		// a specific stampId and only reset those trackers
-		for (const collection of this.collections.values()) {
-			// Reset the tracker to discard all pending transforms
-			collection.tracker.reset();
+	async rollback(stampId: string): Promise<void> {
+		const data = this.stampData.get(stampId);
+		if (!data) return;
+
+		this.stampData.delete(stampId);
+
+		// Collect all remaining stamps to replay
+		const toReplay = [...this.stampData.entries()]
+			.sort(([, a], [, b]) => a.order - b.order);
+
+		// Find the earliest snapshot among the rolled-back stamp and all remaining stamps.
+		// This is necessary because interleaved execution means a lower-order stamp
+		// may have batches applied after a higher-order stamp's snapshot was taken.
+		let earliestSnapshot = data.preSnapshot;
+		let earliestOrder = data.order;
+		for (const [, d] of toReplay) {
+			if (d.order < earliestOrder) {
+				earliestSnapshot = d.preSnapshot;
+				earliestOrder = d.order;
+			}
+		}
+
+		// Restore to the earliest snapshot
+		for (const [collectionId, transforms] of earliestSnapshot) {
+			const collection = this.collections.get(collectionId);
+			if (collection) {
+				collection.tracker.reset(structuredClone(transforms));
+			}
+		}
+
+		// Replay all remaining stamps' batches in order
+		for (const [replayStampId, replayData] of toReplay) {
+			// Update the snapshot to reflect current (post-replay) state
+			const newSnapshot = new Map<CollectionId, Transforms>();
+			for (const [id, col] of this.collections) {
+				newSnapshot.set(id, structuredClone(col.tracker.transforms));
+			}
+			replayData.preSnapshot = newSnapshot;
+
+			for (const actionBatch of replayData.actionBatches) {
+				await this.applyActionsRaw(actionBatch, replayStampId);
+			}
 		}
 	}
 
@@ -190,14 +278,34 @@ export class TransactionCoordinator {
 		}
 	}
 
+	/**
+	 * Collect read dependencies from all participating collections.
+	 */
+	getReadDependencies(): ReadDependency[] {
+		const reads: ReadDependency[] = [];
+		for (const collection of this.collections.values()) {
+			reads.push(...collection.getReadDependencies());
+		}
+		return reads;
+	}
+
+	/**
+	 * Clear read dependencies from all collections.
+	 */
+	clearReadDependencies(): void {
+		for (const collection of this.collections.values()) {
+			collection.clearReadDependencies();
+		}
+	}
+
 	/**
 	 * Compute hash of all operations in a transaction.
 	 * This hash is used for validation - validators re-execute the transaction
 	 * and compare their computed operations hash with this one.
 	 */
-	private hashOperations(operations: readonly Operation[]): string {
+	private async hashOperations(operations: readonly Operation[]): Promise<string> {
 		const operationsData = JSON.stringify(operations);
-		return `ops:${hashString(operationsData)}`;
+		return `ops:${await hashString(operationsData)}`;
 	}
 
 	/**
@@ -223,7 +331,7 @@ export class TransactionCoordinator {
 		const reads = context.getReads();
 
 		// Create stamp from context
-		const stamp = createTransactionStamp(
+		const stamp = await createTransactionStamp(
 			'local', // TODO: Get from context or coordinator
 			Date.now(),
 			'', // TODO: Get from engine
@@ -234,7 +342,7 @@ export class TransactionCoordinator {
 			stamp,
 			statements,
 			reads,
-			id: createTransactionId(stamp.id, statements, reads)
+			id: await createTransactionId(stamp.id, statements, reads)
 		};
 
 		const engine = new ActionsEngine(this);
@@ -254,12 +362,22 @@ export class TransactionCoordinator {
 	 * @returns Execution result with actions and results
 	 */
 	async execute(transaction: Transaction, engine: ITransactionEngine): Promise<ExecutionResult> {
+		const trxId = transaction.id;
+		const t0 = Date.now();
+
+		if (isTransactionExpired(transaction.stamp)) {
+			return { success: false, error: `Transaction expired at ${transaction.stamp.expiration}` };
+		}
+
 		// 1. Validate engine matches transaction
 		// Note: We don't enforce this strictly since the engine is passed in explicitly
 		// The caller is responsible for ensuring the correct engine is used
 
+		const tEngine = Date.now();
 		const result = await engine.execute(transaction);
+		const engineMs = Date.now() - tEngine;
 		if (!result.success) {
+			log('execute:done trxId=%s engine=%dms success=false total=%dms', trxId, engineMs, Date.now() - t0);
 			return result;
 		}
 
@@ -268,6 +386,7 @@ export class TransactionCoordinator {
 		}
 
 		// 2. Apply actions to collections and collect transforms
+		const tApply = Date.now();
 		const collectionTransforms = new Map<CollectionId, Transforms>();
 		const criticalBlocks = new Map<CollectionId, BlockId>();
 		const actionResults = new Map<CollectionId, any[]>();
@@ -301,9 +420,12 @@ export class TransactionCoordinator {
 				({ type: 'delete' as const, collectionId, blockId })
 			)
 		]);
-		const operationsHash = this.hashOperations(allOperations);
+		const operationsHash = await this.hashOperations(allOperations);
+
+		const applyMs = Date.now() - tApply;
 
 		// 4. Coordinate (GATHER if multi-collection)
+		const tCoord = Date.now();
 		const coordResult = await this.coordinateTransaction(
 			transaction,
 			operationsHash,
@@ -311,11 +433,34 @@ export class TransactionCoordinator {
 			criticalBlocks
 		);
 
+		const coordMs = Date.now() - tCoord;
 		if (!coordResult.success) {
+			log('execute:done trxId=%s engine=%dms apply=%dms coordinate=%dms success=false total=%dms', trxId, engineMs, applyMs, coordMs, Date.now() - t0);
 			return coordResult;
 		}
 
-		// 4. Return results from actions
+		// 5. Update actionContext and reset trackers after successful commit
+		for (const collectionActions of result.actions) {
+			const collection = this.collections.get(collectionActions.collectionId);
+			if (collection) {
+				const newRev = (collection['source'].actionContext?.rev ?? 0) + 1;
+				const actionId = transaction.id;
+				collection['source'].actionContext = {
+					committed: [
+						...(collection['source'].actionContext?.committed ?? []),
+						{ actionId, rev: newRev }
+					],
+					rev: newRev,
+				};
+				collection.tracker.reset();
+			}
+		}
+
+		// Clean up stamp tracking data
+		this.stampData.delete(transaction.stamp.id);
+
+		// 6. Return results from actions
+		log('execute:done trxId=%s engine=%dms apply=%dms coordinate=%dms total=%dms', trxId, engineMs, applyMs, coordMs, Date.now() - t0);
 		return {
 			success: true,
 			actions: result.actions,
@@ -399,36 +544,52 @@ export class TransactionCoordinator {
 		collectionTransforms: Map<CollectionId, Transforms>,
 		criticalBlocks: Map<CollectionId, BlockId>
 	): Promise<{ success: boolean; error?: string }> {
+		const trxId = transaction.id;
+		const t0 = Date.now();
+
 		// 1. GATHER phase: collect critical cluster nominees (skip if single collection)
 		const criticalBlockIds = Array.from(criticalBlocks.values());
+		const tGather = Date.now();
 		const superclusterNominees = await this.gatherPhase(criticalBlockIds);
+		const gatherMs = Date.now() - tGather;
 
 		// 2. PEND phase: distribute to all block clusters
+		const tPend = Date.now();
 		const pendResult = await this.pendPhase(
 			transaction,
 			operationsHash,
 			collectionTransforms,
 			superclusterNominees
 		);
+		const pendMs = Date.now() - tPend;
 		if (!pendResult.success) {
+			log('trx:phases trxId=%s gather=%dms pend=%dms (failed) total=%dms', trxId, gatherMs, pendMs, Date.now() - t0);
 			return pendResult;
 		}
 
-		// 3. COMMIT phase: commit to all critical blocks
+		// 3. COMMIT phase: commit to all critical blocks (with retry for forward recovery)
+		const tCommit = Date.now();
 		const commitResult = await this.commitPhase(
 			transaction.id as ActionId,
 			criticalBlockIds,
 			pendResult.pendedBlockIds!
 		);
+		const commitMs = Date.now() - tCommit;
 		if (!commitResult.success) {
-			// Cancel pending actions on failure
-			await this.cancelPhase(transaction.id as ActionId, collectionTransforms);
-			return commitResult;
+			// Targeted cancel: only cancel collections that are still pending (not already committed)
+			await this.cancelPhase(
+				transaction.id as ActionId,
+				pendResult.pendedBlockIds!,
+				commitResult.committedCollections
+			);
+			log('trx:phases trxId=%s gather=%dms pend=%dms commit=%dms (failed) total=%dms', trxId, gatherMs, pendMs, commitMs, Date.now() - t0);
+			return { success: false, error: commitResult.error };
 		}
 
 		// 4. PROPAGATE and CHECKPOINT phases are handled by clusters automatically
 		// (as per user's note: "managed by each cluster, the client doesn't have to worry about them")
 
+		log('trx:phases trxId=%s gather=%dms pend=%dms commit=%dms total=%dms', trxId, gatherMs, pendMs, commitMs, Date.now() - t0);
 		return { success: true };
 	}
 
@@ -518,6 +679,10 @@ export class TransactionCoordinator {
 			// Pend the transaction
 			const pendResult = await this.transactor.pend(pendRequest);
 			if (!pendResult.success) {
+				// Cancel any already-pended collections before returning
+				for (const [pendedCollectionId, pendedBlockIdList] of pendedBlockIds.entries()) {
+					await this.transactor.cancel({ actionId, blockIds: pendedBlockIdList });
+				}
 				return {
 					success: false,
 					error: `Pend failed for collection ${collectionId}: ${pendResult.reason}`
@@ -532,18 +697,36 @@ export class TransactionCoordinator {
 	}
 
 	/**
-	 * COMMIT phase: Commit to all critical blocks.
+	 * COMMIT phase: Commit to all critical blocks with retry for transient failures.
+	 *
+	 * Once all collections are pended (Phase 1 passes), the coordinator has decided
+	 * to commit. Failed commits are retried (forward recovery) before giving up.
+	 * Returns which collections committed vs failed so the caller can do targeted cancel.
 	 */
 	private async commitPhase(
 		actionId: ActionId,
 		criticalBlockIds: BlockId[],
 		pendedBlockIds: Map<CollectionId, BlockId[]>
-	): Promise<{ success: boolean; error?: string }> {
-		// Commit each collection's transaction
+	): Promise<{
+		success: boolean;
+		error?: string;
+		committedCollections: Set<CollectionId>;
+		failedCollections: Set<CollectionId>;
+	}> {
+		const committedCollections = new Set<CollectionId>();
+		const failedCollections = new Set<CollectionId>();
+
+		// Commit each collection's transaction with retry
 		for (const [collectionId, blockIds] of pendedBlockIds.entries()) {
 			const collection = this.collections.get(collectionId);
 			if (!collection) {
-				return { success: false, error: `Collection not found: ${collectionId}` };
+				failedCollections.add(collectionId);
+				return {
+					success: false,
+					error: `Collection not found: ${collectionId}`,
+					committedCollections,
+					failedCollections
+				};
 			}
 
 			// Get revision
@@ -555,9 +738,12 @@ export class TransactionCoordinator {
 			);
 
 			if (!logTailBlockId) {
+				failedCollections.add(collectionId);
 				return {
 					success: false,
-					error: `Log tail block not found for collection ${collectionId}`
+					error: `Log tail block not found for collection ${collectionId}`,
+					committedCollections,
+					failedCollections
 				};
 			}
 
@@ -569,41 +755,42 @@ export class TransactionCoordinator {
 				rev
 			};
 
-			// Commit the transaction
-			const commitResult = await this.transactor.commit(commitRequest);
-			if (!commitResult.success) {
-				return {
-					success: false,
-					error: `Commit failed for collection ${collectionId}`
-				};
+			// Retry up to 3 attempts for transient failures
+			let committed = false;
+			for (let attempt = 0; attempt < 3 && !committed; attempt++) {
+				const commitResult = await this.transactor.commit(commitRequest);
+				if (commitResult.success) {
+					committed = true;
+					committedCollections.add(collectionId);
+				} else if (attempt === 2) {
+					failedCollections.add(collectionId);
+					return {
+						success: false,
+						error: `Commit failed for collection ${collectionId} after 3 attempts`,
+						committedCollections,
+						failedCollections
+					};
+				}
 			}
 		}
 
-		return { success: true };
+		return { success: true, committedCollections, failedCollections };
 	}
 
 	/**
-	 * CANCEL phase: Cancel pending actions on all affected blocks.
+	 * CANCEL phase: Cancel pending actions on affected blocks.
+	 *
+	 * Uses the authoritative pended block IDs from pendPhase rather than
+	 * recomputing from transforms. Optionally skips already-committed collections.
 	 */
 	private async cancelPhase(
 		actionId: ActionId,
-		collectionTransforms: Map<CollectionId, Transforms>
+		pendedBlockIds: Map<CollectionId, BlockId[]>,
+		excludeCollections?: Set<CollectionId>
 	): Promise<void> {
-		// Cancel each collection's pending transaction
-		for (const [collectionId, transforms] of collectionTransforms.entries()) {
-			const collection = this.collections.get(collectionId);
-			if (!collection) {
-				continue; // Skip if collection not found
-			}
-
-			// Get the block IDs from transforms
-			const blockIds = blockIdsForTransforms(transforms);
-
-			// Cancel the transaction
-			await this.transactor.cancel({
-				actionId,
-				blockIds
-			});
+		for (const [collectionId, blockIds] of pendedBlockIds.entries()) {
+			if (excludeCollections?.has(collectionId)) continue;
+			await this.transactor.cancel({ actionId, blockIds });
 		}
 	}
 

package/src/transaction/index.ts

@@ -14,7 +14,9 @@ export type {
 export {
 	createTransactionStamp,
 	createTransactionId,
-	createActionsStatements
+	createActionsStatements,
+	DEFAULT_TRANSACTION_TTL_MS,
+	isTransactionExpired
 } from './transaction.js';
 
 export {
@@ -26,4 +28,4 @@ export { TransactionCoordinator } from './coordinator.js';
 export { TransactionContext } from './context.js';
 export { TransactionSession } from './session.js';
 export { TransactionValidator } from './validator.js';
-export type { EngineRegistration, ValidationCoordinatorFactory } from './validator.js';
+export type { EngineRegistration, ValidationCoordinatorFactory, BlockStateProvider } from './validator.js';

package/src/transaction/session.ts

@@ -1,6 +1,6 @@
 import type { TransactionCoordinator } from "./coordinator.js";
 import type { Transaction, ExecutionResult, ITransactionEngine, TransactionStamp, CollectionActions } from "./transaction.js";
-import { createTransactionStamp, createTransactionId } from "./transaction.js";
+import { createTransactionStamp, createTransactionId, isTransactionExpired } from "./transaction.js";
 
 /**
  * TransactionSession manages incremental transaction building.
@@ -14,7 +14,7 @@ import { createTransactionStamp, createTransactionId } from "./transaction.js";
  * - The Transaction is then committed through coordinator.commit() for PEND/COMMIT orchestration
  *
  * Usage:
- *   const session = new TransactionSession(coordinator, engine);
+ *   const session = await TransactionSession.create(coordinator, engine);
  *   await session.execute('INSERT INTO users (id, name) VALUES (?, ?)', [1, 'Alice']);
  *   await session.execute('SELECT * FROM orders WHERE user_id = ?', [1]);
  *   const result = await session.commit();
@@ -27,19 +27,33 @@ export class TransactionSession {
 	private committed = false;
 	private rolledBack = false;
 
-	constructor(
+	private constructor(
 		private readonly coordinator: TransactionCoordinator,
 		private readonly engine: ITransactionEngine,
-		peerId: string = 'local', // TODO: Get from coordinator or config
-		schemaHash: string = '' // TODO: Get from engine
+		stamp: TransactionStamp
 	) {
-		// Create stamp at BEGIN (stable throughout transaction)
-		this.stamp = createTransactionStamp(
+		this.stamp = stamp;
+	}
+
+	/**
+	 * Create a new TransactionSession.
+	 * Uses async factory because stamp creation requires SHA-256 hashing.
+	 */
+	static async create(
+		coordinator: TransactionCoordinator,
+		engine: ITransactionEngine,
+		peerId: string = 'local',
+		schemaHash: string = '',
+		ttlMs?: number
+	): Promise<TransactionSession> {
+		const stamp = await createTransactionStamp(
 			peerId,
 			Date.now(),
 			schemaHash,
-			'unknown' // TODO: Get engine ID from engine
+			'unknown', // TODO: Get engine ID from engine
+			ttlMs
 		);
+		return new TransactionSession(coordinator, engine, stamp);
 	}
 
 	/**
@@ -107,28 +121,37 @@ export class TransactionSession {
 		if (this.rolledBack) {
 			return { success: false, error: 'Transaction already rolled back' };
 		}
+		if (isTransactionExpired(this.stamp)) {
+			return { success: false, error: `Transaction expired at ${this.stamp.expiration}` };
+		}
+
+		// Collect read dependencies from all participating collections
+		const reads = this.coordinator.getReadDependencies();
 
 		// Create the complete transaction
 		const transaction: Transaction = {
 			stamp: this.stamp,
 			statements: this.statements,
-			reads: [], // TODO: Track reads during statement execution
-			id: createTransactionId(this.stamp.id, this.statements, [])
+			reads,
+			id: await createTransactionId(this.stamp.id, this.statements, reads)
 		};
 
 		// Commit through coordinator (which will orchestrate PEND/COMMIT)
 		await this.coordinator.commit(transaction);
 
+		// Clear read dependencies after successful commit
+		this.coordinator.clearReadDependencies();
+
 		this.committed = true;
 		return { success: true };
 	}
 
 	/**
-	 * Rollback the transaction (discard local state).
+	 * Rollback the transaction (undo this session's applied actions).
 	 *
-	 * Note: Actions have already been applied to collections' trackers.
-	 * Rollback just prevents commit and clears session state.
-	 * Collections will discard tracker state when they sync or update.
+	 * Delegates to coordinator.rollback(stampId) which restores collection
+	 * trackers to the pre-session snapshot and replays any later sessions'
+	 * actions to preserve their transforms.
 	 */
 	async rollback(): Promise<void> {
 		if (this.committed) {
@@ -140,6 +163,7 @@ export class TransactionSession {
 
 		// Rollback through coordinator
 		await this.coordinator.rollback(this.stamp.id);
+		this.coordinator.clearReadDependencies();
 		this.rolledBack = true;
 		this.statements.length = 0;
 	}