@optimystic/db-core 0.5.1 → 0.6.0

package/src/transactor/network-transactor.ts

@@ -5,6 +5,9 @@ import { transformForBlockId, groupBy, concatTransforms, concatTransform, transf
 import { blockIdToBytes } from "../utility/block-id-to-bytes.js";
 import { isRecordEmpty } from "../utility/is-record-empty.js";
 import { type CoordinatorBatch, makeBatchesByPeer, incompleteBatches, everyBatch, allBatches, mergeBlocks, processBatches, createBatchesForPayload } from "../utility/batch-coordinator.js";
+import { createLogger, verbose } from "../logger.js";
+
+const log = createLogger('network-transactor');
 
 type NetworkTransactorInit = {
 	timeoutMs: number;
@@ -31,6 +34,8 @@ export class NetworkTransactor implements ITransactor {
 	async get(blockGets: BlockGets): Promise<GetBlockResults> {
 		// Group by block id
 		const distinctBlockIds = Array.from(new Set(blockGets.blockIds));
+		const t0 = Date.now();
+		log('get blockIds=%d', distinctBlockIds.length);
 
 		const batches = await this.batchesForPayload<BlockId[], GetBlockResults>(
 			distinctBlockIds,
@@ -77,6 +82,7 @@ export class NetworkTransactor implements ITransactor {
 		) as CoordinatorBatch<BlockId[], GetBlockResults>[];
 
 		if (retryable.length > 0 && Date.now() < expiration) {
+			log('get:retry retryable=%d', retryable.length);
 			try {
 				const excludedByRoot = new Map<CoordinatorBatch<BlockId[], GetBlockResults>, Set<PeerId>>();
 				for (const b of retryable) {
@@ -128,6 +134,7 @@ export class NetworkTransactor implements ITransactor {
 		// Ensure we have at least one response per requested block id
 		const missingIds = distinctBlockIds.filter(bid => !resultEntries.has(bid));
 		if (missingIds.length > 0) {
+			log('get:missing blockIds=%o', missingIds);
 			const details = this.formatBatchStatuses(batches,
 				b => (b.request?.isResponse as boolean) ?? false,
 				b => {
@@ -139,6 +146,7 @@ export class NetworkTransactor implements ITransactor {
 			throw aggregate;
 		}
 
+		log('get:done blockIds=%d ms=%d', distinctBlockIds.length, Date.now() - t0);
 		return Object.fromEntries(resultEntries) as GetBlockResults;
 	}
 
@@ -181,24 +189,77 @@ export class NetworkTransactor implements ITransactor {
 		transforms: Transforms,
 		transformForBlock: (payload: Transforms, blockId: BlockId, mergeWith?: Transforms) => Transforms
 	): Promise<CoordinatorBatch<Transforms, PendResult>[]> {
-		const blockCoordinators = await Promise.all(
-			blockIds.map(async bid => ({
+		// Use cluster intersections to minimize the number of coordinators.
+		// For each block, find its full cluster, then greedily assign blocks to
+		// peers that appear in the most clusters — reducing round trips when
+		// blocks share cluster members.
+
+		// Step 1: Get cluster peer sets for each block
+		const blockClusterPeerIds: Map<BlockId, Set<string>> = new Map();
+		const fallbackBlocks: BlockId[] = [];
+
+		await Promise.all(blockIds.map(async bid => {
+			try {
+				const clusterPeers = await this.keyNetwork.findCluster(await blockIdToBytes(bid));
+				blockClusterPeerIds.set(bid, new Set(Object.keys(clusterPeers)));
+			} catch {
+				fallbackBlocks.push(bid);
+			}
+		}));
+
+		// Step 2: Build peer → blocks index (which blocks each peer can coordinate)
+		const peerBlocks = new Map<string, BlockId[]>();
+		for (const [blockId, peerIds] of blockClusterPeerIds) {
+			for (const peerId of peerIds) {
+				const blocks = peerBlocks.get(peerId) ?? [];
+				blocks.push(blockId);
+				peerBlocks.set(peerId, blocks);
+			}
+		}
+
+		// Step 3: Greedy set cover — assign blocks to peers covering the most uncovered blocks
+		const uncovered = new Set(blockClusterPeerIds.keys());
+		const assignments = new Map<string, BlockId[]>(); // peerIdStr → assigned blockIds
+
+		while (uncovered.size > 0) {
+			let bestPeer: string | undefined;
+			let bestCount = 0;
+
+			for (const [peerId, blocks] of peerBlocks) {
+				const coverCount = blocks.filter(bid => uncovered.has(bid)).length;
+				if (coverCount > bestCount) {
+					bestCount = coverCount;
+					bestPeer = peerId;
+				}
+			}
+
+			if (!bestPeer || bestCount === 0) break;
+
+			const covered = peerBlocks.get(bestPeer)!.filter(bid => uncovered.has(bid));
+			assignments.set(bestPeer, covered);
+			for (const bid of covered) uncovered.delete(bid);
+		}
+
+		// Step 4: Any remaining uncovered blocks fall back to findCoordinator
+		for (const bid of uncovered) fallbackBlocks.push(bid);
+
+		const fallbackCoordinators = await Promise.all(
+			fallbackBlocks.map(async bid => ({
 				blockId: bid,
 				coordinator: await this.keyNetwork.findCoordinator(await blockIdToBytes(bid), { excludedPeers: [] })
 			}))
 		);
-
-		const byCoordinator = new Map<string, BlockId[]>();
-		for (const { blockId, coordinator } of blockCoordinators) {
+		for (const { blockId, coordinator } of fallbackCoordinators) {
 			const key = coordinator.toString();
-			const blocks = byCoordinator.get(key) ?? [];
-			blocks.push(blockId);
-			byCoordinator.set(key, blocks);
+			const existing = assignments.get(key) ?? [];
+			existing.push(blockId);
+			assignments.set(key, existing);
 		}
 
+		// Step 5: Convert assignments to batches
 		const batches: CoordinatorBatch<Transforms, PendResult>[] = [];
-		for (const [coordinatorStr, consolidatedBlocks] of byCoordinator) {
-			const coordinator = blockCoordinators.find(bc => bc.coordinator.toString() === coordinatorStr)!.coordinator;
+		for (const [peerIdStr, consolidatedBlocks] of assignments) {
+			const peerId = peerIdFromString(peerIdStr);
 
 			let batchTransforms: Transforms = { inserts: {}, updates: {}, deletes: [] };
 			for (const bid of consolidatedBlocks) {
@@ -207,7 +268,7 @@ export class NetworkTransactor implements ITransactor {
 			}
 
 			batches.push({
-				peerId: coordinator,
+				peerId,
 				payload: batchTransforms,
 				blockId: consolidatedBlocks[0]!,
 				coordinatingBlockIds: consolidatedBlocks,
@@ -219,6 +280,7 @@ export class NetworkTransactor implements ITransactor {
 	}
 
 	async pend(blockAction: PendRequest): Promise<PendResult> {
+		const t0 = Date.now();
 		const transformForBlock = (payload: Transforms, blockId: BlockId, mergeWithPayload: Transforms | undefined): Transforms => {
 			const filteredTransform = transformForBlockId(payload, blockId);
 			return mergeWithPayload
@@ -227,6 +289,17 @@ export class NetworkTransactor implements ITransactor {
 		};
 		const blockIds = blockIdsForTransforms(blockAction.transforms);
 		const batches = await this.consolidateCoordinators(blockIds, blockAction.transforms, transformForBlock);
+		log('pend actionId=%s blockIds=%d batches=%d', blockAction.actionId, blockIds.length, batches.length);
+		if (verbose) {
+			const batchSummary = batches.map(b => ({
+				peer: b.peerId.toString().substring(0, 12),
+				blocks: (b as any).coordinatingBlockIds ?? [b.blockId],
+				inserts: Object.keys(b.payload.inserts ?? {}).length,
+				updates: Object.keys(b.payload.updates ?? {}).length,
+				deletes: b.payload.deletes?.length ?? 0
+			}));
+			log('pend:batches actionId=%s detail=%o', blockAction.actionId, batchSummary);
+		}
 		const expiration = Date.now() + this.timeoutMs;
 
 		let error: Error | undefined;
@@ -274,9 +347,11 @@ export class NetworkTransactor implements ITransactor {
 		}
 
 		if (error) { // If any failures, cancel all pending actions as background microtask
+			log('pend:cancel actionId=%s', blockAction.actionId);
 			void Promise.resolve().then(() => this.cancelBatch(batches, { blockIds, actionId: blockAction.actionId }));
 			const stale = Array.from(allBatches(batches, b => b.request?.isResponse as boolean && !b.request!.response!.success));
 			if (stale.length > 0) {	// Any active stale failures should preempt reporting connection or other potential transient errors (we have information)
+				log('pend:stale actionId=%s staleCount=%d', blockAction.actionId, stale.length);
 				return {
 					success: false,
 					missing: distinctBlockActionTransforms(stale.flatMap(b => (b.request!.response! as StaleFailure).missing).filter((x): x is ActionTransforms => x !== undefined)),
@@ -287,6 +362,7 @@ export class NetworkTransactor implements ITransactor {
 
 		// Collect replies back into result structure
 		const completed = Array.from(allBatches(batches, b => b.request?.isResponse as boolean && b.request!.response!.success));
+		log('pend:done actionId=%s ms=%d batches=%d', blockAction.actionId, Date.now() - t0, batches.length);
 		return {
 			success: true,
 			pending: completed.flatMap(b => (b.request!.response! as PendSuccess).pending),
@@ -295,6 +371,7 @@ export class NetworkTransactor implements ITransactor {
 	}
 
 	async cancel(actionRef: ActionBlocks): Promise<void> {
+		log('cancel actionId=%s blockIds=%d', actionRef.actionId, actionRef.blockIds.length);
 		const batches = await this.batchesForPayload<BlockId[], void>(
 			actionRef.blockIds,
 			actionRef.blockIds,
@@ -320,6 +397,8 @@ export class NetworkTransactor implements ITransactor {
 	}
 
 	async commit(request: CommitRequest): Promise<CommitResult> {
+		const t0 = Date.now();
+		log('commit actionId=%s rev=%d blockIds=%d', request.actionId, request.rev, request.blockIds.length);
 		const allBlockIds = [...new Set([...request.blockIds, request.tailId])];
 
 		// Commit the header block if provided and not already in blockIds
@@ -350,6 +429,7 @@ export class NetworkTransactor implements ITransactor {
 			}
 		}
 
+		log('commit:done actionId=%s ms=%d', request.actionId, Date.now() - t0);
 		return { success: true };
 	}
 
@@ -372,6 +452,7 @@ export class NetworkTransactor implements ITransactor {
 	private async commitBlocks({ blockIds, actionId, rev }: RepoCommitRequest) {
 		const expiration = Date.now() + this.timeoutMs;
 		const batches = await this.batchesForPayload<BlockId[], CommitResult>(blockIds, blockIds, mergeBlocks, []);
+		log('commitBlocks actionId=%s rev=%d batches=%d', actionId, rev, batches.length);
 		let error: Error | undefined;
 		try {
 			await processBatches(

package/src/transactor/transactor-source.ts

@@ -1,8 +1,11 @@
 import { randomBytes } from '@noble/hashes/utils.js'
 import { toString as uint8ArrayToString } from 'uint8arrays/to-string'
 import type { IBlock, BlockId, BlockHeader, ITransactor, ActionId, StaleFailure, ActionContext, BlockType, BlockSource, Transforms } from "../index.js";
+import type { ReadDependency } from "../transaction/transaction.js";
 
 export class TransactorSource<TBlock extends IBlock> implements BlockSource<TBlock> {
+	private readDependencies: ReadDependency[] = [];
+
 	constructor(
 		private readonly collectionId: BlockId,
 		private readonly transactor: ITransactor,
@@ -26,12 +29,22 @@ export class TransactorSource<TBlock extends IBlock> implements BlockSource<TBlo
 		const result = await this.transactor.get({ blockIds: [id], context: this.actionContext });
 		if (result) {
 			const { block, state } = result[id]!;
+			// Record read dependency for optimistic concurrency control
+			this.readDependencies.push({ blockId: id, revision: state.latest?.rev ?? 0 });
 			// TODO: if the state reports that there is a pending action, record this so that we are sure to update before syncing
 			//state.pendings
 			return block as TBlock;
 		}
 	}
 
+	getReadDependencies(): ReadDependency[] {
+		return this.readDependencies;
+	}
+
+	clearReadDependencies(): void {
+		this.readDependencies = [];
+	}
+
 	/**
 	 * Attempts to apply the given transforms in a transactional manner.
 	 * @param transform - The transforms to apply.
@@ -50,15 +63,21 @@ export class TransactorSource<TBlock extends IBlock> implements BlockSource<TBlo
 			return pendResult;
 		}
 		const isNew = transform.inserts && Object.hasOwn(transform.inserts, headerId);
-		const commitResult = await this.transactor.commit({
-			headerId: isNew ? headerId : undefined,
-			tailId,
-			blockIds: pendResult.blockIds,
-			actionId,
-			rev
-		});
-		if (!commitResult.success) {
-			return commitResult;
+		try {
+			const commitResult = await this.transactor.commit({
+				headerId: isNew ? headerId : undefined,
+				tailId,
+				blockIds: pendResult.blockIds,
+				actionId,
+				rev
+			});
+			if (!commitResult.success) {
+				await this.transactor.cancel({ actionId, blockIds: pendResult.blockIds });
+				return commitResult;
+			}
+		} catch (e) {
+			await this.transactor.cancel({ actionId, blockIds: pendResult.blockIds });
+			throw e;
 		}
 	}
 }

package/src/transform/atomic-proxy.ts

@@ -0,0 +1,49 @@
+import { Atomic } from './atomic.js';
+import type { IBlock, BlockId, BlockStore, BlockType, BlockHeader, BlockOperation } from '../index.js';
+
+/**
+ * A BlockStore proxy that enables scoped atomic operations.
+ * Operations normally delegate directly to the underlying store,
+ * but during an `atomic()` call, they route through an Atomic tracker
+ * that commits all-or-nothing on success, or rolls back on error.
+ *
+ * Both the BTree and its trunk should share the same AtomicProxy instance
+ * so that all mutations (including root pointer updates) are part of the
+ * same atomic batch.
+ */
+export class AtomicProxy<T extends IBlock> implements BlockStore<T> {
+	private _base: BlockStore<T>;
+	private _active: BlockStore<T>;
+
+	constructor(store: BlockStore<T>) {
+		this._base = store;
+		this._active = store;
+	}
+
+	async tryGet(id: BlockId): Promise<T | undefined> { return this._active.tryGet(id); }
+	insert(block: T): void { this._active.insert(block); }
+	update(blockId: BlockId, op: BlockOperation): void { this._active.update(blockId, op); }
+	delete(blockId: BlockId): void { this._active.delete(blockId); }
+	generateId(): BlockId { return this._active.generateId(); }
+	createBlockHeader(type: BlockType, newId?: BlockId): BlockHeader { return this._active.createBlockHeader(type, newId); }
+
+	/** Execute fn within an atomic scope. All store mutations are collected
+	 *  and committed on success, or discarded on error. Re-entrant safe. */
+	async atomic<R>(fn: () => Promise<R>): Promise<R> {
+		if (this._active !== this._base) {
+			return fn();	// Already in atomic context
+		}
+		const atomic = new Atomic<T>(this._base);
+		this._active = atomic;
+		try {
+			const result = await fn();
+			atomic.commit();
+			return result;
+		} catch (e) {
+			atomic.reset();
+			throw e;
+		} finally {
+			this._active = this._base;
+		}
+	}
+}

package/src/transform/cache-source.ts

@@ -1,19 +1,33 @@
 import type { IBlock, BlockHeader, BlockId, BlockSource, BlockType, Transforms } from "../index.js";
 import { applyOperation } from "../index.js";
+import { LruMap } from "../utility/lru-map.js";
+import { createLogger } from "../logger.js";
+
+const log = createLogger('cache');
+
+const DefaultMaxSize = 128;
 
 export class CacheSource<T extends IBlock> implements BlockSource<T> {
-	protected cache = new Map<BlockId, T>();
+	protected cache: LruMap<BlockId, T>;
 
 	constructor(
-		protected readonly source: BlockSource<T>
-	) { }
+		protected readonly source: BlockSource<T>,
+		maxSize = DefaultMaxSize
+	) {
+		this.cache = new LruMap(maxSize);
+	}
 
 	async tryGet(id: BlockId): Promise<T | undefined> {
 		let block = this.cache.get(id);
-		if (!block) {
+		if (block) {
+			log('hit id=%s', id);
+		} else {
 			block = await this.source.tryGet(id);
 			if (block) {
 				this.cache.set(id, block);
+				log('miss:loaded id=%s cacheSize=%d', id, this.cache.size);
+			} else {
+				log('miss:absent id=%s', id);
 			}
 		}
 		return structuredClone(block);

package/src/transform/index.ts

@@ -1,4 +1,5 @@
 export * from "./atomic.js";
+export * from "./atomic-proxy.js";
 export * from "./cache-source.js";
 export * from "./helpers.js";
 export * from "./struct.js";

package/src/utility/batch-coordinator.ts

@@ -1,6 +1,9 @@
 import type { PeerId } from "../network/types.js";
 import type { BlockId } from "../index.js";
 import { Pending } from "./pending.js";
+import { createLogger } from "../logger.js";
+
+const log = createLogger('batch-coordinator');
 
 /**
  * Represents a batch of operations for a specific block coordinated by a peer
@@ -122,6 +125,7 @@ export async function processBatches<TPayload, TResponse>(
                 .catch(async e => {
                     if (expiration > Date.now()) {
                         const excludedPeers = [batch.peerId, ...(batch.excludedPeers ?? [])];
+                        log('retry peer=%s excluded=%d', batch.peerId.toString(), excludedPeers.length);
                         const retries = await createBatchesForPayload<TPayload, TResponse>(
                             getBlockIds(batch),
                             batch.payload,
@@ -170,5 +174,7 @@ export async function createBatchesForPayload<TPayload, TResponse>(
 	);
 
 	// Group blocks around their coordinating peers
-	return makeBatchesByPeer<TPayload, TResponse>(blockIdPeerId, payload, getBlockPayload, excludedPeers);
+	const batches = makeBatchesByPeer<TPayload, TResponse>(blockIdPeerId, payload, getBlockPayload, excludedPeers);
+	log('createBatches blockIds=%d batches=%d excluded=%d', distinctBlockIds.size, batches.length, excludedPeers.length);
+	return batches;
 }

package/src/utility/hash-string.ts

@@ -1,17 +1,14 @@
-/**
- * Simple djb2 string hash function.
- * 
- * This is a non-cryptographic hash suitable for generating short identifiers.
- * For security-critical hashing, use SHA-256 from multiformats/hashes/sha2.
- * 
- * @param str - The string to hash
- * @returns A base-36 encoded hash string
- */
-export function hashString(str: string): string {
-	let hash = 5381;
-	for (let i = 0; i < str.length; i++) {
-		hash = ((hash << 5) + hash + str.charCodeAt(i)) | 0;
-	}
-	return Math.abs(hash).toString(36);
-}
-
+import { sha256 } from 'multiformats/hashes/sha2';
+import { toString } from 'uint8arrays/to-string';
+
+/**
+ * SHA-256 string hash function.
+ *
+ * @param str - The string to hash
+ * @returns A base64url-encoded SHA-256 hash string
+ */
+export async function hashString(str: string): Promise<string> {
+	const input = new TextEncoder().encode(str);
+	const mh = await sha256.digest(input);
+	return toString(mh.digest, 'base64url');
+}

package/src/utility/lru-map.ts

@@ -0,0 +1,55 @@
+/**
+ * A simple LRU (Least Recently Used) map backed by JavaScript's Map insertion order.
+ * Accessing or setting an entry refreshes it to the most-recently-used position.
+ * When the map exceeds maxSize, the least-recently-used entry is evicted.
+ */
+export class LruMap<K, V> {
+	private readonly map = new Map<K, V>();
+
+	constructor(private readonly maxSize: number) {
+		if (maxSize < 1) throw new Error('LruMap maxSize must be >= 1');
+	}
+
+	get(key: K): V | undefined {
+		const value = this.map.get(key);
+		if (value !== undefined) {
+			// Refresh: delete and re-insert to move to end (most recent)
+			this.map.delete(key);
+			this.map.set(key, value);
+		}
+		return value;
+	}
+
+	set(key: K, value: V): this {
+		// If already present, delete first to refresh position
+		if (this.map.has(key)) {
+			this.map.delete(key);
+		} else if (this.map.size >= this.maxSize) {
+			// Evict the oldest (first) entry
+			const oldest = this.map.keys().next().value!;
+			this.map.delete(oldest);
+		}
+		this.map.set(key, value);
+		return this;
+	}
+
+	has(key: K): boolean {
+		return this.map.has(key);
+	}
+
+	delete(key: K): boolean {
+		return this.map.delete(key);
+	}
+
+	clear(): void {
+		this.map.clear();
+	}
+
+	get size(): number {
+		return this.map.size;
+	}
+
+	[Symbol.iterator](): IterableIterator<[K, V]> {
+		return this.map[Symbol.iterator]();
+	}
+}