@optimystic/db-core 0.14.0 → 0.16.2

package/src/transform/cache-source.ts

@@ -1,38 +1,102 @@
-import type { IBlock, BlockHeader, BlockId, BlockSource, BlockType, Transforms } from "../index.js";
+import type { IBlock, BlockHeader, BlockId, BlockSource, BlockType, ReadPurpose, Transforms } from "../index.js";
 import { applyOperation } from "../index.js";
 import { LruMap } from "../utility/lru-map.js";
 import { createLogger } from "../logger.js";
+import type { ReadDependencyCollector } from "../transaction/read-dependency-collector.js";
 
 const log = createLogger('cache');
 
 const DefaultMaxSize = 128;
 
+/** The revision a source reports for an id, or undefined if the source can't report one.
+ *  Duck-typed exactly like {@link Tracker}'s getGeneration probe — CacheSource layers over
+ *  arbitrary BlockSources (including test doubles) that need not implement it. */
+function sourceReadRevision(source: unknown, id: BlockId): number | undefined {
+	const src = source as { getReadRevision?: (id: BlockId) => number | undefined };
+	return typeof src.getReadRevision === 'function' ? src.getReadRevision(id) : undefined;
+}
+
 export class CacheSource<T extends IBlock> implements BlockSource<T> {
 	protected cache: LruMap<BlockId, T>;
+	/** Per-id monotonic counter, bumped whenever the cached content for an id changes.
+	 * Consumers (e.g. {@link Tracker}'s materialized-block memo) read it via
+	 * {@link getGeneration} to detect that a cached "source + ops" result has gone stale.
+	 * Over-bumping is safe (it only forces a re-materialize); under-bumping is a correctness
+	 * bug, so every content-changing site bumps. A benign LRU evict + reload also bumps. */
+	// NOTE: generations is never pruned — it retains one small (id → number) entry per distinct id
+	// ever touched, even after LRU eviction from `cache`. Bounded by the number of distinct blocks a
+	// collection sees over its lifetime; if that ever grows large enough to matter, evict alongside
+	// the LRU (dropping a generation is safe — a reload re-bumps from 0/absent, forcing re-materialize).
+	private generations = new Map<BlockId, number>();
+	/** Per-id committed revision of the content currently cached for that id. Learned from the
+	 *  source on a miss-load, advanced by {@link transformCache} when a commit folds new content
+	 *  in, and dropped alongside the cached block on delete/clear. Re-emitted on every cache HIT
+	 *  so a hit records a read dependency at the right revision — the whole point of this map, since
+	 *  the underlying source is never consulted on a hit. */
+	// NOTE: an LRU-evicted id can leave a stale `revisions` entry (eviction drops `cache` but not
+	// this map — see clear()/the LruMap eviction). Benign: the next read of that id is a cache MISS
+	// that re-learns the revision from the source and overwrites the entry before recording anything.
+	private revisions = new Map<BlockId, number>();
 
 	constructor(
 		protected readonly source: BlockSource<T>,
-		maxSize = DefaultMaxSize
+		maxSize = DefaultMaxSize,
+		/** Shared per-transaction read-dependency accumulator (same instance the collection's
+		 *  TransactorSource holds). Optional: log-walk caches that never form a transaction omit it. */
+		private readonly collector?: ReadDependencyCollector,
 	) {
 		this.cache = new LruMap(maxSize);
 	}
 
-	async tryGet(id: BlockId): Promise<T | undefined> {
+	private bump(id: BlockId) {
+		this.generations.set(id, (this.generations.get(id) ?? 0) + 1);
+	}
+
+	/** The current generation for an id — advances every time this cache's content for
+	 * the id changes (miss-load, clear, or transformCache). Stable across pure cache hits. */
+	getGeneration(id: BlockId): number {
+		return this.generations.get(id) ?? 0;
+	}
+
+	async tryGet(id: BlockId, purpose: ReadPurpose = 'value'): Promise<T | undefined> {
 		let block = this.cache.get(id);
 		if (block) {
+			// Cache hit: the source is never consulted, so re-emit the revision we learned when this
+			// id was first loaded/folded. Without this a block served from cache records NO read
+			// dependency (the original bug), so its stale-read check could never fire. Carry the
+			// caller's purpose so a navigation-only cache hit stays droppable from the conflict set.
+			const rev = this.revisions.get(id);
+			if (rev !== undefined) this.collector?.record(id, rev, purpose);
 			log('hit id=%s', id);
 		} else {
-			block = await this.source.tryGet(id);
+			block = await this.source.tryGet(id, purpose);
 			if (block) {
 				this.cache.set(id, block);
+				this.bump(id);
+				// Learn the revision from the source (which just served it) and record it. On a miss the
+				// underlying TransactorSource already recorded the same id@rev/purpose into the shared
+				// collector; max-wins (revision) + value-wins (purpose) collapse the two to one entry.
+				const rev = sourceReadRevision(this.source, id) ?? 0;
+				this.revisions.set(id, rev);
+				this.collector?.record(id, rev, purpose);
 				log('miss:loaded id=%s cacheSize=%d', id, this.cache.size);
 			} else {
+				// Absent block: record nothing (matches TransactorSource, which skips missing blocks).
 				log('miss:absent id=%s', id);
 			}
 		}
 		return structuredClone(block);
 	}
 
+	/** Upgrade an already-captured read of `id` to a `value` read in the shared collector,
+	 *  retaining it in the conflict set. The B-tree point-lookup descent calls this (through the
+	 *  Tracker, which forwards) to pin the terminal leaf after recording the interior nodes as
+	 *  `navigation`. No-op when no collector is wired (log-walk caches) or the id was never
+	 *  recorded. Duck-typed by the Tracker; keep the name in sync with Tracker.markReadValue. */
+	markReadValue(id: BlockId): void {
+		this.collector?.markValue(id);
+	}
+
 	generateId(): BlockId {
 		return this.source.generateId();
 	}
@@ -45,26 +109,40 @@ export class CacheSource<T extends IBlock> implements BlockSource<T> {
 		if (blockIds) {
 			for (const id of blockIds) {
 				this.cache.delete(id);
+				this.revisions.delete(id);
+				this.bump(id);
 			}
 		} else {
+			for (const [id] of this.cache) {
+				this.bump(id);
+			}
 			this.cache.clear();
+			this.revisions.clear();
 		}
 	}
 
-	/** Mutates the cache without affecting the source */
-	transformCache(transform: Transforms) {
+	/** Mutates the cache without affecting the source. `revision` is the committed revision this
+	 *  transform lands at; the stored per-id revision advances to it so a later read records a
+	 *  dependency at the NEW revision (recording the old one would spuriously fail validation). */
+	transformCache(transform: Transforms, revision: number) {
 		for (const blockId of transform.deletes ?? []) {
 			this.cache.delete(blockId);
+			this.revisions.delete(blockId);
+			this.bump(blockId);
 		}
 		for (const [, block] of Object.entries(transform.inserts ?? {})) {
 			this.cache.set(block.header.id, structuredClone(block) as T);
+			this.revisions.set(block.header.id, revision);
+			this.bump(block.header.id);
 		}
 		for (const [blockId, operations] of Object.entries(transform.updates ?? {})) {
-			for (const op of operations) {
-				const block = this.cache.get(blockId);
-				if (block) {
+			const block = this.cache.get(blockId);
+			if (block) {
+				for (const op of operations) {
 					applyOperation(block, op);
+					this.bump(blockId);
 				}
+				this.revisions.set(blockId, revision);
 			}
 		}
 	}

package/src/transform/helpers.ts

@@ -72,10 +72,16 @@ export function copyTransforms(transform: Transforms): Transforms {
 }
 
 export function mergeTransforms(a: Transforms, b: Transforms): Transforms {
+	// updates: for a block id present in both sides, concatenate a's ops then b's ops
+	// (order-preserving) rather than letting b's array replace a's. inserts stay last-wins.
+	const updates: { [blockId: BlockId]: BlockOperation[] } = { ...a.updates };
+	for (const [id, ops] of Object.entries(b.updates ?? {})) {
+		updates[id] = updates[id] ? [...updates[id], ...ops] : ops;
+	}
 	return {
 		inserts: { ...a.inserts, ...b.inserts },
-		updates: { ...a.updates, ...b.updates },
-		deletes: [...(a.deletes ?? []), ...(b.deletes ?? [])]
+		updates,
+		deletes: [...new Set([...(a.deletes ?? []), ...(b.deletes ?? [])])]
 	};
 }
 

package/src/transform/tracker.ts

@@ -1,29 +1,75 @@
-import type { IBlock, BlockId, BlockStore as IBlockStore, BlockHeader, BlockOperation, BlockType, BlockSource as IBlockSource } from "../index.js";
-import { applyOperation, emptyTransforms, blockIdsForTransforms, ensured } from "../index.js";
+import type { IBlock, BlockId, BlockStore as IBlockStore, BlockHeader, BlockOperation, BlockType, BlockSource as IBlockSource, ReadPurpose } from "../index.js";
+import { applyOperation, applyOperations, emptyTransforms, blockIdsForTransforms, ensured } from "../index.js";
 
 /** A block store that collects transformations, without applying them to the underlying source.
  * Transformations are also applied to the retrieved blocks, making it seem like the source has been modified.
  */
 export class Tracker<T extends IBlock> implements IBlockStore<T> {
+	/** Per-id memo of the materialized (source block + all `updates[id]` ops) result, so a
+	 * repeated read of a hot op-carrying block is O(block size) instead of O(block size + ops).
+	 * Kept fresh incrementally on {@link update}, dropped on {@link insert}/{@link delete}/{@link reset},
+	 * and invalidated when the source's generation for the id advances (external cache mutation).
+	 * Only populated for sources that expose `getGeneration` — without a drift signal we cannot
+	 * detect source changes, so those fall back to always-replay. `gen` is the source generation of
+	 * the base block content the memo was built from. */
+	private materialized = new Map<BlockId, { block: T; gen: number }>();
+
 	constructor(
 		private readonly source: IBlockSource<T>,
 		/** The collected set of transformations to be applied. Treat as immutable */
 		public transforms = emptyTransforms(),
 	) { }
 
-	async tryGet(id: BlockId): Promise<T | undefined> {
-		const block = await this.source.tryGet(id);
+	/** The source's generation for an id, or undefined if the source cannot report drift. */
+	private sourceGeneration(id: BlockId): number | undefined {
+		const src = this.source as { getGeneration?: (id: BlockId) => number };
+		return typeof src.getGeneration === 'function' ? src.getGeneration(id) : undefined;
+	}
+
+	async tryGet(id: BlockId, purpose: ReadPurpose = 'value'): Promise<T | undefined> {
+		// NOTE: precedence here is insert > delete > source+updates. In a well-formed transform an id is
+		// never in both `inserts` and `deletes` (insert/delete each clear the other), so order is moot. It
+		// only diverges from the canonical `applyTransform` (delete-last-wins, see struct.ts / helpers.ts:132)
+		// in the malformed insert+delete state reachable via the phantom-delete bug (double-delete then
+		// reinsert). Likewise the insert path intentionally skips `updates[id]` — inserted blocks bake ops
+		// in-place via update(); a stale pre-insert `updates[id]` is discarded here but would be re-applied
+		// on commit. Both are read-vs-commit inconsistencies confined to malformed states; fix the source
+		// bug (phantom delete / stale updates) rather than papering over it here.
+		if (this.transforms.inserts && Object.hasOwn(this.transforms.inserts, id)) {
+			return structuredClone(this.transforms.inserts[id]) as T;
+		}
+		if (this.transforms.deletes?.includes(id)) {
+			return undefined;
+		}
+		const gen = this.sourceGeneration(id);
+		const memo = this.materialized.get(id);
+		if (memo && (gen === undefined || memo.gen === gen)) {
+			return structuredClone(memo.block);           // O(block size), no replay
+		}
+		const block = await this.source.tryGet(id, purpose);
 		if (block) {
 			const ops = this.transforms.updates?.[id] ?? [];
-			ops.forEach(op => applyOperation(block!, op));
-			if (this.transforms.deletes?.includes(id)) {
-				return undefined;
+			if (ops.length > 0) {
+				applyOperations(block, ops);
+				// Memoize only when the source can report drift, and stamp with the generation read
+				// AFTER the load — the source may bump during tryGet (a cache miss-load), and stamping
+				// with the pre-load generation would force a needless reload on the very next read.
+				const freshGen = this.sourceGeneration(id);
+				if (freshGen !== undefined) {
+					this.materialized.set(id, { block, gen: freshGen });
+				}
+				return structuredClone(block);              // clone so callers can't mutate the memo
 			}
-		} else if (this.transforms.inserts && Object.hasOwn(this.transforms.inserts, id)) {
-			return structuredClone(this.transforms.inserts[id]) as T;
 		}
+		return block;                                    // no-ops path unchanged (source already cloned)
+	}
 
-		return block;
+	/** Forward a leaf-value upgrade down to the source's read collector (duck-typed: only the
+	 *  CacheSource layer implements it). Lets the B-tree point-lookup descent, which reads through
+	 *  this tracker, pin its terminal leaf as a `value` read after tagging interior nodes
+	 *  `navigation`. No-op for sources without a collector (test doubles, log-walk caches). */
+	markReadValue(id: BlockId): void {
+		(this.source as { markReadValue?: (id: BlockId) => void }).markReadValue?.(id);
 	}
 
 	generateId(): BlockId {
@@ -37,6 +83,8 @@ export class Tracker<T extends IBlock> implements IBlockStore<T> {
 	insert(block: T) {
 		const inserts = this.transforms.inserts ??= {};
 		inserts[block.header.id] = structuredClone(block);
+		// Served from `inserts` now, not source+updates — the materialized memo no longer applies.
+		this.materialized.delete(block.header.id);
 		const deletes = this.transforms.deletes;
 		const deleteIndex = deletes?.indexOf(block.header.id) ?? -1;
 		if (deleteIndex >= 0) {
@@ -51,12 +99,21 @@ export class Tracker<T extends IBlock> implements IBlockStore<T> {
 		} else {
 			const updates = this.transforms.updates ??= {};
 			ensured(updates, blockId, () => []).push(structuredClone(op));
+			// The memo already equals (base source content + prior ops); applying just the new op
+			// keeps it equal to the full ops list — O(1), no full replay. Leave `gen` untouched: it
+			// still records the base-content generation, so a later external source change still
+			// forces a reload. (Refreshing gen here would mask stale base content.)
+			const memo = this.materialized.get(blockId);
+			if (memo) {
+				applyOperation(memo.block, op);
+			}
 		}
 	}
 
 	delete(blockId: BlockId) {
 		if (this.transforms.inserts) delete this.transforms.inserts[blockId];
 		if (this.transforms.updates) delete this.transforms.updates[blockId];
+		this.materialized.delete(blockId);
 		const deletes = this.transforms.deletes ??= [];
 		deletes.push(blockId);
 	}
@@ -64,11 +121,12 @@ export class Tracker<T extends IBlock> implements IBlockStore<T> {
 	reset(newTransform = emptyTransforms()) {
 		const oldTransform = this.transforms;
 		this.transforms = newTransform;
+		this.materialized.clear();
 		return oldTransform;
 	}
 
 	transformedBlockIds(): BlockId[] {
-		return Array.from(new Set(blockIdsForTransforms(this.transforms)));
+		return blockIdsForTransforms(this.transforms);
 	}
 
 	conflicts(blockIds: Set<BlockId>) {

package/src/utility/backoff.ts

@@ -0,0 +1,95 @@
+import { randomBytes } from '@noble/hashes/utils.js';
+
+/**
+ * Jittered exponential backoff shared by the retry loops in db-core
+ * ({@link Collection.sync} and {@link TransactionCoordinator.commit}).
+ *
+ * The point of the jitter is fairness under contention: when many clients lose the SAME
+ * optimistic-concurrency race at t=0, a bare exponential curve makes them all re-attempt at the
+ * identical next tick — a thundering herd that just re-collides. Multiplying the exponential value
+ * by a random factor spreads those re-attempts across a window so offered load per tick sheds
+ * instead of cascading. This is the same shape as `cohort-topic`'s `backoffRetryMs`, with the
+ * proportional-jitter term added.
+ */
+
+/** A source of uniform randomness in [0, 1). Injected in tests so a deterministic sequence can be
+ * asserted; production uses {@link cryptoRand}. */
+export type RandFn = () => number;
+
+/** Uniform value in [0, 1) drawn from the same CSPRNG (`@noble/hashes` `randomBytes`) the rest of
+ * db-core uses — deliberately NOT `Math.random`, so backoff jitter shares the package's randomness
+ * source rather than introducing a second, weaker one. */
+export function cryptoRand(): number {
+	const b = randomBytes(4);
+	// Assemble a uint32 (>>> 0 clears the sign bit the shifts would otherwise set), then normalise.
+	const u = (((b[0]! << 24) | (b[1]! << 16) | (b[2]! << 8) | b[3]!) >>> 0);
+	return u / 0x1_0000_0000;
+}
+
+export interface JitteredBackoffConfig {
+	/** Base delay (ms) for attempt 0, before jitter. */
+	baseMs: number;
+	/** Hard ceiling (ms) on the pre-jitter exponential value. */
+	capMs: number;
+	/** Geometric growth per attempt. Default 2 (doubling). */
+	factor?: number;
+	/** Fraction of the exponential value that jitter may subtract, in [0, 1]. Default 0.5, so the
+	 * returned delay lands uniformly in `((1 - jitterFraction) · exp, exp]` — i.e. `(0.5·exp, exp]`. */
+	jitterFraction?: number;
+}
+
+/**
+ * Delay (ms) for retry `attempt` (0-based):
+ *
+ *   exp   = min(baseMs · factor^attempt, capMs)
+ *   delay = exp · (1 - jitterFraction · rand())     // rand() ∈ [0, 1)
+ *
+ * With the defaults (factor 2, jitterFraction 0.5) the delay is uniform in `(0.5·exp, exp]`: never
+ * zero (as long as `baseMs > 0`) and never above `capMs`. `attempt` must be a non-negative integer
+ * — pass `consecutiveFailures - 1` so the first retry backs off by ~`baseMs`.
+ */
+export function jitteredBackoffMs(
+	attempt: number,
+	config: JitteredBackoffConfig,
+	rand: RandFn = cryptoRand,
+): number {
+	if (!Number.isInteger(attempt) || attempt < 0) {
+		throw new RangeError(`attempt must be a non-negative integer, got ${attempt}`);
+	}
+	const factor = config.factor ?? 2;
+	const jitterFraction = config.jitterFraction ?? 0.5;
+	const exp = Math.min(config.baseMs * factor ** attempt, config.capMs);
+	return exp * (1 - jitterFraction * rand());
+}
+
+/** Build an AbortError for a cooperatively-aborted wait. Prefers the signal's own reason when it is
+ * an Error (so callers who passed a custom abort reason see it), otherwise a `name='AbortError'`
+ * Error. */
+export function makeAbortError(signal?: AbortSignal): Error {
+	if (signal && signal.reason instanceof Error) {
+		return signal.reason;
+	}
+	const err = new Error('The operation was aborted');
+	err.name = 'AbortError';
+	return err;
+}
+
+/** Sleep for `ms`, resolving early (rejecting with an AbortError) if `signal` aborts — so a retry
+ * loop waiting out a backoff rejects promptly instead of finishing the sleep. */
+export function abortableDelay(ms: number, signal?: AbortSignal): Promise<void> {
+	return new Promise<void>((resolve, reject) => {
+		if (signal?.aborted) {
+			reject(makeAbortError(signal));
+			return;
+		}
+		const onAbort = () => {
+			clearTimeout(timer);
+			reject(makeAbortError(signal!));
+		};
+		const timer = setTimeout(() => {
+			signal?.removeEventListener('abort', onAbort);
+			resolve();
+		}, ms);
+		signal?.addEventListener('abort', onAbort, { once: true });
+	});
+}

package/src/utility/pending.ts

@@ -1,11 +1,12 @@
 export class Pending<T> {
 	response?: T;
 	error?: unknown;
+	private _responded = false;
 	t1 = Date.now();
 	duration?: number;
 
 	get isResponse(): boolean {
-		return this.response !== undefined;
+		return this._responded;
 	}
 
 	get isError(): boolean {
@@ -32,6 +33,7 @@ export class Pending<T> {
 		promise.then(response => {
 			this.duration = Date.now() - this.t1;
 			this.response = response;
+			this._responded = true;
 			return response;
 		}, error => {
 			this.duration = Date.now() - this.t1;

package/src/transaction/context.ts

@@ -1,103 +0,0 @@
-import type { CollectionId } from "../index.js";
-import type { TransactionCoordinator } from "./coordinator.js";
-import type { ReadDependency, ExecutionResult } from "./transaction.js";
-import type { Action } from "../collection/action.js";
-
-/**
- * Transaction context for accumulating actions and reads.
- *
- * Usage:
- *   const txn = coordinator.begin();
- *   txn.addAction('users', { type: 'insert', data: {...} });
- *   txn.addAction('users', { type: 'get', data: { key: 1 } });
- *   const result = await txn.commit();
- */
-export class TransactionContext {
-	private readonly collectionActions: Map<CollectionId, Action<any>[]> = new Map();
-	private readonly reads: ReadDependency[] = [];
-
-	constructor(
-		private readonly coordinator: TransactionCoordinator,
-		public readonly transactionId: string,
-		public readonly engine: string
-	) {}
-
-	/**
-	 * Add an action to a collection.
-	 *
-	 * Actions are collection-specific:
-	 * - Tree: 'insert', 'delete', 'get', 'scan'
-	 * - Diary: 'append', 'read'
-	 * - etc.
-	 */
-	async addAction(collectionId: CollectionId, action: Action<any>): Promise<void> {
-		// Tag action with transaction reference
-		const taggedAction = {
-			...action,
-			transaction: this.transactionId
-		};
-
-		// Get the collection and immediately execute the action to update local snapshot
-		const collection = this.coordinator['collections'].get(collectionId);
-		if (collection) {
-			// Execute through collection to update tracker and pending buffer
-			await collection.act(taggedAction);
-		}
-		// If no collection registered, just buffer the action (for testing or deferred collection creation)
-
-		// Record in transaction context
-		if (!this.collectionActions.has(collectionId)) {
-			this.collectionActions.set(collectionId, []);
-		}
-		this.collectionActions.get(collectionId)!.push(taggedAction);
-	}
-
-	/**
-	 * Add a read dependency for optimistic concurrency control.
-	 */
-	addRead(read: ReadDependency): void {
-		this.reads.push(read);
-	}
-
-	/**
-	 * Commit the transaction.
-	 *
-	 * This executes all accumulated actions across all affected collections,
-	 * coordinating with the network as needed.
-	 */
-	async commit(): Promise<ExecutionResult> {
-		return await this.coordinator.commitTransaction(this);
-	}
-
-	/**
-	 * Rollback the transaction (just discard accumulated state).
-	 */
-	rollback(): void {
-		this.collectionActions.clear();
-		this.reads.length = 0;
-	}
-
-	/**
-	 * Get all accumulated actions by collection.
-	 * Used by coordinator during commit.
-	 */
-	getCollectionActions(): Map<CollectionId, Action<any>[]> {
-		return this.collectionActions;
-	}
-
-	/**
-	 * Get all accumulated read dependencies.
-	 * Used by coordinator during commit.
-	 */
-	getReads(): ReadDependency[] {
-		return this.reads;
-	}
-
-	/**
-	 * Get the set of affected collection IDs.
-	 */
-	getAffectedCollections(): Set<CollectionId> {
-		return new Set(this.collectionActions.keys());
-	}
-}
-