@optimystic/db-p2p 0.13.4 → 0.14.0

package/src/storage/block-storage.ts

@@ -1,5 +1,5 @@
 import type { BlockId, IBlock, Transform, ActionId, ActionRev } from "@optimystic/db-core";
-import { Latches, applyTransform } from "@optimystic/db-core";
+import { Latches, applyTransform, hashString } from "@optimystic/db-core";
 import type { BlockArchive, BlockMetadata, RestoreCallback, RevisionRange } from "./struct.js";
 import type { IRawStorage } from "./i-raw-storage.js";
 import { mergeRanges } from "./helpers.js";
@@ -123,6 +123,105 @@ export class BlockStorage implements IBlockStorage {
 		return { reconciled: false, latest: meta.latest };
 	}
 
+	async saveReplica(block: IBlock, source?: ActionRev): Promise<ActionRev> {
+		const rev = source?.rev ?? 1;
+		// Deterministic fallback id when the sender carried no revision metadata, so a
+		// re-push of the same block resolves to the same (rev, actionId) and stays
+		// idempotent. Never random/time-based — that would mint a new revision per retry.
+		const actionId = source?.actionId ?? await hashString(`${this.blockId}:${JSON.stringify(block)}`);
+
+		// Serialize the read-modify-write on this block's metadata (mirrors ensureRevision).
+		const lockId = `BlockStorage.saveReplica:${this.blockId}`;
+		const release = await Latches.acquire(lockId);
+		try {
+			let meta = await this.storage.getMetadata(this.blockId);
+
+			// Monotonic guard: an equal-or-newer revision is already held. The block is
+			// durably present; do not downgrade `latest` or rewrite the metadata.
+			if (meta?.latest && meta.latest.rev >= rev) {
+				log('replica:skip blockId=%s rev=%d held=%d', this.blockId, rev, meta.latest.rev);
+				return meta.latest;
+			}
+
+			// Write rev → actionId, the action transform, and the materialized block.
+			// `{ insert: block }` satisfies saveRestored's write invariants; on the serving
+			// path materializeBlock returns the materialized block directly (single rev), so
+			// this transform is never applied — see ticket notes.
+			const archive: BlockArchive = {
+				blockId: this.blockId,
+				revisions: {
+					[rev]: {
+						action: { actionId, rev, transform: { insert: block } },
+						block
+					}
+				},
+				range: [rev, rev + 1]
+			};
+			await this.saveRestored(archive);
+
+			// Seed metadata when absent, advance latest, and merge the covered range.
+			if (!meta) {
+				meta = { latest: undefined, ranges: [] };
+			}
+			meta.latest = { rev, actionId };
+			meta.ranges.unshift([rev, rev + 1]);
+			meta.ranges = mergeRanges(meta.ranges);
+			await this.storage.saveMetadata(this.blockId, meta);
+
+			log('replica:save blockId=%s rev=%d actionId=%s', this.blockId, rev, actionId);
+			return meta.latest;
+		} finally {
+			release();
+		}
+	}
+
+	async saveDeletion(source: ActionRev): Promise<ActionRev> {
+		const { rev, actionId } = source;
+
+		// Share the saveReplica latch: both do a read-modify-write of `meta.latest`, so they must be
+		// mutually exclusive on this block to keep the monotonic guard sound.
+		const lockId = `BlockStorage.saveReplica:${this.blockId}`;
+		const release = await Latches.acquire(lockId);
+		try {
+			let meta = await this.storage.getMetadata(this.blockId);
+
+			// Monotonic guard: an equal-or-newer revision is already held. Do not downgrade `latest`
+			// or rewrite metadata — the tombstone (or a later revision) is already durable.
+			if (meta?.latest && meta.latest.rev >= rev) {
+				log('deletion:skip blockId=%s rev=%d held=%d', this.blockId, rev, meta.latest.rev);
+				return meta.latest;
+			}
+
+			// Forward tombstone: a `{ delete: true }` transform and NO materialized block. saveRestored
+			// skips materialization when `block` is absent, so the reverse-apply in materializeBlock
+			// resolves this revision to an absent block (read-back as undefined).
+			const archive: BlockArchive = {
+				blockId: this.blockId,
+				revisions: {
+					[rev]: {
+						action: { actionId, rev, transform: { delete: true } }
+					}
+				},
+				range: [rev, rev + 1]
+			};
+			await this.saveRestored(archive);
+
+			// Seed metadata when absent, advance latest, and merge the covered range.
+			if (!meta) {
+				meta = { latest: undefined, ranges: [] };
+			}
+			meta.latest = { rev, actionId };
+			meta.ranges.unshift([rev, rev + 1]);
+			meta.ranges = mergeRanges(meta.ranges);
+			await this.storage.saveMetadata(this.blockId, meta);
+
+			log('deletion:save blockId=%s rev=%d actionId=%s', this.blockId, rev, actionId);
+			return meta.latest;
+		} finally {
+			release();
+		}
+	}
+
 	private async ensureRevision(meta: BlockMetadata, rev: number): Promise<void> {
 		if (this.inRanges(rev, meta.ranges)) {
 			return;
@@ -154,7 +253,7 @@ export class BlockStorage implements IBlockStorage {
 		}
 	}
 
-	private async materializeBlock(_meta: BlockMetadata, targetRev: number): Promise<{ block: IBlock, actionRev: ActionRev }> {
+	private async materializeBlock(_meta: BlockMetadata, targetRev: number): Promise<{ block: IBlock, actionRev: ActionRev } | undefined> {
 		let block: IBlock | undefined;
 		let materializedActionRev: ActionRev | undefined;
 		const actions: ActionRev[] = [];
@@ -187,7 +286,11 @@ export class BlockStorage implements IBlockStorage {
 		}
 
 		if (!block) {
-			throw new Error(`Block ${this.blockId} has been deleted`);
+			// The reverse-apply collapsed to a tombstone (a `{ delete: true }` revision) — the block
+			// is absent at this revision, not corrupt. Read it back as undefined (matching getBlock's
+			// "no materialized content" contract) rather than throwing. The genuine-truncation throw
+			// ("Failed to find materialized block") above still fires when no materialization exists.
+			return undefined;
 		}
 		if (actions.length) {
 			await this.storage.saveMaterializedBlock(this.blockId, actions[0]!.actionId, block);

package/src/storage/i-block-storage.ts

@@ -44,6 +44,33 @@ export interface IBlockStorage {
     /** Sets the latest revision information */
     setLatest(latest: ActionRev): Promise<void>;
 
+    /**
+     * Persist a replica of a block received out-of-band (churn re-replication).
+     *
+     * Seeds metadata if absent, writes `rev → actionId`, the action transform, and the
+     * materialized block, merges `[rev, rev+1]` into `ranges`, and advances `latest`
+     * monotonically. When `source` is provided its `rev`/`actionId` are used; otherwise
+     * it falls back to `rev = 1` and a deterministic `actionId` derived from the block
+     * (so retries stay idempotent — never random).
+     *
+     * No-op (still durable) when an equal-or-newer revision is already present: `latest`
+     * is never downgraded. Idempotent for a fixed `(rev, actionId)`. Returns the
+     * effective latest `ActionRev`.
+     */
+    saveReplica(block: IBlock, source?: ActionRev): Promise<ActionRev>;
+
+    /**
+     * Writes a forward TOMBSTONE revision that reverses a block creation: persists `rev → actionId`,
+     * a `{ delete: true }` transform, and NO materialized block, then merges `[rev, rev+1]` into
+     * `ranges` and advances `latest` monotonically. The reverse-apply path treats the absent
+     * materialization as a deletion, so a `getBlock()` after a tombstone reads back as *absent*
+     * (`undefined`) while a historical `getBlock(creationRev)` still materializes the created content.
+     *
+     * Idempotent for a fixed `(rev, actionId)`; never downgrades `latest` (a no-op — still durable —
+     * when an equal-or-newer revision is already present). Returns the effective latest `ActionRev`.
+     */
+    saveDeletion(source: ActionRev): Promise<ActionRev>;
+
     /**
      * Reconciles `metadata.latest` with the highest contiguous fully-promoted revision in
      * the revisions table. Intended for post-crash recovery of the Crash-D3 gap, where

package/src/storage/storage-repo.ts

@@ -1,8 +1,9 @@
 import type {
 	IRepo, MessageOptions, BlockId, CommitRequest, CommitResult, GetBlockResults, PendRequest, PendResult, ActionBlocks,
 	ActionId, BlockGets, ActionPending, PendSuccess, ActionTransform, ActionTransforms,
-	GetBlockResult,
-	PendValidationHook
+	GetBlockResult, IBlock, ActionRev,
+	PendValidationHook,
+	CollectionId, IBlockChangeNotifier, CollectionChangeListener, CollectionChangeEvent
 } from "@optimystic/db-core";
 import {
 	Latches, transformForBlockId, applyTransform, groupBy, concatTransform, emptyTransforms,
@@ -10,6 +11,7 @@ import {
 } from "@optimystic/db-core";
 import { asyncIteratorToArray } from "../it-utility.js";
 import type { IBlockStorage } from "./i-block-storage.js";
+import type { IBlockReplicaStore } from "../cluster/block-transfer-service.js";
 import { createLogger } from "../logger.js";
 
 const log = createLogger('storage-repo');
@@ -19,8 +21,12 @@ export type StorageRepoOptions = {
 	validatePend?: PendValidationHook;
 };
 
-export class StorageRepo implements IRepo {
+export class StorageRepo implements IRepo, IBlockChangeNotifier, IBlockReplicaStore {
 	private readonly validatePend?: PendValidationHook;
+	/** Per-collection change listeners; empty sets are pruned on unsubscribe. */
+	private readonly changeListeners = new Map<CollectionId, Set<CollectionChangeListener>>();
+	/** Catch-all change listeners — fire for EVERY collection's commit on this node. */
+	private readonly anyChangeListeners = new Set<CollectionChangeListener>();
 
 	constructor(
 		private readonly createBlockStorage: (blockId: BlockId) => IBlockStorage,
@@ -29,9 +35,96 @@ export class StorageRepo implements IRepo {
 		this.validatePend = options?.validatePend;
 	}
 
+	/**
+	 * Subscribe to commits that mutate `collectionId`'s blocks on this node.
+	 * Returns an idempotent unsubscribe. See {@link IBlockChangeNotifier}.
+	 */
+	onCollectionChange(collectionId: CollectionId, listener: CollectionChangeListener): () => void {
+		let set = this.changeListeners.get(collectionId);
+		if (!set) {
+			set = new Set();
+			this.changeListeners.set(collectionId, set);
+		}
+		set.add(listener);
+		let unsubscribed = false;
+		return () => {
+			if (unsubscribed) return;
+			unsubscribed = true;
+			const current = this.changeListeners.get(collectionId);
+			if (current) {
+				current.delete(listener);
+				if (current.size === 0) {
+					this.changeListeners.delete(collectionId);
+				}
+			}
+		};
+	}
+
+	/**
+	 * Subscribe to commits mutating ANY collection on this node — the catch-all feed the
+	 * cohort-topic origination bridge consumes (it cannot enumerate collection ids ahead of time,
+	 * so a per-collection {@link onCollectionChange} subscription cannot see every commit). Fires for
+	 * the same `(pending → committed)` transitions as {@link onCollectionChange}, but across every
+	 * collection. Returns an idempotent unsubscribe; a throwing listener is isolated + logged.
+	 */
+	onAnyCollectionChange(listener: CollectionChangeListener): () => void {
+		this.anyChangeListeners.add(listener);
+		let unsubscribed = false;
+		return () => {
+			if (unsubscribed) return;
+			unsubscribed = true;
+			this.anyChangeListeners.delete(listener);
+		};
+	}
+
+	/**
+	 * Fire one {@link CollectionChangeEvent} per distinct collection that was
+	 * newly committed. Called AFTER the commit critical section (locks released),
+	 * fire-and-forget synchronous; a throwing listener is isolated and logged. Each event reaches
+	 * both that collection's {@link onCollectionChange} subscribers and every
+	 * {@link onAnyCollectionChange} catch-all subscriber.
+	 *
+	 * `tailId` is the `CommitRequest.tailId` on the commit path; `undefined` on read-driven
+	 * promotions (the get/emitPromotions path has no commit request). A single commit is for one
+	 * collection's chain in practice, so all events from one commit share the same `tailId`.
+	 */
+	private emitCollectionChanges(collectionBlocks: Map<CollectionId, BlockId[]>, actionId: ActionId, rev: number, tailId?: BlockId): void {
+		const hasCatchAll = this.anyChangeListeners.size > 0;
+		for (const [collectionId, blockIds] of collectionBlocks) {
+			const listeners = this.changeListeners.get(collectionId);
+			if ((!listeners || listeners.size === 0) && !hasCatchAll) {
+				continue;
+			}
+			const event: CollectionChangeEvent = { collectionId, blockIds, actionId, rev, tailId };
+			if (listeners && listeners.size > 0) {
+				this.fireChangeListeners(listeners, event);
+			}
+			if (hasCatchAll) {
+				this.fireChangeListeners(this.anyChangeListeners, event);
+			}
+		}
+	}
+
+	/** Dispatch `event` to a snapshot of `listeners` (safe under mid-emit (un)subscribe), isolating + logging any throw. */
+	private fireChangeListeners(listeners: Set<CollectionChangeListener>, event: CollectionChangeEvent): void {
+		for (const listener of Array.from(listeners)) {
+			try {
+				listener(event);
+			} catch (err) {
+				log('onCollectionChange listener threw for collection=%s: %o', event.collectionId, err);
+			}
+		}
+	}
+
 	async get({ blockIds, context }: BlockGets, _options?: MessageOptions): Promise<GetBlockResults> {
 		const distinctBlockIds = Array.from(new Set(blockIds));
 		log('get blockIds=%d', distinctBlockIds.length);
+		// Read-driven promotions that land durably here, captured so we can emit a
+		// change event per durable landing after the parallel reads complete (mirrors
+		// commit's "emit after the work" ordering). The array is shared across the
+		// parallel map closures below — safe because each push happens synchronously
+		// between awaits (single-threaded), never concurrently.
+		const promotions: { collectionId: CollectionId, blockId: BlockId, actionId: ActionId, rev: number }[] = [];
 		const results = await Promise.all(distinctBlockIds.map(async (blockId) => {
 			const blockStorage = this.createBlockStorage(blockId);
 
@@ -44,7 +137,10 @@ export class StorageRepo implements IRepo {
 				for (const { actionId, rev } of missing.sort((a, b) => a.rev - b.rev)) {
 					const pending = await blockStorage.getPendingTransaction(actionId);
 					if (pending) {
-						await this.internalCommit(blockId, actionId, rev, blockStorage);
+						const collectionId = await this.internalCommit(blockId, actionId, rev, blockStorage);
+						if (collectionId !== undefined) {
+							promotions.push({ collectionId, blockId, actionId, rev });
+						}
 					}
 				}
 			}
@@ -82,9 +178,42 @@ export class StorageRepo implements IRepo {
 				}
 			}];
 		}));
+
+		// Emit per durable read-driven landing (Option A — emit eagerly). Done after the
+		// parallel reads complete so emission stays outside the per-block work, matching
+		// commit's ordering. No-op when nothing was promoted.
+		this.emitPromotions(promotions);
+
 		return Object.fromEntries(results);
 	}
 
+	/**
+	 * Emit a {@link CollectionChangeEvent} for each read-driven promotion that landed
+	 * during a {@link get}. A single get() can promote multiple distinct actions, each
+	 * at its own `(actionId, rev)`, so group by `(actionId, rev)` and route each group
+	 * through {@link emitCollectionChanges} once.
+	 */
+	private emitPromotions(promotions: { collectionId: CollectionId, blockId: BlockId, actionId: ActionId, rev: number }[]): void {
+		if (promotions.length === 0) {
+			return;
+		}
+		const groups = new Map<string, { actionId: ActionId, rev: number, collectionBlocks: Map<CollectionId, BlockId[]> }>();
+		for (const { collectionId, blockId, actionId, rev } of promotions) {
+			const key = `${actionId} ${rev}`;
+			let group = groups.get(key);
+			if (!group) {
+				group = { actionId, rev, collectionBlocks: new Map() };
+				groups.set(key, group);
+			}
+			const list = group.collectionBlocks.get(collectionId) ?? [];
+			list.push(blockId);
+			group.collectionBlocks.set(collectionId, list);
+		}
+		for (const { actionId, rev, collectionBlocks } of groups.values()) {
+			this.emitCollectionChanges(collectionBlocks, actionId, rev);
+		}
+	}
+
 	async pend(request: PendRequest, _options?: MessageOptions): Promise<PendResult> {
 		// Validate transaction if present and validation hook is configured
 		if (this.validatePend && request.transaction && request.operationsHash) {
@@ -194,6 +323,14 @@ export class StorageRepo implements IRepo {
 		log('commit actionId=%s rev=%d blockIds=%d', request.actionId, request.rev, request.blockIds.length);
 		const uniqueBlockIds = Array.from(new Set(request.blockIds)).sort();
 		const releases: (() => void)[] = [];
+		// Collects the blocks newly committed in this call, grouped by collection,
+		// so we can emit change events once locks are released. Blocks that land before
+		// a mid-loop failure stay here and are still emitted (Option A — emit eagerly):
+		// they are durably committed and a retry rolls the remainder forward.
+		const collectionBlocks = new Map<CollectionId, BlockId[]>();
+		// Captured when internalCommit throws mid-loop; we break (rather than return)
+		// so locks release and accumulated landings still emit before we report failure.
+		let failure: { reason: string } | undefined;
 
 		try {
 			// Acquire locks sequentially based on sorted IDs to prevent deadlocks
@@ -273,14 +410,19 @@ export class StorageRepo implements IRepo {
 			for (const { blockId, storage } of toCommit) {
 				try {
 					// internalCommit will throw if it encounters an issue
-					await this.internalCommit(blockId, request.actionId, request.rev, storage);
+					const collectionId = await this.internalCommit(blockId, request.actionId, request.rev, storage);
+					if (collectionId !== undefined) {
+						const list = collectionBlocks.get(collectionId) ?? [];
+						list.push(blockId);
+						collectionBlocks.set(collectionId, list);
+					}
 				} catch (err) {
-					// Partial-commit recovery: a retry with the same (actionId, rev) will treat
-					// already-done blocks as idempotent no-ops and advance the remainder.
-					return {
-						success: false,
-						reason: err instanceof Error ? err.message : 'Unknown error during commit'
-					};
+					// Partial-commit recovery: blocks already in collectionBlocks DID land
+					// durably and must still emit; a retry with the same (actionId, rev)
+					// treats them as idempotent no-ops and advances the remainder. Break
+					// instead of returning so locks release and those landings emit below.
+					failure = { reason: err instanceof Error ? err.message : 'Unknown error during commit' };
+					break;
 				}
 			}
 		}
@@ -289,7 +431,12 @@ export class StorageRepo implements IRepo {
 			releases.reverse().forEach(release => release());
 		}
 
-		return { success: true };
+		// Notify after the critical section, for every block newly committed here —
+		// including those that landed before a mid-loop failure (alreadyDone / stale
+		// partitions never reach `collectionBlocks`).
+		this.emitCollectionChanges(collectionBlocks, request.actionId, request.rev, request.tailId);
+
+		return failure ? { success: false, reason: failure.reason } : { success: true };
 	}
 
 	/**
@@ -305,7 +452,50 @@ export class StorageRepo implements IRepo {
 		await storage.recover();
 	}
 
-	private async internalCommit(blockId: BlockId, actionId: ActionId, rev: number, storage: IBlockStorage): Promise<void> {
+	/**
+	 * Persist a replica of a block received out-of-band (churn re-replication) into
+	 * local storage. Distinct from the {@link IRepo} commit funnel: the block arrives
+	 * already materialized from a departing owner, not as a pend/commit. See
+	 * {@link IBlockStorage.saveReplica} for the durability/monotonicity contract.
+	 *
+	 * Held under the same `StorageRepo.commit:<id>` latch as {@link commit} so the
+	 * replica's read-modify-write of `latest` is mutually exclusive with a concurrent
+	 * local commit on the same block — otherwise `saveReplica`'s monotonic guard could
+	 * read a stale `latest` and clobber a commit that advanced it in between.
+	 */
+	async saveReplicatedBlock(blockId: BlockId, block: IBlock, source?: ActionRev): Promise<void> {
+		log('saveReplicatedBlock blockId=%s rev=%s', blockId, source?.rev);
+		const storage = this.createBlockStorage(blockId);
+		const release = await Latches.acquire(`StorageRepo.commit:${blockId}`);
+		// Captured under the latch; emitted after release to match commit's ordering.
+		let landed: { collectionId: CollectionId, actionId: ActionId, rev: number } | undefined;
+		try {
+			const priorLatest = await storage.getLatest();
+			const effective = await storage.saveReplica(block, source);
+			// Advanced iff there was no prior revision or the effective rev moved past it. On the
+			// monotonic no-op, saveReplica returns the held latest unchanged → effective.rev === priorLatest.rev.
+			const advanced = priorLatest === undefined || effective.rev > priorLatest.rev;
+			const collectionId = block.header?.collectionId;
+			if (advanced && collectionId !== undefined) {
+				landed = { collectionId, actionId: effective.actionId, rev: effective.rev };
+			}
+		} finally {
+			release();
+		}
+		// Replica-persist has no CommitRequest, hence no tailId — like a read-driven promotion,
+		// this wakes local onCollectionChange watchers but is cert-gated out of cohort-topic
+		// re-origination downstream (change-bridge selfIsCohortMember treats a tail-less event as
+		// never a member).
+		if (landed) {
+			this.emitCollectionChanges(
+				new Map([[landed.collectionId, [blockId]]]),
+				landed.actionId,
+				landed.rev,
+			);
+		}
+	}
+
+	private async internalCommit(blockId: BlockId, actionId: ActionId, rev: number, storage: IBlockStorage): Promise<CollectionId | undefined> {
 		// Note: This method is called within the locked critical section of commit()
 		// So, operations like getPendingTransaction, getLatest, getBlock, saveMaterializedBlock,
 		// saveRevision, promotePendingTransaction, setLatest are protected against
@@ -340,6 +530,12 @@ export class StorageRepo implements IRepo {
 
 		// Update latest revision *last*
 		await storage.setLatest({ actionId, rev });
+
+		// Report the affected collection for change-event routing. For a delete the
+		// materialized block is undefined, so fall back to the prior block's header.
+		// Either may be absent only for a malformed/headerless block — return
+		// undefined so the caller skips it rather than emitting a bogus event.
+		return newBlock?.header.collectionId ?? priorBlock?.header.collectionId;
 	}
 }
 

package/src/sync/client.ts

@@ -2,6 +2,7 @@ import type { PeerId } from '@libp2p/interface';
 import type { IPeerNetwork } from '@optimystic/db-core';
 import { ProtocolClient } from '../protocol-client.js';
 import { buildSyncProtocol, type SyncRequest, type SyncResponse } from './protocol.js';
+import { withRpcDeadlineDefaults, type RpcDeadlineOptions } from '../rpc-deadline.js';
 
 /**
  * Client for sending sync requests to remote peers.
@@ -25,11 +26,13 @@ export class SyncClient extends ProtocolClient {
 	 * Request a block from the remote peer.
 	 *
 	 * @param request - Sync request specifying block and options
+	 * @param options - Optional per-call deadlines/cancellation. Absent keys fall
+	 *   back to the client-level defaults so a silent peer can't hang the caller.
 	 * @returns Response with archive if successful
 	 * @throws Error if request fails or times out
 	 */
-	async requestBlock(request: SyncRequest): Promise<SyncResponse> {
-		return await this.processMessage<SyncResponse>(request, this.protocol);
+	async requestBlock(request: SyncRequest, options?: RpcDeadlineOptions): Promise<SyncResponse> {
+		return await this.processMessage<SyncResponse>(request, this.protocol, withRpcDeadlineDefaults(options));
 	}
 
 	/**