@optimystic/db-p2p 0.28.0 → 1.0.0-beta.1

package/src/repo/service.ts

@@ -1,22 +1,19 @@
 import { pipe } from 'it-pipe'
 import { decode as lpDecode, encode as lpEncode } from 'it-length-prefixed'
-import type { Startable, Logger, Stream, Connection, StreamHandler, PeerId, Libp2p } from '@libp2p/interface'
+import type { Startable, Stream, Connection, StreamHandler, PeerId, Libp2p } from '@libp2p/interface'
 import type { IRepo, RepoMessage } from '@optimystic/db-core'
 import { blockIdsForTransforms } from '@optimystic/db-core'
 import { peersEqual } from '../peer-utils.js'
 import { encodePeers, type RedirectPayload } from './redirect.js'
 import { MAX_BLOCK_MESSAGE_BYTES } from '../protocol-limits.js'
 import type { Uint8ArrayList } from 'uint8arraylist'
-import { createLogger } from '../logger.js'
+import { createLogger, type Logger } from '../logger.js'
 import { publishableAddrsForPeer, type AddressLog, type DirectionalConnection } from '../peer-address-book.js'
 import { createInboundStreamAuthorization, type InboundStreamAuthorization, type InboundStreamAuthorizationInit } from '../inbound-authorization.js'
 import { registerProtocolHandler } from '../network/register-protocol-handler.js'
 
-const debugLog = createLogger('repo-service')
-
 // Define Components interface
 interface BaseComponents {
-	logger: { forComponent: (name: string) => Logger },
 	registrar: {
 		handle: (protocol: string, handler: StreamHandler, options: any) => Promise<void>
 		unhandle: (protocol: string) => Promise<void>
@@ -52,6 +49,12 @@ export type RepoServiceInit = InboundStreamAuthorizationInit & {
 	protocolPrefix?: string,
 	maxInboundStreams?: number,
 	maxOutboundStreams?: number,
+	/**
+	 * Sub-namespace this service logs under, i.e. the `<x>` in `optimystic:db-p2p:<x>`.
+	 * NOT a full namespace: an embedder cannot use this to move the service's lines outside the
+	 * `optimystic:db-p2p:*` tree `docs/debugging.md` tells operators to filter on.
+	 * Default: `repo-service`.
+	 */
 	logPrefix?: string,
 	kBucketSize?: number,
 	/**
@@ -94,8 +97,8 @@ export class RepoService implements Startable {
 	private readonly authorization: InboundStreamAuthorization | undefined
 	/**
 	 * Sink for this service's `peer-address-book:*` lines — same reasoning as `ClusterService`'s:
-	 * `this.log.error` would strand them under `db-p2p:repo-service:error`, outside the
-	 * `optimystic:db-p2p:*` tree every other address-book line lives in.
+	 * `this.log.error` would scatter one tag family across whichever service happened to be the
+	 * ingress point, instead of keeping it filterable as the single `peer-address-book` namespace.
 	 */
 	private readonly addressLog: AddressLog
 
@@ -105,7 +108,7 @@ export class RepoService implements Startable {
 		this.protocol = computed
 		this.maxInboundStreams = init.maxInboundStreams ?? 32
 		this.maxOutboundStreams = init.maxOutboundStreams ?? 64
-		this.log = components.logger.forComponent(init.logPrefix ?? 'db-p2p:repo-service')
+		this.log = createLogger(init.logPrefix ?? 'repo-service')
 		this.addressLog = createLogger('peer-address-book', components.peerId?.toString())
 		this.repo = components.repo
 		this.running = false
@@ -246,7 +249,7 @@ export class RepoService implements Startable {
 
 		if (!smallMesh && !isMember) {
 			const peers = cluster.filter((p: PeerId) => !peersEqual(p, selfId))
-			debugLog('redirect op=%s blockKey=%s cluster=%d', opName, blockKey, cluster.length)
+			this.log('redirect op=%s blockKey=%s cluster=%d', opName, blockKey, cluster.length)
 			return encodePeers(await Promise.all(peers.map(async (pid: PeerId) => ({
 				id: pid.toString(),
 				addrs: await this.getPeerAddrs(pid)

package/src/storage/storage-repo.ts

@@ -281,6 +281,15 @@ export class StorageRepo implements IRepo, IBlockChangeNotifier, IBlockReplicaSt
 						// Sort a COPY: when `latest` is undefined, `missing` aliases the caller's
 						// `context.committed` array, and an in-place `.sort()` would reorder the shared
 						// request context under the caller's feet.
+						//
+						// NOTE: this loop SKIPS an entry whose pending it does not hold and promotes the
+						// next one anyway — so a member missing an intermediate revision that touched THIS
+						// block forks it exactly as an un-guarded commit would. `internalCommit`'s declared-
+						// base guard cannot help here: `context.committed` is a collection-level list of
+						// (actionId, rev) with no per-block base, and "no pending for that action" is the
+						// normal case for the many actions that never touched this block. Closing it needs
+						// the authored base stored WITH the pended transform — tracked by
+						// `backlog/bug-a-pended-transform-does-not-carry-its-base`.
 						try {
 							for (const { actionId, rev } of [...missing].sort((a, b) => a.rev - b.rev)) {
 								const pending = await blockStorage.getPendingTransaction(actionId);
@@ -905,7 +914,11 @@ export class StorageRepo implements IRepo, IBlockChangeNotifier, IBlockReplicaSt
 				}
 				try {
 					// internalCommit will throw if it encounters an issue
-					const collectionId = await this.internalCommit(blockId, request.actionId, request.rev, storage, latch, proof);
+					// The writer's per-block base declaration (see BlockContentDigest.baseRev) rides on the
+					// commit op and is what lets internalCommit tell a legitimate collection-level rev gap
+					// apart from a genuinely missed update to THIS block. Untrusted wire data — the guard
+					// validates it, this call site only forwards it.
+					const collectionId = await this.internalCommit(blockId, request.actionId, request.rev, storage, latch, proof, request.blockDigests?.[blockId]?.baseRev);
 					if (collectionId !== undefined) {
 						const list = collectionBlocks.get(collectionId) ?? [];
 						list.push(blockId);
@@ -1174,7 +1187,13 @@ export class StorageRepo implements IRepo, IBlockChangeNotifier, IBlockReplicaSt
 		return await this.createBlockStorage(blockId).getBlockProof(rev);
 	}
 
-	private async internalCommit(blockId: BlockId, actionId: ActionId, rev: number, storage: IBlockStorage, latch: BlockWriteLatch, proof?: BlockCommitProof): Promise<CollectionId | undefined> {
+	/**
+	 * @param declaredBaseRev The committed revision of the base the WRITER applied this block's
+	 * transform to, as declared in the commit op's `blockDigests[blockId].baseRev`. Untrusted wire
+	 * data, so it is typed `unknown` and validated below. Absent from the read-driven promotion in
+	 * {@link get}, which has no commit request and therefore nothing to compare against.
+	 */
+	private async internalCommit(blockId: BlockId, actionId: ActionId, rev: number, storage: IBlockStorage, latch: BlockWriteLatch, proof?: BlockCommitProof, declaredBaseRev?: unknown): Promise<CollectionId | undefined> {
 		// Note: This method is called under the block write latch — by commit() (within its locked
 		// critical section) and by the read-driven promotion in get() (which takes the same latch);
 		// `latch` is the proof of that. So, operations like getPendingTransaction, getLatest,
@@ -1194,6 +1213,50 @@ export class StorageRepo implements IRepo, IBlockChangeNotifier, IBlockReplicaSt
 
 		// Get prior materialized block if it exists
 		const latest = await storage.getLatest();
+
+		// FORK GUARD: apply an update-only transform ONLY to the base the writer actually read.
+		// Revisions are allocated per COLLECTION, not per block, so `rev - 1` is meaningless here — a
+		// member legitimately holds block X at rev 1 and receives a commit of X at rev 7 when revs 2-6
+		// touched other blocks. The only sound discriminator is the writer's own per-block declaration:
+		// it read the block at `declaredBaseRev`, so a member holding anything else would be applying
+		// the transform to different bytes and silently forking the block's content at this revision.
+		//
+		// Each clause is deliberate:
+		// - `typeof declaredBaseRev === 'number'` — `blockDigests` is untrusted wire data with no
+		//   ingress schema (same rule as ClusterMember.validateCommitOperations). Missing, malformed,
+		//   or absent-by-design declarations ABSTAIN, preserving today's behavior for pre-upgrade
+		//   writers, undeclarable blocks, and the read-driven promotion in get().
+		// - `!transform.insert` — an insert-carrying transform is base-independent, so there is nothing
+		//   to fork. Keyed on the member's OWN pended transform (as previewCommitDigest does), never on
+		//   the declaration, so a hostile writer cannot flip the arm by attaching a bogus baseRev.
+		// - `latest?.rev !== declaredBaseRev` covers all three unsafe states: BEHIND the declared base
+		//   (missed updates — the fork case), AHEAD of it (this member holds a revision the writer
+		//   never saw — divergent history), and no local revision at all against a numeric declaration.
+		//
+		// Refusing is cheap and self-healing: refuseMissingBase throws MissingBaseRevisionError, which
+		// commit() classifies as divergence and ClusterMember.applyConsensusOperation maps to "behind",
+		// running reconcileDivergentCommit to pull the committed revision from a cohort peer. The
+		// writer's retry then lands on a healed base. A hostile writer declaring a junk numeric baseRev
+		// can force refusals and reconcile churn, but never a fork.
+		//
+		// NOTE: the AHEAD case is reported as "behind" divergence like every other missing-base
+		// refusal, so a cohort where nobody holds `rev` reconciles, fails `no-rev-quorum`, and logs
+		// that rather than a clean stale failure. Correct outcome — the writer read a base the cohort
+		// has moved past, and its retry re-reads — but the log reads as lag when it is the opposite.
+		// If those lines ever have to be triaged in volume, give the ahead arm its own reason string.
+		//
+		// NOTE: this guard only reaches what the writer declared, so two arms of the same fork still
+		// stand — both tracked by `backlog/bug-a-pended-transform-does-not-carry-its-base`:
+		// (1) a commit whose block declares NO digest — pre-upgrade writer, undeclarable block
+		// (read-far-then-update eviction, see db-core transform/digest.ts), or a delete-only transform —
+		// gap-applies exactly as before; (2) the read-driven promotion in `get`, which reaches this
+		// method with no commit request at all (`declaredBaseRev` undefined) and so cannot check. If
+		// forked-content reports persist, those are the residuals to look at.
+		if (typeof declaredBaseRev === 'number' && !transform.insert && latest?.rev !== declaredBaseRev) {
+			return await this.refuseMissingBase(blockId, actionId, rev, storage, latch,
+				`local latest ${latest?.rev ?? 'none'} is not the declared base ${declaredBaseRev} of rev ${rev}`);
+		}
+
 		const priorBlock = await this.readCommitBase(blockId, actionId, rev, storage, latest, latch);
 
 		// Apply transform and save materialized block

package/src/sync/service.ts

@@ -1,4 +1,4 @@
-import type { ComponentLogger, Connection, Startable } from '@libp2p/interface';
+import type { Connection, Startable } from '@libp2p/interface';
 import { buildSyncProtocol, type SyncRequest, type SyncResponse } from './protocol.js';
 import { pipe } from 'it-pipe';
 import { toString as u8ToString } from 'uint8arrays/to-string';
@@ -9,13 +9,13 @@ import { createInboundStreamAuthorization, type InboundStreamAuthorization, type
 import { serveBlockArchive, type ArchiveServingRepo } from '../storage/block-archive.js';
 import { registerProtocolHandler } from '../network/register-protocol-handler.js';
 import type { BlockArchive } from '../storage/struct.js';
+import { createLogger, type Logger } from '../logger.js';
 
 export interface SyncServiceInit extends InboundStreamAuthorizationInit {
 	protocolPrefix?: string;
 }
 
 export interface SyncServiceComponents {
-	logger: ComponentLogger;
 	registrar: { handle: (...args: any[]) => Promise<void>, unhandle: (...args: any[]) => Promise<void> };
 	/**
 	 * The local store this node answers repair fetches out of. Declared as `ArchiveServingRepo`
@@ -28,8 +28,6 @@ export interface SyncServiceComponents {
 	repo: ArchiveServingRepo;
 }
 
-type Logger = ReturnType<ComponentLogger['forComponent']>;
-
 /**
  * Service for handling incoming sync requests from other cluster peers.
  *
@@ -53,7 +51,7 @@ export class SyncService implements Startable {
 		components: SyncServiceComponents,
 		init: SyncServiceInit = {}
 	) {
-		this.log = components.logger.forComponent('db-p2p:sync-service');
+		this.log = createLogger('sync-service');
 		this.protocol = buildSyncProtocol(init.protocolPrefix ?? '');
 		this.repo = components.repo;
 		this.registrar = components.registrar;

package/src/testing/mesh-harness.ts

@@ -485,6 +485,12 @@ export async function nonResponsibleNodes(mesh: Mesh, blockId: string): Promise<
 export interface BuildTransactorOptions {
 	timeoutMs?: number;
 	abortOrCancelTimeoutMs?: number;
+	/**
+	 * Wraps each node's repo before the transactor sees it, so a test can inject a transport-shaped
+	 * failure into one RPC (a dropped sweep commit, a refused pend) and leave every other call — and
+	 * therefore all of the production code after the injection — untouched.
+	 */
+	wrapRepo?: (repo: IRepo, node: MeshNode) => IRepo;
 }
 
 /**
@@ -496,7 +502,8 @@ export interface BuildTransactorOptions {
 export const buildNetworkTransactor = (mesh: Mesh, options: BuildTransactorOptions = {}): ITransactor => {
 	const repoByPeer = new Map<string, IRepo>();
 	for (const node of mesh.nodes) {
-		repoByPeer.set(node.peerId.toString(), node.coordinatorRepo as unknown as IRepo);
+		const repo = node.coordinatorRepo as unknown as IRepo;
+		repoByPeer.set(node.peerId.toString(), options.wrapRepo ? options.wrapRepo(repo, node) : repo);
 	}
 	return new NetworkTransactor({
 		timeoutMs: options.timeoutMs ?? 5_000,