@optimystic/db-p2p 0.25.1 → 0.26.0

package/src/storage/storage-repo.ts

@@ -520,6 +520,8 @@ export class StorageRepo implements IRepo, IBlockChangeNotifier, IBlockReplicaSt
 			};
 		}
 
+		// Already deduped: `blockIdsForTransforms` builds its result through a Set. So the pass-2 save
+		// loop below cannot write one block twice, and the echoed `blockIds` carries no duplicate.
 		const blockIds = blockIdsForTransforms(request.transforms);
 		log('pend actionId=%s blockIds=%d rev=%s', request.actionId, blockIds.length, request.rev);
 		const pendings: ActionPending[] = [];
@@ -531,124 +533,162 @@ export class StorageRepo implements IRepo, IBlockChangeNotifier, IBlockReplicaSt
 		// Blocks this action ALREADY committed at exactly the requested revision — the durable half
 		// of a torn action whose retry reuses the same actionId. Sibling of the `alreadyDone`
 		// partition in `commit` below: satisfied, not merely non-stale, so no pending is recorded
-		// for them (see the fan-out at the end of this method).
+		// for them (see pass 2).
 		const satisfied = new Set<BlockId>();
-
-		// Potential race condition: A concurrent commit operation could complete
-		// between the conflict checks (latest.rev, listPendingTransactions) and the
-		// savePendingTransaction call below. This pend operation might succeed based on
-		// stale information, but the subsequent commit for this pend would likely
-		// fail correctly later if a conflict arose. Locking here could make the initial
-		// check more accurate but adds overhead. The current approach prioritizes
-		// letting the commit be the final arbiter.
-		for (const blockId of blockIds) {
-			const blockStorage = this.createBlockStorage(blockId);
-			const transforms = transformForBlockId(request.transforms, blockId);
-
-			// Handle any conflicting revisions FIRST: a block this same action already committed at
-			// exactly the requested revision is satisfied, and skips both this check and the
-			// pending-action listing below.
-			if (request.rev !== undefined || transforms.insert) {
-				const latest = await blockStorage.getLatest();
-				// Our own already-durable work, met again by a retry (see {@link isOwnRevision}):
-				// treating it as a stale rival would refuse the writer with its own commit.
-				// NOTE: a rev-less pend (`request.rev === undefined`, an insert-only claim) can
-				// never match, so a torn action retried WITHOUT a revision is still refused by its
-				// own insert. No production caller sends one — `TransactorSource.transact` and the
-				// multi-collection coordinator both require a rev — so this is unreachable today;
-				// if a rev-less write path ever appears, match on `latest.actionId` alone here.
-				if (isOwnRevision(latest, request.rev, request.actionId)) {
-					satisfied.add(blockId);
-					continue;
-				}
-				if (latest && latest.rev >= (request.rev ?? 0)) {
-					// Only a real revision race yields a meaningful `staleAt`. When `request.rev` is
-					// undefined this same branch fires for an insert collision (the comparison degrades
-					// to `latest.rev >= 0`, true for any existing block), and reporting that block's
-					// revision would be a number that answers a question nobody asked.
-					if (request.rev !== undefined) {
-						staleAt = highestStaleAt([staleAt, { blockId, rev: latest.rev }]);
+		// Blocks observed at or past the requested revision under a DIFFERENT action — a real stale
+		// loss. Counted separately from `missing` because the two are not the same question: `missing`
+		// is the catch-up the loser is handed, and a node whose revision index is sparse over
+		// [request.rev, latest.rev] hands back an empty one while still having lost. Gating the
+		// refusal on the enumeration would then let a block pass classification that pass 2 cannot
+		// write (`savePendingTransaction` refuses it), turning a stale answer into a throw. `commit`
+		// takes the same position — it pushes a `missedCommits` entry "even if transforms is empty,
+		// because we want to reject the older version".
+		let staleCount = 0;
+
+		// Classifying and saving are ONE atomic step per pend: both passes below run inside a single
+		// multi-block write-latch hold, so no commit can land between deciding a block is pendable
+		// and writing its pending record. That is the whole property — a pend never writes a pending
+		// record for a revision already taken. Such a record could never be promoted (`commit`
+		// partitions the block as already-done or refuses it as stale, and promotion is the only
+		// thing that removes a record on the success path), and would then be reported as a
+		// conflicting in-flight action to every later writer of the block. See docs/repository.md,
+		// Invariant P; `BlockStorage.savePendingTransaction` refuses such a write outright.
+		//
+		// TWO passes, not one interleaved loop: with a single loop a block refused partway through
+		// would leave records already written for its predecessors, and retracting those under the
+		// hold could delete a record an EARLIER pend of the same action legitimately left. Classify
+		// everything before writing anything, and no record is ever written that must be taken back.
+		//
+		// Everything inside the hold is local storage I/O. No network I/O and no caller-supplied
+		// code may enter it — `checkPendValidation` above can call the caller's validation hook,
+		// which is precisely why it stays outside. `commit` keeps the same rule. Acquiring through
+		// `acquireBlockWriteLatches` (deduped, sorted) is what keeps the three multi-latch holders —
+		// this, `commit`, and `applyInvalidation` — free of deadlock, and no caller of `pend`
+		// (`ClusterRepo`, `CoordinatorRepo`, `service.ts`) holds a block latch, so the hold cannot
+		// re-enter itself.
+		//
+		// NOTE: a pend now blocks concurrent commits on its blocks for the span of BOTH passes, not
+		// just its writes. Accepted: every call inside is local storage I/O, and `commit` already
+		// holds the same set for a comparable span. If pend latency on contended blocks ever shows
+		// up in a profile, two things inside the hold scale with width and are the ones to look at:
+		// the policy-'r' arm reads one transform per rival, and pass 2 awaits its saves one block at
+		// a time (where the pre-latch code fanned out with `Promise.all`). Sequential is the
+		// deliberate choice — a throw mid-pass then strands records for FEWER blocks, not more — so
+		// batch or fan out only with that tradeoff in hand.
+		const { latches, release } = await acquireBlockWriteLatches(blockIds);
+		try {
+			// --- Pass 1: classify. Every read below runs under the hold. ---
+			for (const blockId of blockIds) {
+				const blockStorage = this.createBlockStorage(blockId);
+				const transforms = transformForBlockId(request.transforms, blockId);
+
+				// Handle any conflicting revisions FIRST: a block this same action already committed at
+				// exactly the requested revision is satisfied, and skips both this check and the
+				// pending-action listing below.
+				if (request.rev !== undefined || transforms.insert) {
+					const latest = await blockStorage.getLatest();
+					// Our own already-durable work, met again by a retry (see {@link isOwnRevision}):
+					// treating it as a stale rival would refuse the writer with its own commit.
+					// NOTE: a rev-less pend (`request.rev === undefined`, an insert-only claim) can
+					// never match, so a torn action retried WITHOUT a revision is still refused by its
+					// own insert. No production caller sends one — `TransactorSource.transact` and the
+					// multi-collection coordinator both require a rev — so this is unreachable today;
+					// if a rev-less write path ever appears, match on `latest.actionId` alone here.
+					if (isOwnRevision(latest, request.rev, request.actionId)) {
+						satisfied.add(blockId);
+						continue;
 					}
-					const transforms = await asyncIteratorToArray(blockStorage.listRevisions(request.rev ?? 0, latest.rev));
-					for (const actionRev of transforms) {
-						const transform = await blockStorage.getTransaction(actionRev.actionId);
-						if (!transform) {
-							throw new Error(`Missing action ${actionRev.actionId} for block ${blockId}`);
+					if (latest && latest.rev >= (request.rev ?? 0)) {
+						// Only a real revision race yields a meaningful `staleAt`. When `request.rev` is
+						// undefined this same branch fires for an insert collision (the comparison degrades
+						// to `latest.rev >= 0`, true for any existing block), and reporting that block's
+						// revision would be a number that answers a question nobody asked.
+						if (request.rev !== undefined) {
+							staleAt = highestStaleAt([staleAt, { blockId, rev: latest.rev }]);
+						}
+						staleCount++;
+						const missedRevisions = await asyncIteratorToArray(blockStorage.listRevisions(request.rev ?? 0, latest.rev));
+						for (const actionRev of missedRevisions) {
+							const transform = await blockStorage.getTransaction(actionRev.actionId);
+							if (!transform) {
+								throw new Error(`Missing action ${actionRev.actionId} for block ${blockId}`);
+							}
+							missing.push({
+								actionId: actionRev.actionId,
+								rev: actionRev.rev,
+								transforms: transformsFromTransform(transform, blockId)
+							});
 						}
-						missing.push({
-							actionId: actionRev.actionId,
-							rev: actionRev.rev,
-							transforms: transformsFromTransform(transform, blockId)
-						});
 					}
 				}
-			}
 
-			// Then handle any pending actions
-			const pending = await asyncIteratorToArray(blockStorage.listPendingTransactions());
-			pendings.push(...pending.map(actionId => ({ blockId, actionId })));
-		}
-
-		if (missing.length) {
-			log('pend:stale actionId=%s missing=%d', request.actionId, missing.length);
-			return {
-				success: false,
-				conflict: true,
-				missing,
-				...(staleAt === undefined ? {} : { staleAt })
-			};
-		}
+				// Then handle any pending actions
+				const pending = await asyncIteratorToArray(blockStorage.listPendingTransactions());
+				pendings.push(...pending.map(actionId => ({ blockId, actionId })));
+			}
 
-		if (pendings.length > 0) {
-			if (request.policy === 'f') {	// Fail on pending actions
-				return { success: false, conflict: true, pending: pendings };
-			} else if (request.policy === 'r') {	// Return populated pending actions
+			// Every refusal below returns having written ZERO pending records — that is what pass 1
+			// finishing before pass 2 begins buys.
+			if (staleCount > 0) {
+				log('pend:stale actionId=%s stale=%d missing=%d', request.actionId, staleCount, missing.length);
 				return {
 					success: false,
 					conflict: true,
-					pending: await Promise.all(pendings.map(async action => {
-						const blockStorage = this.createBlockStorage(action.blockId);
-						return {
-							blockId: action.blockId,
-							actionId: action.actionId,
-							transform: (await blockStorage.getPendingTransaction(action.actionId))
-								?? (await blockStorage.getTransaction(action.actionId))!	// Possible that since enumeration, the action has been promoted
-						}
-					}))
+					missing,
+					...(staleAt === undefined ? {} : { staleAt })
 				};
 			}
-		}
 
+			if (pendings.length > 0) {
+				if (request.policy === 'f') {	// Fail on pending actions
+					return { success: false, conflict: true, pending: pendings };
+				} else if (request.policy === 'r') {	// Return populated pending actions
+					return {
+						success: false,
+						conflict: true,
+						pending: await Promise.all(pendings.map(async action => {
+							const blockStorage = this.createBlockStorage(action.blockId);
+							return {
+								blockId: action.blockId,
+								actionId: action.actionId,
+								// The fallback stays: a rival enumerated on a block we hold cannot be promoted
+								// out from under us mid-hold, but a partially-overlapping pend can still have
+								// promoted one on a block outside this hold.
+								transform: (await blockStorage.getPendingTransaction(action.actionId))
+									?? (await blockStorage.getTransaction(action.actionId))!
+							}
+						}))
+					};
+				}
+			}
 
-		// Simultaneously save pending action for each block
-		// Note: that this is not atomic, after we checked for conflicts and pending actions
-		// new pending or committed actions may have been added.  This is okay, because
-		// this check during pend is conservative.
-		//
-		// Each block's pending write runs under THAT block's write latch, one latch per branch and
-		// never nested: savePendingTransaction seeds the block's metadata blob when it has none, and
-		// an unlatched seed racing a concurrent commit/replica on a fresh block erases the `latest`
-		// the other writer just landed. Never more than one block latch is held by a branch, so this
-		// cannot deadlock against commit's sorted multi-latch acquisition.
-		//
-		// `satisfied` blocks are skipped: `commit`'s `alreadyDone` arm skips `internalCommit`, the
-		// only thing that promotes (and thereby removes) a pending record, so a pending saved here
-		// would never clear — a permanent durable reservation that the rival-pending checks (this
-		// method's listPendingTransactions scan, and `ClusterMember.validatePendOperations`) refuse
-		// every future writer against, a worse wedge than the one this carve-out fixes. They still
-		// ride in the returned `blockIds` so `cancel` covers them (deleting an absent pending is a
-		// no-op that writes no metadata).
-		await Promise.all(blockIds.filter(blockId => !satisfied.has(blockId)).map(blockId => {
-			const blockStorage = this.createBlockStorage(blockId);
-			const blockTransform = transformForBlockId(request.transforms, blockId);
-			return withBlockWriteLatch(blockId, latch => blockStorage.savePendingTransaction(request.actionId, blockTransform, latch));
-		}));
+			// --- Pass 2: save. Same hold, so nothing advanced a block since pass 1 observed it. ---
+			//
+			// `satisfied` blocks are skipped: `commit`'s `alreadyDone` arm skips `internalCommit`, the
+			// only thing that promotes (and thereby removes) a pending record, so a pending saved here
+			// would never clear — a permanent durable reservation that the rival-pending checks (this
+			// method's listPendingTransactions scan, and `ClusterMember.validatePendOperations`) refuse
+			// every future writer against. They still ride in the returned `blockIds` so `cancel`
+			// covers them (deleting an absent pending is a no-op that writes no metadata).
+			for (const blockId of blockIds) {
+				if (satisfied.has(blockId)) {
+					continue;
+				}
+				const blockStorage = this.createBlockStorage(blockId);
+				const blockTransform = transformForBlockId(request.transforms, blockId);
+				await blockStorage.savePendingTransaction(request.actionId, blockTransform, request.rev, latches.get(blockId)!);
+			}
 
-		return {
-			success: true,
-			pending: pendings,
-			blockIds
-		} as PendSuccess;
+			return {
+				success: true,
+				pending: pendings,
+				blockIds
+			} as PendSuccess;
+		} finally {
+			// Releases on every path, including the early returns above and the
+			// `Missing action … for block …` throw inside pass 1.
+			release();
+		}
 	}
 
 	async cancel(actionRef: ActionBlocks, _options?: MessageOptions): Promise<void> {

package/src/storage/with-read-cache.ts

@@ -102,8 +102,12 @@ class Lease implements ReadCacheLease {
  * Returns the storage **unchanged**, with no `lease`, when caching would not pay:
  * - `MemoryRawStorage` is already in memory; the cache would duplicate every map entry's
  *   bookkeeping with nothing to save (see `CachedStoreDriver`'s class doc).
- * - An already-cached storage (a host that wrapped before handing it over) is not wrapped twice,
- *   and stays the host's to dispose.
+ * - A storage that reports {@link IRawStorage.readCached} (a host that attached a cache before
+ *   handing it over) is not wrapped twice, and stays the host's to dispose. The capability, not
+ *   a class check: BOTH documented constructions — `new CachedRawStorage(inner)` and
+ *   `new KvRawStorage(new CachedStoreDriver(driver))` — report it, and only the first is a
+ *   `CachedRawStorage`. Checking the class instead sent the driver-level shape down the
+ *   construct path, where the pool's identity guard threw and the host's node failed to start.
  *
  * Why this is needed at all: `BlockStorage` re-reads block metadata on essentially every
  * operation and `StorageRepo` builds a fresh `BlockStorage` per block per call, so nothing above
@@ -122,9 +126,9 @@ class Lease implements ReadCacheLease {
  * unwrapped instance handed to two consumers). Identity is one-directional — equal proves
  * sameness, unequal proves nothing — so backends whose identity under-approximates (path
  * aliases, two handles over one database; each backend's `NOTE:` lists its gaps) can still end
- * up with two caches; and a host that builds its own `CachedRawStorage` never enters the
- * registry, so a second consumer wrapping a fresh instance over that store still gets a second
- * cache. What remains of Invariant 5 (`packages/db-p2p/docs/storage.md`) is the cross-process
+ * up with two caches; and a host that builds its own cache — in either construction — never
+ * enters the registry, so a second consumer wrapping a fresh instance over that store still gets
+ * a second cache. What remains of Invariant 5 (`packages/db-p2p/docs/storage.md`) is the cross-process
  * case: the filesystem driver takes no lock, and a second process's writes bypass this cache.
  *
  * On a dedupe hit the FIRST caller's `label` and `pool` stick: `pool.stats()` shows whoever
@@ -148,7 +152,13 @@ class Lease implements ReadCacheLease {
  *               dedupe hit.
  */
 export function withReadCache(storage: IRawStorage, label?: string, pool?: SharedCachePool): ResolvedReadCache {
-	if (storage instanceof MemoryRawStorage || storage instanceof CachedRawStorage) {
+	// `readCached` is a plain property read, which keeps this whole function synchronous — see
+	// the "no `await` between lookup and insert" note above; never make it an accessor that
+	// could become async. `MemoryRawStorage` never carries the marker (its driver is uncached);
+	// it is excluded for a different reason and keeps its own class check. Note a plain
+	// `KvRawStorage` over a bare `MemoryStoreDriver` is NOT a `MemoryRawStorage` and is still
+	// wrapped — deliberate, and pinned by a test.
+	if (storage instanceof MemoryRawStorage || storage.readCached) {
 		return { storage, lease: undefined };
 	}