@optimystic/db-core 0.16.3 → 0.18.0

package/src/transactor/network-transactor.ts

@@ -1,5 +1,7 @@
 import { peerIdFromString } from "../network/types.js";
 import type { PeerId } from "../network/types.js";
+import { isConflictFailure } from "../network/stale-failure.js";
+import { BlockUnavailableError } from "../network/struct.js";
 import type { ActionTransforms, ActionBlocks, BlockActionStatus, ITransactor, PendSuccess, StaleFailure, IKeyNetwork, BlockId, GetBlockResults, PendResult, CommitResult, PendRequest, IRepo, BlockGets, Transforms, CommitRequest, ActionId, RepoCommitRequest, ClusterNomineesResult, CollectionId, IBlock } from "../index.js";
 import type { IBlockChangeNotifier, CollectionChangeListener } from "./change-notifier.js";
 import { transformForBlockId, groupBy, concatTransforms, concatTransform, transformsFromTransform, blockIdsForTransforms, Log, Tracker, CacheSource, TransactorSource } from "../index.js";
@@ -130,28 +132,33 @@ export class NetworkTransactor implements ITransactor, IBlockChangeNotifier {
 		}
 
 		// Second-chance retry: ONLY for a genuine no-response — a batch with no valid
-		// response, or a response missing an entry for a requested block id. An
-		// authoritative "absent" answer (a valid response that carries an entry for
-		// every requested block id, even one whose entry has only `state` and no
-		// materialized `block`) is FINAL and must not retry. A block that genuinely
-		// does not exist yet surfaces as `{ state: {} }` (an entry that is present) —
-		// retrying it doubles the round-trips on the common createOrOpen "does this
-		// block exist?" probe. Cross-member reconciliation for a missing block has
-		// already happened one layer down: CoordinatorRepo.get detects `isMissing` and
-		// consults cluster peers before it responds, so by the time an authoritative
-		// absent reaches here there is nothing left for a transactor-level retry to
-		// discover. See ticket txn-perf-authoritative-notfound.
+		// response, a response missing an entry for a requested block id, or an entry
+		// flagged `unavailable`. An authoritative "absent" answer (a valid response that
+		// carries an entry for every requested block id, even one whose entry has only
+		// `state` and no materialized `block`) is FINAL and must not retry. A block that
+		// genuinely does not exist yet surfaces as `{ state: {} }` (an entry that is
+		// present and unflagged) — retrying it doubles the round-trips on the common
+		// createOrOpen "does this block exist?" probe. Cross-member reconciliation for a
+		// missing block has already happened one layer down: CoordinatorRepo.get detects
+		// `isMissing` and consults cluster peers before it responds — and when that
+		// consult FAILS, the entry now says so via `unavailable` instead of posing as an
+		// authoritative absent. So by the time an unflagged absent reaches here there is
+		// nothing left for a transactor-level retry to discover, while a flagged entry
+		// earns the retry against a different peer that an absent deliberately does not.
+		// See tickets txn-perf-authoritative-notfound and repo-reports-unavailable-vs-absent.
 		const hasValidResponse = (b: CoordinatorBatch<BlockId[], GetBlockResults>) => {
 			return b.request?.isResponse === true && b.request.response != null;
 		};
 
 		// A batch is answered when its response carries an entry for EVERY requested
-		// block id. An entry present with only `state` (no `block`) is an authoritative
-		// "absent", which counts as answered — not a gap.
+		// block id and none of those entries is flagged `unavailable`. An entry present
+		// with only `state` (no `block`) is an authoritative "absent", which counts as
+		// answered — not a gap. An `unavailable` entry is the peer saying its own answer
+		// is a guess, so it does NOT count as answered.
 		const isAuthoritative = (b: CoordinatorBatch<BlockId[], GetBlockResults>) => {
 			if (!hasValidResponse(b)) return false;
 			const resp = b.request!.response! as GetBlockResults;
-			return b.payload.every(bid => resp[bid] !== undefined);
+			return b.payload.every(bid => resp[bid] !== undefined && resp[bid]!.unavailable === undefined);
 		};
 
 		// Retry only genuine no-response / partial-response batches. An authoritative
@@ -199,16 +206,24 @@ export class NetworkTransactor implements ITransactor, IBlockChangeNotifier {
 		// Cache the completed batches that had actual responses (not just coordinator not found)
 		const completedBatches = Array.from(allBatches(batches, b => b.request?.isResponse as boolean && !isRecordEmpty(b.request!.response!)));
 
+		// Three-way ranking per block id: a materialized block beats an authoritative
+		// absent, which beats an `unavailable` guess — one peer that positively knows the
+		// block is absent outranks another that could not find out. Non-object junk ranks
+		// below everything so any real entry replaces it.
+		const rankOf = (r: unknown): number => {
+			if (!r || typeof r !== 'object') return -1;
+			const entry = r as GetBlockResults[BlockId];
+			if (entry.block != null) return 2;
+			return entry.unavailable === undefined ? 1 : 0;
+		};
+
 		// Create a lookup map from successful responses only
 		const resultEntries = new Map<string, any>();
 		for (const batch of completedBatches) {
 			const resp = batch.request!.response! as any;
 			for (const [bid, res] of Object.entries(resp)) {
 				const existing = resultEntries.get(bid);
-				// Prefer responses that include a materialized block
-				const resHasBlock = res && typeof res === 'object' && 'block' in (res as any) && (res as any).block != null;
-				const existingHasBlock = existing && typeof existing === 'object' && 'block' in (existing as any) && (existing as any).block != null;
-				if (!existing || (resHasBlock && !existingHasBlock)) {
+				if (!existing || rankOf(res) > rankOf(existing)) {
 					resultEntries.set(bid, res);
 				}
 			}
@@ -245,6 +260,17 @@ export class NetworkTransactor implements ITransactor, IBlockChangeNotifier {
 		// Get block states from repos
 		const blockStates = await this.get({ blockIds: allBlockIds });
 
+		// A block whose repo could not determine whether it exists carries no status either:
+		// its empty `state` would read below as `aborted`, turning "I could not find out" into
+		// a definite verdict on someone's action. Fail loudly instead, like every other read of
+		// an unavailable block (see BlockUnavailableError).
+		for (const blockId of allBlockIds) {
+			const entry = blockStates[blockId];
+			if (entry?.unavailable !== undefined && entry.block == null) {
+				throw new BlockUnavailableError(blockId, entry.unavailable);
+			}
+		}
+
 		// Determine status for each action ref
 		const results: BlockActionStatus[] = blockActions.map(ref => ({
 			...ref,
@@ -511,8 +537,28 @@ export class NetworkTransactor implements ITransactor, IBlockChangeNotifier {
 			const stale = Array.from(allBatches(batches, b => b.request?.isResponse as boolean && !b.request!.response!.success));
 			if (stale.length > 0) {	// Any active stale failures should preempt reporting connection or other potential transient errors (we have information)
 				log('pend:stale actionId=%s staleCount=%d', blockAction.actionId, stale.length);
+				// Carry the first available reject reason through: `SyncRetryExhaustedError.lastReason`
+				// and the multi-collection writer's failure message both read it, and it is the only
+				// diagnostic that survives an exhausted retry budget.
+				const reason = stale.map(b => (b.request!.response! as StaleFailure).reason).find(r => r !== undefined);
+				// This response is REBUILT from the per-batch ones rather than forwarded, so
+				// retryability has to be carried explicitly or it is lost: a batch whose failure was
+				// a confirmed lost race can arrive with neither `missing` nor `pending` (see
+				// CoordinatorRepo.classifyStaleRejection), and the aggregate would then look like a
+				// hard rejection to `isConflictFailure`. Any conflicting batch makes the aggregate a
+				// conflict — the pend failed as a whole, and a re-read/rebase can clear it.
+				// NOTE: `some`, not `every`, so a pend whose batches mix a lost race with a genuine hard
+				// rejection is reported retryable and burns its (bounded, backed-off) retry budget before
+				// failing. Deliberate: an unclassified reason-only response from an older peer is
+				// indistinguishable from a hard rejection here, and `every` would refuse to retry a real
+				// race whenever one batch came from such a peer. Revisit if every producer sets `conflict`
+				// (then `every` is both safe and tighter), or if mixed-outcome pends show up as wasted
+				// retry latency in practice.
+				const conflict = stale.some(b => isConflictFailure(b.request!.response! as StaleFailure));
 				return {
 					success: false,
+					conflict,
+					...(reason === undefined ? {} : { reason }),
 					missing: distinctBlockActionTransforms(stale.flatMap(b => (b.request!.response! as StaleFailure).missing).filter((x): x is ActionTransforms => x !== undefined)),
 				};
 			}

package/src/transactor/transactor-source.ts

@@ -1,6 +1,7 @@
 import { randomBytes } from '@noble/hashes/utils.js'
 import { toString as uint8ArrayToString } from 'uint8arrays/to-string'
 import type { IBlock, BlockId, BlockHeader, ITransactor, ActionId, StaleFailure, ActionContext, BlockType, BlockSource, ReadPurpose, Transforms } from "../index.js";
+import { BlockUnavailableError } from "../network/struct.js";
 import type { ReadDependency } from "../transaction/transaction.js";
 import { ReadDependencyCollector } from "../transaction/read-dependency-collector.js";
 
@@ -42,7 +43,14 @@ export class TransactorSource<TBlock extends IBlock> implements BlockSource<TBlo
 		// `result[id]` is undefined. Destructuring that would throw a TypeError.
 		const entry = result?.[id];
 		if (entry) {
-			const { block, state } = entry;
+			const { block, state, unavailable } = entry;
+			// An entry flagged `unavailable` with no block is the repo saying "I could not find
+			// out whether this exists" — an answer that must not be read as absent. Throw rather
+			// than return undefined, and record no read dependency (dependencies are recorded
+			// only for blocks that actually exist). A repo that omits the flag stays authoritative.
+			if (!block && unavailable) {
+				throw new BlockUnavailableError(id, unavailable);
+			}
 			// Record a read dependency only for a block that actually exists. A transactor may return a
 			// populated entry with `block: undefined` for a genuinely-missing block (TestTransactor does;
 			// the Network transactor always populates the key); recording there would add a phantom