@optimystic/db-core 0.13.4 → 0.14.0

package/src/transactor/change-notifier.ts

@@ -0,0 +1,80 @@
+import type { BlockId, CollectionId } from '../index.js';
+import type { ActionId } from '../collection/action.js';
+
+/** A commit landed on this node mutating one collection's blocks. */
+export type CollectionChangeEvent = {
+	/** Header/collection id of the affected collection (block.header.collectionId). */
+	readonly collectionId: CollectionId;
+	/** Blocks within that collection mutated by this commit. */
+	readonly blockIds: readonly BlockId[];
+	readonly actionId: ActionId;
+	readonly rev: number;
+	/**
+	 * The collection's chain tail block id at the time of this commit (the `CommitRequest.tailId`).
+	 * Anchors the rotating reactivity topic `H(tailId ‖ "reactivity")`. Present on commit-driven
+	 * events; `undefined` on read-driven promotions (the `StorageRepo.get` path has no commit tail —
+	 * those never originate anyway, and are cert-gated out downstream).
+	 */
+	readonly tailId?: BlockId;
+	/**
+	 * `true` iff this change event is a durable **invalidation** — a compensating revision + appended
+	 * `InvalidationEntry` reversing a previously-committed action proven invalid by dispute
+	 * (`docs/right-is-right.md` §Durable Invalidation), rather than an ordinary commit. An invalidation
+	 * is a committed collection change like any other, so it flows through the same notification path;
+	 * this flag lets a subscriber distinguish it (drop derived results + resubmit) from a plain commit
+	 * (refresh). A hint only — correctness never depends on it: the subscriber always re-reads the
+	 * authoritative reverted state. Absent (falsy) on ordinary commits.
+	 */
+	readonly invalidation?: boolean;
+	/**
+	 * When {@link invalidation} is set, the `actionId` of the original committed action that was
+	 * reversed (the `InvalidationEntry.invalidatedActionId`). Lets an invalidation-aware client
+	 * dedup/coalesce multiple cascade notifications by the action they reverse, and resubmit exactly
+	 * the affected work. Absent on ordinary commits.
+	 */
+	readonly invalidatedActionId?: ActionId;
+};
+
+export type CollectionChangeListener = (event: CollectionChangeEvent) => void;
+
+/**
+ * The cluster-consensus commit certificate for a committed action — the **authoritative** proof
+ * that the cohort agreed the commit. It is the cross-package currency the local change-notifier
+ * bridge forwards into the cohort-topic substrate (`CohortTopicService.onLocalCommit`) so reactivity
+ * can originate notifications **without re-signing**: a notification's signature is bit-for-bit this
+ * `thresholdSig`. The bridge and every downstream consumer treat the bytes as opaque and pass them
+ * through unchanged — only the cluster layer that produced consensus ever mints one.
+ *
+ * Peer ids are carried as their string form (the cluster keys signatures by peer-id string); db-core
+ * stays cross-platform and never references a libp2p `PeerId` type here.
+ */
+export type CommitCert = {
+	/** Threshold signature bytes proving `signers` agreed the commit. Forwarded UNCHANGED; never re-signed. */
+	readonly thresholdSig: Uint8Array;
+	/** Peer-id strings whose signatures compose {@link thresholdSig} (a distinct set of size ≥ {@link minSigs}). */
+	readonly signers: readonly string[];
+	/** Threshold the signer set satisfies (the cluster super-majority / cohort `k − x`). */
+	readonly minSigs: number;
+	/**
+	 * The exact byte preimage each `signers[i]` signed to produce its 64-byte chunk of
+	 * {@link thresholdSig} — the cluster's per-member commit-vote payload `utf8(commitHash + ":approve")`,
+	 * identical across all approving signers. Reactivity sets a notification's `digest` to
+	 * base64url(signedPayload) so a subscriber's threshold-verify over `digest` reproduces the exact
+	 * signed image. Opaque to db-core; minted only by the cluster layer.
+	 */
+	readonly signedPayload: Uint8Array;
+};
+
+export interface IBlockChangeNotifier {
+	/**
+	 * Subscribe to commits that mutate the given collection. Returns an
+	 * idempotent unsubscribe. Listeners are invoked AFTER the commit's critical
+	 * section (locks released), synchronously in commit order; a throwing
+	 * listener must not break the commit or other listeners (log + continue).
+	 */
+	onCollectionChange(collectionId: CollectionId, listener: CollectionChangeListener): () => void;
+}
+
+export function isBlockChangeNotifier(x: unknown): x is IBlockChangeNotifier {
+	return !!x && typeof (x as IBlockChangeNotifier).onCollectionChange === 'function';
+}

package/src/transactor/index.ts

@@ -1,3 +1,4 @@
+export * from "./change-notifier.js";
 export * from "./network-transactor.js";
 export * from "./transactor-source.js";
 export * from "./transactor.js";

package/src/transactor/network-transactor.ts

@@ -1,7 +1,8 @@
 import { peerIdFromString } from "../network/types.js";
 import type { PeerId } from "../network/types.js";
-import type { ActionTransforms, ActionBlocks, BlockActionStatus, ITransactor, PendSuccess, StaleFailure, IKeyNetwork, BlockId, GetBlockResults, PendResult, CommitResult, PendRequest, IRepo, BlockGets, Transforms, CommitRequest, ActionId, RepoCommitRequest, ClusterNomineesResult } from "../index.js";
-import { transformForBlockId, groupBy, concatTransforms, concatTransform, transformsFromTransform, blockIdsForTransforms } from "../index.js";
+import type { ActionTransforms, ActionBlocks, BlockActionStatus, ITransactor, PendSuccess, StaleFailure, IKeyNetwork, BlockId, GetBlockResults, PendResult, CommitResult, PendRequest, IRepo, BlockGets, Transforms, CommitRequest, ActionId, RepoCommitRequest, ClusterNomineesResult, CollectionId, IBlock } from "../index.js";
+import type { IBlockChangeNotifier, CollectionChangeListener } from "./change-notifier.js";
+import { transformForBlockId, groupBy, concatTransforms, concatTransform, transformsFromTransform, blockIdsForTransforms, Log, Tracker, CacheSource, TransactorSource } from "../index.js";
 import { blockIdToBytes } from "../utility/block-id-to-bytes.js";
 import { isRecordEmpty } from "../utility/is-record-empty.js";
 import { type CoordinatorBatch, makeBatchesByPeer, incompleteBatches, everyBatch, allBatches, mergeBlocks, processBatches, createBatchesForPayload } from "../utility/batch-coordinator.js";
@@ -14,13 +15,41 @@ type NetworkTransactorInit = {
 	abortOrCancelTimeoutMs: number;
 	keyNetwork: IKeyNetwork;
 	getRepo: (peerId: PeerId) => IRepo;
+	/**
+	 * Per-peer dial deadline in ms applied to each downstream repo call.
+	 * `timeoutMs` is the overall transaction budget; `dialTimeoutMs` caps how
+	 * long a single peer can hold that budget hostage during its dial. When a
+	 * peer is unreachable, the dial fails fast and the batch-retry loop can
+	 * re-pick a different coordinator within the remaining overall budget.
+	 * Omit to fall back to a sensible default (3s); set 0 / negative to disable.
+	 */
+	dialTimeoutMs?: number;
+	/**
+	 * Optional local change-notifier (e.g. the hosting node's StorageRepo) used to
+	 * satisfy {@link IBlockChangeNotifier}. When supplied, `onCollectionChange`
+	 * delegates to it so consumers can feature-detect change notifications on the
+	 * transactor they already hold rather than reaching into node internals. When
+	 * absent, `onCollectionChange` is a logged no-op.
+	 */
+	localChangeNotifier?: IBlockChangeNotifier;
 }
 
-export class NetworkTransactor implements ITransactor {
+/**
+ * Default per-peer dial deadline. Chosen as a compromise between:
+ *  - long enough for a typical libp2p dial+TLS handshake on a wired LAN
+ *    (sub-second) plus reasonable WAN latency, including circuit-relay hops;
+ *  - short enough that an unreachable cluster member burns ~1/10th of a
+ *    typical 30s transaction budget before the retry loop moves on.
+ */
+const DEFAULT_DIAL_TIMEOUT_MS = 3000;
+
+export class NetworkTransactor implements ITransactor, IBlockChangeNotifier {
 	private readonly keyNetwork: IKeyNetwork;
 	private readonly timeoutMs: number;
 	private readonly abortOrCancelTimeoutMs: number;
+	private readonly dialTimeoutMs: number | undefined;
 	private readonly getRepo: (peerId: PeerId) => IRepo;
+	private readonly localChangeNotifier: IBlockChangeNotifier | undefined;
 
 	constructor(
 		init: NetworkTransactorInit,
@@ -28,7 +57,27 @@ export class NetworkTransactor implements ITransactor {
 		this.keyNetwork = init.keyNetwork;
 		this.timeoutMs = init.timeoutMs;
 		this.abortOrCancelTimeoutMs = init.abortOrCancelTimeoutMs;
+		// A user explicitly passing 0 or negative means "do not bound dials separately".
+		// Undefined falls back to the library default.
+		this.dialTimeoutMs = init.dialTimeoutMs === undefined
+			? DEFAULT_DIAL_TIMEOUT_MS
+			: (init.dialTimeoutMs > 0 ? init.dialTimeoutMs : undefined);
 		this.getRepo = init.getRepo;
+		this.localChangeNotifier = init.localChangeNotifier;
+	}
+
+	/**
+	 * Subscribe to commits landing on the local node for `collectionId`, delegating
+	 * to the `localChangeNotifier` supplied at construction. When no notifier was
+	 * supplied this is a no-op (returns an inert unsubscribe) — a NetworkTransactor
+	 * with no co-located storage cannot observe commits locally.
+	 */
+	onCollectionChange(collectionId: CollectionId, listener: CollectionChangeListener): () => void {
+		if (!this.localChangeNotifier) {
+			log('onCollectionChange: no localChangeNotifier configured; subscription is a no-op for collection=%s', collectionId);
+			return () => { };
+		}
+		return this.localChangeNotifier.onCollectionChange(collectionId, listener);
 	}
 
 	async get(blockGets: BlockGets): Promise<GetBlockResults> {
@@ -50,7 +99,7 @@ export class NetworkTransactor implements ITransactor {
 		try {
 			await processBatches(
 				batches,
-				(batch) => this.getRepo(batch.peerId).get({ blockIds: batch.payload, context: blockGets.context }, { expiration }),
+				(batch) => this.getRepo(batch.peerId).get({ blockIds: batch.payload, context: blockGets.context }, { expiration, dialTimeoutMs: this.dialTimeoutMs }),
 				batch => batch.payload,
 				(gets, blockId, mergeWithGets) => [...(mergeWithGets ?? []), ...gets.filter(bid => bid === blockId)],
 				expiration,
@@ -99,7 +148,7 @@ export class NetworkTransactor implements ITransactor {
 						b.subsumedBy = [...(b.subsumedBy ?? []), ...retries];
 						await processBatches(
 							retries,
-							(batch) => this.getRepo(batch.peerId).get({ blockIds: batch.payload, context: blockGets.context }, { expiration }),
+							(batch) => this.getRepo(batch.peerId).get({ blockIds: batch.payload, context: blockGets.context }, { expiration, dialTimeoutMs: this.dialTimeoutMs }),
 							batch => batch.payload,
 							(gets, blockId, mergeWithGets) => [...(mergeWithGets ?? []), ...gets.filter(id => id === blockId)],
 							expiration,
@@ -164,7 +213,7 @@ export class NetworkTransactor implements ITransactor {
 		const blockStates = await this.get({ blockIds: allBlockIds });
 
 		// Determine status for each action ref
-		return blockActions.map(ref => ({
+		const results: BlockActionStatus[] = blockActions.map(ref => ({
 			...ref,
 			statuses: ref.blockIds.map(blockId => {
 				const result = blockStates[blockId];
@@ -178,12 +227,85 @@ export class NetworkTransactor implements ITransactor {
 				if (state.latest?.actionId === ref.actionId) {
 					return 'committed';
 				}
-				// Action is neither pending nor the latest committed - consider it aborted
-				// Note: This doesn't check historical commits; a more complete implementation
-				// would need to query the revision history
+				// Neither pending nor the latest committed. Block state alone calls this `aborted`, but a
+				// committed action that was later durably **invalidated** also presents this way — the
+				// compensating revision is now the block's latest, so the original action is no longer it.
+				// The `refineInvalidatedStatuses` pass below disambiguates from the authoritative log.
 				return 'aborted';
 			})
 		}));
+
+		// Authoritative `committed-invalidated` from durable state: consult each affected collection's
+		// log for an InvalidationEntry against the queried action (survives a node restart — the in-memory
+		// dispute map is only a fast cache; the log is the source of truth). Only `aborted` slots are
+		// ambiguous, so this is a no-op for ordinary pending/committed queries.
+		await this.refineInvalidatedStatuses(results, blockStates);
+		return results;
+	}
+
+	/**
+	 * Refine the otherwise-`aborted` statuses to `committed-invalidated` for any queried action that has a
+	 * durable {@link import("../log/struct.js").InvalidationEntry} against it. Reads the collection log via
+	 * {@link Log.findInvalidation} — the durable, restart-surviving source of truth (`docs/right-is-right.md`
+	 * §Durable Invalidation) — rather than the per-node, in-memory dispute map. A `pending` slot is left
+	 * untouched: a still-pending transaction whose base was invalidated will be rejected on its own
+	 * validation ("pending → will-be-rejected"), which is not the same as `committed-invalidated`.
+	 *
+	 * Best-effort and isolated: a log-open/read fault leaves the slot `aborted` (logged) rather than
+	 * failing the whole status query. The per-call caches keep one log open and one lookup per
+	 * `(collection, action)` even when many refs share a collection.
+	 */
+	private async refineInvalidatedStatuses(results: BlockActionStatus[], blockStates: GetBlockResults): Promise<void> {
+		const logByCollection = new Map<CollectionId, Log<unknown> | undefined>();
+		const invalidatedByKey = new Map<string, boolean>();
+
+		for (const ref of results) {
+			if (!ref.statuses.some(status => status === 'aborted')) {
+				continue; // no ambiguous slot — an invalidation could not change this ref's answer
+			}
+			const collectionId = collectionIdForRef(ref, blockStates);
+			if (collectionId === undefined) {
+				continue; // genuinely aborted: no fetched block to anchor a collection log on
+			}
+			const key = `${collectionId} ${ref.actionId}`;
+			let invalidated = invalidatedByKey.get(key);
+			if (invalidated === undefined) {
+				invalidated = await this.hasDurableInvalidation(collectionId, ref.actionId, logByCollection);
+				invalidatedByKey.set(key, invalidated);
+			}
+			if (!invalidated) {
+				continue;
+			}
+			for (let i = 0; i < ref.statuses.length; i++) {
+				if (ref.statuses[i] === 'aborted') {
+					ref.statuses[i] = 'committed-invalidated';
+				}
+			}
+		}
+	}
+
+	/** Whether `actionId` has a durable invalidation entry in `collectionId`'s log (opened once, cached). */
+	private async hasDurableInvalidation(
+		collectionId: CollectionId,
+		actionId: ActionId,
+		logByCollection: Map<CollectionId, Log<unknown> | undefined>,
+	): Promise<boolean> {
+		try {
+			let collectionLog = logByCollection.get(collectionId);
+			if (!logByCollection.has(collectionId)) {
+				const source = new TransactorSource<IBlock>(collectionId, this, undefined);
+				const tracker = new Tracker<IBlock>(new CacheSource<IBlock>(source));
+				collectionLog = await Log.open<unknown>(tracker, collectionId);
+				logByCollection.set(collectionId, collectionLog);
+			}
+			if (!collectionLog) {
+				return false;
+			}
+			return (await collectionLog.findInvalidation(actionId)) !== undefined;
+		} catch (err) {
+			log('getStatus: durable invalidation lookup failed collection=%s action=%s: %o', collectionId, actionId, err);
+			return false;
+		}
 	}
 
 	private async consolidateCoordinators(
@@ -313,6 +435,7 @@ export class NetworkTransactor implements ITransactor {
 					{ ...blockAction, transforms: batch.payload },
 					{
 						expiration,
+						dialTimeoutMs: this.dialTimeoutMs,
 						coordinatingBlockIds: batch.coordinatingBlockIds
 					} as any
 				),
@@ -387,7 +510,7 @@ export class NetworkTransactor implements ITransactor {
 		const expiration = Date.now() + this.abortOrCancelTimeoutMs;
 		await processBatches(
 			batches,
-			(batch) => this.getRepo(batch.peerId).cancel({ actionId: actionRef.actionId, blockIds: batch.payload }, { expiration }),
+			(batch) => this.getRepo(batch.peerId).cancel({ actionId: actionRef.actionId, blockIds: batch.payload }, { expiration, dialTimeoutMs: this.dialTimeoutMs }),
 			batch => batch.payload,
 			mergeBlocks,
 			expiration,
@@ -407,16 +530,20 @@ export class NetworkTransactor implements ITransactor {
 		log('commit actionId=%s rev=%d blockIds=%d', request.actionId, request.rev, request.blockIds.length);
 		const allBlockIds = [...new Set([...request.blockIds, request.tailId])];
 
-		// Commit the header block if provided and not already in blockIds
+		// Commit the header block if provided and not already in blockIds.
+		// `request.tailId` is threaded into every per-block commit so the coordinator carries it into the
+		// consensus commit op → each committing node's StorageRepo.commit stamps it onto the emitted
+		// CollectionChangeEvent (the reactivity topic anchor). Without this the per-block RepoCommitRequest
+		// drops the collection tail and reactivity origination is gated off (undefined tail → non-member).
 		if (request.headerId && !request.blockIds.includes(request.headerId)) {
-			const headerResult = await this.commitBlock(request.headerId, allBlockIds, request.actionId, request.rev);
+			const headerResult = await this.commitBlock(request.headerId, allBlockIds, request.actionId, request.rev, request.tailId);
 			if (!headerResult.success) {
 				return headerResult;
 			}
 		}
 
 		// Commit the tail block
-		const tailResult = await this.commitBlock(request.tailId, allBlockIds, request.actionId, request.rev);
+		const tailResult = await this.commitBlock(request.tailId, allBlockIds, request.actionId, request.rev, request.tailId);
 		if (!tailResult.success) {
 			return tailResult;
 		}
@@ -427,7 +554,7 @@ export class NetworkTransactor implements ITransactor {
 			!(request.headerId && bid === request.headerId && !request.blockIds.includes(request.headerId))
 		);
 		if (remainingBlocks.length > 0) {
-			const { error } = await this.commitBlocks({ blockIds: remainingBlocks, actionId: request.actionId, rev: request.rev });
+			const { error } = await this.commitBlocks({ blockIds: remainingBlocks, actionId: request.actionId, rev: request.rev, tailId: request.tailId });
 			if (error) {
 				// Non-tail block commit failures should not fail the overall action once the tail has committed.
 				// Proceed and rely on reconciliation paths (e.g. reads with context) to finalize state on lagging peers.
@@ -439,8 +566,8 @@ export class NetworkTransactor implements ITransactor {
 		return { success: true };
 	}
 
-	private async commitBlock(blockId: BlockId, blockIds: BlockId[], actionId: ActionId, rev: number): Promise<CommitResult> {
-		const { batches: tailBatches, error: tailError } = await this.commitBlocks({ blockIds: [blockId], actionId, rev });
+	private async commitBlock(blockId: BlockId, blockIds: BlockId[], actionId: ActionId, rev: number, tailId?: BlockId): Promise<CommitResult> {
+		const { batches: tailBatches, error: tailError } = await this.commitBlocks({ blockIds: [blockId], actionId, rev, tailId });
 		if (tailError) {
 			// Cancel all pending actions as background microtask
 			Promise.resolve().then(() => this.cancel({ blockIds, actionId }));
@@ -455,7 +582,7 @@ export class NetworkTransactor implements ITransactor {
 	}
 
 	/** Attempts to commit a set of blocks, and handles failures and errors */
-	private async commitBlocks({ blockIds, actionId, rev }: RepoCommitRequest) {
+	private async commitBlocks({ blockIds, actionId, rev, tailId }: RepoCommitRequest) {
 		const expiration = Date.now() + this.timeoutMs;
 		const batches = await this.batchesForPayload<BlockId[], CommitResult>(blockIds, blockIds, mergeBlocks, []);
 		log('commitBlocks actionId=%s rev=%d batches=%d', actionId, rev, batches.length);
@@ -463,7 +590,7 @@ export class NetworkTransactor implements ITransactor {
 		try {
 			await processBatches(
 				batches,
-				(batch) => this.getRepo(batch.peerId).commit({ actionId, blockIds: batch.payload, rev }, { expiration }),
+				(batch) => this.getRepo(batch.peerId).commit({ actionId, blockIds: batch.payload, rev, tailId }, { expiration, dialTimeoutMs: this.dialTimeoutMs }),
 				batch => batch.payload,
 				mergeBlocks,
 				expiration,
@@ -521,7 +648,7 @@ export class NetworkTransactor implements ITransactor {
 		);
 		await processBatches(
 			operationBatches,
-			(batch) => this.getRepo(batch.peerId).cancel({ actionId: actionRef.actionId, blockIds: batch.payload }, { expiration }),
+			(batch) => this.getRepo(batch.peerId).cancel({ actionId: actionRef.actionId, blockIds: batch.payload }, { expiration, dialTimeoutMs: this.dialTimeoutMs }),
 			batch => batch.payload,
 			mergeBlocks,
 			expiration,
@@ -544,6 +671,22 @@ export class NetworkTransactor implements ITransactor {
 }
 
 
+/**
+ * The owning collection id for an action ref, read from any fetched block's header. A
+ * committed-then-invalidated action still has a materialized (compensating) block whose header carries
+ * the collection id; a genuinely-aborted action has no fetched block, so this returns `undefined` and
+ * the status stays `aborted`.
+ */
+function collectionIdForRef(ref: ActionBlocks, blockStates: GetBlockResults): CollectionId | undefined {
+	for (const blockId of ref.blockIds) {
+		const collectionId = blockStates[blockId]?.block?.header.collectionId;
+		if (collectionId !== undefined) {
+			return collectionId;
+		}
+	}
+	return undefined;
+}
+
 /** Returns a readable message for an unknown error value. */
 function errorMessage(err: unknown): string {
 	if (err instanceof Error) return err.message;