@optimystic/db-core 0.14.1 → 0.16.3

package/src/transaction/errors.ts

@@ -0,0 +1,91 @@
+import type { CollectionId } from "../collection/index.js";
+
+/**
+ * Thrown by {@link TransactionCoordinator.commit} when a multi-collection commit
+ * fails AFTER at least one collection has already DURABLY committed through the
+ * distributed consensus path (GATHER/PEND/COMMIT).
+ *
+ * ## Why this exists (and why we can't just "roll back")
+ *
+ * The COMMIT phase commits each collection's pended blocks independently (see
+ * `commitPhase`). A per-collection commit can *permanently* fail — e.g. a racing
+ * transaction advanced that collection's log tail between PEND and COMMIT (a stale
+ * loss) — while the other collections commit successfully. Those durable commits
+ * are per-collection and there is no cross-collection undo, so a failure on one
+ * collection cannot un-commit the ones that already landed.
+ *
+ * Uniformly restoring every collection's pre-commit local state (as a clean
+ * rollback would) is exactly wrong here: for a collection that DID durably commit
+ * it would re-stage its already-durable actions as still-pending, making local
+ * tracker memory disagree with cluster storage. Instead the coordinator gives the
+ * committed collections the success-path local treatment (fold to cache + reset)
+ * and only reverts the failed/never-committed collections, then surfaces THIS error
+ * naming both sets so the caller knows reconciliation is required and does NOT
+ * falsely report a clean rollback.
+ *
+ * This is the session-mode / distributed-consensus analog of the plugin's legacy
+ * `PartialCommitError` (single-node, per-tree `sync()`).
+ *
+ * ## The design decision is settled (not "still open")
+ *
+ * The default multi-collection guarantee is formally **atomicity of intent + eventual,
+ * reported visibility**, NOT all-or-nothing — see `docs/correctness.md` **Theorem 3** and
+ * `docs/transactions.md` (§ "Session-mode (distributed) commit is not atomic across
+ * collections"). This error IS that guarantee's reporting surface, not a placeholder for a
+ * stronger one. Genuine cross-collection all-or-nothing is a future opt-in strong mode
+ * (backlog `feat-cross-collection-atomic-commit`).
+ *
+ * ## Reconcile contract for the catcher
+ *
+ * A caller receiving this error MUST NOT blindly retry the whole transaction and MUST NOT
+ * treat it as a clean abort: `committedCollections` are durable and cannot be rolled back,
+ * so a whole-transaction retry would double-apply them. Reconcile the named committed set
+ * against `failedCollections` (re-drive only the failed collections, or repair the split).
+ */
+export class CoordinatorPartialCommitError extends Error {
+	constructor(
+		/** Collections durably committed via consensus before the failure (NOT rolled back). */
+		public readonly committedCollections: readonly CollectionId[],
+		/** Collections that never committed this attempt (local state reverted for retry). */
+		public readonly failedCollections: readonly CollectionId[],
+		/** The underlying commit-phase failure that aborted the commit. */
+		public readonly reason?: unknown,
+	) {
+		super(
+			`Multi-collection commit was not atomic: ${committedCollections.length} collection(s) ` +
+			`durably committed via distributed consensus before the commit failed and CANNOT be ` +
+			`rolled back — reconciliation is required. ` +
+			`Committed (durable, now out of sync with the failed collections): [${committedCollections.join(', ')}]. ` +
+			`Failed (never committed; local state reverted for retry): [${failedCollections.join(', ')}]. ` +
+			`Underlying failure: ${reason instanceof Error ? reason.message : String(reason)}`
+		);
+		this.name = 'CoordinatorPartialCommitError';
+	}
+}
+
+/**
+ * Thrown by {@link TransactionCoordinator.commit} when a multi-collection commit failed as a
+ * CLEAN stale loss — an optimistic-concurrency conflict (a racing transaction advanced a log tail)
+ * in which NOTHING durably committed, so every participating collection's local tracker was
+ * restored to its pre-append state and the transaction is safe to re-drive.
+ *
+ * This is the retryable counterpart to {@link CoordinatorPartialCommitError}: a partial landing
+ * cannot be blindly retried (it would double-apply the durable half), but a clean loss can. The
+ * coordinator's built-in backoff+jitter retry catches this internally and re-drives after re-reading
+ * fresh revisions; it only escapes to the caller once the retry budget (`maxAttempts` / `deadlineMs`)
+ * is exhausted, at which point it signals "gave up after a clean loss" rather than a partial split.
+ */
+export class CoordinatorStaleLossError extends Error {
+	constructor(
+		/** Collections that lost the race this attempt (all had their local state reverted for retry). */
+		public readonly failedCollections: readonly CollectionId[],
+		/** The underlying stale/conflict reason surfaced by the failed pend/commit phase. */
+		public readonly reason?: string,
+	) {
+		super(
+			`Multi-collection commit failed on a clean stale loss (no collection durably committed) ` +
+			`for [${failedCollections.join(', ')}]` + (reason ? ` — ${reason}` : '')
+		);
+		this.name = 'CoordinatorStaleLossError';
+	}
+}

package/src/transaction/index.ts

@@ -1,6 +1,7 @@
 export type {
 	Transaction,
 	TransactionStamp,
+	StampFields,
 	ReadDependency,
 	TransactionRef,
 	ITransactionEngine,
@@ -8,15 +9,22 @@ export type {
 	CollectionActions,
 	ValidationResult,
 	ITransactionValidator,
-	ActionsStatement
+	ActionsStatement,
+	TransactionSigner,
+	ClientSignatureVerifier
 } from './transaction.js';
 
 export {
 	createTransactionStamp,
+	computeStampId,
 	createTransactionId,
 	createActionsStatements,
 	DEFAULT_TRANSACTION_TTL_MS,
-	isTransactionExpired
+	isTransactionExpired,
+	clientSignaturePayload,
+	CLIENT_SIG_VERSION,
+	MaxPriority,
+	clampPriority
 } from './transaction.js';
 
 export {
@@ -24,8 +32,20 @@ export {
 	ACTIONS_ENGINE_ID,
 } from './actions-engine.js';
 
+export { ReadDependencyCollector } from './read-dependency-collector.js';
 export { TransactionCoordinator } from './coordinator.js';
-export { TransactionContext } from './context.js';
+export { CoordinatorPartialCommitError, CoordinatorStaleLossError } from './errors.js';
 export { TransactionSession } from './session.js';
 export { TransactionValidator } from './validator.js';
 export type { EngineRegistration, ValidationCoordinatorFactory, BlockStateProvider } from './validator.js';
+
+export {
+	collectOperations,
+	hashOperations,
+	canonicalStringify,
+	canonicalOperationsPayload,
+	opsHashVersion,
+	OPS_HASH_VERSION,
+	OPS_HASH_PREFIX,
+} from './operations-hash.js';
+export type { Operation } from './operations-hash.js';

package/src/transaction/operations-hash.ts

@@ -0,0 +1,196 @@
+import type { BlockId, IBlock, BlockOperations } from '../blocks/structs.js';
+import type { CollectionId } from '../collection/struct.js';
+import type { Transforms } from '../transform/struct.js';
+import { hashString } from '../utility/hash-string.js';
+
+/**
+ * Represents an operation on a block within a collection.
+ *
+ * This is the SINGLE source of truth for the operation shape used by the
+ * transaction "operations hash" — the fingerprint a coordinator sends and a
+ * validator recomputes. Both {@link TransactionCoordinator} and
+ * {@link TransactionValidator} import this type; it must never be duplicated,
+ * because the two sides disagreeing is exactly the bug this module prevents.
+ */
+export type Operation =
+	| { readonly type: 'insert'; readonly collectionId: CollectionId; readonly blockId: BlockId; readonly block: IBlock }
+	| { readonly type: 'update'; readonly collectionId: CollectionId; readonly blockId: BlockId; readonly operations: BlockOperations }
+	| { readonly type: 'delete'; readonly collectionId: CollectionId; readonly blockId: BlockId };
+
+/**
+ * Rank of each operation type, used ONLY as the final tiebreaker when the same
+ * (collectionId, blockId) legitimately carries more than one operation — a block
+ * staged, then mutated, then deleted within one transform (Transforms apply order
+ * is insert → update → delete; see transform/struct.ts). This is the semantic
+ * apply order. The exact ranking is arbitrary for correctness (both sides use it),
+ * but it MUST be defined in exactly one place — here.
+ */
+const TYPE_RANK: Record<Operation['type'], number> = {
+	insert: 0,
+	update: 1,
+	delete: 2,
+};
+
+/**
+ * Collect every block operation across all collections into a flat list.
+ *
+ * The returned order reflects Map/object insertion order and is therefore NOT
+ * canonical — {@link hashOperations} sorts before hashing, so callers do not need
+ * to pre-sort. Shared by the coordinator (both commit() and execute()) and the
+ * validator so all three collect sites produce the identical logical set.
+ */
+export function collectOperations(transforms: Map<CollectionId, Transforms>): Operation[] {
+	const operations: Operation[] = [];
+	for (const [collectionId, t] of transforms) {
+		for (const [blockId, block] of Object.entries(t.inserts ?? {})) {
+			operations.push({ type: 'insert', collectionId, blockId, block });
+		}
+		for (const [blockId, ops] of Object.entries(t.updates ?? {})) {
+			operations.push({ type: 'update', collectionId, blockId, operations: ops });
+		}
+		for (const blockId of t.deletes ?? []) {
+			operations.push({ type: 'delete', collectionId, blockId });
+		}
+	}
+	return operations;
+}
+
+/**
+ * Total order over operations by the tuple (collectionId, blockId, type), using
+ * plain string comparison on the first two components and {@link TYPE_RANK} on the
+ * third. This is the cross-node ordering contract: two honest nodes that see the
+ * same logical set MUST produce the same sorted sequence regardless of the order
+ * they happened to collect operations in.
+ */
+function compareOperations(a: Operation, b: Operation): number {
+	if (a.collectionId !== b.collectionId) return a.collectionId < b.collectionId ? -1 : 1;
+	if (a.blockId !== b.blockId) return a.blockId < b.blockId ? -1 : 1;
+	return TYPE_RANK[a.type] - TYPE_RANK[b.type];
+}
+
+/**
+ * Canonical JSON encoder: recursively sorts object keys but PRESERVES array element
+ * order, and matches JSON.stringify leaf semantics.
+ *
+ * - Object keys are emitted in ascending (sorted) order, so two objects with the
+ *   same content but different key-insertion order encode identically.
+ * - Array element order is preserved because BlockOperations is an ordered list of
+ *   [entity, index, deleteCount, inserted] tuples whose order is semantically
+ *   meaningful (as are any data arrays nested inside an IBlock).
+ * - Leaf semantics mirror JSON.stringify: `undefined`/function/symbol object-values
+ *   are dropped, the same in arrays encode as `null`, `null` encodes as `null`,
+ *   non-finite numbers as `null`, and other primitives via JSON.stringify.
+ *
+ * Exported so the order-independence test can exercise it directly.
+ *
+ * NOTE: no toJSON/Date special-casing — a value carrying toJSON (e.g. a Date)
+ * encodes as its plain enumerable keys ({} for a Date), not JSON.stringify's
+ * toJSON string. Harmless today (blocks hold plain JSON) and still deterministic
+ * across nodes (both sides run this same encoder); if IBlock content ever grows
+ * toJSON-bearing values and matching JSON.stringify exactly matters, add the hook.
+ */
+export function canonicalStringify(value: unknown): string {
+	if (value === null) return 'null';
+
+	const type = typeof value;
+	if (type === 'number') return Number.isFinite(value) ? JSON.stringify(value) : 'null';
+	if (type === 'string' || type === 'boolean') return JSON.stringify(value);
+	if (type === 'bigint') throw new TypeError('Do not know how to serialize a BigInt');
+	if (type === 'undefined' || type === 'function' || type === 'symbol') return 'null';
+
+	if (Array.isArray(value)) {
+		const items = value.map(element => {
+			const et = typeof element;
+			// In arrays, undefined/function/symbol serialize as null (JSON.stringify semantics).
+			if (element === undefined || et === 'function' || et === 'symbol') return 'null';
+			return canonicalStringify(element);
+		});
+		return `[${items.join(',')}]`;
+	}
+
+	// Plain object: sort keys ascending, drop undefined/function/symbol values.
+	const obj = value as Record<string, unknown>;
+	const parts: string[] = [];
+	for (const key of Object.keys(obj).sort()) {
+		const v = obj[key];
+		const vt = typeof v;
+		if (v === undefined || vt === 'function' || vt === 'symbol') continue;
+		parts.push(`${JSON.stringify(key)}:${canonicalStringify(v)}`);
+	}
+	return `{${parts.join(',')}}`;
+}
+
+/**
+ * Current operations-hash FORMAT VERSION. Versions the *serialization* of operations
+ * into bytes (the sort key, {@link canonicalStringify} rules, and the SHA-256/base64url
+ * step) — distinct from `engineId` in the TransactionStamp, which versions the operation
+ * *content* an engine produces from the same statements. Two honest nodes must agree on
+ * BOTH dimensions to produce the same ops-hash.
+ *
+ * Bump this whenever a change to the canonical serialization alters the emitted bytes,
+ * so a peer running the old format is *detected* (a legible version-skew error) rather
+ * than mistaken for a content disagreement or a Byzantine lie. See docs/transactions.md
+ * ("Operations Hash — Canonical Serialization").
+ */
+export const OPS_HASH_VERSION = 'v1';
+
+/**
+ * Wire-token prefix carrying {@link OPS_HASH_VERSION}: `ops.v1:`. The `.` delimiter is
+ * outside the base64url alphabet (A–Z a–z 0–9 `-` `_`) that follows the trailing `:`, so
+ * the version segment can always be sliced back out unambiguously — see {@link opsHashVersion}.
+ */
+export const OPS_HASH_PREFIX = `ops.${OPS_HASH_VERSION}:`;
+
+/**
+ * Extract the format-version segment from an ops-hash token — the `v1` in `ops.v1:<hash>` —
+ * or `null` if the string is not a recognizable versioned ops-hash token.
+ *
+ * Total: never throws. A bare legacy `ops:<hash>` token (no `.` delimiter), an empty
+ * string, or any garbage all return `null`. A validator treats a `null` (or a version it
+ * does not recognize) as an unsupported/foreign format — a legible version-skew error —
+ * never as an accidental content match.
+ */
+export function opsHashVersion(token: string): string | null {
+	if (typeof token !== 'string') return null;
+	if (!token.startsWith('ops.')) return null;
+	const colon = token.indexOf(':', 4);
+	if (colon <= 4) return null; // no version characters between "ops." and ":"
+	return token.slice(4, colon);
+}
+
+/**
+ * The EXACT canonical byte-string {@link hashOperations} feeds into SHA-256: the operations
+ * sorted by {@link compareOperations} and run through {@link canonicalStringify}. The version
+ * token is NOT part of this preimage — it wraps the resulting hash, it is not hashed.
+ *
+ * Exposed so a future client signature (design-client-transaction-signatures) can bind the
+ * IDENTICAL bytes the validators hash, rather than re-deriving the serialization and risking
+ * drift. Pair it with {@link OPS_HASH_VERSION} to record which format the bytes belong to.
+ */
+export function canonicalOperationsPayload(operations: readonly Operation[]): string {
+	return canonicalStringify([...operations].sort(compareOperations));
+}
+
+/**
+ * Compute the transaction operations hash: sort into canonical order, canonically
+ * stringify, SHA-256 (base64url) via {@link hashString}, then prefix the versioned
+ * {@link OPS_HASH_PREFIX} token (`ops.v1:`) so the wire string is self-describing.
+ *
+ * This is the fingerprint the coordinator sends in PendRequest.operationsHash and
+ * the validator recomputes; equality of this string is the cross-node agreement
+ * that a transaction's operations match.
+ *
+ * NOTE: the ops-hash carries a format-version token, so a mixed-version cluster now
+ * *detects* skew — a validator seeing a foreign/legacy version emits a distinct
+ * "unsupported operations-hash format version" error instead of the ambiguous
+ * "Operations hash mismatch" (see TransactionValidator). What it does NOT do is
+ * *cross-compute* older formats: a node only recognizes that a peer speaks a different
+ * version, it cannot reproduce that peer's historical bytes. Mixed-version clusters
+ * therefore fail LEGIBLY, not interoperably; multi-version compatibility is future
+ * work if rolling upgrades ship. The ops-hash is recomputed fresh on both sides and
+ * never persisted, so bumping the version cannot retroactively invalidate committed
+ * transactions (unlike transaction.id, which IS persisted — leave it untouched).
+ */
+export async function hashOperations(operations: readonly Operation[]): Promise<string> {
+	return `${OPS_HASH_PREFIX}${await hashString(canonicalOperationsPayload(operations))}`;
+}

package/src/transaction/read-dependency-collector.ts

@@ -0,0 +1,78 @@
+import type { BlockId, ReadPurpose } from "../blocks/index.js";
+import type { ReadDependency } from "./transaction.js";
+
+/** One captured read: the highest revision observed for the block, plus its {@link ReadPurpose}. */
+type ReadEntry = { revision: number; purpose: ReadPurpose };
+
+/** Accumulates the read dependencies of one transaction. Keyed by block id. For each id it keeps:
+ *
+ *  - the HIGHEST revision observed (never downgrades — a re-read from cache must not overwrite a
+ *    higher revision seen earlier), and
+ *  - the read's PURPOSE with value-wins semantics: once a block is recorded as a `value` read from
+ *    ANY path it stays `value`; `navigation` sticks only for a block that was never read as a value.
+ *    This is the exact analogue of the max-wins rule on revision, and it makes the classification
+ *    order-independent (so it is deterministic regardless of cache/timing — required for the
+ *    transaction id and client signature to match across a re-executing validator; Theorem 4).
+ *
+ *  {@link getReadDependencies} returns only the CONFLICT set (value reads); purely-structural
+ *  `navigation` reads — interior B-tree branches walked through to reach a captured leaf — are
+ *  dropped. See {@link markValue} and docs/correctness.md Theorem 5.
+ *
+ *  One instance is shared by a collection's {@link TransactorSource} (direct structural reads —
+ *  bootstrap, header) and its {@link CacheSource} (every cache hit/miss), so a block read from
+ *  either layer produces a dependency. Because both feed the same collector, a cache miss records
+ *  the id once from each layer at the same revision/purpose; the merge collapses those to one
+ *  entry. Cleared at each txn boundary. */
+export class ReadDependencyCollector {
+	private reads = new Map<BlockId, ReadEntry>();
+
+	/** Record a read of `blockId` at `revision`. `purpose` defaults to `value` (retained); pass
+	 *  `navigation` for an interior structural read that a later {@link markValue} may or may not
+	 *  upgrade. Revision is max-wins; purpose is value-wins (see class doc). */
+	record(blockId: BlockId, revision: number, purpose: ReadPurpose = 'value'): void {
+		const prev = this.reads.get(blockId);
+		if (prev === undefined) {
+			this.reads.set(blockId, { revision, purpose });
+			return;
+		}
+		if (revision > prev.revision) {
+			prev.revision = revision;
+		}
+		// value-wins: a value read anywhere pins the block as value; navigation never downgrades it.
+		if (purpose === 'value') {
+			prev.purpose = 'value';
+		}
+	}
+
+	/** Upgrade an already-recorded read to `value` (keeping its revision), retaining it in the
+	 *  conflict set. The B-tree point-lookup path uses this to mark the terminal leaf — the
+	 *  load-bearing read whose content the result depends on — after its interior descent nodes
+	 *  were recorded as `navigation`. No-op if the id was never recorded (e.g. a leaf served from
+	 *  an uncommitted staged insert, which records no dependency at all). */
+	markValue(blockId: BlockId): void {
+		const prev = this.reads.get(blockId);
+		if (prev) {
+			prev.purpose = 'value';
+		}
+	}
+
+	/** The optimistic-concurrency conflict (read) set: every `value` read, with `navigation`
+	 *  reads excluded. Dropping a covered navigation read cannot admit a lost update — any
+	 *  concurrent change to the queried result also bumps a retained value read (the target
+	 *  leaf) — but it removes the false-positive stale rejections that structural block reads
+	 *  otherwise caused (Theorem 5 Bound). Deterministic in query shape, so a coordinator and a
+	 *  re-executing validator derive the identical set. */
+	getReadDependencies(): ReadDependency[] {
+		const result: ReadDependency[] = [];
+		for (const [blockId, entry] of this.reads) {
+			if (entry.purpose === 'value') {
+				result.push({ blockId, revision: entry.revision });
+			}
+		}
+		return result;
+	}
+
+	clear(): void {
+		this.reads.clear();
+	}
+}

package/src/transaction/session.ts

@@ -1,6 +1,7 @@
 import type { TransactionCoordinator } from "./coordinator.js";
-import type { Transaction, ExecutionResult, ITransactionEngine, TransactionStamp, CollectionActions } from "./transaction.js";
-import { createTransactionStamp, createTransactionId, isTransactionExpired } from "./transaction.js";
+import type { SyncOptions } from "../collection/index.js";
+import type { Transaction, ExecutionResult, ITransactionEngine, TransactionStamp, CollectionActions, TransactionSigner } from "./transaction.js";
+import { createTransactionStamp, createTransactionId, isTransactionExpired, clientSignaturePayload } from "./transaction.js";
 
 /**
  * TransactionSession manages incremental transaction building.
@@ -30,7 +31,8 @@ export class TransactionSession {
 	private constructor(
 		private readonly coordinator: TransactionCoordinator,
 		private readonly engine: ITransactionEngine,
-		stamp: TransactionStamp
+		stamp: TransactionStamp,
+		private readonly signer?: TransactionSigner
 	) {
 		this.stamp = stamp;
 	}
@@ -38,22 +40,27 @@ export class TransactionSession {
 	/**
 	 * Create a new TransactionSession.
 	 * Uses async factory because stamp creation requires SHA-256 hashing.
+	 *
+	 * @param signer - Optional client signer. When supplied, commit() stamps the built
+	 *   transaction's `signature` over its canonical payload; when omitted the
+	 *   transaction is left unsigned (accepted only by nodes with no verifier wired).
 	 */
 	static async create(
 		coordinator: TransactionCoordinator,
 		engine: ITransactionEngine,
 		peerId: string = 'local',
 		schemaHash: string = '',
-		ttlMs?: number
+		ttlMs?: number,
+		signer?: TransactionSigner
 	): Promise<TransactionSession> {
 		const stamp = await createTransactionStamp(
 			peerId,
 			Date.now(),
 			schemaHash,
-			'unknown', // TODO: Get engine ID from engine
+			engine.id,
 			ttlMs
 		);
-		return new TransactionSession(coordinator, engine, stamp);
+		return new TransactionSession(coordinator, engine, stamp, signer);
 	}
 
 	/**
@@ -113,8 +120,11 @@ export class TransactionSession {
 	 * Commit the transaction.
 	 *
 	 * Compiles all statements into a complete Transaction and commits through coordinator.
+	 *
+	 * @param options - Optional retry knobs forwarded to {@link TransactionCoordinator.commit}
+	 *   (bounded backoff+jitter retry of a clean stale loss). Defaults are safe out of the box.
 	 */
-	async commit(): Promise<ExecutionResult> {
+	async commit(options?: SyncOptions): Promise<ExecutionResult> {
 		if (this.committed) {
 			return { success: false, error: 'Transaction already committed' };
 		}
@@ -136,8 +146,26 @@ export class TransactionSession {
 			id: await createTransactionId(this.stamp.id, this.statements, reads)
 		};
 
-		// Commit through coordinator (which will orchestrate PEND/COMMIT)
-		await this.coordinator.commit(transaction);
+		// Sign the canonical client-signature payload if a signer was supplied. The
+		// payload covers stamp.id + statements + reads, which are all final now. A signer failure
+		// (e.g. a wrong-key-type libp2p sign) is caught and surfaced as a clean commit failure BEFORE
+		// coordinator.commit runs — the transaction object is local and discarded, so a failed sign can
+		// never leak a partially-built unsigned transaction into consensus.
+		if (this.signer) {
+			const payload = clientSignaturePayload(this.stamp.id, this.statements, reads);
+			try {
+				transaction.signature = await this.signer(payload);
+			} catch (error) {
+				return {
+					success: false,
+					error: `Failed to sign transaction: ${error instanceof Error ? error.message : String(error)}`
+				};
+			}
+		}
+
+		// Commit through coordinator (which will orchestrate PEND/COMMIT, with bounded
+		// backoff+jitter retry of a clean stale loss)
+		await this.coordinator.commit(transaction, options);
 
 		// Clear read dependencies after successful commit
 		this.coordinator.clearReadDependencies();