@optimystic/db-p2p 0.21.0 → 0.24.0

package/src/storage/i-block-storage.ts

@@ -47,7 +47,18 @@ export interface IBlockStorage {
     /** Saves a revision */
     saveRevision(rev: number, actionId: ActionId): Promise<void>;
 
-    /** Promotes a pending action to committed */
+    /**
+     * Promotes a pending action to committed, MOVING the record from the pending namespace to the
+     * committed one in a single atomic step.
+     *
+     * **Invariant P** — a block never holds a pending record and a committed record for the same
+     * action id at the same time. This method maintains it on the commit path; every OTHER writer of
+     * a committed transform for a block ({@link saveReplica}, {@link saveDeletion}, and any forward
+     * path added later) must maintain it too, by deleting that action's pending record when it
+     * writes the committed one. A pending record left beside a committed one can never be promoted,
+     * and is reported as a phantom conflicting action by `StorageRepo.pend` on every later write to
+     * the block.
+     */
     promotePendingTransaction(actionId: ActionId): Promise<void>;
 
     /** Sets the latest revision information */
@@ -62,6 +73,11 @@ export interface IBlockStorage {
      * it falls back to `rev = 1` and a deterministic `actionId` derived from the block
      * (so retries stay idempotent — never random).
      *
+     * Maintains **Invariant P** (see {@link promotePendingTransaction}): writing the committed
+     * transform for `actionId` also deletes that action's pending record on this block, so a node
+     * that pended the action but diverged before committing it does not keep an unpromotable record.
+     * Only on the write path — the monotonic no-op below deletes nothing.
+     *
      * No-op (still durable) when an equal-or-newer revision is already present: `latest`
      * is never downgraded. Idempotent for a fixed `(rev, actionId)`. Returns the
      * effective latest `ActionRev`.
@@ -75,6 +91,9 @@ export interface IBlockStorage {
      * materialization as a deletion, so a `getBlock()` after a tombstone reads back as *absent*
      * (`undefined`) while a historical `getBlock(creationRev)` still materializes the created content.
      *
+     * Maintains **Invariant P** (see {@link promotePendingTransaction}) on the write path, exactly as
+     * {@link saveReplica} does: the tombstone's `actionId` loses its pending record on this block.
+     *
      * Idempotent for a fixed `(rev, actionId)`; never downgrades `latest` (a no-op — still durable —
      * when an equal-or-newer revision is already present). Returns the effective latest `ActionRev`.
      */

package/src/storage/i-kv-store.ts

@@ -1,8 +1,8 @@
-/** Portable async key-value store. Platform packages provide implementations. */
-export interface IKVStore {
-	get(key: string): Promise<string | undefined>;
-	set(key: string, value: string): Promise<void>;
-	delete(key: string): Promise<void>;
-	/** Return all keys matching the given prefix */
-	list(prefix: string): Promise<string[]>;
-}
+/** Portable async key-value store. Platform packages provide implementations. */
+export interface IKVStore {
+	get(key: string): Promise<string | undefined>;
+	set(key: string, value: string): Promise<void>;
+	delete(key: string): Promise<void>;
+	/** Return all keys matching the given prefix */
+	list(prefix: string): Promise<string[]>;
+}

package/src/storage/i-raw-storage.ts

@@ -39,11 +39,18 @@ export interface IRawStorage {
 	getApproximateBytesUsed?(): Promise<number>;
 
 	/**
-	 * Enumerate the block ids that currently have durable state in this backend
-	 * (one id per block that has committed/replicated metadata). Used at node
-	 * startup to seed the resilience monitors' owned-block tracked set from blocks
-	 * already on disk from a previous run, so churn-spread / rebalance protection
-	 * covers them without waiting for each to be touched again.
+	 * Enumerate the block ids that currently have durable state in this backend —
+	 * one id per block with ANY durable metadata. That includes a block whose only
+	 * durable state is an uncommitted pending transform, since
+	 * `BlockStorage.savePendingTransaction` writes metadata for a block that has none
+	 * before storing the transform. Implementations enumerate metadata keys only; they
+	 * must NOT read or decode each block's metadata to filter by committed revision
+	 * (that would turn a cheap key scan into a per-block read).
+	 *
+	 * Used at node startup to seed the resilience monitors' owned-block tracked set
+	 * from blocks already on disk from a previous run, so churn-spread / rebalance
+	 * protection covers them without waiting for each to be touched again. See the
+	 * NOTE in `seedOwnedBlocksFromStorage` for why the over-inclusion is accepted.
 	 *
 	 * Streamed (AsyncIterable) so a large store does not force the whole id list
 	 * into memory at once. Order is unspecified. Optional: a backend that omits it

package/src/storage/kv-raw-storage.ts

@@ -1,135 +1,135 @@
-import type { BlockId, ActionId, ActionRev, Transform, IBlock } from "@optimystic/db-core";
-import type { BlockMetadata } from "./struct.js";
-import type { IRawStorage } from "./i-raw-storage.js";
-import type { RawStoreDriver } from "./raw-store-driver.js";
-import { encodeJson, decodeJson, encodeActionId, decodeActionId } from "./raw-store-codec.js";
-
-/**
- * Shared ordered-KV storage kernel. Implements the full {@link IRawStorage}
- * surface over a {@link RawStoreDriver}, owning all value (de)serialization and
- * call orchestration so each backend only has to expose its five logical stores
- * as bytes-valued maps over its native mechanism.
- *
- * The genuinely-shared logic lives ABOVE the storage primitive: the JSON/string
- * codec for the four value types, `listRevisions`' lo/hi/reverse bound
- * computation, `saveMaterializedBlock`'s put-or-delete branch, and the
- * passthroughs. Key layout / storage topology stays in the driver, because the
- * backends do NOT share one — LevelDB is a single ordered byte keyspace, SQLite
- * five relational tables, IndexedDB five object stores, the filesystem a
- * directory tree.
- */
-export class KvRawStorage implements IRawStorage {
-	/**
-	 * Optional passthroughs are wired in the constructor ONLY when the driver
-	 * provides them, so a `StorageMonitor` / owned-block seed that feature-detects
-	 * (`typeof storage.listBlockIds === 'function'`) sees the driver's true
-	 * capability instead of a stub that silently reports 0 / no seed.
-	 */
-	listBlockIds?: () => AsyncIterable<BlockId>;
-	getApproximateBytesUsed?: () => Promise<number>;
-
-	constructor(private readonly driver: RawStoreDriver) {
-		if (driver.listBlockIds) {
-			this.listBlockIds = () => driver.listBlockIds!();
-		}
-		if (driver.approximateBytesUsed) {
-			this.getApproximateBytesUsed = () => driver.approximateBytesUsed!();
-		}
-	}
-
-	// --- Metadata ---
-
-	async getMetadata(blockId: BlockId): Promise<BlockMetadata | undefined> {
-		const bytes = await this.driver.getMetadata(blockId);
-		return bytes === undefined ? undefined : decodeJson<BlockMetadata>(bytes);
-	}
-
-	// NOTE: every value write funnels through the driver put/delete calls in the
-	// methods below (saveMetadata / saveRevision / save*Transaction /
-	// saveMaterializedBlock). This is the single choke point where an incremental
-	// byte counter would hook in, replacing the per-driver full-scan
-	// getApproximateBytesUsed. The chosen capacity-estimate mechanism is instead a
-	// short-TTL cache over the full scan in StorageMonitor (see storage-monitor.ts
-	// `usedBytesCacheTtlMs`) — backend-agnostic, no write-path changes, and ring
-	// selection tolerates the bounded staleness by design. The counter remains a
-	// future option only if TTL staleness ever proves insufficient; do NOT
-	// implement it here — leave the write path a single seam.
-	async saveMetadata(blockId: BlockId, metadata: BlockMetadata): Promise<void> {
-		await this.driver.putMetadata(blockId, encodeJson(metadata));
-	}
-
-	// --- Revisions ---
-
-	async getRevision(blockId: BlockId, rev: number): Promise<ActionId | undefined> {
-		const bytes = await this.driver.getRevision(blockId, rev);
-		return bytes === undefined ? undefined : decodeActionId(bytes);
-	}
-
-	async saveRevision(blockId: BlockId, rev: number, actionId: ActionId): Promise<void> {
-		await this.driver.putRevision(blockId, rev, encodeActionId(actionId));
-	}
-
-	async *listRevisions(blockId: BlockId, startRev: number, endRev: number): AsyncIterable<ActionRev> {
-		// Both bounds inclusive; ordering follows startRev→endRev direction. Empty
-		// revs are skipped by the driver's range. Preserves the exact semantics of
-		// every existing backend.
-		const ascending = startRev <= endRev;
-		const lo = ascending ? startRev : endRev;
-		const hi = ascending ? endRev : startRev;
-		for await (const [rev, bytes] of this.driver.rangeRevisions(blockId, lo, hi, !ascending)) {
-			yield { rev, actionId: decodeActionId(bytes) };
-		}
-	}
-
-	// --- Pending transactions ---
-
-	async getPendingTransaction(blockId: BlockId, actionId: ActionId): Promise<Transform | undefined> {
-		const bytes = await this.driver.getPending(blockId, actionId);
-		return bytes === undefined ? undefined : decodeJson<Transform>(bytes);
-	}
-
-	async savePendingTransaction(blockId: BlockId, actionId: ActionId, transform: Transform): Promise<void> {
-		await this.driver.putPending(blockId, actionId, encodeJson(transform));
-	}
-
-	async deletePendingTransaction(blockId: BlockId, actionId: ActionId): Promise<void> {
-		await this.driver.deletePending(blockId, actionId);
-	}
-
-	async *listPendingTransactions(blockId: BlockId): AsyncIterable<ActionId> {
-		yield* this.driver.listPendingActionIds(blockId);
-	}
-
-	// --- Committed transactions ---
-
-	async getTransaction(blockId: BlockId, actionId: ActionId): Promise<Transform | undefined> {
-		const bytes = await this.driver.getTransaction(blockId, actionId);
-		return bytes === undefined ? undefined : decodeJson<Transform>(bytes);
-	}
-
-	async saveTransaction(blockId: BlockId, actionId: ActionId, transform: Transform): Promise<void> {
-		await this.driver.putTransaction(blockId, actionId, encodeJson(transform));
-	}
-
-	// --- Materialized blocks ---
-
-	async getMaterializedBlock(blockId: BlockId, actionId: ActionId): Promise<IBlock | undefined> {
-		const bytes = await this.driver.getMaterialized(blockId, actionId);
-		return bytes === undefined ? undefined : decodeJson<IBlock>(bytes);
-	}
-
-	async saveMaterializedBlock(blockId: BlockId, actionId: ActionId, block?: IBlock): Promise<void> {
-		// Present ⇒ put the materialized block; absent ⇒ delete (tombstone / truncation).
-		if (block) {
-			await this.driver.putMaterialized(blockId, actionId, encodeJson(block));
-		} else {
-			await this.driver.deleteMaterialized(blockId, actionId);
-		}
-	}
-
-	// --- Promote (the only cross-key atomic op; driver owns atomicity) ---
-
-	async promotePendingTransaction(blockId: BlockId, actionId: ActionId): Promise<void> {
-		await this.driver.promote(blockId, actionId);
-	}
-}
+import type { BlockId, ActionId, ActionRev, Transform, IBlock } from "@optimystic/db-core";
+import type { BlockMetadata } from "./struct.js";
+import type { IRawStorage } from "./i-raw-storage.js";
+import type { RawStoreDriver } from "./raw-store-driver.js";
+import { encodeJson, decodeJson, encodeActionId, decodeActionId } from "./raw-store-codec.js";
+
+/**
+ * Shared ordered-KV storage kernel. Implements the full {@link IRawStorage}
+ * surface over a {@link RawStoreDriver}, owning all value (de)serialization and
+ * call orchestration so each backend only has to expose its five logical stores
+ * as bytes-valued maps over its native mechanism.
+ *
+ * The genuinely-shared logic lives ABOVE the storage primitive: the JSON/string
+ * codec for the four value types, `listRevisions`' lo/hi/reverse bound
+ * computation, `saveMaterializedBlock`'s put-or-delete branch, and the
+ * passthroughs. Key layout / storage topology stays in the driver, because the
+ * backends do NOT share one — LevelDB is a single ordered byte keyspace, SQLite
+ * five relational tables, IndexedDB five object stores, the filesystem a
+ * directory tree.
+ */
+export class KvRawStorage implements IRawStorage {
+	/**
+	 * Optional passthroughs are wired in the constructor ONLY when the driver
+	 * provides them, so a `StorageMonitor` / owned-block seed that feature-detects
+	 * (`typeof storage.listBlockIds === 'function'`) sees the driver's true
+	 * capability instead of a stub that silently reports 0 / no seed.
+	 */
+	listBlockIds?: () => AsyncIterable<BlockId>;
+	getApproximateBytesUsed?: () => Promise<number>;
+
+	constructor(private readonly driver: RawStoreDriver) {
+		if (driver.listBlockIds) {
+			this.listBlockIds = () => driver.listBlockIds!();
+		}
+		if (driver.approximateBytesUsed) {
+			this.getApproximateBytesUsed = () => driver.approximateBytesUsed!();
+		}
+	}
+
+	// --- Metadata ---
+
+	async getMetadata(blockId: BlockId): Promise<BlockMetadata | undefined> {
+		const bytes = await this.driver.getMetadata(blockId);
+		return bytes === undefined ? undefined : decodeJson<BlockMetadata>(bytes);
+	}
+
+	// NOTE: every value write funnels through the driver put/delete calls in the
+	// methods below (saveMetadata / saveRevision / save*Transaction /
+	// saveMaterializedBlock). This is the single choke point where an incremental
+	// byte counter would hook in, replacing the per-driver full-scan
+	// getApproximateBytesUsed. The chosen capacity-estimate mechanism is instead a
+	// short-TTL cache over the full scan in StorageMonitor (see storage-monitor.ts
+	// `usedBytesCacheTtlMs`) — backend-agnostic, no write-path changes, and ring
+	// selection tolerates the bounded staleness by design. The counter remains a
+	// future option only if TTL staleness ever proves insufficient; do NOT
+	// implement it here — leave the write path a single seam.
+	async saveMetadata(blockId: BlockId, metadata: BlockMetadata): Promise<void> {
+		await this.driver.putMetadata(blockId, encodeJson(metadata));
+	}
+
+	// --- Revisions ---
+
+	async getRevision(blockId: BlockId, rev: number): Promise<ActionId | undefined> {
+		const bytes = await this.driver.getRevision(blockId, rev);
+		return bytes === undefined ? undefined : decodeActionId(bytes);
+	}
+
+	async saveRevision(blockId: BlockId, rev: number, actionId: ActionId): Promise<void> {
+		await this.driver.putRevision(blockId, rev, encodeActionId(actionId));
+	}
+
+	async *listRevisions(blockId: BlockId, startRev: number, endRev: number): AsyncIterable<ActionRev> {
+		// Both bounds inclusive; ordering follows startRev→endRev direction. Empty
+		// revs are skipped by the driver's range. Preserves the exact semantics of
+		// every existing backend.
+		const ascending = startRev <= endRev;
+		const lo = ascending ? startRev : endRev;
+		const hi = ascending ? endRev : startRev;
+		for await (const [rev, bytes] of this.driver.rangeRevisions(blockId, lo, hi, !ascending)) {
+			yield { rev, actionId: decodeActionId(bytes) };
+		}
+	}
+
+	// --- Pending transactions ---
+
+	async getPendingTransaction(blockId: BlockId, actionId: ActionId): Promise<Transform | undefined> {
+		const bytes = await this.driver.getPending(blockId, actionId);
+		return bytes === undefined ? undefined : decodeJson<Transform>(bytes);
+	}
+
+	async savePendingTransaction(blockId: BlockId, actionId: ActionId, transform: Transform): Promise<void> {
+		await this.driver.putPending(blockId, actionId, encodeJson(transform));
+	}
+
+	async deletePendingTransaction(blockId: BlockId, actionId: ActionId): Promise<void> {
+		await this.driver.deletePending(blockId, actionId);
+	}
+
+	async *listPendingTransactions(blockId: BlockId): AsyncIterable<ActionId> {
+		yield* this.driver.listPendingActionIds(blockId);
+	}
+
+	// --- Committed transactions ---
+
+	async getTransaction(blockId: BlockId, actionId: ActionId): Promise<Transform | undefined> {
+		const bytes = await this.driver.getTransaction(blockId, actionId);
+		return bytes === undefined ? undefined : decodeJson<Transform>(bytes);
+	}
+
+	async saveTransaction(blockId: BlockId, actionId: ActionId, transform: Transform): Promise<void> {
+		await this.driver.putTransaction(blockId, actionId, encodeJson(transform));
+	}
+
+	// --- Materialized blocks ---
+
+	async getMaterializedBlock(blockId: BlockId, actionId: ActionId): Promise<IBlock | undefined> {
+		const bytes = await this.driver.getMaterialized(blockId, actionId);
+		return bytes === undefined ? undefined : decodeJson<IBlock>(bytes);
+	}
+
+	async saveMaterializedBlock(blockId: BlockId, actionId: ActionId, block?: IBlock): Promise<void> {
+		// Present ⇒ put the materialized block; absent ⇒ delete (tombstone / truncation).
+		if (block) {
+			await this.driver.putMaterialized(blockId, actionId, encodeJson(block));
+		} else {
+			await this.driver.deleteMaterialized(blockId, actionId);
+		}
+	}
+
+	// --- Promote (the only cross-key atomic op; driver owns atomicity) ---
+
+	async promotePendingTransaction(blockId: BlockId, actionId: ActionId): Promise<void> {
+		await this.driver.promote(blockId, actionId);
+	}
+}

package/src/storage/memory-kv-store.ts

@@ -1,28 +1,28 @@
-import type { IKVStore } from "./i-kv-store.js";
-
-/** In-memory IKVStore backed by a Map. Used for testing. */
-export class MemoryKVStore implements IKVStore {
-	private readonly store = new Map<string, string>();
-
-	async get(key: string): Promise<string | undefined> {
-		return this.store.get(key);
-	}
-
-	async set(key: string, value: string): Promise<void> {
-		this.store.set(key, value);
-	}
-
-	async delete(key: string): Promise<void> {
-		this.store.delete(key);
-	}
-
-	async list(prefix: string): Promise<string[]> {
-		const result: string[] = [];
-		for (const key of this.store.keys()) {
-			if (key.startsWith(prefix)) {
-				result.push(key);
-			}
-		}
-		return result;
-	}
-}
+import type { IKVStore } from "./i-kv-store.js";
+
+/** In-memory IKVStore backed by a Map. Used for testing. */
+export class MemoryKVStore implements IKVStore {
+	private readonly store = new Map<string, string>();
+
+	async get(key: string): Promise<string | undefined> {
+		return this.store.get(key);
+	}
+
+	async set(key: string, value: string): Promise<void> {
+		this.store.set(key, value);
+	}
+
+	async delete(key: string): Promise<void> {
+		this.store.delete(key);
+	}
+
+	async list(prefix: string): Promise<string[]> {
+		const result: string[] = [];
+		for (const key of this.store.keys()) {
+			if (key.startsWith(prefix)) {
+				result.push(key);
+			}
+		}
+		return result;
+	}
+}

package/src/storage/memory-storage.ts

@@ -1,25 +1,25 @@
-import { KvRawStorage } from "./kv-raw-storage.js";
-import { MemoryStoreDriver } from "./memory-store-driver.js";
-
-/**
- * In-memory {@link IRawStorage}, now a thin shell over the shared
- * {@link KvRawStorage} kernel backed by a {@link MemoryStoreDriver}.
- *
- * @pitfall The old hand-rolled version had to `structuredClone` on every get and
- * put — otherwise callers mutating a returned/stored object corrupted the store
- * (see docs/internals.md "Storage Returns References"). That discipline is GONE
- * and must not be reintroduced: values cross the driver boundary as
- * `Uint8Array` produced by `JSON`-encode and consumed by `JSON`-decode, so every
- * read decodes a fresh object and every write stored an independent byte
- * snapshot BY CONSTRUCTION. The clone-on-store / clone-on-read invariant is now
- * structural, not a rule. The conformance suite asserts it so a future driver
- * that shortcuts the byte boundary is caught.
- *
- * The name is kept stable so existing imports (`from '@optimystic/db-p2p'`)
- * continue to resolve.
- */
-export class MemoryRawStorage extends KvRawStorage {
-	constructor() {
-		super(new MemoryStoreDriver());
-	}
-}
+import { KvRawStorage } from "./kv-raw-storage.js";
+import { MemoryStoreDriver } from "./memory-store-driver.js";
+
+/**
+ * In-memory {@link IRawStorage}, now a thin shell over the shared
+ * {@link KvRawStorage} kernel backed by a {@link MemoryStoreDriver}.
+ *
+ * @pitfall The old hand-rolled version had to `structuredClone` on every get and
+ * put — otherwise callers mutating a returned/stored object corrupted the store
+ * (see docs/internals.md "Storage Returns References"). That discipline is GONE
+ * and must not be reintroduced: values cross the driver boundary as
+ * `Uint8Array` produced by `JSON`-encode and consumed by `JSON`-decode, so every
+ * read decodes a fresh object and every write stored an independent byte
+ * snapshot BY CONSTRUCTION. The clone-on-store / clone-on-read invariant is now
+ * structural, not a rule. The conformance suite asserts it so a future driver
+ * that shortcuts the byte boundary is caught.
+ *
+ * The name is kept stable so existing imports (`from '@optimystic/db-p2p'`)
+ * continue to resolve.
+ */
+export class MemoryRawStorage extends KvRawStorage {
+	constructor() {
+		super(new MemoryStoreDriver());
+	}
+}