@optimystic/db-p2p 0.11.2 → 0.11.3

package/dist/src/testing/mesh-harness.js

@@ -0,0 +1,220 @@
+import { NetworkTransactor } from '@optimystic/db-core';
+import { peerIdFromPrivateKey } from '@libp2p/peer-id';
+import { generateKeyPair } from '@libp2p/crypto/keys';
+import { clusterMember } from '../cluster/cluster-repo.js';
+import { StorageRepo } from '../storage/storage-repo.js';
+import { MemoryRawStorage } from '../storage/memory-storage.js';
+import { BlockStorage } from '../storage/block-storage.js';
+import { coordinatorRepo } from '../repo/coordinator-repo.js';
+import { sortPeersByDistance } from '../routing/responsibility.js';
+import { toString as u8ToString } from 'uint8arrays';
+class MockPeerNetwork {
+    async connect(_peerId, _protocol) {
+        return {};
+    }
+}
+/**
+ * Mock IKeyNetwork that returns peers based on XOR distance.
+ * With responsibilityK >= nodeCount, all nodes are returned.
+ * Otherwise, K-nearest by XOR distance are returned.
+ */
+class MockMeshKeyNetwork {
+    nodes;
+    responsibilityK;
+    failures;
+    constructor(nodes, responsibilityK, failures = {}) {
+        this.nodes = nodes;
+        this.responsibilityK = responsibilityK;
+        this.failures = failures;
+    }
+    async findCoordinator(key, options) {
+        const excluded = new Set((options?.excludedPeers ?? []).map(p => p.toString()));
+        const sorted = this.sortedByDistance(key);
+        const pick = sorted.find(n => !excluded.has(n.peerId.toString()));
+        if (!pick) {
+            throw new Error('No coordinator available for key (all candidates excluded)');
+        }
+        return pick.peerId;
+    }
+    async findCluster(key) {
+        if (this.failures.findClusterFails) {
+            return {};
+        }
+        const sorted = this.sortedByDistance(key);
+        const k = Math.min(this.responsibilityK, sorted.length);
+        const selected = sorted.slice(0, k);
+        const peers = {};
+        for (const node of selected) {
+            peers[node.peerId.toString()] = {
+                multiaddrs: ['/ip4/127.0.0.1/tcp/8000'],
+                publicKey: u8ToString(node.peerId.publicKey.raw, 'base64url')
+            };
+        }
+        return peers;
+    }
+    sortedByDistance(key) {
+        const knownPeers = this.nodes.map(n => ({
+            id: n.peerId,
+            addrs: ['/ip4/127.0.0.1/tcp/8000']
+        }));
+        const sorted = sortPeersByDistance(knownPeers, key);
+        return sorted.map(kp => this.nodes.find(n => n.peerId.equals(kp.id)));
+    }
+}
+/**
+ * Creates N interconnected mesh nodes with real components and mock transport.
+ * ClusterClient calls route directly to target ClusterMember instances.
+ */
+export async function createMesh(nodeCount, options) {
+    const failures = {};
+    // Generate key pairs for all nodes
+    const keyPairs = await Promise.all(Array.from({ length: nodeCount }, async () => {
+        const privateKey = await generateKeyPair('Ed25519');
+        return { peerId: peerIdFromPrivateKey(privateKey), privateKey };
+    }));
+    // Build nodes array (partially — coordinatorRepo added after keyNetwork is ready)
+    const nodes = [];
+    const peerNetwork = new MockPeerNetwork();
+    // Map peerId → rawStorage for data sync simulation in clusterLatestCallback
+    const rawStorages = new Map();
+    // Phase 1: create storage + cluster members
+    let nodeIndex = 0;
+    for (const { peerId, privateKey } of keyPairs) {
+        const rawStorage = options.rawStorageFactory
+            ? options.rawStorageFactory(nodeIndex)
+            : new MemoryRawStorage();
+        rawStorages.set(peerId.toString(), rawStorage);
+        nodeIndex++;
+        const storageRepo = new StorageRepo((blockId) => new BlockStorage(blockId, rawStorage));
+        const consensusConfig = {
+            superMajorityThreshold: options.superMajorityThreshold ?? 0.75,
+            simpleMajorityThreshold: 0.51,
+            minAbsoluteClusterSize: 2,
+            allowClusterDownsize: options.allowClusterDownsize ?? true,
+            clusterSizeTolerance: 0.5,
+            partitionDetectionWindow: 60000
+        };
+        const member = clusterMember({
+            storageRepo,
+            peerNetwork,
+            peerId,
+            privateKey,
+            consensusConfig
+        });
+        nodes.push({
+            peerId,
+            privateKey,
+            storageRepo,
+            clusterMember: member,
+            coordinatorRepo: undefined // filled in phase 2
+        });
+    }
+    // Phase 2: create key network and coordinator repos (needs all nodes for routing)
+    const keyNetwork = new MockMeshKeyNetwork(nodes, options.responsibilityK, failures);
+    const createClusterClient = (targetPeerId) => {
+        const target = nodes.find(n => n.peerId.equals(targetPeerId));
+        if (!target) {
+            throw new Error(`Unknown peer: ${targetPeerId.toString()}`);
+        }
+        return {
+            async update(record) {
+                if (failures.failingPeers?.has(targetPeerId.toString())) {
+                    throw new Error(`Peer ${targetPeerId.toString()} is unreachable`);
+                }
+                return target.clusterMember.update(record);
+            }
+        };
+    };
+    for (const node of nodes) {
+        const localRawStorage = rawStorages.get(node.peerId.toString());
+        // Per-node callback: queries remote peer and replicates committed data locally
+        // (simulates what SyncClient does in production)
+        const clusterLatestCallback = async (peerId, blockId, context) => {
+            const target = nodes.find(n => n.peerId.equals(peerId));
+            if (!target)
+                return undefined;
+            const result = await target.storageRepo.get({ blockIds: [blockId], context }, { skipClusterFetch: true });
+            const entry = result[blockId];
+            const latest = entry?.state?.latest;
+            // Simulate data sync: replicate committed block data to local storage
+            if (latest && entry?.block) {
+                const localBlockStorage = new BlockStorage(blockId, localRawStorage);
+                const localLatest = await localBlockStorage.getLatest();
+                if (!localLatest || localLatest.rev < latest.rev) {
+                    // Ensure metadata exists
+                    const meta = await localRawStorage.getMetadata(blockId);
+                    if (!meta) {
+                        await localRawStorage.saveMetadata(blockId, { latest: undefined, ranges: [[0]] });
+                    }
+                    await localBlockStorage.saveMaterializedBlock(latest.actionId, entry.block);
+                    await localBlockStorage.saveRevision(latest.rev, latest.actionId);
+                    await localBlockStorage.setLatest(latest);
+                }
+            }
+            return latest;
+        };
+        // Wrap key network to include self in findCluster (matches real Libp2pKeyPeerNetwork behavior)
+        const nodeKeyNetwork = {
+            findCoordinator: (key, opts) => keyNetwork.findCoordinator(key, opts),
+            async findCluster(key) {
+                const peers = await keyNetwork.findCluster(key);
+                const selfStr = node.peerId.toString();
+                if (!(selfStr in peers)) {
+                    peers[selfStr] = {
+                        multiaddrs: ['/ip4/127.0.0.1/tcp/8000'],
+                        publicKey: u8ToString(node.peerId.publicKey.raw, 'base64url')
+                    };
+                }
+                return peers;
+            }
+        };
+        const factory = coordinatorRepo(nodeKeyNetwork, (peerId) => createClusterClient(peerId), {
+            clusterSize: options.clusterSize ?? nodeCount,
+            superMajorityThreshold: options.superMajorityThreshold ?? 0.75,
+            allowClusterDownsize: options.allowClusterDownsize ?? true
+        });
+        node.coordinatorRepo = factory({
+            storageRepo: node.storageRepo,
+            localCluster: node.clusterMember,
+            localPeerId: node.peerId,
+            clusterLatestCallback
+        });
+    }
+    return { nodes, failures, keyNetwork };
+}
+/**
+ * Builds a NetworkTransactor over a mesh. All nodes share the same mock
+ * infrastructure so a single transactor routes to every peer via `getRepo`.
+ * Suitable for solo-mesh tests; for multi-node tests prefer
+ * `buildNetworkTransactors` to label "which node is driving".
+ */
+export const buildNetworkTransactor = (mesh, options = {}) => {
+    const repoByPeer = new Map();
+    for (const node of mesh.nodes) {
+        repoByPeer.set(node.peerId.toString(), node.coordinatorRepo);
+    }
+    return new NetworkTransactor({
+        timeoutMs: options.timeoutMs ?? 5_000,
+        abortOrCancelTimeoutMs: options.abortOrCancelTimeoutMs ?? 5_000,
+        keyNetwork: mesh.keyNetwork,
+        getRepo: (peerId) => {
+            const repo = repoByPeer.get(peerId.toString());
+            if (!repo)
+                throw new Error(`Unknown peer ${peerId.toString()}`);
+            return repo;
+        }
+    });
+};
+/**
+ * Builds one NetworkTransactor per mesh node, keyed by peer-id string. Each
+ * transactor shares the mesh's key network and peer→repo map — the separate
+ * instances exist so tests can semantically say "driven by node A".
+ */
+export const buildNetworkTransactors = (mesh, options = {}) => {
+    const transactors = new Map();
+    for (const node of mesh.nodes) {
+        transactors.set(node.peerId.toString(), buildNetworkTransactor(mesh, options));
+    }
+    return transactors;
+};
+//# sourceMappingURL=mesh-harness.js.map

package/dist/src/testing/mesh-harness.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"mesh-harness.js","sourceRoot":"","sources":["../../../src/testing/mesh-harness.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AACxD,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AACvD,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACtD,OAAO,EAAiB,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAC1E,OAAO,EAAE,WAAW,EAAE,MAAM,4BAA4B,CAAC;AACzD,OAAO,EAAE,gBAAgB,EAAE,MAAM,8BAA8B,CAAC;AAChE,OAAO,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAC;AAE3D,OAAO,EAAE,eAAe,EAA8B,MAAM,6BAA6B,CAAC;AAE1F,OAAO,EAAE,mBAAmB,EAAkB,MAAM,8BAA8B,CAAC;AACnF,OAAO,EAAE,QAAQ,IAAI,UAAU,EAAE,MAAM,aAAa,CAAC;AA+BrD,MAAM,eAAe;IACpB,KAAK,CAAC,OAAO,CAAC,OAAe,EAAE,SAAiB;QAC/C,OAAO,EAAE,CAAC;IACX,CAAC;CACD;AAED;;;;GAIG;AACH,MAAM,kBAAkB;IAEL;IACA;IACA;IAHlB,YACkB,KAAiB,EACjB,eAAuB,EACvB,WAA8B,EAAE;QAFhC,UAAK,GAAL,KAAK,CAAY;QACjB,oBAAe,GAAf,eAAe,CAAQ;QACvB,aAAQ,GAAR,QAAQ,CAAwB;IAC/C,CAAC;IAEJ,KAAK,CAAC,eAAe,CAAI,GAAe,EAAE,OAAyC;QAClF,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,CAAC,OAAO,EAAE,aAAa,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;QAChF,MAAM,MAAM,GAAG,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;QAC1C,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;QAClE,IAAI,CAAC,IAAI,EAAE,CAAC;YACX,MAAM,IAAI,KAAK,CAAC,4DAA4D,CAAC,CAAC;QAC/E,CAAC;QACD,OAAO,IAAI,CAAC,MAAM,CAAC;IACpB,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,GAAe;QAChC,IAAI,IAAI,CAAC,QAAQ,CAAC,gBAAgB,EAAE,CAAC;YACpC,OAAO,EAAkB,CAAC;QAC3B,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;QAC1C,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,eAAe,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;QACxD,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAEpC,MAAM,KAAK,GAAiB,EAAE,CAAC;QAC/B,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;YAC7B,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,GAAG;gBAC/B,UAAU,EAAE,CAAC,yBAAyB,CAAC;gBACvC,SAAS,EAAE,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,SAAU,CAAC,GAAG,EAAE,WAAW,CAAC;aAC9D,CAAC;QACH,CAAC;QACD,OAAO,KAAK,CAAC;IACd,CAAC;IAEO,gBAAgB,CAAC,GAAe;QACvC,MAAM,UAAU,GAAgB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACpD,EAAE,EAAE,CAAC,CAAC,MAAM;YACZ,KAAK,EAAE,CAAC,yBAAyB,CAAC;SAClC,CAAC,CAAC,CAAC;QACJ,MAAM,MAAM,GAAG,mBAAmB,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC;QACpD,OAAO,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC,CAAE,CAAC,CAAC;IACxE,CAAC;CACD;AAQD;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,SAAiB,EAAE,OAAoB;IACvE,MAAM,QAAQ,GAAsB,EAAE,CAAC;IAEvC,mCAAmC;IACnC,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,GAAG,CACjC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,EAAE,KAAK,IAAI,EAAE;QAC5C,MAAM,UAAU,GAAG,MAAM,eAAe,CAAC,SAAS,CAAC,CAAC;QACpD,OAAO,EAAE,MAAM,EAAE,oBAAoB,CAAC,UAAU,CAAC,EAAE,UAAU,EAAE,CAAC;IACjE,CAAC,CAAC,CACF,CAAC;IAEF,kFAAkF;IAClF,MAAM,KAAK,GAAe,EAAE,CAAC;IAC7B,MAAM,WAAW,GAAG,IAAI,eAAe,EAAE,CAAC;IAC1C,4EAA4E;IAC5E,MAAM,WAAW,GAAG,IAAI,GAAG,EAAuB,CAAC;IAEnD,4CAA4C;IAC5C,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,KAAK,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,IAAI,QAAQ,EAAE,CAAC;QAC/C,MAAM,UAAU,GAAG,OAAO,CAAC,iBAAiB;YAC3C,CAAC,CAAC,OAAO,CAAC,iBAAiB,CAAC,SAAS,CAAC;YACtC,CAAC,CAAC,IAAI,gBAAgB,EAAE,CAAC;QAC1B,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,EAAE,EAAE,UAAU,CAAC,CAAC;QAC/C,SAAS,EAAE,CAAC;QACZ,MAAM,WAAW,GAAG,IAAI,WAAW,CAClC,CAAC,OAAgB,EAAE,EAAE,CAAC,IAAI,YAAY,CAAC,OAAO,EAAE,UAAU,CAAC,CAC3D,CAAC;QAEF,MAAM,eAAe,GAA2B;YAC/C,sBAAsB,EAAE,OAAO,CAAC,sBAAsB,IAAI,IAAI;YAC9D,uBAAuB,EAAE,IAAI;YAC7B,sBAAsB,EAAE,CAAC;YACzB,oBAAoB,EAAE,OAAO,CAAC,oBAAoB,IAAI,IAAI;YAC1D,oBAAoB,EAAE,GAAG;YACzB,wBAAwB,EAAE,KAAK;SAC/B,CAAC;QAEF,MAAM,MAAM,GAAG,aAAa,CAAC;YAC5B,WAAW;YACX,WAAW;YACX,MAAM;YACN,UAAU;YACV,eAAe;SACf,CAAC,CAAC;QAEH,KAAK,CAAC,IAAI,CAAC;YACV,MAAM;YACN,UAAU;YACV,WAAW;YACX,aAAa,EAAE,MAAM;YACrB,eAAe,EAAE,SAAgB,CAAC,oBAAoB;SACtD,CAAC,CAAC;IACJ,CAAC;IAED,kFAAkF;IAClF,MAAM,UAAU,GAAG,IAAI,kBAAkB,CAAC,KAAK,EAAE,OAAO,CAAC,eAAe,EAAE,QAAQ,CAAC,CAAC;IAEpF,MAAM,mBAAmB,GAAG,CAAC,YAAoB,EAAY,EAAE;QAC9D,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC;QAC9D,IAAI,CAAC,MAAM,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CAAC,iBAAiB,YAAY,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;QAC7D,CAAC;QACD,OAAO;YACN,KAAK,CAAC,MAAM,CAAC,MAAqB;gBACjC,IAAI,QAAQ,CAAC,YAAY,EAAE,GAAG,CAAC,YAAY,CAAC,QAAQ,EAAE,CAAC,EAAE,CAAC;oBACzD,MAAM,IAAI,KAAK,CAAC,QAAQ,YAAY,CAAC,QAAQ,EAAE,iBAAiB,CAAC,CAAC;gBACnE,CAAC;gBACD,OAAO,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YAC5C,CAAC;SACD,CAAC;IACH,CAAC,CAAC;IAEF,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QAC1B,MAAM,eAAe,GAAG,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAE,CAAC;QAEjE,+EAA+E;QAC/E,iDAAiD;QACjD,MAAM,qBAAqB,GAA0B,KAAK,EAAE,MAAc,EAAE,OAAgB,EAAE,OAAQ,EAAkC,EAAE;YACzI,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;YACxD,IAAI,CAAC,MAAM;gBAAE,OAAO,SAAS,CAAC;YAC9B,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,WAAW,CAAC,GAAG,CAC1C,EAAE,QAAQ,EAAE,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,EAChC,EAAE,gBAAgB,EAAE,IAAI,EAAS,CACjC,CAAC;YACF,MAAM,KAAK,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC;YAC9B,MAAM,MAAM,GAAG,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC;YAEpC,sEAAsE;YACtE,IAAI,MAAM,IAAI,KAAK,EAAE,KAAK,EAAE,CAAC;gBAC5B,MAAM,iBAAiB,GAAG,IAAI,YAAY,CAAC,OAAkB,EAAE,eAAe,CAAC,CAAC;gBAChF,MAAM,WAAW,GAAG,MAAM,iBAAiB,CAAC,SAAS,EAAE,CAAC;gBACxD,IAAI,CAAC,WAAW,IAAI,WAAW,CAAC,GAAG,GAAG,MAAM,CAAC,GAAG,EAAE,CAAC;oBAClD,yBAAyB;oBACzB,MAAM,IAAI,GAAG,MAAM,eAAe,CAAC,WAAW,CAAC,OAAkB,CAAC,CAAC;oBACnE,IAAI,CAAC,IAAI,EAAE,CAAC;wBACX,MAAM,eAAe,CAAC,YAAY,CAAC,OAAkB,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;oBAC9F,CAAC;oBACD,MAAM,iBAAiB,CAAC,qBAAqB,CAAC,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC;oBAC5E,MAAM,iBAAiB,CAAC,YAAY,CAAC,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC;oBAClE,MAAM,iBAAiB,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;gBAC3C,CAAC;YACF,CAAC;YAED,OAAO,MAAM,CAAC;QACf,CAAC,CAAC;QACF,+FAA+F;QAC/F,MAAM,cAAc,GAAgB;YACnC,eAAe,EAAE,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,UAAU,CAAC,eAAe,CAAC,GAAG,EAAE,IAAI,CAAC;YACrE,KAAK,CAAC,WAAW,CAAC,GAAG;gBACpB,MAAM,KAAK,GAAG,MAAM,UAAU,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;gBAChD,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC;gBACvC,IAAI,CAAC,CAAC,OAAO,IAAI,KAAK,CAAC,EAAE,CAAC;oBACzB,KAAK,CAAC,OAAO,CAAC,GAAG;wBAChB,UAAU,EAAE,CAAC,yBAAyB,CAAC;wBACvC,SAAS,EAAE,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,SAAU,CAAC,GAAG,EAAE,WAAW,CAAC;qBAC9D,CAAC;gBACH,CAAC;gBACD,OAAO,KAAK,CAAC;YACd,CAAC;SACD,CAAC;QACF,MAAM,OAAO,GAAG,eAAe,CAC9B,cAAc,EACd,CAAC,MAAc,EAAE,EAAE,CAAC,mBAAmB,CAAC,MAAM,CAAQ,EACtD;YACC,WAAW,EAAE,OAAO,CAAC,WAAW,IAAI,SAAS;YAC7C,sBAAsB,EAAE,OAAO,CAAC,sBAAsB,IAAI,IAAI;YAC9D,oBAAoB,EAAE,OAAO,CAAC,oBAAoB,IAAI,IAAI;SAC1D,CACD,CAAC;QACF,IAAI,CAAC,eAAe,GAAG,OAAO,CAAC;YAC9B,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,YAAY,EAAE,IAAI,CAAC,aAAa;YAChC,WAAW,EAAE,IAAI,CAAC,MAAM;YACxB,qBAAqB;SACrB,CAAC,CAAC;IACJ,CAAC;IAED,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,UAAU,EAAE,CAAC;AACxC,CAAC;AAOD;;;;;GAKG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC,IAAU,EAAE,UAAkC,EAAE,EAAe,EAAE;IACvG,MAAM,UAAU,GAAG,IAAI,GAAG,EAAiB,CAAC;IAC5C,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;QAC/B,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,EAAE,IAAI,CAAC,eAAmC,CAAC,CAAC;IAClF,CAAC;IACD,OAAO,IAAI,iBAAiB,CAAC;QAC5B,SAAS,EAAE,OAAO,CAAC,SAAS,IAAI,KAAK;QACrC,sBAAsB,EAAE,OAAO,CAAC,sBAAsB,IAAI,KAAK;QAC/D,UAAU,EAAE,IAAI,CAAC,UAAU;QAC3B,OAAO,EAAE,CAAC,MAAgB,EAAE,EAAE;YAC7B,MAAM,IAAI,GAAG,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;YAC/C,IAAI,CAAC,IAAI;gBAAE,MAAM,IAAI,KAAK,CAAC,gBAAgB,MAAM,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;YAChE,OAAO,IAAI,CAAC;QACb,CAAC;KACD,CAAC,CAAC;AACJ,CAAC,CAAC;AAEF;;;;GAIG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAC,IAAU,EAAE,UAAkC,EAAE,EAA4B,EAAE;IACrH,MAAM,WAAW,GAAG,IAAI,GAAG,EAAuB,CAAC;IACnD,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;QAC/B,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,EAAE,sBAAsB,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC;IAChF,CAAC;IACD,OAAO,WAAW,CAAC;AACpB,CAAC,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@optimystic/db-p2p",
-  "version": "0.11.2",
+  "version": "0.11.3",
   "type": "module",
   "description": "P2P database functionality for Optimystic",
   "main": "dist/src/index.js",
@@ -23,6 +23,10 @@
     "./rn": {
       "types": "./dist/src/rn.d.ts",
       "import": "./dist/src/rn.js"
+    },
+    "./testing": {
+      "types": "./dist/src/testing/index.d.ts",
+      "import": "./dist/src/testing/index.js"
     }
   },
   "repository": {
@@ -46,7 +50,8 @@
     "clean": "rimraf dist",
     "build": "tsc",
     "test": "node --import ./register.mjs node_modules/mocha/bin/mocha.js \"test/**/*.spec.ts\" --colors --reporter min",
-    "test:verbose": "node --import ./register.mjs node_modules/mocha/bin/mocha.js \"test/**/*.spec.ts\" --colors --reporter spec"
+    "test:verbose": "node --import ./register.mjs node_modules/mocha/bin/mocha.js \"test/**/*.spec.ts\" --colors --reporter spec",
+    "test:integration": "OPTIMYSTIC_INTEGRATION=1 node --import ./register.mjs node_modules/mocha/bin/mocha.js \"test/**/*.integration.spec.ts\" --colors --reporter spec"
   },
   "devDependencies": {
     "@types/chai": "^5.2.3",
@@ -74,7 +79,7 @@
     "@libp2p/tcp": "^11.0.10",
     "@libp2p/websockets": "^10.1.3",
     "@multiformats/multiaddr": "^13.0.1",
-    "@optimystic/db-core": "^0.11.2",
+    "@optimystic/db-core": "^0.11.3",
     "async-mutex": "^0.5.0",
     "debug": "^4.4.3",
     "it-all": "^3.0.9",

package/src/repo/coordinator-repo.ts

@@ -1,5 +1,5 @@
-import type { PendRequest, ActionBlocks, IRepo, MessageOptions, CommitResult, GetBlockResults, PendResult, BlockGets, CommitRequest, RepoMessage, IKeyNetwork, ICluster, ClusterConsensusConfig, BlockId, ActionRev, ActionContext } from "@optimystic/db-core";
-import { LruMap } from "@optimystic/db-core";
+import type { PendRequest, ActionBlocks, IRepo, MessageOptions, CommitResult, GetBlockResults, PendResult, BlockGets, CommitRequest, RepoMessage, IKeyNetwork, ICluster, ClusterConsensusConfig, BlockId, ActionRev, ActionContext, ClusterRecord } from "@optimystic/db-core";
+import { LruMap, blockIdsForTransforms } from "@optimystic/db-core";
 import { ClusterCoordinator } from "./cluster-coordinator.js";
 import type { ClusterClient } from "../cluster/client.js";
 import type { PeerId } from "@libp2p/interface";
@@ -237,7 +237,7 @@ export class CoordinatorRepo implements IRepo {
 	}
 
 	async pend(request: PendRequest, options?: MessageOptions): Promise<PendResult> {
-		const allBlockIds = Object.keys(request.transforms);
+		const allBlockIds = blockIdsForTransforms(request.transforms);
 		await this.verifyResponsibility(allBlockIds);
 		const coordinatingBlockIds = (options as any)?.coordinatingBlockIds ?? allBlockIds;
 
@@ -273,7 +273,7 @@ export class CoordinatorRepo implements IRepo {
 			return {
 				success: true,
 				pending: [],
-				blockIds: Object.keys(request.transforms)
+				blockIds: allBlockIds
 			};
 		} catch (error) {
 			log('coordinator-repo:pend-error', { actionId: request.actionId, error: (error as Error).message });
@@ -326,16 +326,38 @@ export class CoordinatorRepo implements IRepo {
 		};
 
 		try {
-			const { localExecuted } = await this.coordinator.executeClusterTransaction(blockIds[0]!, message, options);
-			// Only call storageRepo if local cluster didn't already execute during consensus
-			if (!localExecuted) {
+			const { record, localExecuted } = await this.coordinator.executeClusterTransaction(blockIds[0]!, message, options);
+			if (localExecuted) {
+				return { success: true };
+			}
+			// Local cluster didn't execute during consensus. Attempt a local commit,
+			// but tolerate failure (e.g., "pending action not found") when the cluster
+			// already reached consensus — this coordinator was likely picked for commit
+			// after missing the pend phase (unreachable during pend, fresh join, etc.).
+			// The cluster's majority is authoritative; this peer will catch up via sync.
+			try {
 				return await this.storageRepo.commit(request, options);
+			} catch (err) {
+				if (clusterReachedCommitConsensus(record)) {
+					log('coordinator-repo:commit-local-failed-cluster-succeeded', {
+						actionId: request.actionId,
+						error: (err as Error).message
+					});
+					return { success: true };
+				}
+				throw err;
 			}
-			// Local cluster already executed - return success
-			return { success: true };
 		} catch (error) {
 			log('coordinator-repo:commit-error', { actionId: request.actionId, error: (error as Error).message });
 			throw error;
 		}
 	}
 }
+
+/** True if a simple majority of cluster peers signed an approving commit. */
+function clusterReachedCommitConsensus(record: ClusterRecord): boolean {
+	const peerCount = Object.keys(record.peers).length;
+	if (peerCount === 0) return false;
+	const approvedCommits = Object.values(record.commits).filter(s => s.type === 'approve').length;
+	return approvedCommits > peerCount / 2;
+}

package/src/storage/block-storage.ts

@@ -91,6 +91,38 @@ export class BlockStorage implements IBlockStorage {
 		await this.storage.saveMetadata(this.blockId, meta);
 	}
 
+	async recover(): Promise<{ reconciled: boolean; latest?: ActionRev }> {
+		const meta = await this.storage.getMetadata(this.blockId);
+		if (!meta) {
+			return { reconciled: false };
+		}
+
+		const currentRev = meta.latest?.rev ?? 0;
+		let maxRev = currentRev;
+		let maxActionId = meta.latest?.actionId;
+
+		// Probe forward until we hit a gap or a revision whose action is not yet
+		// in the committed log (Crash-D2 state — retry-commit owns that advance).
+		for (let next = currentRev + 1; ; next++) {
+			const actionId = await this.storage.getRevision(this.blockId, next);
+			if (actionId === undefined) break;
+			const promoted = await this.storage.getTransaction(this.blockId, actionId);
+			if (promoted === undefined) break;
+			maxRev = next;
+			maxActionId = actionId;
+		}
+
+		if (maxRev > currentRev && maxActionId !== undefined) {
+			const advanced: ActionRev = { rev: maxRev, actionId: maxActionId };
+			meta.latest = advanced;
+			await this.storage.saveMetadata(this.blockId, meta);
+			log('recover blockId=%s advanced latest from rev=%d to rev=%d', this.blockId, currentRev, maxRev);
+			return { reconciled: true, latest: advanced };
+		}
+
+		return { reconciled: false, latest: meta.latest };
+	}
+
 	private async ensureRevision(meta: BlockMetadata, rev: number): Promise<void> {
 		if (this.inRanges(rev, meta.ranges)) {
 			return;

package/src/storage/i-block-storage.ts

@@ -43,4 +43,19 @@ export interface IBlockStorage {
 
     /** Sets the latest revision information */
     setLatest(latest: ActionRev): Promise<void>;
+
+    /**
+     * Reconciles `metadata.latest` with the highest contiguous fully-promoted revision in
+     * the revisions table. Intended for post-crash recovery of the Crash-D3 gap, where
+     * `promotePendingTransaction` succeeded but `setLatest` did not: the revision and
+     * committed-log entry are durable, but `meta.latest` still points at the prior rev
+     * (or is undefined), and retry-commit is rejected because the pending record is gone.
+     *
+     * Stops at the first rev whose action is not yet in the committed log, preserving the
+     * Crash-D2 invariant that retry-commit — not recovery — owns advancement past a half-
+     * promoted state.
+     *
+     * Idempotent and monotonic (latest only advances forward).
+     */
+    recover(): Promise<{ reconciled: boolean; latest?: ActionRev }>;
 }

package/src/storage/memory-storage.ts

@@ -17,12 +17,30 @@ export class MemoryRawStorage implements IRawStorage {
 		return `${blockId}:${actionId}`;
 	}
 
+	/**
+	 * Retrieves metadata for a block.
+	 *
+	 * @pitfall **MUST return a clone** - `BlockStorage.setLatest` mutates the returned
+	 * metadata in place (`meta.latest = latest`) before calling `saveMetadata`. Returning
+	 * the stored reference leaks that mutation into RAM even when a subsequent
+	 * `saveMetadata` call fails, masking mid-commit crashes that a persistent store
+	 * (file/sqlite/leveldb) would surface correctly.
+	 * @see docs/internals.md "Storage Returns References" pitfall
+	 */
 	async getMetadata(blockId: BlockId): Promise<BlockMetadata | undefined> {
-		return this.metadata.get(blockId);
+		const meta = this.metadata.get(blockId);
+		return meta ? structuredClone(meta) : undefined;
 	}
 
+	/**
+	 * Stores metadata for a block.
+	 *
+	 * @pitfall **MUST store a clone** - callers may continue mutating the metadata object
+	 * after saving; storing the reference lets those mutations corrupt persisted state.
+	 * @see docs/internals.md "Storage Returns References" pitfall
+	 */
 	async saveMetadata(blockId: BlockId, metadata: BlockMetadata): Promise<void> {
-		this.metadata.set(blockId, metadata);
+		this.metadata.set(blockId, structuredClone(metadata));
 	}
 
 	async getRevision(blockId: BlockId, rev: number): Promise<ActionId | undefined> {

package/src/storage/storage-repo.ts

@@ -211,11 +211,23 @@ export class StorageRepo implements IRepo {
 				storage: this.createBlockStorage(blockId)
 			}));
 
-			// Check for stale revisions and collect missing actions
+			// Partition blocks into:
+			//   - alreadyDone: latest.rev === request.rev && latest.actionId === request.actionId
+			//     (idempotent retry — a prior commit of this same action already landed here;
+			//     skip rather than treat as a conflict. Needed to rollforward stranded blocks
+			//     after a mid-batch crash committed some but not all blocks.)
+			//   - missedCommits: latest.rev >= request.rev but not the same actionId → real stale conflict.
+			//   - toCommit: latest.rev < request.rev or no latest yet → run internalCommit.
+			const toCommit: { blockId: BlockId, storage: IBlockStorage }[] = [];
 			const missedCommits: { blockId: BlockId, transforms: ActionTransform[] }[] = [];
-			for (const { blockId, storage } of blockStorages) {
+			for (const entry of blockStorages) {
+				const { blockId, storage } = entry;
 				const latest = await storage.getLatest();
 				if (latest && latest.rev >= request.rev) {
+					if (latest.rev === request.rev && latest.actionId === request.actionId) {
+						// Idempotent no-op for this block — already committed with this exact (actionId, rev).
+						continue;
+					}
 					const transforms: ActionTransform[] = [];
 					for await (const actionRev of storage.listRevisions(request.rev, latest.rev)) {
 						const transform = await storage.getTransaction(actionRev.actionId);
@@ -229,7 +241,9 @@ export class StorageRepo implements IRepo {
 						});
 					}
 					missedCommits.push({ blockId, transforms });	// Push, even if transforms is empty, because we want to reject the older version
+					continue;
 				}
+				toCommit.push(entry);
 			}
 
 			if (missedCommits.length) {
@@ -240,9 +254,11 @@ export class StorageRepo implements IRepo {
 				};
 			}
 
-			// Check for missing pending actions
+			// Check for missing pending actions only on blocks that still need to commit.
+			// Already-done blocks will have had their pending promoted, so skipping them here
+			// is what makes the idempotent rollforward work.
 			const missingPends: { blockId: BlockId, actionId: ActionId }[] = [];
-			for (const { blockId, storage } of blockStorages) {
+			for (const { blockId, storage } of toCommit) {
 				const pendingAction = await storage.getPendingTransaction(request.actionId);
 				if (!pendingAction) {
 					missingPends.push({ blockId, actionId: request.actionId });
@@ -253,14 +269,15 @@ export class StorageRepo implements IRepo {
 				throw new Error(`Pending action ${request.actionId} not found for block(s): ${missingPends.map(p => p.blockId).join(', ')}`);
 			}
 
-			// Commit the action for each block
-			// This loop will execute atomically for all blocks due to the acquired locks
-			for (const { blockId, storage } of blockStorages) {
+			// Commit the action for each block that still needs it.
+			// This loop will execute atomically for all blocks due to the acquired locks.
+			for (const { blockId, storage } of toCommit) {
 				try {
 					// internalCommit will throw if it encounters an issue
 					await this.internalCommit(blockId, request.actionId, request.rev, storage);
 				} catch (err) {
-					// TODO: Recover as best we can. Rollback or handle partial commit? For now, return failure.
+					// Partial-commit recovery: a retry with the same (actionId, rev) will treat
+					// already-done blocks as idempotent no-ops and advance the remainder.
 					return {
 						success: false,
 						reason: err instanceof Error ? err.message : 'Unknown error during commit'
@@ -276,6 +293,19 @@ export class StorageRepo implements IRepo {
 		return { success: true };
 	}
 
+	/**
+	 * Reconciles `metadata.latest` for a single block with the highest contiguous
+	 * fully-promoted revision in durable storage. Use after a crash between
+	 * `promotePendingTransaction` and `setLatest` when retry-commit cannot help
+	 * (the pending record is already gone) but the revision and committed-log entry
+	 * are durable. Idempotent and monotonic.
+	 */
+	async recoverBlock(blockId: BlockId): Promise<void> {
+		log('recoverBlock blockId=%s', blockId);
+		const storage = this.createBlockStorage(blockId);
+		await storage.recover();
+	}
+
 	private async internalCommit(blockId: BlockId, actionId: ActionId, rev: number, storage: IBlockStorage): Promise<void> {
 		// Note: This method is called within the locked critical section of commit()
 		// So, operations like getPendingTransaction, getLatest, getBlock, saveMaterializedBlock,

package/src/testing/index.ts

@@ -0,0 +1 @@
+export * from './mesh-harness.js';