@optimystic/db-p2p 0.11.2 → 0.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (99) hide show
  1. package/dist/src/cluster/block-transfer-service.d.ts +0 -1
  2. package/dist/src/cluster/block-transfer-service.d.ts.map +1 -1
  3. package/dist/src/cluster/block-transfer-service.js +0 -2
  4. package/dist/src/cluster/block-transfer-service.js.map +1 -1
  5. package/dist/src/cluster/cluster-repo.d.ts +1 -2
  6. package/dist/src/cluster/cluster-repo.d.ts.map +1 -1
  7. package/dist/src/cluster/cluster-repo.js +5 -5
  8. package/dist/src/cluster/cluster-repo.js.map +1 -1
  9. package/dist/src/cluster/partition-detector.d.ts.map +1 -1
  10. package/dist/src/cluster/partition-detector.js +0 -1
  11. package/dist/src/cluster/partition-detector.js.map +1 -1
  12. package/dist/src/cluster/rebalance-monitor.d.ts.map +1 -1
  13. package/dist/src/cluster/service.d.ts +0 -6
  14. package/dist/src/cluster/service.d.ts.map +1 -1
  15. package/dist/src/cluster/service.js +0 -13
  16. package/dist/src/cluster/service.js.map +1 -1
  17. package/dist/src/dispute/dispute-service.d.ts.map +1 -1
  18. package/dist/src/dispute/dispute-service.js +1 -1
  19. package/dist/src/dispute/dispute-service.js.map +1 -1
  20. package/dist/src/libp2p-key-network.js.map +1 -1
  21. package/dist/src/libp2p-node-base.d.ts.map +1 -1
  22. package/dist/src/libp2p-node-base.js +14 -1
  23. package/dist/src/libp2p-node-base.js.map +1 -1
  24. package/dist/src/network/network-manager-service.js +1 -1
  25. package/dist/src/network/network-manager-service.js.map +1 -1
  26. package/dist/src/repo/cluster-coordinator.d.ts +1 -1
  27. package/dist/src/repo/cluster-coordinator.d.ts.map +1 -1
  28. package/dist/src/repo/cluster-coordinator.js +3 -3
  29. package/dist/src/repo/cluster-coordinator.js.map +1 -1
  30. package/dist/src/repo/coordinator-repo.d.ts.map +1 -1
  31. package/dist/src/repo/coordinator-repo.js +48 -17
  32. package/dist/src/repo/coordinator-repo.js.map +1 -1
  33. package/dist/src/repo/service.d.ts +0 -1
  34. package/dist/src/repo/service.d.ts.map +1 -1
  35. package/dist/src/repo/service.js +0 -2
  36. package/dist/src/repo/service.js.map +1 -1
  37. package/dist/src/routing/simple-cluster-coordinator.d.ts +2 -2
  38. package/dist/src/routing/simple-cluster-coordinator.d.ts.map +1 -1
  39. package/dist/src/routing/simple-cluster-coordinator.js +2 -2
  40. package/dist/src/routing/simple-cluster-coordinator.js.map +1 -1
  41. package/dist/src/storage/block-storage.d.ts +4 -0
  42. package/dist/src/storage/block-storage.d.ts.map +1 -1
  43. package/dist/src/storage/block-storage.js +29 -0
  44. package/dist/src/storage/block-storage.js.map +1 -1
  45. package/dist/src/storage/i-block-storage.d.ts +17 -0
  46. package/dist/src/storage/i-block-storage.d.ts.map +1 -1
  47. package/dist/src/storage/i-raw-storage.d.ts +9 -0
  48. package/dist/src/storage/i-raw-storage.d.ts.map +1 -1
  49. package/dist/src/storage/memory-storage.d.ts +18 -0
  50. package/dist/src/storage/memory-storage.d.ts.map +1 -1
  51. package/dist/src/storage/memory-storage.js +47 -2
  52. package/dist/src/storage/memory-storage.js.map +1 -1
  53. package/dist/src/storage/ring-selector.d.ts.map +1 -1
  54. package/dist/src/storage/ring-selector.js +23 -5
  55. package/dist/src/storage/ring-selector.js.map +1 -1
  56. package/dist/src/storage/storage-monitor.d.ts.map +1 -1
  57. package/dist/src/storage/storage-monitor.js +10 -14
  58. package/dist/src/storage/storage-monitor.js.map +1 -1
  59. package/dist/src/storage/storage-repo.d.ts +10 -2
  60. package/dist/src/storage/storage-repo.d.ts.map +1 -1
  61. package/dist/src/storage/storage-repo.js +39 -10
  62. package/dist/src/storage/storage-repo.js.map +1 -1
  63. package/dist/src/sync/service.d.ts +0 -1
  64. package/dist/src/sync/service.d.ts.map +1 -1
  65. package/dist/src/sync/service.js +0 -2
  66. package/dist/src/sync/service.js.map +1 -1
  67. package/dist/src/testing/index.d.ts +2 -0
  68. package/dist/src/testing/index.d.ts.map +1 -0
  69. package/dist/src/testing/index.js +2 -0
  70. package/dist/src/testing/index.js.map +1 -0
  71. package/dist/src/testing/mesh-harness.d.ts +60 -0
  72. package/dist/src/testing/mesh-harness.d.ts.map +1 -0
  73. package/dist/src/testing/mesh-harness.js +220 -0
  74. package/dist/src/testing/mesh-harness.js.map +1 -0
  75. package/package.json +8 -3
  76. package/src/cluster/block-transfer-service.ts +1 -1
  77. package/src/cluster/cluster-repo.ts +5 -3
  78. package/src/cluster/partition-detector.ts +0 -2
  79. package/src/cluster/rebalance-monitor.ts +1 -1
  80. package/src/cluster/service.ts +0 -13
  81. package/src/dispute/dispute-service.ts +1 -2
  82. package/src/dispute/service.ts +1 -1
  83. package/src/libp2p-key-network.ts +1 -1
  84. package/src/libp2p-node-base.ts +13 -3
  85. package/src/network/network-manager-service.ts +1 -1
  86. package/src/repo/cluster-coordinator.ts +4 -4
  87. package/src/repo/coordinator-repo.ts +51 -19
  88. package/src/repo/service.ts +0 -2
  89. package/src/routing/simple-cluster-coordinator.ts +2 -2
  90. package/src/storage/block-storage.ts +32 -0
  91. package/src/storage/i-block-storage.ts +15 -0
  92. package/src/storage/i-raw-storage.ts +10 -0
  93. package/src/storage/memory-storage.ts +48 -2
  94. package/src/storage/ring-selector.ts +26 -5
  95. package/src/storage/storage-monitor.ts +10 -17
  96. package/src/storage/storage-repo.ts +40 -11
  97. package/src/sync/service.ts +2 -3
  98. package/src/testing/index.ts +1 -0
  99. package/src/testing/mesh-harness.ts +294 -0
@@ -1,5 +1,5 @@
1
1
  import { peerIdFromString } from "@libp2p/peer-id";
2
- import type { ClusterRecord, IKeyNetwork, RepoMessage, BlockId, ClusterPeers, MessageOptions, Signature, ClusterConsensusConfig } from "@optimystic/db-core";
2
+ import type { ClusterRecord, IKeyNetwork, RepoMessage, BlockId, ClusterPeers, MessageOptions, ClusterConsensusConfig } from "@optimystic/db-core";
3
3
  import { base58btc } from "multiformats/bases/base58";
4
4
  import { sha256 } from "multiformats/hashes/sha2";
5
5
  import { ClusterClient } from "../cluster/client.js";
@@ -123,7 +123,7 @@ export class ClusterCoordinator {
123
123
  * Initiates a 2-phase transaction for a specific block ID.
124
124
  * Returns the cluster record and whether the local cluster already executed the operations.
125
125
  */
126
- async executeClusterTransaction(blockId: BlockId, message: RepoMessage, options?: MessageOptions): Promise<{
126
+ async executeClusterTransaction(blockId: BlockId, message: RepoMessage, _options?: MessageOptions): Promise<{
127
127
  record: ClusterRecord;
128
128
  localExecuted: boolean;
129
129
  }> {
@@ -298,7 +298,7 @@ export class ClusterCoordinator {
298
298
  * Validate that a small cluster size is legitimate by querying remote peers
299
299
  * for their network size estimates. Returns true if estimates roughly agree.
300
300
  */
301
- private async validateSmallCluster(localSize: number, peers: ClusterPeers): Promise<boolean> {
301
+ private async validateSmallCluster(localSize: number, _peers: ClusterPeers): Promise<boolean> {
302
302
  // If we have FRET and it shows confident estimate
303
303
  if (this.fretService) {
304
304
  try {
@@ -581,7 +581,7 @@ export class ClusterCoordinator {
581
581
  });
582
582
  }
583
583
 
584
- private scheduleCommitRetry(messageHash: string, record: ClusterRecord, missingPeers: string[]): void {
584
+ private scheduleCommitRetry(messageHash: string, _record: ClusterRecord, missingPeers: string[]): void {
585
585
  const state = this.transactions.get(messageHash);
586
586
  if (!state) {
587
587
  return;
@@ -1,5 +1,5 @@
1
- import type { PendRequest, ActionBlocks, IRepo, MessageOptions, CommitResult, GetBlockResults, PendResult, BlockGets, CommitRequest, RepoMessage, IKeyNetwork, ICluster, ClusterConsensusConfig, BlockId, ActionRev, ActionContext } from "@optimystic/db-core";
2
- import { LruMap } from "@optimystic/db-core";
1
+ import type { PendRequest, ActionBlocks, IRepo, MessageOptions, CommitResult, GetBlockResults, PendResult, BlockGets, CommitRequest, RepoMessage, IKeyNetwork, ICluster, ClusterConsensusConfig, BlockId, ActionRev, ActionContext, ClusterRecord } from "@optimystic/db-core";
2
+ import { LruMap, blockIdsForTransforms } from "@optimystic/db-core";
3
3
  import { ClusterCoordinator } from "./cluster-coordinator.js";
4
4
  import type { ClusterClient } from "../cluster/client.js";
5
5
  import type { PeerId } from "@libp2p/interface";
@@ -187,8 +187,25 @@ export class CoordinatorRepo implements IRepo {
187
187
  private async fetchBlockFromCluster(blockId: BlockId, context?: ActionContext): Promise<void> {
188
188
  if (!this.clusterLatestCallback) return;
189
189
 
190
- // Query cluster for the block
191
- const clusterLatest = await this.queryClusterForLatest(blockId, context);
190
+ const blockIdBytes = new TextEncoder().encode(blockId);
191
+ const peers = await this.keyNetwork.findCluster(blockIdBytes);
192
+ const peerIds = peers ? Object.keys(peers) : [];
193
+ if (peerIds.length === 0) return;
194
+
195
+ // Solo-cluster short-circuit: the only responsible peer is us. There is no
196
+ // remote to sync from, so skip the callback entirely. Querying ourselves
197
+ // would dial self via SyncClient — pointless at best, and on nodes without
198
+ // listen addresses (e.g. solo WebSocket-only) the dial can hang.
199
+ if (
200
+ peerIds.length === 1
201
+ && this.localPeerId
202
+ && peerIds[0] === this.localPeerId.toString()
203
+ ) {
204
+ log('cluster-fetch:solo-self-skip', { blockId });
205
+ return;
206
+ }
207
+
208
+ const clusterLatest = await this.queryClusterForLatest(peerIds, blockId, context);
192
209
  if (clusterLatest) {
193
210
  // Found on cluster - trigger restoration to sync the block
194
211
  await this.storageRepo.get({ blockIds: [blockId], context: { committed: [clusterLatest], rev: clusterLatest.rev } });
@@ -199,14 +216,7 @@ export class CoordinatorRepo implements IRepo {
199
216
  /**
200
217
  * Query cluster peers to find the maximum latest revision for a block.
201
218
  */
202
- private async queryClusterForLatest(blockId: BlockId, context?: ActionContext): Promise<ActionRev | undefined> {
203
- const blockIdBytes = new TextEncoder().encode(blockId);
204
- const peers = await this.keyNetwork.findCluster(blockIdBytes);
205
- if (!peers || Object.keys(peers).length === 0) {
206
- return undefined;
207
- }
208
-
209
- const peerIds = Object.keys(peers);
219
+ private async queryClusterForLatest(peerIds: string[], blockId: BlockId, context?: ActionContext): Promise<ActionRev | undefined> {
210
220
  let maxLatest: ActionRev | undefined;
211
221
 
212
222
  // Add timeout wrapper to prevent hanging on unresponsive peers
@@ -237,7 +247,7 @@ export class CoordinatorRepo implements IRepo {
237
247
  }
238
248
 
239
249
  async pend(request: PendRequest, options?: MessageOptions): Promise<PendResult> {
240
- const allBlockIds = Object.keys(request.transforms);
250
+ const allBlockIds = blockIdsForTransforms(request.transforms);
241
251
  await this.verifyResponsibility(allBlockIds);
242
252
  const coordinatingBlockIds = (options as any)?.coordinatingBlockIds ?? allBlockIds;
243
253
 
@@ -273,7 +283,7 @@ export class CoordinatorRepo implements IRepo {
273
283
  return {
274
284
  success: true,
275
285
  pending: [],
276
- blockIds: Object.keys(request.transforms)
286
+ blockIds: allBlockIds
277
287
  };
278
288
  } catch (error) {
279
289
  log('coordinator-repo:pend-error', { actionId: request.actionId, error: (error as Error).message });
@@ -326,16 +336,38 @@ export class CoordinatorRepo implements IRepo {
326
336
  };
327
337
 
328
338
  try {
329
- const { localExecuted } = await this.coordinator.executeClusterTransaction(blockIds[0]!, message, options);
330
- // Only call storageRepo if local cluster didn't already execute during consensus
331
- if (!localExecuted) {
339
+ const { record, localExecuted } = await this.coordinator.executeClusterTransaction(blockIds[0]!, message, options);
340
+ if (localExecuted) {
341
+ return { success: true };
342
+ }
343
+ // Local cluster didn't execute during consensus. Attempt a local commit,
344
+ // but tolerate failure (e.g., "pending action not found") when the cluster
345
+ // already reached consensus — this coordinator was likely picked for commit
346
+ // after missing the pend phase (unreachable during pend, fresh join, etc.).
347
+ // The cluster's majority is authoritative; this peer will catch up via sync.
348
+ try {
332
349
  return await this.storageRepo.commit(request, options);
350
+ } catch (err) {
351
+ if (clusterReachedCommitConsensus(record)) {
352
+ log('coordinator-repo:commit-local-failed-cluster-succeeded', {
353
+ actionId: request.actionId,
354
+ error: (err as Error).message
355
+ });
356
+ return { success: true };
357
+ }
358
+ throw err;
333
359
  }
334
- // Local cluster already executed - return success
335
- return { success: true };
336
360
  } catch (error) {
337
361
  log('coordinator-repo:commit-error', { actionId: request.actionId, error: (error as Error).message });
338
362
  throw error;
339
363
  }
340
364
  }
341
365
  }
366
+
367
+ /** True if a simple majority of cluster peers signed an approving commit. */
368
+ function clusterReachedCommitConsensus(record: ClusterRecord): boolean {
369
+ const peerCount = Object.keys(record.peers).length;
370
+ if (peerCount === 0) return false;
371
+ const approvedCommits = Object.values(record.commits).filter(s => s.type === 'approve').length;
372
+ return approvedCommits > peerCount / 2;
373
+ }
@@ -63,7 +63,6 @@ export class RepoService implements Startable {
63
63
  private readonly repo: IRepo
64
64
  private readonly components: RepoServiceComponents
65
65
  private running: boolean
66
- private readonly k: number
67
66
  /** Responsibility K - how many peers are responsible for a key (for redirect decisions) */
68
67
  private readonly responsibilityK: number
69
68
 
@@ -76,7 +75,6 @@ export class RepoService implements Startable {
76
75
  this.log = components.logger.forComponent(init.logPrefix ?? 'db-p2p:repo-service')
77
76
  this.repo = components.repo
78
77
  this.running = false
79
- this.k = init.kBucketSize ?? 10
80
78
  this.responsibilityK = init.responsibilityK ?? 1
81
79
  }
82
80
 
@@ -57,13 +57,13 @@ export class ModuloCoordinator implements SimpleClusterCoordinator {
57
57
  * For very small clusters, just replicate everywhere
58
58
  */
59
59
  export class FullReplicationCoordinator implements SimpleClusterCoordinator {
60
- async selectCoordinator(key: Uint8Array, peers: PeerId[]): Promise<PeerId> {
60
+ async selectCoordinator(_key: Uint8Array, peers: PeerId[]): Promise<PeerId> {
61
61
  // Always select first peer as primary
62
62
  if (peers.length === 0) throw new Error('No peers available')
63
63
  return peers[0]!
64
64
  }
65
65
 
66
- async selectReplicas(key: Uint8Array, peers: PeerId[], replicationFactor: number): Promise<PeerId[]> {
66
+ async selectReplicas(_key: Uint8Array, peers: PeerId[], _replicationFactor: number): Promise<PeerId[]> {
67
67
  // Replicate to all peers in small clusters
68
68
  return [...peers]
69
69
  }
@@ -91,6 +91,38 @@ export class BlockStorage implements IBlockStorage {
91
91
  await this.storage.saveMetadata(this.blockId, meta);
92
92
  }
93
93
 
94
+ async recover(): Promise<{ reconciled: boolean; latest?: ActionRev }> {
95
+ const meta = await this.storage.getMetadata(this.blockId);
96
+ if (!meta) {
97
+ return { reconciled: false };
98
+ }
99
+
100
+ const currentRev = meta.latest?.rev ?? 0;
101
+ let maxRev = currentRev;
102
+ let maxActionId = meta.latest?.actionId;
103
+
104
+ // Probe forward until we hit a gap or a revision whose action is not yet
105
+ // in the committed log (Crash-D2 state — retry-commit owns that advance).
106
+ for (let next = currentRev + 1; ; next++) {
107
+ const actionId = await this.storage.getRevision(this.blockId, next);
108
+ if (actionId === undefined) break;
109
+ const promoted = await this.storage.getTransaction(this.blockId, actionId);
110
+ if (promoted === undefined) break;
111
+ maxRev = next;
112
+ maxActionId = actionId;
113
+ }
114
+
115
+ if (maxRev > currentRev && maxActionId !== undefined) {
116
+ const advanced: ActionRev = { rev: maxRev, actionId: maxActionId };
117
+ meta.latest = advanced;
118
+ await this.storage.saveMetadata(this.blockId, meta);
119
+ log('recover blockId=%s advanced latest from rev=%d to rev=%d', this.blockId, currentRev, maxRev);
120
+ return { reconciled: true, latest: advanced };
121
+ }
122
+
123
+ return { reconciled: false, latest: meta.latest };
124
+ }
125
+
94
126
  private async ensureRevision(meta: BlockMetadata, rev: number): Promise<void> {
95
127
  if (this.inRanges(rev, meta.ranges)) {
96
128
  return;
@@ -43,4 +43,19 @@ export interface IBlockStorage {
43
43
 
44
44
  /** Sets the latest revision information */
45
45
  setLatest(latest: ActionRev): Promise<void>;
46
+
47
+ /**
48
+ * Reconciles `metadata.latest` with the highest contiguous fully-promoted revision in
49
+ * the revisions table. Intended for post-crash recovery of the Crash-D3 gap, where
50
+ * `promotePendingTransaction` succeeded but `setLatest` did not: the revision and
51
+ * committed-log entry are durable, but `meta.latest` still points at the prior rev
52
+ * (or is undefined), and retry-commit is rejected because the pending record is gone.
53
+ *
54
+ * Stops at the first rev whose action is not yet in the committed log, preserving the
55
+ * Crash-D2 invariant that retry-commit — not recovery — owns advancement past a half-
56
+ * promoted state.
57
+ *
58
+ * Idempotent and monotonic (latest only advances forward).
59
+ */
60
+ recover(): Promise<{ reconciled: boolean; latest?: ActionRev }>;
46
61
  }
@@ -27,4 +27,14 @@ export interface IRawStorage {
27
27
 
28
28
  // Promote a pending action to a committed action
29
29
  promotePendingTransaction(blockId: BlockId, actionId: ActionId): Promise<void>;
30
+
31
+ /**
32
+ * Approximate bytes currently stored by this backend.
33
+ *
34
+ * Used by `StorageMonitor` to feed real used-space figures into ring selection.
35
+ * Implementations should return their best cheap estimate (e.g. on-disk size for
36
+ * filesystem backends, tracked footprint for in-memory backends). The result is
37
+ * advisory — `StorageMonitor` treats a missing implementation as 0.
38
+ */
39
+ getApproximateBytesUsed?(): Promise<number>;
30
40
  }
@@ -17,12 +17,30 @@ export class MemoryRawStorage implements IRawStorage {
17
17
  return `${blockId}:${actionId}`;
18
18
  }
19
19
 
20
+ /**
21
+ * Retrieves metadata for a block.
22
+ *
23
+ * @pitfall **MUST return a clone** - `BlockStorage.setLatest` mutates the returned
24
+ * metadata in place (`meta.latest = latest`) before calling `saveMetadata`. Returning
25
+ * the stored reference leaks that mutation into RAM even when a subsequent
26
+ * `saveMetadata` call fails, masking mid-commit crashes that a persistent store
27
+ * (file/sqlite/leveldb) would surface correctly.
28
+ * @see docs/internals.md "Storage Returns References" pitfall
29
+ */
20
30
  async getMetadata(blockId: BlockId): Promise<BlockMetadata | undefined> {
21
- return this.metadata.get(blockId);
31
+ const meta = this.metadata.get(blockId);
32
+ return meta ? structuredClone(meta) : undefined;
22
33
  }
23
34
 
35
+ /**
36
+ * Stores metadata for a block.
37
+ *
38
+ * @pitfall **MUST store a clone** - callers may continue mutating the metadata object
39
+ * after saving; storing the reference lets those mutations corrupt persisted state.
40
+ * @see docs/internals.md "Storage Returns References" pitfall
41
+ */
24
42
  async saveMetadata(blockId: BlockId, metadata: BlockMetadata): Promise<void> {
25
- this.metadata.set(blockId, metadata);
43
+ this.metadata.set(blockId, structuredClone(metadata));
26
44
  }
27
45
 
28
46
  async getRevision(blockId: BlockId, rev: number): Promise<ActionId | undefined> {
@@ -124,4 +142,32 @@ export class MemoryRawStorage implements IRawStorage {
124
142
  this.pendingActions.delete(key);
125
143
  }
126
144
  }
145
+
146
+ async getApproximateBytesUsed(): Promise<number> {
147
+ let total = 0;
148
+ for (const [blockId, meta] of this.metadata) {
149
+ total += blockId.length + jsonByteSize(meta);
150
+ }
151
+ for (const [key, actionId] of this.revisions) {
152
+ total += key.length + actionId.length;
153
+ }
154
+ for (const [key, transform] of this.pendingActions) {
155
+ total += key.length + jsonByteSize(transform);
156
+ }
157
+ for (const [key, transform] of this.actions) {
158
+ total += key.length + jsonByteSize(transform);
159
+ }
160
+ for (const [key, block] of this.materializedBlocks) {
161
+ total += key.length + jsonByteSize(block);
162
+ }
163
+ return total;
164
+ }
165
+ }
166
+
167
+ function jsonByteSize(value: unknown): number {
168
+ try {
169
+ return JSON.stringify(value)?.length ?? 0;
170
+ } catch {
171
+ return 0;
172
+ }
127
173
  }
@@ -42,11 +42,32 @@ export class RingSelector {
42
42
  return -1;
43
43
  }
44
44
 
45
- // Estimate total network size from FRET
46
- // We use a simple heuristic: assume average block size and typical data distribution
47
- const avgBlockSize = 100 * 1024; // 100KB typical block
48
- const estimatedTotalBlocks = 1000; // Conservative estimate
49
- const estimatedTotalData = estimatedTotalBlocks * avgBlockSize;
45
+ // Estimate total network spare capacity by aggregating observed per-ring stats.
46
+ // See docs/arachnode.md §"Capacity Management and Ring Adjustment": ring selection
47
+ // should reflect observed network demand, not a hard-coded constant. We treat
48
+ // Σ (peerCount × avgCapacity) over known rings as a proxy for total network spare
49
+ // capacity, so `coverage` becomes our share of it. More peers (or richer peers) →
50
+ // smaller share → higher ring depth (more specialization). On bootstrap, when no
51
+ // ring stats are observed yet, we fall back to a conservative fixed estimate so
52
+ // first-boot behavior is deterministic.
53
+ const ringStats = this.fretAdapter.getRingStats();
54
+ let estimatedTotalData: number;
55
+ if (ringStats.length > 0) {
56
+ estimatedTotalData = ringStats.reduce(
57
+ (sum, stat) => sum + stat.peerCount * stat.avgCapacity,
58
+ 0
59
+ );
60
+ } else {
61
+ const avgBlockSize = 100 * 1024; // 100KB typical block
62
+ const estimatedTotalBlocks = 1000; // Conservative estimate
63
+ estimatedTotalData = estimatedTotalBlocks * avgBlockSize;
64
+ }
65
+
66
+ // Guard against pathological zero-aggregate (e.g. all peers report 0 available);
67
+ // fall back to local capacity so coverage = 1.0 (ring 0).
68
+ if (estimatedTotalData <= 0) {
69
+ estimatedTotalData = capacity.available;
70
+ }
50
71
 
51
72
  // Calculate what fraction of keyspace we can cover
52
73
  const coverage = capacity.available / estimatedTotalData;
@@ -28,22 +28,15 @@ export class StorageMonitor {
28
28
  const usedOverride = this.config.usedBytes;
29
29
  const availableOverride = this.config.availableBytes;
30
30
 
31
- if (
32
- usedOverride !== undefined ||
33
- availableOverride !== undefined ||
34
- this.config.totalBytes !== undefined
35
- ) {
36
- const used = usedOverride ?? Math.max(0, total - (availableOverride ?? total));
37
- const available = availableOverride ?? Math.max(0, total - used);
38
- return {
39
- total,
40
- used: Math.min(total, Math.max(0, used)),
41
- available: Math.max(0, Math.min(total, available))
42
- };
43
- }
44
-
45
- const used = await this.estimateUsedSpace();
46
- const available = Math.max(0, total - used);
31
+ // Only short-circuit the backend when used/available is explicitly supplied;
32
+ // a bare `totalBytes` is just the disk-size hint and should still let the
33
+ // backend report actual used bytes.
34
+ const rawUsed = usedOverride
35
+ ?? (availableOverride !== undefined ? total - availableOverride : await this.estimateUsedSpace());
36
+ const used = Math.min(total, Math.max(0, rawUsed));
37
+ const available = availableOverride !== undefined
38
+ ? Math.max(0, Math.min(total, availableOverride))
39
+ : Math.max(0, total - used);
47
40
 
48
41
  return {
49
42
  total,
@@ -53,7 +46,7 @@ export class StorageMonitor {
53
46
  }
54
47
 
55
48
  private async estimateUsedSpace(): Promise<number> {
56
- return 0; // TODO: Implement actual space calculation
49
+ return (await this.storage.getApproximateBytesUsed?.()) ?? 0;
57
50
  }
58
51
  }
59
52
 
@@ -1,7 +1,6 @@
1
1
  import type {
2
2
  IRepo, MessageOptions, BlockId, CommitRequest, CommitResult, GetBlockResults, PendRequest, PendResult, ActionBlocks,
3
3
  ActionId, BlockGets, ActionPending, PendSuccess, ActionTransform, ActionTransforms,
4
- Transforms,
5
4
  GetBlockResult,
6
5
  PendValidationHook
7
6
  } from "@optimystic/db-core";
@@ -30,7 +29,7 @@ export class StorageRepo implements IRepo {
30
29
  this.validatePend = options?.validatePend;
31
30
  }
32
31
 
33
- async get({ blockIds, context }: BlockGets, options?: MessageOptions): Promise<GetBlockResults> {
32
+ async get({ blockIds, context }: BlockGets, _options?: MessageOptions): Promise<GetBlockResults> {
34
33
  const distinctBlockIds = Array.from(new Set(blockIds));
35
34
  log('get blockIds=%d', distinctBlockIds.length);
36
35
  const results = await Promise.all(distinctBlockIds.map(async (blockId) => {
@@ -191,7 +190,7 @@ export class StorageRepo implements IRepo {
191
190
  }));
192
191
  }
193
192
 
194
- async commit(request: CommitRequest, options?: MessageOptions): Promise<CommitResult> {
193
+ async commit(request: CommitRequest, _options?: MessageOptions): Promise<CommitResult> {
195
194
  log('commit actionId=%s rev=%d blockIds=%d', request.actionId, request.rev, request.blockIds.length);
196
195
  const uniqueBlockIds = Array.from(new Set(request.blockIds)).sort();
197
196
  const releases: (() => void)[] = [];
@@ -211,11 +210,23 @@ export class StorageRepo implements IRepo {
211
210
  storage: this.createBlockStorage(blockId)
212
211
  }));
213
212
 
214
- // Check for stale revisions and collect missing actions
213
+ // Partition blocks into:
214
+ // - alreadyDone: latest.rev === request.rev && latest.actionId === request.actionId
215
+ // (idempotent retry — a prior commit of this same action already landed here;
216
+ // skip rather than treat as a conflict. Needed to rollforward stranded blocks
217
+ // after a mid-batch crash committed some but not all blocks.)
218
+ // - missedCommits: latest.rev >= request.rev but not the same actionId → real stale conflict.
219
+ // - toCommit: latest.rev < request.rev or no latest yet → run internalCommit.
220
+ const toCommit: { blockId: BlockId, storage: IBlockStorage }[] = [];
215
221
  const missedCommits: { blockId: BlockId, transforms: ActionTransform[] }[] = [];
216
- for (const { blockId, storage } of blockStorages) {
222
+ for (const entry of blockStorages) {
223
+ const { blockId, storage } = entry;
217
224
  const latest = await storage.getLatest();
218
225
  if (latest && latest.rev >= request.rev) {
226
+ if (latest.rev === request.rev && latest.actionId === request.actionId) {
227
+ // Idempotent no-op for this block — already committed with this exact (actionId, rev).
228
+ continue;
229
+ }
219
230
  const transforms: ActionTransform[] = [];
220
231
  for await (const actionRev of storage.listRevisions(request.rev, latest.rev)) {
221
232
  const transform = await storage.getTransaction(actionRev.actionId);
@@ -229,7 +240,9 @@ export class StorageRepo implements IRepo {
229
240
  });
230
241
  }
231
242
  missedCommits.push({ blockId, transforms }); // Push, even if transforms is empty, because we want to reject the older version
243
+ continue;
232
244
  }
245
+ toCommit.push(entry);
233
246
  }
234
247
 
235
248
  if (missedCommits.length) {
@@ -240,9 +253,11 @@ export class StorageRepo implements IRepo {
240
253
  };
241
254
  }
242
255
 
243
- // Check for missing pending actions
256
+ // Check for missing pending actions only on blocks that still need to commit.
257
+ // Already-done blocks will have had their pending promoted, so skipping them here
258
+ // is what makes the idempotent rollforward work.
244
259
  const missingPends: { blockId: BlockId, actionId: ActionId }[] = [];
245
- for (const { blockId, storage } of blockStorages) {
260
+ for (const { blockId, storage } of toCommit) {
246
261
  const pendingAction = await storage.getPendingTransaction(request.actionId);
247
262
  if (!pendingAction) {
248
263
  missingPends.push({ blockId, actionId: request.actionId });
@@ -253,14 +268,15 @@ export class StorageRepo implements IRepo {
253
268
  throw new Error(`Pending action ${request.actionId} not found for block(s): ${missingPends.map(p => p.blockId).join(', ')}`);
254
269
  }
255
270
 
256
- // Commit the action for each block
257
- // This loop will execute atomically for all blocks due to the acquired locks
258
- for (const { blockId, storage } of blockStorages) {
271
+ // Commit the action for each block that still needs it.
272
+ // This loop will execute atomically for all blocks due to the acquired locks.
273
+ for (const { blockId, storage } of toCommit) {
259
274
  try {
260
275
  // internalCommit will throw if it encounters an issue
261
276
  await this.internalCommit(blockId, request.actionId, request.rev, storage);
262
277
  } catch (err) {
263
- // TODO: Recover as best we can. Rollback or handle partial commit? For now, return failure.
278
+ // Partial-commit recovery: a retry with the same (actionId, rev) will treat
279
+ // already-done blocks as idempotent no-ops and advance the remainder.
264
280
  return {
265
281
  success: false,
266
282
  reason: err instanceof Error ? err.message : 'Unknown error during commit'
@@ -276,6 +292,19 @@ export class StorageRepo implements IRepo {
276
292
  return { success: true };
277
293
  }
278
294
 
295
+ /**
296
+ * Reconciles `metadata.latest` for a single block with the highest contiguous
297
+ * fully-promoted revision in durable storage. Use after a crash between
298
+ * `promotePendingTransaction` and `setLatest` when retry-commit cannot help
299
+ * (the pending record is already gone) but the revision and committed-log entry
300
+ * are durable. Idempotent and monotonic.
301
+ */
302
+ async recoverBlock(blockId: BlockId): Promise<void> {
303
+ log('recoverBlock blockId=%s', blockId);
304
+ const storage = this.createBlockStorage(blockId);
305
+ await storage.recover();
306
+ }
307
+
279
308
  private async internalCommit(blockId: BlockId, actionId: ActionId, rev: number, storage: IBlockStorage): Promise<void> {
280
309
  // Note: This method is called within the locked critical section of commit()
281
310
  // So, operations like getPendingTransaction, getLatest, getBlock, saveMaterializedBlock,
@@ -1,8 +1,7 @@
1
- import type { ComponentLogger, Startable, Stream } from '@libp2p/interface';
1
+ import type { ComponentLogger, Startable } from '@libp2p/interface';
2
2
  import type { IRepo } from '@optimystic/db-core';
3
3
  import { buildSyncProtocol, type SyncRequest, type SyncResponse } from './protocol.js';
4
4
  import { pipe } from 'it-pipe';
5
- import { fromString as u8FromString } from 'uint8arrays/from-string';
6
5
  import { toString as u8ToString } from 'uint8arrays/to-string';
7
6
  import * as lp from 'it-length-prefixed';
8
7
  import type { Uint8ArrayList } from 'uint8arraylist';
@@ -37,7 +36,7 @@ export class SyncService implements Startable {
37
36
  private readonly registrar: { handle: (...args: any[]) => Promise<void>, unhandle: (...args: any[]) => Promise<void> };
38
37
 
39
38
  constructor(
40
- private readonly components: SyncServiceComponents,
39
+ components: SyncServiceComponents,
41
40
  init: SyncServiceInit = {}
42
41
  ) {
43
42
  this.log = components.logger.forComponent('db-p2p:sync-service');
@@ -0,0 +1 @@
1
+ export * from './mesh-harness.js';