@optimystic/db-p2p 0.24.1 → 0.25.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (257) hide show
  1. package/{readme.md → README.md} +688 -668
  2. package/dist/src/cluster/block-transfer-service.d.ts +133 -10
  3. package/dist/src/cluster/block-transfer-service.d.ts.map +1 -1
  4. package/dist/src/cluster/block-transfer-service.js +155 -9
  5. package/dist/src/cluster/block-transfer-service.js.map +1 -1
  6. package/dist/src/cluster/block-transfer.d.ts +67 -7
  7. package/dist/src/cluster/block-transfer.d.ts.map +1 -1
  8. package/dist/src/cluster/block-transfer.js +127 -24
  9. package/dist/src/cluster/block-transfer.js.map +1 -1
  10. package/dist/src/cluster/certified-claims.d.ts +170 -0
  11. package/dist/src/cluster/certified-claims.d.ts.map +1 -0
  12. package/dist/src/cluster/certified-claims.js +200 -0
  13. package/dist/src/cluster/certified-claims.js.map +1 -0
  14. package/dist/src/cluster/client-signature-verifier.d.ts +23 -0
  15. package/dist/src/cluster/client-signature-verifier.d.ts.map +1 -0
  16. package/dist/src/cluster/client-signature-verifier.js +33 -0
  17. package/dist/src/cluster/client-signature-verifier.js.map +1 -0
  18. package/dist/src/cluster/cluster-policy.d.ts +9 -3
  19. package/dist/src/cluster/cluster-policy.d.ts.map +1 -1
  20. package/dist/src/cluster/cluster-policy.js +111 -27
  21. package/dist/src/cluster/cluster-policy.js.map +1 -1
  22. package/dist/src/cluster/cluster-repo.d.ts +140 -11
  23. package/dist/src/cluster/cluster-repo.d.ts.map +1 -1
  24. package/dist/src/cluster/cluster-repo.js +485 -29
  25. package/dist/src/cluster/cluster-repo.js.map +1 -1
  26. package/dist/src/cluster/commit-proof.d.ts +134 -0
  27. package/dist/src/cluster/commit-proof.d.ts.map +1 -0
  28. package/dist/src/cluster/commit-proof.js +263 -0
  29. package/dist/src/cluster/commit-proof.js.map +1 -0
  30. package/dist/src/cluster/quorum-restore.d.ts +95 -8
  31. package/dist/src/cluster/quorum-restore.d.ts.map +1 -1
  32. package/dist/src/cluster/quorum-restore.js +113 -14
  33. package/dist/src/cluster/quorum-restore.js.map +1 -1
  34. package/dist/src/cluster/rebalance-monitor.d.ts +84 -0
  35. package/dist/src/cluster/rebalance-monitor.d.ts.map +1 -1
  36. package/dist/src/cluster/rebalance-monitor.js +207 -5
  37. package/dist/src/cluster/rebalance-monitor.js.map +1 -1
  38. package/dist/src/cluster/reconcile-block.d.ts +27 -7
  39. package/dist/src/cluster/reconcile-block.d.ts.map +1 -1
  40. package/dist/src/cluster/reconcile-block.js +179 -38
  41. package/dist/src/cluster/reconcile-block.js.map +1 -1
  42. package/dist/src/cluster/service.d.ts +21 -4
  43. package/dist/src/cluster/service.d.ts.map +1 -1
  44. package/dist/src/cluster/service.js +28 -17
  45. package/dist/src/cluster/service.js.map +1 -1
  46. package/dist/src/cluster/spread-on-churn.d.ts +8 -2
  47. package/dist/src/cluster/spread-on-churn.d.ts.map +1 -1
  48. package/dist/src/cluster/spread-on-churn.js +7 -7
  49. package/dist/src/cluster/spread-on-churn.js.map +1 -1
  50. package/dist/src/cohort-topic/host.d.ts.map +1 -1
  51. package/dist/src/cohort-topic/host.js +8 -7
  52. package/dist/src/cohort-topic/host.js.map +1 -1
  53. package/dist/src/cohort-topic/stream-util.d.ts +7 -6
  54. package/dist/src/cohort-topic/stream-util.d.ts.map +1 -1
  55. package/dist/src/cohort-topic/stream-util.js +14 -41
  56. package/dist/src/cohort-topic/stream-util.js.map +1 -1
  57. package/dist/src/dispute/cascade.d.ts +7 -8
  58. package/dist/src/dispute/cascade.d.ts.map +1 -1
  59. package/dist/src/dispute/cascade.js +24 -7
  60. package/dist/src/dispute/cascade.js.map +1 -1
  61. package/dist/src/dispute/dispute-service.d.ts +5 -0
  62. package/dist/src/dispute/dispute-service.d.ts.map +1 -1
  63. package/dist/src/dispute/dispute-service.js +13 -3
  64. package/dist/src/dispute/dispute-service.js.map +1 -1
  65. package/dist/src/dispute/invalidation.d.ts +33 -15
  66. package/dist/src/dispute/invalidation.d.ts.map +1 -1
  67. package/dist/src/dispute/invalidation.js +127 -34
  68. package/dist/src/dispute/invalidation.js.map +1 -1
  69. package/dist/src/dispute/service.d.ts.map +1 -1
  70. package/dist/src/dispute/service.js +2 -1
  71. package/dist/src/dispute/service.js.map +1 -1
  72. package/dist/src/index.d.ts +7 -0
  73. package/dist/src/index.d.ts.map +1 -1
  74. package/dist/src/index.js +7 -0
  75. package/dist/src/index.js.map +1 -1
  76. package/dist/src/libp2p-key-network.d.ts +78 -6
  77. package/dist/src/libp2p-key-network.d.ts.map +1 -1
  78. package/dist/src/libp2p-key-network.js +136 -57
  79. package/dist/src/libp2p-key-network.js.map +1 -1
  80. package/dist/src/libp2p-node-base.d.ts +38 -7
  81. package/dist/src/libp2p-node-base.d.ts.map +1 -1
  82. package/dist/src/libp2p-node-base.js +150 -62
  83. package/dist/src/libp2p-node-base.js.map +1 -1
  84. package/dist/src/logger.d.ts +2 -2
  85. package/dist/src/logger.js +2 -2
  86. package/dist/src/network/open-protocol-stream.d.ts +56 -0
  87. package/dist/src/network/open-protocol-stream.d.ts.map +1 -0
  88. package/dist/src/network/open-protocol-stream.js +60 -0
  89. package/dist/src/network/open-protocol-stream.js.map +1 -0
  90. package/dist/src/network/register-protocol-handler.d.ts +66 -0
  91. package/dist/src/network/register-protocol-handler.d.ts.map +1 -0
  92. package/dist/src/network/register-protocol-handler.js +51 -0
  93. package/dist/src/network/register-protocol-handler.js.map +1 -0
  94. package/dist/src/peer-address-book.d.ts +128 -1
  95. package/dist/src/peer-address-book.d.ts.map +1 -1
  96. package/dist/src/peer-address-book.js +198 -15
  97. package/dist/src/peer-address-book.js.map +1 -1
  98. package/dist/src/pend-validation.d.ts +65 -0
  99. package/dist/src/pend-validation.d.ts.map +1 -0
  100. package/dist/src/pend-validation.js +69 -0
  101. package/dist/src/pend-validation.js.map +1 -0
  102. package/dist/src/reactivity/notify-transport.d.ts.map +1 -1
  103. package/dist/src/reactivity/notify-transport.js +2 -1
  104. package/dist/src/reactivity/notify-transport.js.map +1 -1
  105. package/dist/src/reactivity/push-state-gossip.d.ts.map +1 -1
  106. package/dist/src/reactivity/push-state-gossip.js +2 -1
  107. package/dist/src/reactivity/push-state-gossip.js.map +1 -1
  108. package/dist/src/repo/cluster-coordinator.d.ts +32 -6
  109. package/dist/src/repo/cluster-coordinator.d.ts.map +1 -1
  110. package/dist/src/repo/cluster-coordinator.js +44 -10
  111. package/dist/src/repo/cluster-coordinator.js.map +1 -1
  112. package/dist/src/repo/coordinator-repo.d.ts +181 -25
  113. package/dist/src/repo/coordinator-repo.d.ts.map +1 -1
  114. package/dist/src/repo/coordinator-repo.js +650 -62
  115. package/dist/src/repo/coordinator-repo.js.map +1 -1
  116. package/dist/src/repo/served-repo-proxy.d.ts +30 -0
  117. package/dist/src/repo/served-repo-proxy.d.ts.map +1 -0
  118. package/dist/src/repo/served-repo-proxy.js +50 -0
  119. package/dist/src/repo/served-repo-proxy.js.map +1 -0
  120. package/dist/src/repo/service.d.ts +22 -1
  121. package/dist/src/repo/service.d.ts.map +1 -1
  122. package/dist/src/repo/service.js +25 -13
  123. package/dist/src/repo/service.js.map +1 -1
  124. package/dist/src/reputation/types.d.ts +3 -1
  125. package/dist/src/reputation/types.d.ts.map +1 -1
  126. package/dist/src/reputation/types.js +3 -1
  127. package/dist/src/reputation/types.js.map +1 -1
  128. package/dist/src/rn.d.ts +24 -2
  129. package/dist/src/rn.d.ts.map +1 -1
  130. package/dist/src/rn.js +33 -4
  131. package/dist/src/rn.js.map +1 -1
  132. package/dist/src/routing/libp2p-known-peers.d.ts.map +1 -1
  133. package/dist/src/routing/libp2p-known-peers.js +5 -0
  134. package/dist/src/routing/libp2p-known-peers.js.map +1 -1
  135. package/dist/src/storage/block-archive.d.ts +153 -0
  136. package/dist/src/storage/block-archive.d.ts.map +1 -0
  137. package/dist/src/storage/block-archive.js +194 -0
  138. package/dist/src/storage/block-archive.js.map +1 -0
  139. package/dist/src/storage/block-latch.d.ts +92 -0
  140. package/dist/src/storage/block-latch.d.ts.map +1 -0
  141. package/dist/src/storage/block-latch.js +132 -0
  142. package/dist/src/storage/block-latch.js.map +1 -0
  143. package/dist/src/storage/block-storage.d.ts +169 -12
  144. package/dist/src/storage/block-storage.d.ts.map +1 -1
  145. package/dist/src/storage/block-storage.js +415 -152
  146. package/dist/src/storage/block-storage.js.map +1 -1
  147. package/dist/src/storage/cached-raw-storage.d.ts +5 -1
  148. package/dist/src/storage/cached-raw-storage.d.ts.map +1 -1
  149. package/dist/src/storage/cached-raw-storage.js +16 -1
  150. package/dist/src/storage/cached-raw-storage.js.map +1 -1
  151. package/dist/src/storage/cached-store-driver.d.ts +14 -4
  152. package/dist/src/storage/cached-store-driver.d.ts.map +1 -1
  153. package/dist/src/storage/cached-store-driver.js +39 -5
  154. package/dist/src/storage/cached-store-driver.js.map +1 -1
  155. package/dist/src/storage/i-block-storage.d.ts +70 -19
  156. package/dist/src/storage/i-block-storage.d.ts.map +1 -1
  157. package/dist/src/storage/i-block-storage.js +21 -1
  158. package/dist/src/storage/i-block-storage.js.map +1 -1
  159. package/dist/src/storage/i-raw-storage.d.ts +38 -0
  160. package/dist/src/storage/i-raw-storage.d.ts.map +1 -1
  161. package/dist/src/storage/kv-raw-storage.d.ts +14 -8
  162. package/dist/src/storage/kv-raw-storage.d.ts.map +1 -1
  163. package/dist/src/storage/kv-raw-storage.js +29 -9
  164. package/dist/src/storage/kv-raw-storage.js.map +1 -1
  165. package/dist/src/storage/memory-store-driver.d.ts +4 -1
  166. package/dist/src/storage/memory-store-driver.d.ts.map +1 -1
  167. package/dist/src/storage/memory-store-driver.js +22 -2
  168. package/dist/src/storage/memory-store-driver.js.map +1 -1
  169. package/dist/src/storage/raw-store-codec.js +2 -2
  170. package/dist/src/storage/raw-store-driver.d.ts +44 -3
  171. package/dist/src/storage/raw-store-driver.d.ts.map +1 -1
  172. package/dist/src/storage/shared-cache-pool.d.ts +60 -3
  173. package/dist/src/storage/shared-cache-pool.d.ts.map +1 -1
  174. package/dist/src/storage/shared-cache-pool.js +68 -4
  175. package/dist/src/storage/shared-cache-pool.js.map +1 -1
  176. package/dist/src/storage/storage-repo.d.ts +204 -26
  177. package/dist/src/storage/storage-repo.d.ts.map +1 -1
  178. package/dist/src/storage/storage-repo.js +422 -119
  179. package/dist/src/storage/storage-repo.js.map +1 -1
  180. package/dist/src/storage/store-identity.d.ts +21 -0
  181. package/dist/src/storage/store-identity.d.ts.map +1 -0
  182. package/dist/src/storage/store-identity.js +33 -0
  183. package/dist/src/storage/store-identity.js.map +1 -0
  184. package/dist/src/storage/struct.d.ts +15 -0
  185. package/dist/src/storage/struct.d.ts.map +1 -1
  186. package/dist/src/storage/with-read-cache.d.ts +92 -0
  187. package/dist/src/storage/with-read-cache.d.ts.map +1 -0
  188. package/dist/src/storage/with-read-cache.js +132 -0
  189. package/dist/src/storage/with-read-cache.js.map +1 -0
  190. package/dist/src/sync/service.d.ts +13 -3
  191. package/dist/src/sync/service.d.ts.map +1 -1
  192. package/dist/src/sync/service.js +7 -30
  193. package/dist/src/sync/service.js.map +1 -1
  194. package/dist/src/testing/mesh-harness.d.ts +75 -2
  195. package/dist/src/testing/mesh-harness.d.ts.map +1 -1
  196. package/dist/src/testing/mesh-harness.js +193 -81
  197. package/dist/src/testing/mesh-harness.js.map +1 -1
  198. package/dist/src/testing/raw-storage-conformance.d.ts +8 -0
  199. package/dist/src/testing/raw-storage-conformance.d.ts.map +1 -1
  200. package/dist/src/testing/raw-storage-conformance.js +97 -2
  201. package/dist/src/testing/raw-storage-conformance.js.map +1 -1
  202. package/package.json +5 -6
  203. package/src/cluster/block-transfer-service.ts +528 -312
  204. package/src/cluster/block-transfer.ts +547 -424
  205. package/src/cluster/certified-claims.ts +319 -0
  206. package/src/cluster/client-signature-verifier.ts +34 -0
  207. package/src/cluster/cluster-policy.ts +294 -203
  208. package/src/cluster/cluster-repo.ts +2501 -2034
  209. package/src/cluster/commit-proof.ts +344 -0
  210. package/src/cluster/quorum-restore.ts +348 -223
  211. package/src/cluster/rebalance-monitor.ts +539 -254
  212. package/src/cluster/reconcile-block.ts +404 -203
  213. package/src/cluster/service.ts +308 -293
  214. package/src/cluster/spread-on-churn.ts +352 -346
  215. package/src/cohort-topic/host.ts +8 -7
  216. package/src/cohort-topic/stream-util.ts +14 -43
  217. package/src/dispute/cascade.ts +31 -15
  218. package/src/dispute/dispute-service.ts +706 -696
  219. package/src/dispute/invalidation.ts +162 -52
  220. package/src/dispute/service.ts +126 -125
  221. package/src/index.ts +7 -0
  222. package/src/libp2p-key-network.ts +142 -58
  223. package/src/libp2p-node-base.ts +1793 -1675
  224. package/src/logger.ts +27 -27
  225. package/src/network/open-protocol-stream.ts +87 -0
  226. package/src/network/register-protocol-handler.ts +85 -0
  227. package/src/peer-address-book.ts +360 -149
  228. package/src/pend-validation.ts +89 -0
  229. package/src/reactivity/notify-transport.ts +145 -144
  230. package/src/reactivity/push-state-gossip.ts +292 -291
  231. package/src/repo/cluster-coordinator.ts +65 -11
  232. package/src/repo/coordinator-repo.ts +729 -71
  233. package/src/repo/served-repo-proxy.ts +55 -0
  234. package/src/repo/service.ts +332 -313
  235. package/src/reputation/types.ts +3 -1
  236. package/src/rn.ts +34 -4
  237. package/src/routing/libp2p-known-peers.ts +31 -26
  238. package/src/storage/block-archive.ts +242 -0
  239. package/src/storage/block-latch.ts +144 -0
  240. package/src/storage/block-storage.ts +787 -498
  241. package/src/storage/cached-raw-storage.ts +21 -1
  242. package/src/storage/cached-store-driver.ts +898 -859
  243. package/src/storage/i-block-storage.ts +172 -116
  244. package/src/storage/i-raw-storage.ts +40 -0
  245. package/src/storage/kv-raw-storage.ts +34 -9
  246. package/src/storage/memory-store-driver.ts +26 -2
  247. package/src/storage/raw-store-codec.ts +42 -42
  248. package/src/storage/raw-store-driver.ts +45 -3
  249. package/src/storage/shared-cache-pool.ts +516 -452
  250. package/src/storage/storage-repo.ts +1421 -1014
  251. package/src/storage/store-identity.ts +41 -0
  252. package/src/storage/struct.ts +52 -34
  253. package/src/storage/with-read-cache.ts +182 -0
  254. package/src/sync/service.ts +178 -198
  255. package/src/testing/cohort-topic-mesh-harness.ts +673 -673
  256. package/src/testing/mesh-harness.ts +524 -344
  257. package/src/testing/raw-storage-conformance.ts +114 -2
@@ -1,313 +1,332 @@
1
- import { pipe } from 'it-pipe'
2
- import { decode as lpDecode, encode as lpEncode } from 'it-length-prefixed'
3
- import type { Startable, Logger, Stream, Connection, StreamHandler, PeerId, Libp2p } from '@libp2p/interface'
4
- import type { IRepo, RepoMessage } from '@optimystic/db-core'
5
- import { blockIdsForTransforms } from '@optimystic/db-core'
6
- import { peersEqual } from '../peer-utils.js'
7
- import { encodePeers, type RedirectPayload } from './redirect.js'
8
- import { MAX_BLOCK_MESSAGE_BYTES } from '../protocol-limits.js'
9
- import type { Uint8ArrayList } from 'uint8arraylist'
10
- import { createLogger } from '../logger.js'
11
- import { createInboundStreamAuthorization, type InboundStreamAuthorization, type InboundStreamAuthorizationInit } from '../inbound-authorization.js'
12
-
13
- const debugLog = createLogger('repo-service')
14
-
15
- // Define Components interface
16
- interface BaseComponents {
17
- logger: { forComponent: (name: string) => Logger },
18
- registrar: {
19
- handle: (protocol: string, handler: StreamHandler, options: any) => Promise<void>
20
- unhandle: (protocol: string) => Promise<void>
21
- }
22
- }
23
-
24
- export interface NetworkManagerLike {
25
- getCluster(key: Uint8Array): Promise<PeerId[]>
26
- }
27
-
28
- export type RepoServiceComponents = BaseComponents & {
29
- repo: IRepo
30
- networkManager?: NetworkManagerLike
31
- peerId?: PeerId
32
- getConnectionAddrs?: (peerId: PeerId) => string[]
33
- /**
34
- * Optional libp2p node. The production wiring injects the node post-construction
35
- * via {@link RepoService.setLibp2p} (the `components.libp2p` proxy does not
36
- * reliably resolve from inside a service at request time); this field is a
37
- * best-effort fallback resolver used only when no node has been injected.
38
- */
39
- libp2p?: Libp2p
40
- }
41
-
42
- export type RepoServiceInit = InboundStreamAuthorizationInit & {
43
- protocol?: string,
44
- protocolPrefix?: string,
45
- maxInboundStreams?: number,
46
- maxOutboundStreams?: number,
47
- logPrefix?: string,
48
- kBucketSize?: number,
49
- /**
50
- * Responsibility K - the replica set size for determining cluster membership.
51
- * This is distinct from kBucketSize (DHT routing).
52
- * When set, this determines how many peers (by XOR distance) are considered
53
- * responsible for a key. If this node is not in the top responsibilityK peers,
54
- * it will redirect requests to closer peers.
55
- * Default: 1 (only the closest peer handles requests)
56
- */
57
- responsibilityK?: number,
58
- }
59
-
60
- export function repoService(init: RepoServiceInit = {}): (components: RepoServiceComponents) => RepoService {
61
- return (components: RepoServiceComponents) => new RepoService(components, init);
62
- }
63
-
64
- /**
65
- * A libp2p service that handles repo protocol messages
66
- */
67
- export class RepoService implements Startable {
68
- private readonly protocol: string
69
- private readonly maxInboundStreams: number
70
- private readonly maxOutboundStreams: number
71
- private readonly log: Logger
72
- private readonly repo: IRepo
73
- private readonly components: RepoServiceComponents
74
- private running: boolean
75
- /** Responsibility K - how many peers are responsible for a key (for redirect decisions) */
76
- private readonly responsibilityK: number
77
- /**
78
- * The libp2p node, injected post-construction by the node wiring (see
79
- * libp2p-node-base.ts, mirroring how `networkManager`/`fret` receive theirs).
80
- * The libp2p `components.libp2p` proxy does NOT reliably resolve from inside a
81
- * service at request time, so the redirect path resolves the network manager,
82
- * self identity, and connection addrs through this explicitly-set reference.
83
- */
84
- private libp2pRef: Libp2p | undefined
85
- /** Optional embedder authorization gate; `undefined` (the default) means no check runs. */
86
- private readonly authorization: InboundStreamAuthorization | undefined
87
-
88
- constructor(components: RepoServiceComponents, init: RepoServiceInit = {}) {
89
- this.components = components
90
- const computed = init.protocol ?? (init.protocolPrefix ?? '/db-p2p') + '/repo/1.0.0'
91
- this.protocol = computed
92
- this.maxInboundStreams = init.maxInboundStreams ?? 32
93
- this.maxOutboundStreams = init.maxOutboundStreams ?? 64
94
- this.log = components.logger.forComponent(init.logPrefix ?? 'db-p2p:repo-service')
95
- this.repo = components.repo
96
- this.running = false
97
- this.responsibilityK = init.responsibilityK ?? 1
98
- this.authorization = createInboundStreamAuthorization(init, this.protocol, (msg, ...args) => this.log.error(msg, ...args))
99
- }
100
-
101
- readonly [Symbol.toStringTag] = '@libp2p/repo-service'
102
-
103
- /**
104
- * Inject the running libp2p node. Called once post-construction by the node
105
- * wiring so the redirect path can resolve the network manager / self id / addrs.
106
- */
107
- setLibp2p(libp2p: Libp2p): void {
108
- this.libp2pRef = libp2p
109
- }
110
-
111
- /** Resolve the libp2p node: the injected ref first, then the (best-effort) components proxy. */
112
- private getLibp2p(): Libp2p | undefined {
113
- return this.libp2pRef ?? (this.components as any).libp2p
114
- }
115
-
116
- /**
117
- * Start the service
118
- */
119
- async start(): Promise<void> {
120
- if (this.running) {
121
- return
122
- }
123
-
124
- await this.components.registrar.handle(this.protocol, this.handleIncomingStream.bind(this), {
125
- maxInboundStreams: this.maxInboundStreams,
126
- maxOutboundStreams: this.maxOutboundStreams
127
- })
128
-
129
- this.running = true
130
- }
131
-
132
- /**
133
- * Stop the service
134
- */
135
- async stop(): Promise<void> {
136
- if (!this.running) {
137
- return
138
- }
139
-
140
- await this.components.registrar.unhandle(this.protocol)
141
- this.running = false
142
- }
143
-
144
- private getNetworkManager(): NetworkManagerLike | undefined {
145
- if (this.components.networkManager) return this.components.networkManager
146
- return (this.getLibp2p() as any)?.services?.networkManager as NetworkManagerLike | undefined
147
- }
148
-
149
- private getSelfId(): PeerId | undefined {
150
- if (this.components.peerId) return this.components.peerId
151
- return this.getLibp2p()?.peerId as PeerId | undefined
152
- }
153
-
154
- private getPeerAddrs(peerId: PeerId): string[] {
155
- if (this.components.getConnectionAddrs) return this.components.getConnectionAddrs(peerId)
156
- const libp2p = this.getLibp2p() as any
157
- if (!libp2p?.getConnections) return []
158
- const conns: any[] = libp2p.getConnections(peerId) ?? []
159
- const addrs: string[] = []
160
- for (const c of conns) {
161
- const addr = c.remoteAddr?.toString?.()
162
- if (addr) addrs.push(addr)
163
- }
164
- return addrs
165
- }
166
-
167
- /**
168
- * Derive the redirect routing key and op name for a single operation.
169
- *
170
- * The key MUST be the block the corresponding handler actually coordinates and
171
- * verifies responsibility on, so redirect routing stays consistent with where the
172
- * op is executed:
173
- * - get → blockIds[0]
174
- * - pend → blockIdsForTransforms(transforms)[0]
175
- * - cancel actionRef.blockIds[0]
176
- * - commit blockIds[0] (CoordinatorRepo.commit anchors consensus on
177
- * getClusterSize(blockIds[0]) / executeClusterTransaction(blockIds[0]) and guards
178
- * with verifyResponsibility(blockIds) NOT tailId; for a per-block commit batch
179
- * whose blockIds[0] !== tailId, keying on tailId redirected the commit to the
180
- * collection tail's cluster, which then fails verifyResponsibility for the non-tail block.)
181
- *
182
- * Returns blockKey === undefined when the op carries no routable key (e.g. a cancel
183
- * with an empty blockIds list), in which case the caller handles it locally without a
184
- * redirect check.
185
- */
186
- deriveBlockKey(operation: RepoMessage['operations'][number]): { blockKey: string | undefined, opName: string } {
187
- if ('get' in operation) {
188
- return { blockKey: operation.get.blockIds[0], opName: 'get' }
189
- }
190
- if ('pend' in operation) {
191
- return { blockKey: blockIdsForTransforms(operation.pend.transforms)[0], opName: 'pend' }
192
- }
193
- if ('cancel' in operation) {
194
- return { blockKey: operation.cancel.actionRef.blockIds[0], opName: 'cancel' }
195
- }
196
- if ('commit' in operation) {
197
- return { blockKey: operation.commit.blockIds[0], opName: 'commit' }
198
- }
199
- return { blockKey: undefined, opName: 'unknown' }
200
- }
201
-
202
- /**
203
- * Check if this node should redirect the request for a given key.
204
- * Returns a RedirectPayload if not responsible, null if should handle locally.
205
- * Also attaches cluster info to the message for downstream use.
206
- */
207
- async checkRedirect(blockKey: string, opName: string, message: RepoMessage): Promise<RedirectPayload | null> {
208
- const nm = this.getNetworkManager()
209
- if (!nm) return null
210
-
211
- // Pass the RAW encoded block-key bytes to getCluster. getCluster hashes
212
- // internally (hashKey == sha256), so the responsible-set coordinate becomes
213
- // hashKey(encode(blockKey)) identical to how the cluster coordinator
214
- // derives it (ClusterCoordinator.getClusterForBlock → findCluster(encode(blockId))).
215
- // Pre-hashing here would double-hash (hashKey(sha256(encode(blockKey)))), placing
216
- // the cohort at an unrelated ring coordinate and redirecting requests the
217
- // coordinator legitimately routed to this peer.
218
- const key = new TextEncoder().encode(blockKey)
219
- const cluster = await nm.getCluster(key)
220
- ;(message as any).cluster = cluster.map((p: PeerId) => p.toString?.() ?? String(p))
221
-
222
- const selfId = this.getSelfId()
223
- if (!selfId) return null
224
-
225
- const isMember = cluster.some((p: PeerId) => peersEqual(p, selfId))
226
- const smallMesh = cluster.length < this.responsibilityK
227
-
228
- if (!smallMesh && !isMember) {
229
- const peers = cluster.filter((p: PeerId) => !peersEqual(p, selfId))
230
- debugLog('redirect op=%s blockKey=%s cluster=%d', opName, blockKey, cluster.length)
231
- return encodePeers(peers.map((pid: PeerId) => ({
232
- id: pid.toString(),
233
- addrs: this.getPeerAddrs(pid)
234
- })))
235
- }
236
-
237
- return null
238
- }
239
-
240
- /**
241
- * Handle incoming streams on the repo protocol
242
- */
243
- private handleIncomingStream(stream: Stream, connection?: Connection): void {
244
- const peerId = connection?.remotePeer
245
-
246
- const processStream = async function* (this: RepoService, source: AsyncIterable<Uint8ArrayList>) {
247
- for await (const msg of source) {
248
- // Decode the message
249
- const decoded = new TextDecoder().decode(msg.subarray())
250
- const message = JSON.parse(decoded) as RepoMessage
251
-
252
- // Process each operation. Derive the redirect routing key once (keyed on the
253
- // block the handler actually coordinates), redirect-check it, then dispatch.
254
- const operation = message.operations[0]
255
- const { blockKey, opName } = this.deriveBlockKey(operation)
256
- const redirect = blockKey !== undefined
257
- ? await this.checkRedirect(blockKey, opName, message)
258
- : null
259
-
260
- let response: any
261
- if (redirect) {
262
- response = redirect
263
- } else if ('get' in operation) {
264
- // No `skipClusterFetch` here: a read on this protocol comes from ANOTHER node, so
265
- // it must reach `CoordinatorRepo`'s cohort consult answering a bare absent for
266
- // a block a cohort peer holds is an authoritative lie the transactor never
267
- // retries. Only the sync protocol keeps the flag (`sync/service.ts`, where the
268
- // consult itself lands), and that is what stops the recursion.
269
- // NOTE: this also puts lazy read-repair on remote reads of locally-present blocks
270
- // — one consult per block per `readRepairWindowMs`, damped by a 1000-entry LRU of
271
- // block ids. If a working set wider than that LRU ever shows a consult on every
272
- // read, widen the LRU rather than reinstating the skip.
273
- response = await this.repo.get(operation.get, { expiration: message.expiration })
274
- } else if ('pend' in operation) {
275
- response = await this.repo.pend(operation.pend, { expiration: message.expiration })
276
- } else if ('cancel' in operation) {
277
- response = await this.repo.cancel(operation.cancel.actionRef, { expiration: message.expiration })
278
- } else if ('commit' in operation) {
279
- response = await this.repo.commit(operation.commit, { expiration: message.expiration })
280
- }
281
-
282
- // Encode and yield the response
283
- yield new TextEncoder().encode(JSON.stringify(response))
284
- // One request per stream: every real RepoClient sends exactly one request
285
- // per dial (see ProtocolClient.processMessage), so complete the generator
286
- // after the first response. A second frame a peer queued is then never read
287
- // or parsed. Mirrors sync/block-transfer.
288
- return
289
- }
290
- }
291
-
292
- void (async () => {
293
- try {
294
- // Authorization runs before ANY decoding or execution. Guarded on the field so a
295
- // node without a predicate keeps the original path untouched.
296
- if (this.authorization && await this.authorization.deny(stream, peerId?.toString())) return
297
- const responses = pipe(
298
- stream,
299
- (source) => lpDecode(source, { maxDataLength: MAX_BLOCK_MESSAGE_BYTES }),
300
- processStream.bind(this),
301
- (source) => lpEncode(source)
302
- )
303
- for await (const chunk of responses) {
304
- stream.send(chunk)
305
- }
306
- await stream.close()
307
- } catch (err) {
308
- this.log.error('error handling repo protocol message from %p - %e', peerId, err)
309
- stream.abort(err instanceof Error ? err : new Error(String(err)))
310
- }
311
- })()
312
- }
313
- }
1
+ import { pipe } from 'it-pipe'
2
+ import { decode as lpDecode, encode as lpEncode } from 'it-length-prefixed'
3
+ import type { Startable, Logger, Stream, Connection, StreamHandler, PeerId, Libp2p } from '@libp2p/interface'
4
+ import type { IRepo, RepoMessage } from '@optimystic/db-core'
5
+ import { blockIdsForTransforms } from '@optimystic/db-core'
6
+ import { peersEqual } from '../peer-utils.js'
7
+ import { encodePeers, type RedirectPayload } from './redirect.js'
8
+ import { MAX_BLOCK_MESSAGE_BYTES } from '../protocol-limits.js'
9
+ import type { Uint8ArrayList } from 'uint8arraylist'
10
+ import { createLogger } from '../logger.js'
11
+ import { publishableAddrsForPeer, type AddressLog, type DirectionalConnection } from '../peer-address-book.js'
12
+ import { createInboundStreamAuthorization, type InboundStreamAuthorization, type InboundStreamAuthorizationInit } from '../inbound-authorization.js'
13
+ import { registerProtocolHandler } from '../network/register-protocol-handler.js'
14
+
15
+ const debugLog = createLogger('repo-service')
16
+
17
+ // Define Components interface
18
+ interface BaseComponents {
19
+ logger: { forComponent: (name: string) => Logger },
20
+ registrar: {
21
+ handle: (protocol: string, handler: StreamHandler, options: any) => Promise<void>
22
+ unhandle: (protocol: string) => Promise<void>
23
+ }
24
+ }
25
+
26
+ export interface NetworkManagerLike {
27
+ getCluster(key: Uint8Array): Promise<PeerId[]>
28
+ }
29
+
30
+ export type RepoServiceComponents = BaseComponents & {
31
+ repo: IRepo
32
+ networkManager?: NetworkManagerLike
33
+ peerId?: PeerId
34
+ /**
35
+ * Optional resolver for the addresses this node may publish for a redirect target. Async
36
+ * because the answer includes the peer's own advertised addresses, which live in the
37
+ * peerStore see `publishableAddrsForPeer`. A synchronous `string[]` is still accepted so an
38
+ * embedder's connections-only stub keeps working.
39
+ */
40
+ getConnectionAddrs?: (peerId: PeerId) => string[] | Promise<string[]>
41
+ /**
42
+ * Optional libp2p node. The production wiring injects the node post-construction
43
+ * via {@link RepoService.setLibp2p} (the `components.libp2p` proxy does not
44
+ * reliably resolve from inside a service at request time); this field is a
45
+ * best-effort fallback resolver used only when no node has been injected.
46
+ */
47
+ libp2p?: Libp2p
48
+ }
49
+
50
+ export type RepoServiceInit = InboundStreamAuthorizationInit & {
51
+ protocol?: string,
52
+ protocolPrefix?: string,
53
+ maxInboundStreams?: number,
54
+ maxOutboundStreams?: number,
55
+ logPrefix?: string,
56
+ kBucketSize?: number,
57
+ /**
58
+ * Responsibility K - the replica set size for determining cluster membership.
59
+ * This is distinct from kBucketSize (DHT routing).
60
+ * When set, this determines how many peers (by XOR distance) are considered
61
+ * responsible for a key. If this node is not in the top responsibilityK peers,
62
+ * it will redirect requests to closer peers.
63
+ * Default: 1 (only the closest peer handles requests)
64
+ */
65
+ responsibilityK?: number,
66
+ }
67
+
68
+ export function repoService(init: RepoServiceInit = {}): (components: RepoServiceComponents) => RepoService {
69
+ return (components: RepoServiceComponents) => new RepoService(components, init);
70
+ }
71
+
72
+ /**
73
+ * A libp2p service that handles repo protocol messages
74
+ */
75
+ export class RepoService implements Startable {
76
+ private readonly protocol: string
77
+ private readonly maxInboundStreams: number
78
+ private readonly maxOutboundStreams: number
79
+ private readonly log: Logger
80
+ private readonly repo: IRepo
81
+ private readonly components: RepoServiceComponents
82
+ private running: boolean
83
+ /** Responsibility K - how many peers are responsible for a key (for redirect decisions) */
84
+ private readonly responsibilityK: number
85
+ /**
86
+ * The libp2p node, injected post-construction by the node wiring (see
87
+ * libp2p-node-base.ts, mirroring how `networkManager`/`fret` receive theirs).
88
+ * The libp2p `components.libp2p` proxy does NOT reliably resolve from inside a
89
+ * service at request time, so the redirect path resolves the network manager,
90
+ * self identity, and connection addrs through this explicitly-set reference.
91
+ */
92
+ private libp2pRef: Libp2p | undefined
93
+ /** Optional embedder authorization gate; `undefined` (the default) means no check runs. */
94
+ private readonly authorization: InboundStreamAuthorization | undefined
95
+ /**
96
+ * Sink for this service's `peer-address-book:*` lines — same reasoning as `ClusterService`'s:
97
+ * `this.log.error` would strand them under `db-p2p:repo-service:error`, outside the
98
+ * `optimystic:db-p2p:*` tree every other address-book line lives in.
99
+ */
100
+ private readonly addressLog: AddressLog
101
+
102
+ constructor(components: RepoServiceComponents, init: RepoServiceInit = {}) {
103
+ this.components = components
104
+ const computed = init.protocol ?? (init.protocolPrefix ?? '/db-p2p') + '/repo/1.0.0'
105
+ this.protocol = computed
106
+ this.maxInboundStreams = init.maxInboundStreams ?? 32
107
+ this.maxOutboundStreams = init.maxOutboundStreams ?? 64
108
+ this.log = components.logger.forComponent(init.logPrefix ?? 'db-p2p:repo-service')
109
+ this.addressLog = createLogger('peer-address-book', components.peerId?.toString())
110
+ this.repo = components.repo
111
+ this.running = false
112
+ this.responsibilityK = init.responsibilityK ?? 1
113
+ this.authorization = createInboundStreamAuthorization(init, this.protocol, (msg, ...args) => this.log.error(msg, ...args))
114
+ }
115
+
116
+ readonly [Symbol.toStringTag] = '@libp2p/repo-service'
117
+
118
+ /**
119
+ * Inject the running libp2p node. Called once post-construction by the node
120
+ * wiring so the redirect path can resolve the network manager / self id / addrs.
121
+ */
122
+ setLibp2p(libp2p: Libp2p): void {
123
+ this.libp2pRef = libp2p
124
+ }
125
+
126
+ /** Resolve the libp2p node: the injected ref first, then the (best-effort) components proxy. */
127
+ private getLibp2p(): Libp2p | undefined {
128
+ return this.libp2pRef ?? (this.components as any).libp2p
129
+ }
130
+
131
+ /**
132
+ * Start the service
133
+ */
134
+ async start(): Promise<void> {
135
+ if (this.running) {
136
+ return
137
+ }
138
+
139
+ await registerProtocolHandler(this.components.registrar, this.protocol, this.handleIncomingStream.bind(this), {
140
+ maxInboundStreams: this.maxInboundStreams,
141
+ maxOutboundStreams: this.maxOutboundStreams
142
+ })
143
+
144
+ this.running = true
145
+ }
146
+
147
+ /**
148
+ * Stop the service
149
+ */
150
+ async stop(): Promise<void> {
151
+ if (!this.running) {
152
+ return
153
+ }
154
+
155
+ await this.components.registrar.unhandle(this.protocol)
156
+ this.running = false
157
+ }
158
+
159
+ private getNetworkManager(): NetworkManagerLike | undefined {
160
+ if (this.components.networkManager) return this.components.networkManager
161
+ return (this.getLibp2p() as any)?.services?.networkManager as NetworkManagerLike | undefined
162
+ }
163
+
164
+ private getSelfId(): PeerId | undefined {
165
+ if (this.components.peerId) return this.components.peerId
166
+ return this.getLibp2p()?.peerId as PeerId | undefined
167
+ }
168
+
169
+ /**
170
+ * The addresses this node may publish for `peerId` in a redirect payload.
171
+ *
172
+ * This fallback is the PRODUCTION source for the repo service: `libp2p-node-base` injects no
173
+ * `getConnectionAddrs` here (the node arrives later, via `setLibp2p`), and unlike a cluster
174
+ * redirect there is no record whose embedded multiaddrs could stand in. A redirect goes to a
175
+ * THIRD party, so it asks `publishableAddrsForPeer` — the one definition, shared with
176
+ * `ClusterService` and `findCluster` rather than reading connections alone.
177
+ */
178
+ private async getPeerAddrs(peerId: PeerId): Promise<string[]> {
179
+ if (this.components.getConnectionAddrs) return await this.components.getConnectionAddrs(peerId)
180
+ const libp2p = this.getLibp2p() as any
181
+ if (!libp2p?.getConnections) return []
182
+ const conns: DirectionalConnection[] = libp2p.getConnections(peerId) ?? []
183
+ return await publishableAddrsForPeer(libp2p, conns, peerId, this.addressLog)
184
+ }
185
+
186
+ /**
187
+ * Derive the redirect routing key and op name for a single operation.
188
+ *
189
+ * The key MUST be the block the corresponding handler actually coordinates and
190
+ * verifies responsibility on, so redirect routing stays consistent with where the
191
+ * op is executed:
192
+ * - get → blockIds[0]
193
+ * - pend → blockIdsForTransforms(transforms)[0]
194
+ * - cancel actionRef.blockIds[0]
195
+ * - commit → blockIds[0] (CoordinatorRepo.commit anchors consensus on
196
+ * getClusterSize(blockIds[0]) / executeClusterTransaction(blockIds[0]) and guards
197
+ * with verifyResponsibility(blockIds) NOT tailId; for a per-block commit batch
198
+ * whose blockIds[0] !== tailId, keying on tailId redirected the commit to the
199
+ * collection tail's cluster, which then fails verifyResponsibility for the non-tail block.)
200
+ *
201
+ * Returns blockKey === undefined when the op carries no routable key (e.g. a cancel
202
+ * with an empty blockIds list), in which case the caller handles it locally without a
203
+ * redirect check.
204
+ */
205
+ deriveBlockKey(operation: RepoMessage['operations'][number]): { blockKey: string | undefined, opName: string } {
206
+ if ('get' in operation) {
207
+ return { blockKey: operation.get.blockIds[0], opName: 'get' }
208
+ }
209
+ if ('pend' in operation) {
210
+ return { blockKey: blockIdsForTransforms(operation.pend.transforms)[0], opName: 'pend' }
211
+ }
212
+ if ('cancel' in operation) {
213
+ return { blockKey: operation.cancel.actionRef.blockIds[0], opName: 'cancel' }
214
+ }
215
+ if ('commit' in operation) {
216
+ return { blockKey: operation.commit.blockIds[0], opName: 'commit' }
217
+ }
218
+ return { blockKey: undefined, opName: 'unknown' }
219
+ }
220
+
221
+ /**
222
+ * Check if this node should redirect the request for a given key.
223
+ * Returns a RedirectPayload if not responsible, null if should handle locally.
224
+ * Also attaches cluster info to the message for downstream use.
225
+ */
226
+ async checkRedirect(blockKey: string, opName: string, message: RepoMessage): Promise<RedirectPayload | null> {
227
+ const nm = this.getNetworkManager()
228
+ if (!nm) return null
229
+
230
+ // Pass the RAW encoded block-key bytes to getCluster. getCluster hashes
231
+ // internally (hashKey == sha256), so the responsible-set coordinate becomes
232
+ // hashKey(encode(blockKey)) — identical to how the cluster coordinator
233
+ // derives it (ClusterCoordinator.getClusterForBlock → findCluster(encode(blockId))).
234
+ // Pre-hashing here would double-hash (hashKey(sha256(encode(blockKey)))), placing
235
+ // the cohort at an unrelated ring coordinate and redirecting requests the
236
+ // coordinator legitimately routed to this peer.
237
+ const key = new TextEncoder().encode(blockKey)
238
+ const cluster = await nm.getCluster(key)
239
+ ;(message as any).cluster = cluster.map((p: PeerId) => p.toString?.() ?? String(p))
240
+
241
+ const selfId = this.getSelfId()
242
+ if (!selfId) return null
243
+
244
+ const isMember = cluster.some((p: PeerId) => peersEqual(p, selfId))
245
+ const smallMesh = cluster.length < this.responsibilityK
246
+
247
+ if (!smallMesh && !isMember) {
248
+ const peers = cluster.filter((p: PeerId) => !peersEqual(p, selfId))
249
+ debugLog('redirect op=%s blockKey=%s cluster=%d', opName, blockKey, cluster.length)
250
+ return encodePeers(await Promise.all(peers.map(async (pid: PeerId) => ({
251
+ id: pid.toString(),
252
+ addrs: await this.getPeerAddrs(pid)
253
+ }))))
254
+ }
255
+
256
+ return null
257
+ }
258
+
259
+ /**
260
+ * Handle incoming streams on the repo protocol
261
+ */
262
+ private handleIncomingStream(stream: Stream, connection?: Connection): void {
263
+ const peerId = connection?.remotePeer
264
+
265
+ const processStream = async function* (this: RepoService, source: AsyncIterable<Uint8ArrayList>) {
266
+ for await (const msg of source) {
267
+ // Decode the message
268
+ const decoded = new TextDecoder().decode(msg.subarray())
269
+ const message = JSON.parse(decoded) as RepoMessage
270
+
271
+ // Process each operation. Derive the redirect routing key once (keyed on the
272
+ // block the handler actually coordinates), redirect-check it, then dispatch.
273
+ const operation = message.operations[0]
274
+ const { blockKey, opName } = this.deriveBlockKey(operation)
275
+ const redirect = blockKey !== undefined
276
+ ? await this.checkRedirect(blockKey, opName, message)
277
+ : null
278
+
279
+ let response: any
280
+ if (redirect) {
281
+ response = redirect
282
+ } else if ('get' in operation) {
283
+ // No `skipClusterFetch` here: a read on this protocol comes from ANOTHER node, so
284
+ // it must reach `CoordinatorRepo`'s cohort consult answering a bare absent for
285
+ // a block a cohort peer holds is an authoritative lie the transactor never
286
+ // retries. Only the sync protocol keeps the flag (`sync/service.ts`, where the
287
+ // consult itself lands), and that is what stops the recursion.
288
+ // NOTE: this also puts lazy read-repair on remote reads of locally-present blocks
289
+ // — one consult per block per `readRepairWindowMs`, damped by a 1000-entry LRU of
290
+ // block ids. If a working set wider than that LRU ever shows a consult on every
291
+ // read, widen the LRU rather than reinstating the skip.
292
+ response = await this.repo.get(operation.get, { expiration: message.expiration })
293
+ } else if ('pend' in operation) {
294
+ response = await this.repo.pend(operation.pend, { expiration: message.expiration })
295
+ } else if ('cancel' in operation) {
296
+ response = await this.repo.cancel(operation.cancel.actionRef, { expiration: message.expiration })
297
+ } else if ('commit' in operation) {
298
+ response = await this.repo.commit(operation.commit, { expiration: message.expiration })
299
+ }
300
+
301
+ // Encode and yield the response
302
+ yield new TextEncoder().encode(JSON.stringify(response))
303
+ // One request per stream: every real RepoClient sends exactly one request
304
+ // per dial (see ProtocolClient.processMessage), so complete the generator
305
+ // after the first response. A second frame a peer queued is then never read
306
+ // or parsed. Mirrors sync/block-transfer.
307
+ return
308
+ }
309
+ }
310
+
311
+ void (async () => {
312
+ try {
313
+ // Authorization runs before ANY decoding or execution. Guarded on the field so a
314
+ // node without a predicate keeps the original path untouched.
315
+ if (this.authorization && await this.authorization.deny(stream, peerId?.toString())) return
316
+ const responses = pipe(
317
+ stream,
318
+ (source) => lpDecode(source, { maxDataLength: MAX_BLOCK_MESSAGE_BYTES }),
319
+ processStream.bind(this),
320
+ (source) => lpEncode(source)
321
+ )
322
+ for await (const chunk of responses) {
323
+ stream.send(chunk)
324
+ }
325
+ await stream.close()
326
+ } catch (err) {
327
+ this.log.error('error handling repo protocol message from %p - %e', peerId, err)
328
+ stream.abort(err instanceof Error ? err : new Error(String(err)))
329
+ }
330
+ })()
331
+ }
332
+ }