@optimystic/db-p2p 0.24.2 → 0.25.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/{readme.md → README.md} +688 -668
- package/dist/src/cluster/block-transfer-service.d.ts +133 -10
- package/dist/src/cluster/block-transfer-service.d.ts.map +1 -1
- package/dist/src/cluster/block-transfer-service.js +155 -9
- package/dist/src/cluster/block-transfer-service.js.map +1 -1
- package/dist/src/cluster/block-transfer.d.ts +67 -7
- package/dist/src/cluster/block-transfer.d.ts.map +1 -1
- package/dist/src/cluster/block-transfer.js +127 -24
- package/dist/src/cluster/block-transfer.js.map +1 -1
- package/dist/src/cluster/certified-claims.d.ts +170 -0
- package/dist/src/cluster/certified-claims.d.ts.map +1 -0
- package/dist/src/cluster/certified-claims.js +200 -0
- package/dist/src/cluster/certified-claims.js.map +1 -0
- package/dist/src/cluster/client-signature-verifier.d.ts +23 -0
- package/dist/src/cluster/client-signature-verifier.d.ts.map +1 -0
- package/dist/src/cluster/client-signature-verifier.js +33 -0
- package/dist/src/cluster/client-signature-verifier.js.map +1 -0
- package/dist/src/cluster/cluster-policy.d.ts +9 -3
- package/dist/src/cluster/cluster-policy.d.ts.map +1 -1
- package/dist/src/cluster/cluster-policy.js +111 -27
- package/dist/src/cluster/cluster-policy.js.map +1 -1
- package/dist/src/cluster/cluster-repo.d.ts +140 -11
- package/dist/src/cluster/cluster-repo.d.ts.map +1 -1
- package/dist/src/cluster/cluster-repo.js +485 -29
- package/dist/src/cluster/cluster-repo.js.map +1 -1
- package/dist/src/cluster/commit-proof.d.ts +134 -0
- package/dist/src/cluster/commit-proof.d.ts.map +1 -0
- package/dist/src/cluster/commit-proof.js +263 -0
- package/dist/src/cluster/commit-proof.js.map +1 -0
- package/dist/src/cluster/quorum-restore.d.ts +95 -8
- package/dist/src/cluster/quorum-restore.d.ts.map +1 -1
- package/dist/src/cluster/quorum-restore.js +113 -14
- package/dist/src/cluster/quorum-restore.js.map +1 -1
- package/dist/src/cluster/rebalance-monitor.d.ts +84 -0
- package/dist/src/cluster/rebalance-monitor.d.ts.map +1 -1
- package/dist/src/cluster/rebalance-monitor.js +207 -5
- package/dist/src/cluster/rebalance-monitor.js.map +1 -1
- package/dist/src/cluster/reconcile-block.d.ts +27 -7
- package/dist/src/cluster/reconcile-block.d.ts.map +1 -1
- package/dist/src/cluster/reconcile-block.js +179 -38
- package/dist/src/cluster/reconcile-block.js.map +1 -1
- package/dist/src/cluster/service.d.ts +13 -4
- package/dist/src/cluster/service.d.ts.map +1 -1
- package/dist/src/cluster/service.js +17 -18
- package/dist/src/cluster/service.js.map +1 -1
- package/dist/src/cluster/spread-on-churn.d.ts +8 -2
- package/dist/src/cluster/spread-on-churn.d.ts.map +1 -1
- package/dist/src/cluster/spread-on-churn.js +7 -7
- package/dist/src/cluster/spread-on-churn.js.map +1 -1
- package/dist/src/cohort-topic/host.d.ts.map +1 -1
- package/dist/src/cohort-topic/host.js +8 -7
- package/dist/src/cohort-topic/host.js.map +1 -1
- package/dist/src/cohort-topic/stream-util.d.ts +7 -6
- package/dist/src/cohort-topic/stream-util.d.ts.map +1 -1
- package/dist/src/cohort-topic/stream-util.js +14 -41
- package/dist/src/cohort-topic/stream-util.js.map +1 -1
- package/dist/src/dispute/cascade.d.ts +7 -8
- package/dist/src/dispute/cascade.d.ts.map +1 -1
- package/dist/src/dispute/cascade.js +24 -7
- package/dist/src/dispute/cascade.js.map +1 -1
- package/dist/src/dispute/dispute-service.d.ts +5 -0
- package/dist/src/dispute/dispute-service.d.ts.map +1 -1
- package/dist/src/dispute/dispute-service.js +13 -3
- package/dist/src/dispute/dispute-service.js.map +1 -1
- package/dist/src/dispute/invalidation.d.ts +33 -15
- package/dist/src/dispute/invalidation.d.ts.map +1 -1
- package/dist/src/dispute/invalidation.js +127 -34
- package/dist/src/dispute/invalidation.js.map +1 -1
- package/dist/src/dispute/service.d.ts.map +1 -1
- package/dist/src/dispute/service.js +2 -1
- package/dist/src/dispute/service.js.map +1 -1
- package/dist/src/index.d.ts +7 -0
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +7 -0
- package/dist/src/index.js.map +1 -1
- package/dist/src/libp2p-key-network.d.ts +14 -10
- package/dist/src/libp2p-key-network.d.ts.map +1 -1
- package/dist/src/libp2p-key-network.js +28 -58
- package/dist/src/libp2p-key-network.js.map +1 -1
- package/dist/src/libp2p-node-base.d.ts +38 -7
- package/dist/src/libp2p-node-base.d.ts.map +1 -1
- package/dist/src/libp2p-node-base.js +148 -63
- package/dist/src/libp2p-node-base.js.map +1 -1
- package/dist/src/network/open-protocol-stream.d.ts +56 -0
- package/dist/src/network/open-protocol-stream.d.ts.map +1 -0
- package/dist/src/network/open-protocol-stream.js +60 -0
- package/dist/src/network/open-protocol-stream.js.map +1 -0
- package/dist/src/network/register-protocol-handler.d.ts +66 -0
- package/dist/src/network/register-protocol-handler.d.ts.map +1 -0
- package/dist/src/network/register-protocol-handler.js +51 -0
- package/dist/src/network/register-protocol-handler.js.map +1 -0
- package/dist/src/peer-address-book.d.ts +60 -2
- package/dist/src/peer-address-book.d.ts.map +1 -1
- package/dist/src/peer-address-book.js +89 -1
- package/dist/src/peer-address-book.js.map +1 -1
- package/dist/src/pend-validation.d.ts +65 -0
- package/dist/src/pend-validation.d.ts.map +1 -0
- package/dist/src/pend-validation.js +69 -0
- package/dist/src/pend-validation.js.map +1 -0
- package/dist/src/reactivity/notify-transport.d.ts.map +1 -1
- package/dist/src/reactivity/notify-transport.js +2 -1
- package/dist/src/reactivity/notify-transport.js.map +1 -1
- package/dist/src/reactivity/push-state-gossip.d.ts.map +1 -1
- package/dist/src/reactivity/push-state-gossip.js +2 -1
- package/dist/src/reactivity/push-state-gossip.js.map +1 -1
- package/dist/src/repo/cluster-coordinator.d.ts +32 -6
- package/dist/src/repo/cluster-coordinator.d.ts.map +1 -1
- package/dist/src/repo/cluster-coordinator.js +44 -10
- package/dist/src/repo/cluster-coordinator.js.map +1 -1
- package/dist/src/repo/coordinator-repo.d.ts +181 -25
- package/dist/src/repo/coordinator-repo.d.ts.map +1 -1
- package/dist/src/repo/coordinator-repo.js +650 -62
- package/dist/src/repo/coordinator-repo.js.map +1 -1
- package/dist/src/repo/served-repo-proxy.d.ts +30 -0
- package/dist/src/repo/served-repo-proxy.d.ts.map +1 -0
- package/dist/src/repo/served-repo-proxy.js +50 -0
- package/dist/src/repo/served-repo-proxy.js.map +1 -0
- package/dist/src/repo/service.d.ts +16 -1
- package/dist/src/repo/service.d.ts.map +1 -1
- package/dist/src/repo/service.js +18 -16
- package/dist/src/repo/service.js.map +1 -1
- package/dist/src/reputation/types.d.ts +3 -1
- package/dist/src/reputation/types.d.ts.map +1 -1
- package/dist/src/reputation/types.js +3 -1
- package/dist/src/reputation/types.js.map +1 -1
- package/dist/src/rn.d.ts +24 -2
- package/dist/src/rn.d.ts.map +1 -1
- package/dist/src/rn.js +33 -4
- package/dist/src/rn.js.map +1 -1
- package/dist/src/storage/block-archive.d.ts +153 -0
- package/dist/src/storage/block-archive.d.ts.map +1 -0
- package/dist/src/storage/block-archive.js +194 -0
- package/dist/src/storage/block-archive.js.map +1 -0
- package/dist/src/storage/block-latch.d.ts +92 -0
- package/dist/src/storage/block-latch.d.ts.map +1 -0
- package/dist/src/storage/block-latch.js +132 -0
- package/dist/src/storage/block-latch.js.map +1 -0
- package/dist/src/storage/block-storage.d.ts +169 -12
- package/dist/src/storage/block-storage.d.ts.map +1 -1
- package/dist/src/storage/block-storage.js +415 -152
- package/dist/src/storage/block-storage.js.map +1 -1
- package/dist/src/storage/cached-raw-storage.d.ts +5 -1
- package/dist/src/storage/cached-raw-storage.d.ts.map +1 -1
- package/dist/src/storage/cached-raw-storage.js +16 -1
- package/dist/src/storage/cached-raw-storage.js.map +1 -1
- package/dist/src/storage/cached-store-driver.d.ts +14 -4
- package/dist/src/storage/cached-store-driver.d.ts.map +1 -1
- package/dist/src/storage/cached-store-driver.js +39 -5
- package/dist/src/storage/cached-store-driver.js.map +1 -1
- package/dist/src/storage/i-block-storage.d.ts +70 -19
- package/dist/src/storage/i-block-storage.d.ts.map +1 -1
- package/dist/src/storage/i-block-storage.js +21 -1
- package/dist/src/storage/i-block-storage.js.map +1 -1
- package/dist/src/storage/i-raw-storage.d.ts +38 -0
- package/dist/src/storage/i-raw-storage.d.ts.map +1 -1
- package/dist/src/storage/kv-raw-storage.d.ts +14 -8
- package/dist/src/storage/kv-raw-storage.d.ts.map +1 -1
- package/dist/src/storage/kv-raw-storage.js +29 -9
- package/dist/src/storage/kv-raw-storage.js.map +1 -1
- package/dist/src/storage/memory-store-driver.d.ts +4 -1
- package/dist/src/storage/memory-store-driver.d.ts.map +1 -1
- package/dist/src/storage/memory-store-driver.js +22 -2
- package/dist/src/storage/memory-store-driver.js.map +1 -1
- package/dist/src/storage/raw-store-codec.js +2 -2
- package/dist/src/storage/raw-store-driver.d.ts +44 -3
- package/dist/src/storage/raw-store-driver.d.ts.map +1 -1
- package/dist/src/storage/shared-cache-pool.d.ts +60 -3
- package/dist/src/storage/shared-cache-pool.d.ts.map +1 -1
- package/dist/src/storage/shared-cache-pool.js +68 -4
- package/dist/src/storage/shared-cache-pool.js.map +1 -1
- package/dist/src/storage/storage-repo.d.ts +204 -26
- package/dist/src/storage/storage-repo.d.ts.map +1 -1
- package/dist/src/storage/storage-repo.js +422 -119
- package/dist/src/storage/storage-repo.js.map +1 -1
- package/dist/src/storage/store-identity.d.ts +21 -0
- package/dist/src/storage/store-identity.d.ts.map +1 -0
- package/dist/src/storage/store-identity.js +33 -0
- package/dist/src/storage/store-identity.js.map +1 -0
- package/dist/src/storage/struct.d.ts +15 -0
- package/dist/src/storage/struct.d.ts.map +1 -1
- package/dist/src/storage/with-read-cache.d.ts +92 -0
- package/dist/src/storage/with-read-cache.d.ts.map +1 -0
- package/dist/src/storage/with-read-cache.js +132 -0
- package/dist/src/storage/with-read-cache.js.map +1 -0
- package/dist/src/sync/service.d.ts +13 -3
- package/dist/src/sync/service.d.ts.map +1 -1
- package/dist/src/sync/service.js +7 -30
- package/dist/src/sync/service.js.map +1 -1
- package/dist/src/testing/mesh-harness.d.ts +75 -2
- package/dist/src/testing/mesh-harness.d.ts.map +1 -1
- package/dist/src/testing/mesh-harness.js +193 -81
- package/dist/src/testing/mesh-harness.js.map +1 -1
- package/dist/src/testing/raw-storage-conformance.d.ts +8 -0
- package/dist/src/testing/raw-storage-conformance.d.ts.map +1 -1
- package/dist/src/testing/raw-storage-conformance.js +97 -2
- package/dist/src/testing/raw-storage-conformance.js.map +1 -1
- package/package.json +5 -6
- package/src/cluster/block-transfer-service.ts +528 -312
- package/src/cluster/block-transfer.ts +547 -424
- package/src/cluster/certified-claims.ts +319 -0
- package/src/cluster/client-signature-verifier.ts +34 -0
- package/src/cluster/cluster-policy.ts +294 -203
- package/src/cluster/cluster-repo.ts +2501 -2034
- package/src/cluster/commit-proof.ts +344 -0
- package/src/cluster/quorum-restore.ts +348 -223
- package/src/cluster/rebalance-monitor.ts +539 -254
- package/src/cluster/reconcile-block.ts +404 -203
- package/src/cluster/service.ts +23 -20
- package/src/cluster/spread-on-churn.ts +352 -346
- package/src/cohort-topic/host.ts +8 -7
- package/src/cohort-topic/stream-util.ts +14 -43
- package/src/dispute/cascade.ts +31 -15
- package/src/dispute/dispute-service.ts +706 -696
- package/src/dispute/invalidation.ts +162 -52
- package/src/dispute/service.ts +126 -125
- package/src/index.ts +7 -0
- package/src/libp2p-key-network.ts +1204 -1235
- package/src/libp2p-node-base.ts +185 -70
- package/src/network/open-protocol-stream.ts +87 -0
- package/src/network/register-protocol-handler.ts +85 -0
- package/src/peer-address-book.ts +96 -2
- package/src/pend-validation.ts +89 -0
- package/src/reactivity/notify-transport.ts +2 -1
- package/src/reactivity/push-state-gossip.ts +2 -1
- package/src/repo/cluster-coordinator.ts +65 -11
- package/src/repo/coordinator-repo.ts +729 -71
- package/src/repo/served-repo-proxy.ts +55 -0
- package/src/repo/service.ts +25 -16
- package/src/reputation/types.ts +3 -1
- package/src/rn.ts +34 -4
- package/src/storage/block-archive.ts +242 -0
- package/src/storage/block-latch.ts +144 -0
- package/src/storage/block-storage.ts +787 -498
- package/src/storage/cached-raw-storage.ts +21 -1
- package/src/storage/cached-store-driver.ts +898 -859
- package/src/storage/i-block-storage.ts +172 -116
- package/src/storage/i-raw-storage.ts +40 -0
- package/src/storage/kv-raw-storage.ts +34 -9
- package/src/storage/memory-store-driver.ts +26 -2
- package/src/storage/raw-store-codec.ts +42 -42
- package/src/storage/raw-store-driver.ts +45 -3
- package/src/storage/shared-cache-pool.ts +516 -452
- package/src/storage/storage-repo.ts +1421 -1014
- package/src/storage/store-identity.ts +41 -0
- package/src/storage/struct.ts +52 -34
- package/src/storage/with-read-cache.ts +182 -0
- package/src/sync/service.ts +178 -198
- package/src/testing/mesh-harness.ts +524 -344
- package/src/testing/raw-storage-conformance.ts +114 -2
|
@@ -1,1014 +1,1421 @@
|
|
|
1
|
-
import type {
|
|
2
|
-
IRepo, MessageOptions, BlockId, CommitRequest, CommitResult, GetBlockResults, PendRequest, PendResult, ActionBlocks,
|
|
3
|
-
ActionId, BlockGets, ActionPending, PendSuccess, ActionTransform, ActionTransforms,
|
|
4
|
-
GetBlockResult, IBlock, ActionRev, BlockUnavailableReason,
|
|
5
|
-
PendValidationHook,
|
|
6
|
-
CollectionId, IBlockChangeNotifier, CollectionChangeListener, CollectionChangeEvent,
|
|
7
|
-
StaleFailure
|
|
8
|
-
} from "@optimystic/db-core";
|
|
9
|
-
import {
|
|
10
|
-
|
|
11
|
-
blockIdsForTransforms, transformsFromTransform, highestStaleAt
|
|
12
|
-
} from "@optimystic/db-core";
|
|
13
|
-
import { asyncIteratorToArray } from "../it-utility.js";
|
|
14
|
-
import type { IBlockStorage } from "./i-block-storage.js";
|
|
15
|
-
import type { IBlockReplicaStore } from "../cluster/block-transfer-service.js";
|
|
16
|
-
import {
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
*
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
*
|
|
34
|
-
*
|
|
35
|
-
*
|
|
36
|
-
*
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
*
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
export
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
//
|
|
256
|
-
//
|
|
257
|
-
//
|
|
258
|
-
//
|
|
259
|
-
//
|
|
260
|
-
//
|
|
261
|
-
//
|
|
262
|
-
//
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
//
|
|
421
|
-
//
|
|
422
|
-
//
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
}
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
//
|
|
832
|
-
//
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
}
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
//
|
|
902
|
-
//
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
*
|
|
926
|
-
*
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
): Promise<
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
*
|
|
969
|
-
*
|
|
970
|
-
*
|
|
971
|
-
*
|
|
972
|
-
*
|
|
973
|
-
*
|
|
974
|
-
*
|
|
975
|
-
*
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1
|
+
import type {
|
|
2
|
+
IRepo, MessageOptions, BlockId, CommitRequest, CommitResult, GetBlockResults, PendRequest, PendResult, ActionBlocks,
|
|
3
|
+
ActionId, BlockGets, ActionPending, PendSuccess, ActionTransform, ActionTransforms,
|
|
4
|
+
GetBlockResult, IBlock, ActionRev, BlockUnavailableReason,
|
|
5
|
+
PendValidationHook, UnvalidatablePendPolicy,
|
|
6
|
+
CollectionId, IBlockChangeNotifier, CollectionChangeListener, CollectionChangeEvent,
|
|
7
|
+
StaleFailure
|
|
8
|
+
} from "@optimystic/db-core";
|
|
9
|
+
import {
|
|
10
|
+
transformForBlockId, applyTransform, groupBy, concatTransform, emptyTransforms,
|
|
11
|
+
blockIdsForTransforms, transformsFromTransform, highestStaleAt, isOwnRevision, canonicalBlockHash
|
|
12
|
+
} from "@optimystic/db-core";
|
|
13
|
+
import { asyncIteratorToArray } from "../it-utility.js";
|
|
14
|
+
import type { IBlockStorage } from "./i-block-storage.js";
|
|
15
|
+
import type { IBlockReplicaStore } from "../cluster/block-transfer-service.js";
|
|
16
|
+
import { proofDeclaredDigest, type BlockCommitProof } from "../cluster/commit-proof.js";
|
|
17
|
+
import { RevisionNotCoveredError } from "./i-block-storage.js";
|
|
18
|
+
import { acquireBlockWriteLatches, withBlockWriteLatch, type BlockWriteLatch } from "./block-latch.js";
|
|
19
|
+
import { createLogger } from "../logger.js";
|
|
20
|
+
import { checkPendValidation } from "../pend-validation.js";
|
|
21
|
+
|
|
22
|
+
const log = createLogger('storage-repo');
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* Stable, greppable prefix on the failure reason a commit carries when this node cannot materialize
|
|
26
|
+
* the revision it was asked to record. It is a STRING marker rather than only an error class because
|
|
27
|
+
* {@link StorageRepo.commit} reports per-block faults as `StaleFailure.reason` (a plain string that
|
|
28
|
+
* also crosses the wire), so the class identity is lost by the time a caller inspects the result.
|
|
29
|
+
*/
|
|
30
|
+
export const MISSING_BASE_REVISION_REASON = 'missing-base-revision';
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* The two stable reject-reason prefixes a validating receiver emits, re-exported here (and from
|
|
34
|
+
* `cluster/cluster-repo.ts`) next to their siblings so a caller inspecting a `PendResult` reason
|
|
35
|
+
* need not know which module defines them. Both tiers refuse with the same prefixes because both
|
|
36
|
+
* run the same {@link checkPendValidation}.
|
|
37
|
+
*/
|
|
38
|
+
export { PEND_NOT_VALIDATABLE, VALIDATOR_FAULT } from "../pend-validation.js";
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* This node was asked to commit revision N of a block it holds no materializable base for, so
|
|
42
|
+
* applying the transform would materialize nothing while `latest` advanced to N — a block that is
|
|
43
|
+
* then unreadable locally, unservable to peers, and that rejects every later write (see
|
|
44
|
+
* {@link StorageRepo.internalCommit}). The commit is refused instead; the caller heals the node
|
|
45
|
+
* out-of-band (`ClusterMember` pulls the committed revision from a cohort peer) and retries.
|
|
46
|
+
*/
|
|
47
|
+
export class MissingBaseRevisionError extends Error {
|
|
48
|
+
constructor(readonly blockId: BlockId, readonly rev: number, detail: string) {
|
|
49
|
+
super(`${MISSING_BASE_REVISION_REASON}: block ${blockId} cannot materialize rev ${rev} — ${detail}`);
|
|
50
|
+
this.name = 'MissingBaseRevisionError';
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* True when a {@link CommitResult} failed because this node holds no materializable base for one of
|
|
56
|
+
* the committed blocks. Distinguishes that recoverable divergence (heal by fetching the block from a
|
|
57
|
+
* cohort peer) from a genuine storage fault, which must still propagate.
|
|
58
|
+
*/
|
|
59
|
+
export function isMissingBaseRevisionFailure(result: CommitResult): boolean {
|
|
60
|
+
return !result.success && (result.reason?.startsWith(MISSING_BASE_REVISION_REASON) ?? false);
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
export type StorageRepoOptions = {
|
|
64
|
+
/** Optional hook to validate transactions in PendRequests */
|
|
65
|
+
validatePend?: PendValidationHook;
|
|
66
|
+
/**
|
|
67
|
+
* What this repo does — when a `validatePend` hook IS configured — with a pend that carries no
|
|
68
|
+
* `validation` payload and therefore nothing to re-check. Default 'accept'; see
|
|
69
|
+
* {@link UnvalidatablePendPolicy}. The cluster tier's mirror of this knob is
|
|
70
|
+
* `ClusterConsensusConfig.unvalidatablePendPolicy`, and both are enforced by the one
|
|
71
|
+
* `checkPendValidation`.
|
|
72
|
+
*
|
|
73
|
+
* NOTE: the two tiers are configured INDEPENDENTLY, so a node set to 'accept' at the cluster tier
|
|
74
|
+
* and 'reject' here would vote approve on a pend its own storage then refuses at apply — burning
|
|
75
|
+
* a consensus round to reach a verdict it already knew. Harmless today because no composition
|
|
76
|
+
* root supplies a checker at either tier (backlog
|
|
77
|
+
* `feat-no-deployment-validates-transactions-at-pend`); when one does, resolve both knobs from a
|
|
78
|
+
* single operator field rather than letting a deployment set them apart.
|
|
79
|
+
*/
|
|
80
|
+
unvalidatablePendPolicy?: UnvalidatablePendPolicy;
|
|
81
|
+
};
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* What {@link StorageRepo.previewCommitDigest} predicts a commit would materialize. `digest` is the
|
|
85
|
+
* {@link canonicalBlockHash} of the materialized content, or `undefined` when the transform
|
|
86
|
+
* materializes nothing (a delete/tombstone, updates with no base to apply them to) or the base
|
|
87
|
+
* exists but cannot be materialized locally. `baseRev` is the local committed revision the preview
|
|
88
|
+
* was computed against (absent when there is none, or when the transform is base-independent and no
|
|
89
|
+
* base was read). `baseIndependent` is true when the pended transform carries an `insert`, making
|
|
90
|
+
* the result identical on every member regardless of what base it holds.
|
|
91
|
+
*/
|
|
92
|
+
export type CommitDigestPreview = {
|
|
93
|
+
digest?: string;
|
|
94
|
+
baseRev?: number;
|
|
95
|
+
baseIndependent: boolean;
|
|
96
|
+
};
|
|
97
|
+
|
|
98
|
+
/**
|
|
99
|
+
* The capability {@link ClusterMember.validateCommitOperations} probes its `storageRepo` for. Named
|
|
100
|
+
* (rather than written inline at the probe) so there is ONE definition of the contract and so a repo
|
|
101
|
+
* decorator wrapping the member's storage seam has something to `implements` and forward — a wrapper
|
|
102
|
+
* that drops the method silently disables the commit content-digest check on that node.
|
|
103
|
+
*/
|
|
104
|
+
export interface ICommitDigestPreviewer {
|
|
105
|
+
previewCommitDigest(blockId: BlockId, actionId: ActionId, rev: number): Promise<CommitDigestPreview | undefined>;
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
/**
|
|
109
|
+
* The capability `ClusterMember.applyConsensusOperation` casts its `storageRepo` to when handing a
|
|
110
|
+
* {@link BlockCommitProof} down the commit path. Named for the same reason as
|
|
111
|
+
* {@link ICommitDigestPreviewer}: one definition of the widened contract, and something a repo
|
|
112
|
+
* decorator can `implements` and forward. `IRepo.commit` takes two arguments; the third is
|
|
113
|
+
* harmless at runtime for a plain `IRepo` implementation (the extra argument is ignored), so
|
|
114
|
+
* callers cast rather than structurally probe — but a decorator that narrows back to `IRepo`
|
|
115
|
+
* silently stops persisting proofs on that node.
|
|
116
|
+
*/
|
|
117
|
+
export interface ICommitProofPersister {
|
|
118
|
+
commit(request: CommitRequest, options?: MessageOptions, proof?: BlockCommitProof): Promise<CommitResult>;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
/**
|
|
122
|
+
* The capability that answers "which action committed revision N of this block?" — the question the
|
|
123
|
+
* commit-tier stale checks need when local `latest` has already advanced PAST a contested revision,
|
|
124
|
+
* so `latest.actionId` alone can no longer distinguish "my commit landed and history moved on"
|
|
125
|
+
* (abstain / not a conflict) from "a rival took my revision" (reject / retryable conflict).
|
|
126
|
+
* Consumed by the cluster member's promise-round stale-commit check
|
|
127
|
+
* (`ClusterMember.validateCommitRevisions`) and by `CoordinatorRepo`'s commit rejection classifier.
|
|
128
|
+
* Named (rather than probed inline) for the same reason as {@link ICommitDigestPreviewer}: one
|
|
129
|
+
* definition of the contract, and something a repo decorator can `implements` and forward — a
|
|
130
|
+
* wrapper that drops the method silently degrades both checks to an abstain on that node.
|
|
131
|
+
*/
|
|
132
|
+
export interface IRevisionActionReader {
|
|
133
|
+
/**
|
|
134
|
+
* The action id recorded for `rev` of `blockId`, or `undefined` when this node holds no revision
|
|
135
|
+
* record for it (never seen, or history truncated below `rev`). Read-only; never takes the block
|
|
136
|
+
* write latch (callers are on vote/classification paths and must treat a throw as "unknown").
|
|
137
|
+
*/
|
|
138
|
+
getRevisionAction(blockId: BlockId, rev: number): Promise<ActionId | undefined>;
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
export class StorageRepo implements IRepo, IBlockChangeNotifier, IBlockReplicaStore, ICommitDigestPreviewer, ICommitProofPersister, IRevisionActionReader {
|
|
142
|
+
private readonly validatePend?: PendValidationHook;
|
|
143
|
+
private readonly unvalidatablePendPolicy: UnvalidatablePendPolicy;
|
|
144
|
+
/** Per-collection change listeners; empty sets are pruned on unsubscribe. */
|
|
145
|
+
private readonly changeListeners = new Map<CollectionId, Set<CollectionChangeListener>>();
|
|
146
|
+
/** Catch-all change listeners — fire for EVERY collection's commit on this node. */
|
|
147
|
+
private readonly anyChangeListeners = new Set<CollectionChangeListener>();
|
|
148
|
+
|
|
149
|
+
constructor(
|
|
150
|
+
private readonly createBlockStorage: (blockId: BlockId) => IBlockStorage,
|
|
151
|
+
options?: StorageRepoOptions
|
|
152
|
+
) {
|
|
153
|
+
this.validatePend = options?.validatePend;
|
|
154
|
+
this.unvalidatablePendPolicy = options?.unvalidatablePendPolicy ?? 'accept';
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
/**
|
|
158
|
+
* Subscribe to commits that mutate `collectionId`'s blocks on this node.
|
|
159
|
+
* Returns an idempotent unsubscribe. See {@link IBlockChangeNotifier}.
|
|
160
|
+
*/
|
|
161
|
+
onCollectionChange(collectionId: CollectionId, listener: CollectionChangeListener): () => void {
|
|
162
|
+
let set = this.changeListeners.get(collectionId);
|
|
163
|
+
if (!set) {
|
|
164
|
+
set = new Set();
|
|
165
|
+
this.changeListeners.set(collectionId, set);
|
|
166
|
+
}
|
|
167
|
+
set.add(listener);
|
|
168
|
+
let unsubscribed = false;
|
|
169
|
+
return () => {
|
|
170
|
+
if (unsubscribed) return;
|
|
171
|
+
unsubscribed = true;
|
|
172
|
+
const current = this.changeListeners.get(collectionId);
|
|
173
|
+
if (current) {
|
|
174
|
+
current.delete(listener);
|
|
175
|
+
if (current.size === 0) {
|
|
176
|
+
this.changeListeners.delete(collectionId);
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
};
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
/**
|
|
183
|
+
* Subscribe to commits mutating ANY collection on this node — the catch-all feed the
|
|
184
|
+
* cohort-topic origination bridge consumes (it cannot enumerate collection ids ahead of time,
|
|
185
|
+
* so a per-collection {@link onCollectionChange} subscription cannot see every commit). Fires for
|
|
186
|
+
* the same `(pending → committed)` transitions as {@link onCollectionChange}, but across every
|
|
187
|
+
* collection. Returns an idempotent unsubscribe; a throwing listener is isolated + logged.
|
|
188
|
+
*/
|
|
189
|
+
onAnyCollectionChange(listener: CollectionChangeListener): () => void {
|
|
190
|
+
this.anyChangeListeners.add(listener);
|
|
191
|
+
let unsubscribed = false;
|
|
192
|
+
return () => {
|
|
193
|
+
if (unsubscribed) return;
|
|
194
|
+
unsubscribed = true;
|
|
195
|
+
this.anyChangeListeners.delete(listener);
|
|
196
|
+
};
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
/**
|
|
200
|
+
* Fire one {@link CollectionChangeEvent} per distinct collection that was
|
|
201
|
+
* newly committed. Called AFTER the commit critical section (locks released),
|
|
202
|
+
* fire-and-forget synchronous; a throwing listener is isolated and logged. Each event reaches
|
|
203
|
+
* both that collection's {@link onCollectionChange} subscribers and every
|
|
204
|
+
* {@link onAnyCollectionChange} catch-all subscriber.
|
|
205
|
+
*
|
|
206
|
+
* `tailId` is the `CommitRequest.tailId` on the commit path; `undefined` on read-driven
|
|
207
|
+
* promotions (the get/emitPromotions path has no commit request). A single commit is for one
|
|
208
|
+
* collection's chain in practice, so all events from one commit share the same `tailId`.
|
|
209
|
+
*/
|
|
210
|
+
private emitCollectionChanges(collectionBlocks: Map<CollectionId, BlockId[]>, actionId: ActionId, rev: number, tailId?: BlockId): void {
|
|
211
|
+
const hasCatchAll = this.anyChangeListeners.size > 0;
|
|
212
|
+
for (const [collectionId, blockIds] of collectionBlocks) {
|
|
213
|
+
const listeners = this.changeListeners.get(collectionId);
|
|
214
|
+
if ((!listeners || listeners.size === 0) && !hasCatchAll) {
|
|
215
|
+
continue;
|
|
216
|
+
}
|
|
217
|
+
const event: CollectionChangeEvent = { collectionId, blockIds, actionId, rev, tailId };
|
|
218
|
+
if (listeners && listeners.size > 0) {
|
|
219
|
+
this.fireChangeListeners(listeners, event);
|
|
220
|
+
}
|
|
221
|
+
if (hasCatchAll) {
|
|
222
|
+
this.fireChangeListeners(this.anyChangeListeners, event);
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
/** Dispatch `event` to a snapshot of `listeners` (safe under mid-emit (un)subscribe), isolating + logging any throw. */
|
|
228
|
+
private fireChangeListeners(listeners: Set<CollectionChangeListener>, event: CollectionChangeEvent): void {
|
|
229
|
+
for (const listener of Array.from(listeners)) {
|
|
230
|
+
try {
|
|
231
|
+
listener(event);
|
|
232
|
+
} catch (err) {
|
|
233
|
+
log('onCollectionChange listener threw for collection=%s: %o', event.collectionId, err);
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
async get({ blockIds, context }: BlockGets, _options?: MessageOptions): Promise<GetBlockResults> {
|
|
239
|
+
const distinctBlockIds = Array.from(new Set(blockIds));
|
|
240
|
+
log('get blockIds=%d', distinctBlockIds.length);
|
|
241
|
+
// Read-driven promotions that land durably here, captured so we can emit a
|
|
242
|
+
// change event per durable landing after the parallel reads complete (mirrors
|
|
243
|
+
// commit's "emit after the work" ordering). The array is shared across the
|
|
244
|
+
// parallel map closures below — safe because each push happens synchronously
|
|
245
|
+
// between awaits (single-threaded), never concurrently.
|
|
246
|
+
const promotions: { collectionId: CollectionId, blockId: BlockId, actionId: ActionId, rev: number }[] = [];
|
|
247
|
+
const results = await Promise.all(distinctBlockIds.map(async (blockId) => {
|
|
248
|
+
const blockStorage = this.createBlockStorage(blockId);
|
|
249
|
+
// Set when this node KNOWS its answer for the block is a guess: the promotion
|
|
250
|
+
// below refused for a missing base, or getBlock() threw (truncated history /
|
|
251
|
+
// failed restore). An absent-reading block then reports `unavailable` instead of
|
|
252
|
+
// posing as an authoritative "never existed" — see BlockUnavailableReason.
|
|
253
|
+
let unavailable: BlockUnavailableReason | undefined;
|
|
254
|
+
|
|
255
|
+
// Ensure that all outstanding transactions in the context are committed.
|
|
256
|
+
// This promotes a landed-elsewhere pending via internalCommit, which writes the
|
|
257
|
+
// block's metadata — the same read-modify-write commit()/saveReplicatedBlock guard
|
|
258
|
+
// with the per-block write latch. It MUST hold that latch too, or a promotion
|
|
259
|
+
// racing a concurrent commit on the block regresses latest non-monotonically /
|
|
260
|
+
// cross-writes a revision. Cheap unlatched pre-scan first so the common
|
|
261
|
+
// contextless read and no-pending read never pay for latch acquisition; the
|
|
262
|
+
// authoritative decision is re-made inside the latch.
|
|
263
|
+
if (context) {
|
|
264
|
+
const preLatest = await blockStorage.getLatest();
|
|
265
|
+
const preMissing = preLatest
|
|
266
|
+
? context.committed.filter(c => c.rev > preLatest.rev)
|
|
267
|
+
: context.committed;
|
|
268
|
+
if (preMissing.length > 0) {
|
|
269
|
+
await withBlockWriteLatch(blockId, async (latch) => {
|
|
270
|
+
// Re-read authoritative state under the latch: a concurrent commit may have
|
|
271
|
+
// promoted or superseded a pending between the unlatched pre-scan and here.
|
|
272
|
+
// Recompute which committed entries are still ahead of `latest` (drops the
|
|
273
|
+
// superseded, rev <= latest.rev) and re-fetch each pending inside the loop
|
|
274
|
+
// (skips the already-promoted, pending gone). This makes read-driven
|
|
275
|
+
// promotion idempotent under races, mirroring commit()'s alreadyDone/stale
|
|
276
|
+
// partitioning.
|
|
277
|
+
const latest = await blockStorage.getLatest();
|
|
278
|
+
const missing = latest
|
|
279
|
+
? context.committed.filter(c => c.rev > latest.rev)
|
|
280
|
+
: context.committed;
|
|
281
|
+
// Sort a COPY: when `latest` is undefined, `missing` aliases the caller's
|
|
282
|
+
// `context.committed` array, and an in-place `.sort()` would reorder the shared
|
|
283
|
+
// request context under the caller's feet.
|
|
284
|
+
try {
|
|
285
|
+
for (const { actionId, rev } of [...missing].sort((a, b) => a.rev - b.rev)) {
|
|
286
|
+
const pending = await blockStorage.getPendingTransaction(actionId);
|
|
287
|
+
if (pending) {
|
|
288
|
+
const collectionId = await this.internalCommit(blockId, actionId, rev, blockStorage, latch);
|
|
289
|
+
if (collectionId !== undefined) {
|
|
290
|
+
promotions.push({ collectionId, blockId, actionId, rev });
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
} catch (err) {
|
|
295
|
+
// This node holds no materializable base for the block, so NO context revision
|
|
296
|
+
// can be promoted here (each builds on the one before). Leave `latest` where it
|
|
297
|
+
// is — the invariant internalCommit just enforced — and let the commit-path
|
|
298
|
+
// healing supply the content; a read must not fail for it. Every other fault
|
|
299
|
+
// still propagates.
|
|
300
|
+
if (!(err instanceof MissingBaseRevisionError)) {
|
|
301
|
+
throw err;
|
|
302
|
+
}
|
|
303
|
+
// This node holds records PROVING the block exists (a pending it could not
|
|
304
|
+
// promote); if the block then reads as absent below, the answer is a guess,
|
|
305
|
+
// not an authoritative "never existed".
|
|
306
|
+
unavailable = 'unmaterializable';
|
|
307
|
+
log('get:promote-skipped-missing-base blockId=%s rev=%d reason=%s', blockId, err.rev, err.message);
|
|
308
|
+
}
|
|
309
|
+
});
|
|
310
|
+
}
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
// NOTE: a Crash-D3 block (durably promoted + revision saved, but the setLatest lost so
|
|
314
|
+
// meta.latest is stale and the pending record is gone) reads as empty/stale here — a
|
|
315
|
+
// context-driven get skips promotion (pending gone) and a default getBlock() sees the
|
|
316
|
+
// stale latest. It is soft-wedged (stale), not hard-wedged: the next commit-retry for
|
|
317
|
+
// (actionId, rev) self-heals it via storage.recover() in commit(). Not repaired lazily on
|
|
318
|
+
// the read path because the plain read below holds no write latch; if stale reads on
|
|
319
|
+
// unwritten blocks ever become a problem, add a latched lazy recover() here.
|
|
320
|
+
//
|
|
321
|
+
// readBlockHealing() THROWS when this node holds a `latest` it cannot materialize
|
|
322
|
+
// (truncated history: "Failed to find materialized block", or a failed restore). Caught
|
|
323
|
+
// PER BLOCK so one broken block cannot fail the whole batch's Promise.all and take healthy
|
|
324
|
+
// siblings down with it. The read still fails for THIS block — TransactorSource throws
|
|
325
|
+
// BlockUnavailableError on the flagged entry — so nothing is swallowed.
|
|
326
|
+
let blockRev: Awaited<ReturnType<IBlockStorage['getBlock']>>;
|
|
327
|
+
try {
|
|
328
|
+
blockRev = await this.readBlockHealing(blockId, blockStorage, context?.rev);
|
|
329
|
+
} catch (err) {
|
|
330
|
+
// NOTE: the entry drops `state.latest`, which this node does know (getLatest() does not
|
|
331
|
+
// materialize, so it does not throw). Empty state is what makes CoordinatorRepo treat the
|
|
332
|
+
// block as missing and consult the cohort — exactly the repair this block needs. If a
|
|
333
|
+
// consumer ever needs the revision behind an unavailable answer (e.g. to ask the cohort
|
|
334
|
+
// for a specific rev instead of the whole block), carry `latest` here and widen the
|
|
335
|
+
// coordinator's consult trigger to `isMissing || unavailable` so repair still fires.
|
|
336
|
+
log('get:unmaterializable blockId=%s error=%s', blockId,
|
|
337
|
+
err instanceof Error ? err.message : String(err));
|
|
338
|
+
return [blockId, { state: {}, unavailable: 'unmaterializable' } as GetBlockResult];
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
// Include pending action if requested, applying the pending transform over whatever
|
|
342
|
+
// committed base getBlock() resolved (possibly none — a pending-only insert has no
|
|
343
|
+
// committed revision under it and getBlock reports that as an absent base, not a fault).
|
|
344
|
+
if (context?.actionId !== undefined) {
|
|
345
|
+
const pendingTransform = await blockStorage.getPendingTransaction(context.actionId);
|
|
346
|
+
if (!pendingTransform) {
|
|
347
|
+
if (unavailable !== undefined) {
|
|
348
|
+
// The promotion refusal above deleted this very pending record
|
|
349
|
+
// (`refuseMissingBase` drops the pending it cannot promote). This node DID hold
|
|
350
|
+
// the record and dropped it, so the honest answer is an availability one — not
|
|
351
|
+
// a caller-contract violation, and never a throw that would fail the whole batch.
|
|
352
|
+
return [blockId, { state: {}, unavailable } as GetBlockResult];
|
|
353
|
+
}
|
|
354
|
+
// Caller-contract violation (the caller asserted a pending this repo never had, or
|
|
355
|
+
// cancelled) — an error, not an availability question. Deliberately NOT `unavailable`.
|
|
356
|
+
//
|
|
357
|
+
// It is NOT the only way to reach here. A context that both PROVES its own action
|
|
358
|
+
// (`committed` names it) and names it as the pending overlay (`actionId`) is
|
|
359
|
+
// self-contradictory, and the two halves of that contradiction land differently: if
|
|
360
|
+
// the read-driven promotion above REFUSED, the arm above answers gracefully; if it
|
|
361
|
+
// SUCCEEDED, `promotePendingTransaction` moved the record and we throw here — failing
|
|
362
|
+
// the whole batch for a request the refusal path tolerates. No production code sets
|
|
363
|
+
// `ActionContext.actionId` at all today, so neither is reachable except from tests or
|
|
364
|
+
// a peer that crafts the field on the wire. See
|
|
365
|
+
// tickets/blocked/repo-pending-overlay-has-no-producer.
|
|
366
|
+
throw new Error(`Pending action ${context.actionId} not found`);
|
|
367
|
+
}
|
|
368
|
+
const block = applyTransform(blockRev?.block, pendingTransform);
|
|
369
|
+
return [blockId, {
|
|
370
|
+
block,
|
|
371
|
+
state: {
|
|
372
|
+
latest: await blockStorage.getLatest(),
|
|
373
|
+
pendings: [context.actionId]
|
|
374
|
+
},
|
|
375
|
+
// The COMMITTED revision underneath the pending overlay. A pending has no revision
|
|
376
|
+
// of its own, so the honest answer is the base it was applied to. Absent when there
|
|
377
|
+
// was no base at all — a pending-only insert served over an absent committed base,
|
|
378
|
+
// where fabricating a revision would claim content this node never committed.
|
|
379
|
+
...(blockRev ? { materialized: blockRev.actionRev } : {}),
|
|
380
|
+
// A pending applied to a missing base can materialize nothing (applyTransform drops
|
|
381
|
+
// updates with no block to apply them to) — that absence is a guess, and is flagged.
|
|
382
|
+
// A materialized block is a real answer regardless of the earlier refusal. TWO ways
|
|
383
|
+
// an empty result is a guess: the promotion refusal fired (`unavailable` set), or
|
|
384
|
+
// there was no committed base under the overlay at all (`blockRev === undefined`) —
|
|
385
|
+
// this node holds a pending record PROVING the block exists and produced nothing.
|
|
386
|
+
// The second clause's ABSENCE in the other direction is equally load-bearing: a
|
|
387
|
+
// pending DELETE over a real committed base also lands here with no block, and that
|
|
388
|
+
// is an authoritative tombstone which must stay unflagged.
|
|
389
|
+
...(block === undefined && (unavailable !== undefined || blockRev === undefined)
|
|
390
|
+
? { unavailable: unavailable ?? 'unmaterializable' }
|
|
391
|
+
: {})
|
|
392
|
+
} as GetBlockResult];
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
if (!blockRev) {
|
|
396
|
+
// `unavailable` distinguishes "never existed" (the common insert-probe case, no flag)
|
|
397
|
+
// from "this node cannot reconstruct it" (the promotion above refused for a missing
|
|
398
|
+
// base). A tombstoned block also lands here with meta.latest set, but it never enters
|
|
399
|
+
// the missing-base catch, so it stays an authoritative absent — keyed off the explicit
|
|
400
|
+
// flag, not off "no block".
|
|
401
|
+
return [blockId, { state: {}, ...(unavailable !== undefined ? { unavailable } : {}) } as GetBlockResult];
|
|
402
|
+
}
|
|
403
|
+
|
|
404
|
+
const pendings = await asyncIteratorToArray(blockStorage.listPendingTransactions());
|
|
405
|
+
return [blockId, {
|
|
406
|
+
block: blockRev.block,
|
|
407
|
+
// `getBlock(context?.rev)` materialized the content at the highest committed revision
|
|
408
|
+
// at or below the pin, and reports it as `actionRev` — report THAT alongside the
|
|
409
|
+
// content. `state.latest` deliberately stays the node's newest revision for the block
|
|
410
|
+
// (StorageRepo.get's own promotion pre-scan and CoordinatorRepo's read-repair compare
|
|
411
|
+
// against it), so the two disagree exactly when a pinned read is serving older content.
|
|
412
|
+
materialized: blockRev.actionRev,
|
|
413
|
+
state: {
|
|
414
|
+
latest: await blockStorage.getLatest(),
|
|
415
|
+
pendings
|
|
416
|
+
}
|
|
417
|
+
}];
|
|
418
|
+
}));
|
|
419
|
+
|
|
420
|
+
// Emit per durable read-driven landing (Option A — emit eagerly). Done after the
|
|
421
|
+
// parallel reads complete so emission stays outside the per-block work, matching
|
|
422
|
+
// commit's ordering. No-op when nothing was promoted.
|
|
423
|
+
this.emitPromotions(promotions);
|
|
424
|
+
|
|
425
|
+
return Object.fromEntries(results);
|
|
426
|
+
}
|
|
427
|
+
|
|
428
|
+
/**
|
|
429
|
+
* The one place a local coverage gap is healed from a peer. `getBlock` is local-only; when it
|
|
430
|
+
* reports the target revision as not covered ({@link RevisionNotCoveredError}) this fetches it
|
|
431
|
+
* through `restoreRevision` under the block's write latch — the restore writes revision records
|
|
432
|
+
* and merges coverage into the metadata blob, so it must serialize against every other writer of
|
|
433
|
+
* the block — and re-reads. Only the restore is latched; the reads on either side are not, and
|
|
434
|
+
* the latch is never held across the two.
|
|
435
|
+
*
|
|
436
|
+
* A restore that fails on a **pending-only** block (metadata seeded by a pend, no committed
|
|
437
|
+
* revision) reads as ABSENT, not as a fault: the named revision was a guess about content this
|
|
438
|
+
* node never held, and the caller's insert-probe / pending-overlay logic already treats an absent
|
|
439
|
+
* base as "nothing committed here". A failed restore on a block that DOES hold a `latest` is a
|
|
440
|
+
* real fault (a `latest` this node cannot serve) and propagates, so the caller reports the block
|
|
441
|
+
* as unavailable. Any throw from the second read (records restored but nothing materializable
|
|
442
|
+
* under them) propagates the same way.
|
|
443
|
+
*/
|
|
444
|
+
private async readBlockHealing(
|
|
445
|
+
blockId: BlockId,
|
|
446
|
+
storage: IBlockStorage,
|
|
447
|
+
rev: number | undefined
|
|
448
|
+
): Promise<{ block: IBlock, actionRev: ActionRev } | undefined> {
|
|
449
|
+
try {
|
|
450
|
+
return await storage.getBlock(rev);
|
|
451
|
+
} catch (err) {
|
|
452
|
+
if (!(err instanceof RevisionNotCoveredError)) {
|
|
453
|
+
throw err;
|
|
454
|
+
}
|
|
455
|
+
try {
|
|
456
|
+
// NOTE: the peer fetch inside restoreRevision runs UNDER the block's write latch, so a
|
|
457
|
+
// slow restore queues every commit/pend/replica on this block behind one network
|
|
458
|
+
// round-trip. Fine at today's restore rates (a gap is healed once, then served
|
|
459
|
+
// locally); if restore latency ever shows up delaying commits, fetch + vet OUTSIDE the
|
|
460
|
+
// latch and take it only to write, re-checking coverage inside.
|
|
461
|
+
await withBlockWriteLatch(blockId, latch => storage.restoreRevision(err.rev, latch));
|
|
462
|
+
} catch (restoreErr) {
|
|
463
|
+
if (await storage.getLatest() === undefined) {
|
|
464
|
+
log('get:restore-failed-pending-only blockId=%s rev=%d error=%s', blockId, err.rev,
|
|
465
|
+
restoreErr instanceof Error ? restoreErr.message : String(restoreErr));
|
|
466
|
+
return undefined;
|
|
467
|
+
}
|
|
468
|
+
throw restoreErr;
|
|
469
|
+
}
|
|
470
|
+
return await storage.getBlock(rev);
|
|
471
|
+
}
|
|
472
|
+
}
|
|
473
|
+
|
|
474
|
+
/**
|
|
475
|
+
* Emit a {@link CollectionChangeEvent} for each read-driven promotion that landed
|
|
476
|
+
* during a {@link get}. A single get() can promote multiple distinct actions, each
|
|
477
|
+
* at its own `(actionId, rev)`, so group by `(actionId, rev)` and route each group
|
|
478
|
+
* through {@link emitCollectionChanges} once.
|
|
479
|
+
*/
|
|
480
|
+
private emitPromotions(promotions: { collectionId: CollectionId, blockId: BlockId, actionId: ActionId, rev: number }[]): void {
|
|
481
|
+
if (promotions.length === 0) {
|
|
482
|
+
return;
|
|
483
|
+
}
|
|
484
|
+
const groups = new Map<string, { actionId: ActionId, rev: number, collectionBlocks: Map<CollectionId, BlockId[]> }>();
|
|
485
|
+
for (const { collectionId, blockId, actionId, rev } of promotions) {
|
|
486
|
+
const key = `${actionId} ${rev}`;
|
|
487
|
+
let group = groups.get(key);
|
|
488
|
+
if (!group) {
|
|
489
|
+
group = { actionId, rev, collectionBlocks: new Map() };
|
|
490
|
+
groups.set(key, group);
|
|
491
|
+
}
|
|
492
|
+
const list = group.collectionBlocks.get(collectionId) ?? [];
|
|
493
|
+
list.push(blockId);
|
|
494
|
+
group.collectionBlocks.set(collectionId, list);
|
|
495
|
+
}
|
|
496
|
+
for (const { actionId, rev, collectionBlocks } of groups.values()) {
|
|
497
|
+
this.emitCollectionChanges(collectionBlocks, actionId, rev);
|
|
498
|
+
}
|
|
499
|
+
}
|
|
500
|
+
|
|
501
|
+
async pend(request: PendRequest, _options?: MessageOptions): Promise<PendResult> {
|
|
502
|
+
// Re-check the transaction when a validation hook is configured — the unvalidatable-pend
|
|
503
|
+
// policy and the throwing-hook catch both live in the shared `checkPendValidation`, so this
|
|
504
|
+
// tier and the cluster tier cannot drift apart on what they refuse.
|
|
505
|
+
const hook = this.validatePend;
|
|
506
|
+
const validation = await checkPendValidation(
|
|
507
|
+
request,
|
|
508
|
+
hook && (({ transaction, operationsHash }) => hook(transaction, operationsHash)),
|
|
509
|
+
this.unvalidatablePendPolicy,
|
|
510
|
+
event => event.kind === 'unvalidatable'
|
|
511
|
+
? log('pend-unvalidatable actionId=%s policy=%s', request.actionId, event.policy)
|
|
512
|
+
: log('pend validator-fault actionId=%s error=%s', request.actionId, event.error)
|
|
513
|
+
);
|
|
514
|
+
if (!validation.valid) {
|
|
515
|
+
// Hard rejection: no `conflict` flag, because re-driving the same request fails the same
|
|
516
|
+
// way and would only burn the writer's retry budget.
|
|
517
|
+
return {
|
|
518
|
+
success: false,
|
|
519
|
+
reason: validation.reason ?? 'Transaction validation failed'
|
|
520
|
+
};
|
|
521
|
+
}
|
|
522
|
+
|
|
523
|
+
const blockIds = blockIdsForTransforms(request.transforms);
|
|
524
|
+
log('pend actionId=%s blockIds=%d rev=%s', request.actionId, blockIds.length, request.rev);
|
|
525
|
+
const pendings: ActionPending[] = [];
|
|
526
|
+
const missing: ActionTransforms[] = [];
|
|
527
|
+
// Highest revision this node confirms holding among the blocks that are at or past the
|
|
528
|
+
// requested one — reported as StaleFailure.staleAt so a losing writer learns the number
|
|
529
|
+
// instead of parsing prose. Confirmed-local only: we read it from our own storage below.
|
|
530
|
+
let staleAt: StaleFailure['staleAt'];
|
|
531
|
+
// Blocks this action ALREADY committed at exactly the requested revision — the durable half
|
|
532
|
+
// of a torn action whose retry reuses the same actionId. Sibling of the `alreadyDone`
|
|
533
|
+
// partition in `commit` below: satisfied, not merely non-stale, so no pending is recorded
|
|
534
|
+
// for them (see the fan-out at the end of this method).
|
|
535
|
+
const satisfied = new Set<BlockId>();
|
|
536
|
+
|
|
537
|
+
// Potential race condition: A concurrent commit operation could complete
|
|
538
|
+
// between the conflict checks (latest.rev, listPendingTransactions) and the
|
|
539
|
+
// savePendingTransaction call below. This pend operation might succeed based on
|
|
540
|
+
// stale information, but the subsequent commit for this pend would likely
|
|
541
|
+
// fail correctly later if a conflict arose. Locking here could make the initial
|
|
542
|
+
// check more accurate but adds overhead. The current approach prioritizes
|
|
543
|
+
// letting the commit be the final arbiter.
|
|
544
|
+
for (const blockId of blockIds) {
|
|
545
|
+
const blockStorage = this.createBlockStorage(blockId);
|
|
546
|
+
const transforms = transformForBlockId(request.transforms, blockId);
|
|
547
|
+
|
|
548
|
+
// Handle any conflicting revisions FIRST: a block this same action already committed at
|
|
549
|
+
// exactly the requested revision is satisfied, and skips both this check and the
|
|
550
|
+
// pending-action listing below.
|
|
551
|
+
if (request.rev !== undefined || transforms.insert) {
|
|
552
|
+
const latest = await blockStorage.getLatest();
|
|
553
|
+
// Our own already-durable work, met again by a retry (see {@link isOwnRevision}):
|
|
554
|
+
// treating it as a stale rival would refuse the writer with its own commit.
|
|
555
|
+
// NOTE: a rev-less pend (`request.rev === undefined`, an insert-only claim) can
|
|
556
|
+
// never match, so a torn action retried WITHOUT a revision is still refused by its
|
|
557
|
+
// own insert. No production caller sends one — `TransactorSource.transact` and the
|
|
558
|
+
// multi-collection coordinator both require a rev — so this is unreachable today;
|
|
559
|
+
// if a rev-less write path ever appears, match on `latest.actionId` alone here.
|
|
560
|
+
if (isOwnRevision(latest, request.rev, request.actionId)) {
|
|
561
|
+
satisfied.add(blockId);
|
|
562
|
+
continue;
|
|
563
|
+
}
|
|
564
|
+
if (latest && latest.rev >= (request.rev ?? 0)) {
|
|
565
|
+
// Only a real revision race yields a meaningful `staleAt`. When `request.rev` is
|
|
566
|
+
// undefined this same branch fires for an insert collision (the comparison degrades
|
|
567
|
+
// to `latest.rev >= 0`, true for any existing block), and reporting that block's
|
|
568
|
+
// revision would be a number that answers a question nobody asked.
|
|
569
|
+
if (request.rev !== undefined) {
|
|
570
|
+
staleAt = highestStaleAt([staleAt, { blockId, rev: latest.rev }]);
|
|
571
|
+
}
|
|
572
|
+
const transforms = await asyncIteratorToArray(blockStorage.listRevisions(request.rev ?? 0, latest.rev));
|
|
573
|
+
for (const actionRev of transforms) {
|
|
574
|
+
const transform = await blockStorage.getTransaction(actionRev.actionId);
|
|
575
|
+
if (!transform) {
|
|
576
|
+
throw new Error(`Missing action ${actionRev.actionId} for block ${blockId}`);
|
|
577
|
+
}
|
|
578
|
+
missing.push({
|
|
579
|
+
actionId: actionRev.actionId,
|
|
580
|
+
rev: actionRev.rev,
|
|
581
|
+
transforms: transformsFromTransform(transform, blockId)
|
|
582
|
+
});
|
|
583
|
+
}
|
|
584
|
+
}
|
|
585
|
+
}
|
|
586
|
+
|
|
587
|
+
// Then handle any pending actions
|
|
588
|
+
const pending = await asyncIteratorToArray(blockStorage.listPendingTransactions());
|
|
589
|
+
pendings.push(...pending.map(actionId => ({ blockId, actionId })));
|
|
590
|
+
}
|
|
591
|
+
|
|
592
|
+
if (missing.length) {
|
|
593
|
+
log('pend:stale actionId=%s missing=%d', request.actionId, missing.length);
|
|
594
|
+
return {
|
|
595
|
+
success: false,
|
|
596
|
+
conflict: true,
|
|
597
|
+
missing,
|
|
598
|
+
...(staleAt === undefined ? {} : { staleAt })
|
|
599
|
+
};
|
|
600
|
+
}
|
|
601
|
+
|
|
602
|
+
if (pendings.length > 0) {
|
|
603
|
+
if (request.policy === 'f') { // Fail on pending actions
|
|
604
|
+
return { success: false, conflict: true, pending: pendings };
|
|
605
|
+
} else if (request.policy === 'r') { // Return populated pending actions
|
|
606
|
+
return {
|
|
607
|
+
success: false,
|
|
608
|
+
conflict: true,
|
|
609
|
+
pending: await Promise.all(pendings.map(async action => {
|
|
610
|
+
const blockStorage = this.createBlockStorage(action.blockId);
|
|
611
|
+
return {
|
|
612
|
+
blockId: action.blockId,
|
|
613
|
+
actionId: action.actionId,
|
|
614
|
+
transform: (await blockStorage.getPendingTransaction(action.actionId))
|
|
615
|
+
?? (await blockStorage.getTransaction(action.actionId))! // Possible that since enumeration, the action has been promoted
|
|
616
|
+
}
|
|
617
|
+
}))
|
|
618
|
+
};
|
|
619
|
+
}
|
|
620
|
+
}
|
|
621
|
+
|
|
622
|
+
|
|
623
|
+
// Simultaneously save pending action for each block
|
|
624
|
+
// Note: that this is not atomic, after we checked for conflicts and pending actions
|
|
625
|
+
// new pending or committed actions may have been added. This is okay, because
|
|
626
|
+
// this check during pend is conservative.
|
|
627
|
+
//
|
|
628
|
+
// Each block's pending write runs under THAT block's write latch, one latch per branch and
|
|
629
|
+
// never nested: savePendingTransaction seeds the block's metadata blob when it has none, and
|
|
630
|
+
// an unlatched seed racing a concurrent commit/replica on a fresh block erases the `latest`
|
|
631
|
+
// the other writer just landed. Never more than one block latch is held by a branch, so this
|
|
632
|
+
// cannot deadlock against commit's sorted multi-latch acquisition.
|
|
633
|
+
//
|
|
634
|
+
// `satisfied` blocks are skipped: `commit`'s `alreadyDone` arm skips `internalCommit`, the
|
|
635
|
+
// only thing that promotes (and thereby removes) a pending record, so a pending saved here
|
|
636
|
+
// would never clear — a permanent durable reservation that the rival-pending checks (this
|
|
637
|
+
// method's listPendingTransactions scan, and `ClusterMember.validatePendOperations`) refuse
|
|
638
|
+
// every future writer against, a worse wedge than the one this carve-out fixes. They still
|
|
639
|
+
// ride in the returned `blockIds` so `cancel` covers them (deleting an absent pending is a
|
|
640
|
+
// no-op that writes no metadata).
|
|
641
|
+
await Promise.all(blockIds.filter(blockId => !satisfied.has(blockId)).map(blockId => {
|
|
642
|
+
const blockStorage = this.createBlockStorage(blockId);
|
|
643
|
+
const blockTransform = transformForBlockId(request.transforms, blockId);
|
|
644
|
+
return withBlockWriteLatch(blockId, latch => blockStorage.savePendingTransaction(request.actionId, blockTransform, latch));
|
|
645
|
+
}));
|
|
646
|
+
|
|
647
|
+
return {
|
|
648
|
+
success: true,
|
|
649
|
+
pending: pendings,
|
|
650
|
+
blockIds
|
|
651
|
+
} as PendSuccess;
|
|
652
|
+
}
|
|
653
|
+
|
|
654
|
+
async cancel(actionRef: ActionBlocks, _options?: MessageOptions): Promise<void> {
|
|
655
|
+
log('cancel actionId=%s blockIds=%d', actionRef.actionId, actionRef.blockIds.length);
|
|
656
|
+
await Promise.all(actionRef.blockIds.map(blockId => {
|
|
657
|
+
const blockStorage = this.createBlockStorage(blockId);
|
|
658
|
+
return withBlockWriteLatch(blockId, latch => blockStorage.deletePendingTransaction(actionRef.actionId, latch));
|
|
659
|
+
}));
|
|
660
|
+
}
|
|
661
|
+
|
|
662
|
+
/**
|
|
663
|
+
* Commit a previously-pended action across its blocks, under the block write latches.
|
|
664
|
+
*
|
|
665
|
+
* **Divergence vs genuine fault.** When the batch cannot be completed, the reason decides what
|
|
666
|
+
* happens to the pending records the pend left behind. `ClusterMember.applyConsensusOperation`
|
|
667
|
+
* makes the same split one layer up — it *tolerates* a divergence (and reconciles every
|
|
668
|
+
* `commit.blockIds` entry from a cohort peer) but *propagates* a genuine fault for retry — so this
|
|
669
|
+
* method must agree with it:
|
|
670
|
+
*
|
|
671
|
+
* - **Divergence** — this node is behind the agreed history, either because it holds no
|
|
672
|
+
* materializable base ({@link MissingBaseRevisionError}) or because it never received the pend
|
|
673
|
+
* (the `Pending action … not found` throw). Reconcile is guaranteed to follow and will advance
|
|
674
|
+
* every block in the batch past `request.rev`, so no pending record here can ever be promoted:
|
|
675
|
+
* {@link dropUnpromotablePendings} deletes them (see {@link refuseMissingBase}, which already
|
|
676
|
+
* accepts this tradeoff for the single refusing block).
|
|
677
|
+
* - **Genuine fault** — any other throw out of {@link internalCommit} (a raw-storage error, …).
|
|
678
|
+
* `ClusterMember` propagates it and the commit is retried, and a retry can still replay the
|
|
679
|
+
* pendings, so they are KEPT.
|
|
680
|
+
*
|
|
681
|
+
* The stale/`missedCommits` early return (this node is AHEAD — it already holds a revision at or
|
|
682
|
+
* past `request.rev`, committed under a different action) deliberately keeps pendings too, and its
|
|
683
|
+
* cure is the losing client's `cancel`: `CoordinatorRepo.cancel` runs through consensus, so every
|
|
684
|
+
* member drops the record, not just the coordinator. Replication cannot be the cure here — this
|
|
685
|
+
* node is already ahead, and a later forward write carries a DIFFERENT action id, which is not
|
|
686
|
+
* what `BlockStorage.saveForwardRevision` deletes. A client that dies between the stale result and
|
|
687
|
+
* its `cancel` therefore still strands the record; that is pre-existing and orthogonal to the
|
|
688
|
+
* divergence split above.
|
|
689
|
+
*/
|
|
690
|
+
async commit(request: CommitRequest, _options?: MessageOptions, proof?: BlockCommitProof): Promise<CommitResult> {
|
|
691
|
+
log('commit actionId=%s rev=%d blockIds=%d', request.actionId, request.rev, request.blockIds.length);
|
|
692
|
+
// Deduped ONCE, in request order — the order blocks are committed and reported in. The latches
|
|
693
|
+
// are acquired in sorted order by `acquireBlockWriteLatches` over this same set, so every
|
|
694
|
+
// `latches.get(blockId)!` below resolves.
|
|
695
|
+
const blockIds = Array.from(new Set(request.blockIds));
|
|
696
|
+
// Collects the blocks newly committed in this call, grouped by collection,
|
|
697
|
+
// so we can emit change events once locks are released. Blocks that land before
|
|
698
|
+
// a mid-loop failure stay here and are still emitted (Option A — emit eagerly):
|
|
699
|
+
// they are durably committed and a retry rolls the remainder forward.
|
|
700
|
+
const collectionBlocks = new Map<CollectionId, BlockId[]>();
|
|
701
|
+
// Captured when internalCommit throws mid-loop; we break (rather than return)
|
|
702
|
+
// so locks release and accumulated landings still emit before we report failure.
|
|
703
|
+
let failure: { reason: string } | undefined;
|
|
704
|
+
|
|
705
|
+
// Every block's token is kept so each write below can prove it runs inside that block's latch.
|
|
706
|
+
const { latches, release } = await acquireBlockWriteLatches(blockIds);
|
|
707
|
+
|
|
708
|
+
try {
|
|
709
|
+
// --- Start of Critical Section ---
|
|
710
|
+
|
|
711
|
+
// Request order, deduped (NOT the sorted acquisition order): the order here is the order
|
|
712
|
+
// blocks are committed and reported in change events, which callers may observe.
|
|
713
|
+
const blockStorages = blockIds.map(blockId => ({
|
|
714
|
+
blockId,
|
|
715
|
+
storage: this.createBlockStorage(blockId),
|
|
716
|
+
latch: latches.get(blockId)!
|
|
717
|
+
}));
|
|
718
|
+
|
|
719
|
+
// Partition blocks into:
|
|
720
|
+
// - alreadyDone: latest.rev === request.rev && latest.actionId === request.actionId
|
|
721
|
+
// (idempotent retry — a prior commit of this same action already landed here;
|
|
722
|
+
// skip rather than treat as a conflict. Needed to rollforward stranded blocks
|
|
723
|
+
// after a mid-batch crash committed some but not all blocks.)
|
|
724
|
+
// - missedCommits: latest.rev >= request.rev but not the same actionId → real stale conflict.
|
|
725
|
+
// - toCommit: latest.rev < request.rev or no latest yet → run internalCommit.
|
|
726
|
+
const toCommit: { blockId: BlockId, storage: IBlockStorage, latch: BlockWriteLatch }[] = [];
|
|
727
|
+
const missedCommits: { blockId: BlockId, transforms: ActionTransform[] }[] = [];
|
|
728
|
+
// Highest revision among the blocks confirmed lost to a newer one — reported as
|
|
729
|
+
// StaleFailure.staleAt. The idempotent-retry `continue` below is a no-op, not a loss,
|
|
730
|
+
// so it never seeds this.
|
|
731
|
+
let staleAt: StaleFailure['staleAt'];
|
|
732
|
+
for (const entry of blockStorages) {
|
|
733
|
+
const { blockId, storage, latch } = entry;
|
|
734
|
+
const latest = await storage.getLatest();
|
|
735
|
+
if (latest && latest.rev >= request.rev) {
|
|
736
|
+
if (isOwnRevision(latest, request.rev, request.actionId)) {
|
|
737
|
+
// Idempotent no-op for this block — already committed with this exact (actionId, rev).
|
|
738
|
+
// A retry can carry a proof the original commit lacked (or crashed before writing):
|
|
739
|
+
// back-fill it, strictly additively, under the same digest-match retention rule the
|
|
740
|
+
// original commit applies. Runs inside the latched critical section.
|
|
741
|
+
await this.backFillProof(blockId, storage, request.rev, request.actionId, proof, latch);
|
|
742
|
+
continue;
|
|
743
|
+
}
|
|
744
|
+
staleAt = highestStaleAt([staleAt, { blockId, rev: latest.rev }]);
|
|
745
|
+
const transforms: ActionTransform[] = [];
|
|
746
|
+
for await (const actionRev of storage.listRevisions(request.rev, latest.rev)) {
|
|
747
|
+
const transform = await storage.getTransaction(actionRev.actionId);
|
|
748
|
+
if (!transform) {
|
|
749
|
+
throw new Error(`Missing action ${actionRev.actionId} for block ${blockId}`);
|
|
750
|
+
}
|
|
751
|
+
transforms.push({
|
|
752
|
+
actionId: actionRev.actionId,
|
|
753
|
+
rev: actionRev.rev,
|
|
754
|
+
transform
|
|
755
|
+
});
|
|
756
|
+
}
|
|
757
|
+
missedCommits.push({ blockId, transforms }); // Push, even if transforms is empty, because we want to reject the older version
|
|
758
|
+
continue;
|
|
759
|
+
}
|
|
760
|
+
toCommit.push(entry);
|
|
761
|
+
}
|
|
762
|
+
|
|
763
|
+
if (missedCommits.length) {
|
|
764
|
+
log('commit:stale actionId=%s missed=%d', request.actionId, missedCommits.length);
|
|
765
|
+
return { // Return directly, locks will be released in finally
|
|
766
|
+
success: false,
|
|
767
|
+
missing: perBlockActionTransformsToPerAction(missedCommits),
|
|
768
|
+
...(staleAt === undefined ? {} : { staleAt })
|
|
769
|
+
};
|
|
770
|
+
}
|
|
771
|
+
|
|
772
|
+
// Check for missing pending actions only on blocks that still need to commit.
|
|
773
|
+
// Already-done blocks will have had their pending promoted, so skipping them here
|
|
774
|
+
// is what makes the idempotent rollforward work.
|
|
775
|
+
//
|
|
776
|
+
// A toCommit block whose pending is absent is one of two states:
|
|
777
|
+
// - Crash-D3: the action was durably promoted and its revision saved, but the crash
|
|
778
|
+
// lost the setLatest, so meta.latest is still < request.rev and the pending record
|
|
779
|
+
// is gone. getTransaction(actionId) returns the promoted transform. Self-heal here
|
|
780
|
+
// via storage.recover() (redoes the lost setLatest, advancing latest to the highest
|
|
781
|
+
// contiguous promoted rev, >= request.rev). recover() is idempotent + monotonic, so
|
|
782
|
+
// calling it under the already-held block write latch is safe. Recovered blocks are then
|
|
783
|
+
// excluded from the internalCommit loop below — their pending is gone, so
|
|
784
|
+
// internalCommit would throw.
|
|
785
|
+
// - Genuine missing pend: the action was never promoted (getTransaction → undefined),
|
|
786
|
+
// so the pend is truly missing. Throw exactly as before.
|
|
787
|
+
// Crash-D2 never reaches this branch: its pending record is still present.
|
|
788
|
+
const missingPends: { blockId: BlockId, actionId: ActionId }[] = [];
|
|
789
|
+
const recovered = new Set<BlockId>();
|
|
790
|
+
for (const { blockId, storage, latch } of toCommit) {
|
|
791
|
+
const pendingAction = await storage.getPendingTransaction(request.actionId);
|
|
792
|
+
if (pendingAction) {
|
|
793
|
+
continue;
|
|
794
|
+
}
|
|
795
|
+
const promoted = await storage.getTransaction(request.actionId);
|
|
796
|
+
if (!promoted) {
|
|
797
|
+
missingPends.push({ blockId, actionId: request.actionId });
|
|
798
|
+
continue;
|
|
799
|
+
}
|
|
800
|
+
// Crash-D3 signature (pending absent + action durably promoted). Redo the lost setLatest.
|
|
801
|
+
const result = await storage.recover(latch);
|
|
802
|
+
if (result.latest !== undefined && result.latest.rev >= request.rev) {
|
|
803
|
+
recovered.add(blockId);
|
|
804
|
+
} else {
|
|
805
|
+
// Torn/partial state: recover() could not advance latest to request.rev (metadata
|
|
806
|
+
// absent, or a revision entry missing despite the promoted transaction). Fall back
|
|
807
|
+
// to treating the block as a genuine missing-pend error rather than silently succeeding.
|
|
808
|
+
missingPends.push({ blockId, actionId: request.actionId });
|
|
809
|
+
}
|
|
810
|
+
}
|
|
811
|
+
|
|
812
|
+
// NOTE: if a batch ever held BOTH a recovered D3 block and a genuine missing-pend block,
|
|
813
|
+
// this throw fires after recover() already advanced the D3 block durably, so that block's
|
|
814
|
+
// change event is skipped (the retry then treats it as alreadyDone and never re-emits;
|
|
815
|
+
// durable state stays correct, only the emit is lost). Judged unreachable today: a single
|
|
816
|
+
// crash mid-internalCommit leaves exactly one D3 block, with the rest alreadyDone or
|
|
817
|
+
// pending-present — a never-pended block cannot coexist with it in one retry. If a path
|
|
818
|
+
// ever produces that mix, emit recovered blocks' events before throwing here.
|
|
819
|
+
if (missingPends.length) {
|
|
820
|
+
// Divergence (this node is behind): `ClusterMember` treats this throw as the canonical
|
|
821
|
+
// "behind" signal and reconciles EVERY block in the batch, advancing each past
|
|
822
|
+
// `request.rev`. Nothing can promote the pendings the other blocks still hold, so drop
|
|
823
|
+
// them here — while the latches are still held — before reporting. The thrown message
|
|
824
|
+
// must stay byte-identical: `ClusterMember.isMissingPendingActionError` matches on it.
|
|
825
|
+
await this.dropUnpromotablePendings(toCommit, request.actionId);
|
|
826
|
+
throw new Error(`Pending action ${request.actionId} not found for block(s): ${missingPends.map(p => p.blockId).join(', ')}`);
|
|
827
|
+
}
|
|
828
|
+
|
|
829
|
+
// The original commit crashed before setLatest, so it also never emitted a change event
|
|
830
|
+
// for a recovered (Crash-D3) block. Now that recover() has committed it at request.rev,
|
|
831
|
+
// report its collection so downstream watchers wake — mirroring internalCommit. Resolve
|
|
832
|
+
// the collectionId from the now-materialized block; a delete materializes to a tombstone
|
|
833
|
+
// (getBlock → undefined), so fall back to the prior materialized block's header exactly as
|
|
834
|
+
// internalCommit does — otherwise a recovered delete would silently fail to wake watchers.
|
|
835
|
+
// Only when neither resolves (a delete-only block with no prior materialization) is the
|
|
836
|
+
// emit skipped, the same terminal fallback internalCommit uses.
|
|
837
|
+
for (const { blockId, storage, latch } of toCommit) {
|
|
838
|
+
if (!recovered.has(blockId)) {
|
|
839
|
+
continue;
|
|
840
|
+
}
|
|
841
|
+
const collectionId = (await storage.getBlock(request.rev))?.block.header.collectionId
|
|
842
|
+
?? (await storage.getBlock(request.rev - 1))?.block.header.collectionId;
|
|
843
|
+
if (collectionId !== undefined) {
|
|
844
|
+
const list = collectionBlocks.get(collectionId) ?? [];
|
|
845
|
+
list.push(blockId);
|
|
846
|
+
collectionBlocks.set(collectionId, list);
|
|
847
|
+
}
|
|
848
|
+
// The recovered block IS committed at request.rev, but it is excluded from the
|
|
849
|
+
// internalCommit loop below — so without this it would be the one landing path that
|
|
850
|
+
// never retains the cohort's proof, even though this very call is carrying it.
|
|
851
|
+
await this.backFillProof(blockId, storage, request.rev, request.actionId, proof, latch);
|
|
852
|
+
}
|
|
853
|
+
|
|
854
|
+
// Commit the action for each block that still needs it.
|
|
855
|
+
// This loop will execute atomically for all blocks due to the acquired locks.
|
|
856
|
+
// Recovered (Crash-D3) blocks are already committed at request.rev and their pending is
|
|
857
|
+
// gone, so skip them — internalCommit would throw on the missing pending record.
|
|
858
|
+
//
|
|
859
|
+
// Set when the mid-loop failure was a divergence rather than a genuine fault — the split
|
|
860
|
+
// documented on commit() above, which decides the fate of the batch's pending records.
|
|
861
|
+
let divergentFailure = false;
|
|
862
|
+
for (const { blockId, storage, latch } of toCommit) {
|
|
863
|
+
if (recovered.has(blockId)) {
|
|
864
|
+
continue;
|
|
865
|
+
}
|
|
866
|
+
try {
|
|
867
|
+
// internalCommit will throw if it encounters an issue
|
|
868
|
+
const collectionId = await this.internalCommit(blockId, request.actionId, request.rev, storage, latch, proof);
|
|
869
|
+
if (collectionId !== undefined) {
|
|
870
|
+
const list = collectionBlocks.get(collectionId) ?? [];
|
|
871
|
+
list.push(blockId);
|
|
872
|
+
collectionBlocks.set(collectionId, list);
|
|
873
|
+
}
|
|
874
|
+
} catch (err) {
|
|
875
|
+
// Partial-commit recovery: blocks already in collectionBlocks DID land
|
|
876
|
+
// durably and must still emit; a retry with the same (actionId, rev)
|
|
877
|
+
// treats them as idempotent no-ops and advances the remainder. Break
|
|
878
|
+
// instead of returning so locks release and those landings emit below.
|
|
879
|
+
failure = { reason: err instanceof Error ? err.message : 'Unknown error during commit' };
|
|
880
|
+
divergentFailure = err instanceof MissingBaseRevisionError;
|
|
881
|
+
break;
|
|
882
|
+
}
|
|
883
|
+
}
|
|
884
|
+
|
|
885
|
+
// The break left every not-yet-reached block still holding its pending record. Whether
|
|
886
|
+
// that record is still usable depends ENTIRELY on why we stopped — see the table on
|
|
887
|
+
// commit() above. Runs inside the try, so the per-block latches are still held.
|
|
888
|
+
// NOTE: a non-divergence fault deliberately KEEPS the batch's pendings so a retry can
|
|
889
|
+
// replay them. If ClusterMember ever stops retrying propagated commit faults, this arm
|
|
890
|
+
// becomes dead weight and the discriminator can collapse to "always drop".
|
|
891
|
+
if (divergentFailure) {
|
|
892
|
+
await this.dropUnpromotablePendings(toCommit, request.actionId);
|
|
893
|
+
}
|
|
894
|
+
}
|
|
895
|
+
finally {
|
|
896
|
+
// Releases every block latch, in reverse acquisition order.
|
|
897
|
+
release();
|
|
898
|
+
}
|
|
899
|
+
|
|
900
|
+
// Notify after the critical section, for every block newly committed here —
|
|
901
|
+
// including those that landed before a mid-loop failure (alreadyDone / stale
|
|
902
|
+
// partitions never reach `collectionBlocks`).
|
|
903
|
+
this.emitCollectionChanges(collectionBlocks, request.actionId, request.rev, request.tailId);
|
|
904
|
+
|
|
905
|
+
return failure ? { success: false, reason: failure.reason } : { success: true };
|
|
906
|
+
}
|
|
907
|
+
|
|
908
|
+
/**
|
|
909
|
+
* Delete `actionId`'s pending record from every given block, tolerating absence.
|
|
910
|
+
*
|
|
911
|
+
* Called by {@link commit} when it abandons a batch **because this node has diverged from the
|
|
912
|
+
* agreed history** — the caller has already made that determination; this helper does not
|
|
913
|
+
* re-derive it. Once `ClusterMember` reconciles the batch, every one of these blocks sits at or
|
|
914
|
+
* past `request.rev`, so a commit retry partitions them as already-done/stale and never revisits
|
|
915
|
+
* their pendings; left in place they are reported as phantom conflicting actions by {@link pend}
|
|
916
|
+
* for every later write to the block (under `policy: 'f'`, forever).
|
|
917
|
+
*
|
|
918
|
+
* No special-casing is needed for blocks that already landed (record promoted), that were
|
|
919
|
+
* `recovered` (record already gone), or for the refusing block itself
|
|
920
|
+
* ({@link refuseMissingBase} deleted its record): deleting an absent pending record is a no-op on
|
|
921
|
+
* every backend.
|
|
922
|
+
*
|
|
923
|
+
* Per-block failures are logged and swallowed rather than propagated: this cleanup must never
|
|
924
|
+
* replace the failure the caller is about to report — the pre-loop throw's message is pattern-
|
|
925
|
+
* matched by `ClusterMember.isMissingPendingActionError`, and a swapped error would misroute
|
|
926
|
+
* consensus. A leftover record only degrades this node's participation in that one block.
|
|
927
|
+
*/
|
|
928
|
+
private async dropUnpromotablePendings(
|
|
929
|
+
blocks: { blockId: BlockId, storage: IBlockStorage, latch: BlockWriteLatch }[],
|
|
930
|
+
actionId: ActionId
|
|
931
|
+
): Promise<void> {
|
|
932
|
+
if (blocks.length === 0) {
|
|
933
|
+
return;
|
|
934
|
+
}
|
|
935
|
+
log('commit:drop-unpromotable-pendings actionId=%s blockIds=%d', actionId, blocks.length);
|
|
936
|
+
await Promise.all(blocks.map(async ({ blockId, storage, latch }) => {
|
|
937
|
+
try {
|
|
938
|
+
await storage.deletePendingTransaction(actionId, latch);
|
|
939
|
+
} catch (err) {
|
|
940
|
+
log('commit:drop-unpromotable-pending-failed blockId=%s actionId=%s error=%s', blockId, actionId,
|
|
941
|
+
err instanceof Error ? err.message : String(err));
|
|
942
|
+
}
|
|
943
|
+
}));
|
|
944
|
+
}
|
|
945
|
+
|
|
946
|
+
/**
|
|
947
|
+
* Reconciles `metadata.latest` for a single block with the highest contiguous
|
|
948
|
+
* fully-promoted revision in durable storage. Use after a crash between
|
|
949
|
+
* `promotePendingTransaction` and `setLatest` when retry-commit cannot help
|
|
950
|
+
* (the pending record is already gone) but the revision and committed-log entry
|
|
951
|
+
* are durable. Idempotent and monotonic.
|
|
952
|
+
*/
|
|
953
|
+
async recoverBlock(blockId: BlockId): Promise<void> {
|
|
954
|
+
log('recoverBlock blockId=%s', blockId);
|
|
955
|
+
const storage = this.createBlockStorage(blockId);
|
|
956
|
+
// Hold the block write latch: recover() is a read-modify-write of the metadata blob that
|
|
957
|
+
// blindly writes back the object it read, so its "advance only" guard is TOCTOU — racing a
|
|
958
|
+
// concurrent commit()/saveReplicatedBlock that advanced latest in between would clobber it
|
|
959
|
+
// (a non-monotonic regression). Same latching invariant as every other metadata writer.
|
|
960
|
+
// commit() calls storage.recover(latch) directly under its own held latch, so it never
|
|
961
|
+
// routes through here — no double-acquire / deadlock.
|
|
962
|
+
await withBlockWriteLatch(blockId, latch => storage.recover(latch));
|
|
963
|
+
}
|
|
964
|
+
|
|
965
|
+
/**
|
|
966
|
+
* Persist a replica of a block received out-of-band (churn re-replication) into
|
|
967
|
+
* local storage. Distinct from the {@link IRepo} commit funnel: the block arrives
|
|
968
|
+
* already materialized from a departing owner, not as a pend/commit. See
|
|
969
|
+
* {@link IBlockStorage.saveReplica} for the durability/monotonicity contract.
|
|
970
|
+
*
|
|
971
|
+
* Held under the same block write latch as {@link commit} so the replica's
|
|
972
|
+
* read-modify-write of the metadata blob is mutually exclusive with a concurrent
|
|
973
|
+
* local commit on the same block — otherwise `saveReplica`'s monotonic guard could
|
|
974
|
+
* read a stale `latest` and clobber a commit that advanced it in between.
|
|
975
|
+
*
|
|
976
|
+
* `verifiedProof` is retained when supplied: both the reconcile path
|
|
977
|
+
* (`cluster/reconcile-block.ts`) and the certified push path (`BlockTransferService.handlePush`)
|
|
978
|
+
* pass the {@link BlockCommitProof} they verified against these exact bytes (`certifyContent`'s
|
|
979
|
+
* digest check), so a repaired replica serves the proof onward and certification no longer decays
|
|
980
|
+
* across repair hops.
|
|
981
|
+
*
|
|
982
|
+
* When the push does NOT advance `latest` (this node already holds that revision), `saveReplica`
|
|
983
|
+
* is a no-op and persists nothing — so the proof is back-filled here instead, through
|
|
984
|
+
* {@link backFillProof}'s digest-match rule. It is deliberately NOT persisted inside
|
|
985
|
+
* `saveReplica`: the proof was verified against the PUSHED bytes, while a back-fill attaches it to
|
|
986
|
+
* this node's HELD materialization, and a diverged holder's bytes at the same `(rev, actionId)`
|
|
987
|
+
* may differ. Storing a proof whose declared digest contradicts local content would make this node
|
|
988
|
+
* serve content that fails its own proof — `digest-mismatch` is an ATTRIBUTABLE fault in
|
|
989
|
+
* `certified-claims.ts`, so every receiver would penalize it.
|
|
990
|
+
*/
|
|
991
|
+
async saveReplicatedBlock(blockId: BlockId, block: IBlock, source?: ActionRev, verifiedProof?: BlockCommitProof): Promise<void> {
|
|
992
|
+
log('saveReplicatedBlock blockId=%s rev=%s', blockId, source?.rev);
|
|
993
|
+
const storage = this.createBlockStorage(blockId);
|
|
994
|
+
// Captured under the latch; emitted after release to match commit's ordering.
|
|
995
|
+
let landed: { collectionId: CollectionId, actionId: ActionId, rev: number } | undefined;
|
|
996
|
+
await withBlockWriteLatch(blockId, async (latch) => {
|
|
997
|
+
const priorLatest = await storage.getLatest();
|
|
998
|
+
const effective = await storage.saveReplica(block, source, verifiedProof, latch);
|
|
999
|
+
// Advanced iff there was no prior revision or the effective rev moved past it. On the
|
|
1000
|
+
// monotonic no-op, saveReplica returns the held latest unchanged → effective.rev === priorLatest.rev.
|
|
1001
|
+
const advanced = priorLatest === undefined || effective.rev > priorLatest.rev;
|
|
1002
|
+
const collectionId = block.header?.collectionId;
|
|
1003
|
+
if (advanced && collectionId !== undefined) {
|
|
1004
|
+
landed = { collectionId, actionId: effective.actionId, rev: effective.rev };
|
|
1005
|
+
}
|
|
1006
|
+
if (!advanced && verifiedProof !== undefined && source !== undefined
|
|
1007
|
+
&& effective.rev === source.rev && effective.actionId === source.actionId) {
|
|
1008
|
+
// The push named exactly the revision this node already holds, and carried a verified
|
|
1009
|
+
// proof for it. Back-fill so a proof-lessly-landed revision stops being corroboration-only
|
|
1010
|
+
// the moment valid evidence for it arrives. Requires agreement on BOTH rev and actionId:
|
|
1011
|
+
// same rev under a different action is a divergence, not the same revision.
|
|
1012
|
+
//
|
|
1013
|
+
// A held revision NEWER than the pushed one is deliberately not back-filled: `servableProof`
|
|
1014
|
+
// only ever serves the proof for `latest.rev`, so the proof would be keyed to a revision
|
|
1015
|
+
// this node will never serve, for content it may not even materialize.
|
|
1016
|
+
//
|
|
1017
|
+
// Runs under the block write latch already held here — the same latch the commit-path
|
|
1018
|
+
// back-fill sites hold, so no new latch interaction. `backFillProof` never throws: the
|
|
1019
|
+
// revision is already durable, and a proof-persist fault must not turn a no-op into a
|
|
1020
|
+
// failure.
|
|
1021
|
+
//
|
|
1022
|
+
// NOTE: once a proof IS retained this costs one key lookup per duplicate push
|
|
1023
|
+
// (`backFillProof` returns before materializing). A holder whose bytes diverge from the
|
|
1024
|
+
// cohort's never retains one, so it re-materializes and re-hashes the block on EVERY
|
|
1025
|
+
// certified push of that revision. Bounded by push frequency and fine at spread-on-churn
|
|
1026
|
+
// rates; if a diverged holder under repeated push ever shows up in a profile, remember the
|
|
1027
|
+
// withheld `(rev, actionId)` and skip the re-check.
|
|
1028
|
+
await this.backFillProof(blockId, storage, effective.rev, effective.actionId, verifiedProof, latch);
|
|
1029
|
+
}
|
|
1030
|
+
});
|
|
1031
|
+
// Replica-persist has no CommitRequest, hence no tailId — like a read-driven promotion,
|
|
1032
|
+
// this wakes local onCollectionChange watchers but is cert-gated out of cohort-topic
|
|
1033
|
+
// re-origination downstream (change-bridge selfIsCohortMember treats a tail-less event as
|
|
1034
|
+
// never a member).
|
|
1035
|
+
if (landed) {
|
|
1036
|
+
this.emitCollectionChanges(
|
|
1037
|
+
new Map([[landed.collectionId, [blockId]]]),
|
|
1038
|
+
landed.actionId,
|
|
1039
|
+
landed.rev,
|
|
1040
|
+
);
|
|
1041
|
+
}
|
|
1042
|
+
}
|
|
1043
|
+
|
|
1044
|
+
/**
|
|
1045
|
+
* The digest the block WOULD materialize to if `actionId`'s pending transform committed at `rev`,
|
|
1046
|
+
* plus the base revision it was computed from. Read-only: touches no durable state and takes no
|
|
1047
|
+
* block write latch.
|
|
1048
|
+
*
|
|
1049
|
+
* Mirrors {@link internalCommit}'s reads (pending transform → latest → base → applyTransform) so
|
|
1050
|
+
* the prediction and the eventual commit cannot drift. Consumed by the cluster member's
|
|
1051
|
+
* promise-round content-digest check (`ClusterMember.validateCommitOperations`), which compares it
|
|
1052
|
+
* against the digest the transaction author declared on the commit request.
|
|
1053
|
+
*
|
|
1054
|
+
* Deliberately does NOT take the block write latch: this runs on the vote path, ahead of the
|
|
1055
|
+
* commit that will take it, so taking it here would serialize voting behind commits and risks
|
|
1056
|
+
* deadlocking against commit's sorted up-front multi-block latch acquisition. The price is that a
|
|
1057
|
+
* concurrent commit can move `latest` mid-preview; the caller's checkable rule (base-independent,
|
|
1058
|
+
* or `baseRev` agreement) makes a torn read at worst an abstain, never a false reject of honest
|
|
1059
|
+
* content.
|
|
1060
|
+
*
|
|
1061
|
+
* `rev` is accepted for parity/logging with the commit that would follow; materialization does not
|
|
1062
|
+
* depend on it (internalCommit only records it).
|
|
1063
|
+
*
|
|
1064
|
+
* Returns `undefined` when this node holds no pending transform for the action (it never saw the
|
|
1065
|
+
* pend) — distinct from a defined preview with `digest: undefined` (see {@link CommitDigestPreview}).
|
|
1066
|
+
*/
|
|
1067
|
+
async previewCommitDigest(blockId: BlockId, actionId: ActionId, rev: number): Promise<CommitDigestPreview | undefined> {
|
|
1068
|
+
const storage = this.createBlockStorage(blockId);
|
|
1069
|
+
const transform = await storage.getPendingTransaction(actionId);
|
|
1070
|
+
if (!transform) {
|
|
1071
|
+
return undefined;
|
|
1072
|
+
}
|
|
1073
|
+
|
|
1074
|
+
// An insert replaces the block wholesale before updates apply, so the result is the same on
|
|
1075
|
+
// every member no matter what base it holds — do not read a base at all (the block may even be
|
|
1076
|
+
// locally wedged/unmaterializable, which must not degrade a base-independent preview).
|
|
1077
|
+
const baseIndependent = transform.insert !== undefined;
|
|
1078
|
+
let base: IBlock | undefined;
|
|
1079
|
+
let baseRev: number | undefined;
|
|
1080
|
+
if (!baseIndependent) {
|
|
1081
|
+
const latest = await storage.getLatest();
|
|
1082
|
+
if (latest) {
|
|
1083
|
+
baseRev = latest.rev;
|
|
1084
|
+
try {
|
|
1085
|
+
base = (await storage.getBlock(latest.rev))?.block;
|
|
1086
|
+
} catch (err) {
|
|
1087
|
+
// This node holds a `latest` it cannot materialize (see readCommitBase). That is a
|
|
1088
|
+
// local deficiency, not a content mismatch — report "cannot check" so the caller
|
|
1089
|
+
// abstains. Unlike the commit path's refuseMissingBase, this must NOT delete the
|
|
1090
|
+
// pending record or throw: preview is read-only and runs before any commit exists.
|
|
1091
|
+
log('previewCommitDigest:unmaterializable-base blockId=%s baseRev=%d rev=%d error=%s',
|
|
1092
|
+
blockId, latest.rev, rev, err instanceof Error ? err.message : String(err));
|
|
1093
|
+
return { baseIndependent: false, baseRev, digest: undefined };
|
|
1094
|
+
}
|
|
1095
|
+
}
|
|
1096
|
+
}
|
|
1097
|
+
|
|
1098
|
+
// Clone both: applyTransform assigns `transform.insert` into the result by reference and
|
|
1099
|
+
// applyOperations mutates the block in place, so materializing on live storage/pending objects
|
|
1100
|
+
// would corrupt them for the real commit that follows.
|
|
1101
|
+
const newBlock = applyTransform(structuredClone(base), structuredClone(transform));
|
|
1102
|
+
// `undefined` covers the tombstone (delete transform) and updates-with-no-base (applyTransform
|
|
1103
|
+
// drops updates when there is no block to apply them to) — both materialize nothing.
|
|
1104
|
+
const digest = newBlock ? await canonicalBlockHash(newBlock) : undefined;
|
|
1105
|
+
return { digest, baseRev, baseIndependent };
|
|
1106
|
+
}
|
|
1107
|
+
|
|
1108
|
+
/**
|
|
1109
|
+
* See {@link IRevisionActionReader}. Reads the block's revision index directly
|
|
1110
|
+
* (`listRevisions(rev, rev)` — both bounds inclusive per the `IBlockStorage` contract); an empty
|
|
1111
|
+
* range means this node holds no record for that revision.
|
|
1112
|
+
*/
|
|
1113
|
+
async getRevisionAction(blockId: BlockId, rev: number): Promise<ActionId | undefined> {
|
|
1114
|
+
const storage = this.createBlockStorage(blockId);
|
|
1115
|
+
for await (const actionRev of storage.listRevisions(rev, rev)) {
|
|
1116
|
+
return actionRev.actionId;
|
|
1117
|
+
}
|
|
1118
|
+
return undefined;
|
|
1119
|
+
}
|
|
1120
|
+
|
|
1121
|
+
/**
|
|
1122
|
+
* The {@link BlockCommitProof} this node retained for `blockId` at `rev`, or `undefined` when it
|
|
1123
|
+
* kept none — a revision committed before proofs were persisted, a member whose materialization
|
|
1124
|
+
* diverged from the declared digest (see {@link persistProofIfContentMatches}), or simply a
|
|
1125
|
+
* revision this node never landed.
|
|
1126
|
+
*
|
|
1127
|
+
* Public because a peer answering a block-repair fetch serves the proof alongside the revision
|
|
1128
|
+
* (`serveBlockArchive`), which is the only way a requester can check a lone holder's claim
|
|
1129
|
+
* without a second holder to corroborate it. Read-only and unlatched: a proof is written once
|
|
1130
|
+
* and never mutated, so a concurrent commit can only make this return a proof for a revision
|
|
1131
|
+
* that just became stale — which the caller pairs with the revision it actually read.
|
|
1132
|
+
*/
|
|
1133
|
+
async getBlockProof(blockId: BlockId, rev: number): Promise<BlockCommitProof | undefined> {
|
|
1134
|
+
return await this.createBlockStorage(blockId).getBlockProof(rev);
|
|
1135
|
+
}
|
|
1136
|
+
|
|
1137
|
+
private async internalCommit(blockId: BlockId, actionId: ActionId, rev: number, storage: IBlockStorage, latch: BlockWriteLatch, proof?: BlockCommitProof): Promise<CollectionId | undefined> {
|
|
1138
|
+
// Note: This method is called under the block write latch — by commit() (within its locked
|
|
1139
|
+
// critical section) and by the read-driven promotion in get() (which takes the same latch);
|
|
1140
|
+
// `latch` is the proof of that. So, operations like getPendingTransaction, getLatest,
|
|
1141
|
+
// getBlock, saveMaterializedBlock, saveRevision, promotePendingTransaction, setLatest are
|
|
1142
|
+
// protected against concurrent writers for the *same blockId*.
|
|
1143
|
+
//
|
|
1144
|
+
// `getBlock` here (via readCommitBase) is LOCAL-ONLY: the commit path never fetches from a
|
|
1145
|
+
// peer while holding N block latches. A coverage gap reads as a missing base, which the
|
|
1146
|
+
// healing path repairs by replication instead.
|
|
1147
|
+
|
|
1148
|
+
const transform = await storage.getPendingTransaction(actionId);
|
|
1149
|
+
// No need to check if !transform here, as the caller (commit) already verified this.
|
|
1150
|
+
// If it's null here, it indicates a logic error or race condition bypassed the lock (unlikely).
|
|
1151
|
+
if (!transform) {
|
|
1152
|
+
throw new Error(`Consistency Error: Pending action ${actionId} disappeared for block ${blockId} within critical section.`);
|
|
1153
|
+
}
|
|
1154
|
+
|
|
1155
|
+
// Get prior materialized block if it exists
|
|
1156
|
+
const latest = await storage.getLatest();
|
|
1157
|
+
const priorBlock = await this.readCommitBase(blockId, actionId, rev, storage, latest, latch);
|
|
1158
|
+
|
|
1159
|
+
// Apply transform and save materialized block
|
|
1160
|
+
// applyTransform handles undefined priorBlock correctly for inserts
|
|
1161
|
+
const newBlock = applyTransform(priorBlock, transform);
|
|
1162
|
+
|
|
1163
|
+
// INVARIANT: `latest` must never advance past a revision this node can materialize.
|
|
1164
|
+
// `applyTransform` silently drops `updates` when there is no block to apply them to, so a
|
|
1165
|
+
// member that missed the block's CREATING revision would otherwise record rev N while storing
|
|
1166
|
+
// nothing to serve it from. `latest === undefined` is precisely the "nothing below to fall
|
|
1167
|
+
// back to" case: materializeBlock's descending walk needs some materialization at or below the
|
|
1168
|
+
// target, and with no prior revision there is none. With a prior `latest` an absent newBlock is
|
|
1169
|
+
// a legitimate tombstone (the walk resolves to an earlier materialization), so it stays allowed.
|
|
1170
|
+
if (!newBlock && latest === undefined) {
|
|
1171
|
+
return await this.refuseMissingBase(blockId, actionId, rev, storage, latch,
|
|
1172
|
+
'no committed revision to apply the transform to');
|
|
1173
|
+
}
|
|
1174
|
+
|
|
1175
|
+
if (newBlock) {
|
|
1176
|
+
await storage.saveMaterializedBlock(actionId, newBlock, latch);
|
|
1177
|
+
}
|
|
1178
|
+
|
|
1179
|
+
// Save revision and promote action *before* updating latest
|
|
1180
|
+
// This ensures that if the process crashes between these steps,
|
|
1181
|
+
// the 'latest' pointer doesn't point to a revision that hasn't been fully recorded.
|
|
1182
|
+
await storage.saveRevision(rev, actionId, latch);
|
|
1183
|
+
await storage.promotePendingTransaction(actionId, latch);
|
|
1184
|
+
|
|
1185
|
+
// Update latest revision *last*
|
|
1186
|
+
await storage.setLatest({ actionId, rev }, latch);
|
|
1187
|
+
|
|
1188
|
+
// Persist the cohort's commit proof AFTER the commit is durably latest — the proof is
|
|
1189
|
+
// evidence about a landed revision, never a precondition of landing it. The retention rule
|
|
1190
|
+
// (persist only when the LOCAL materialization matches the digest the commit op declared)
|
|
1191
|
+
// and its failure logging live in the shared helper; a proof-persist fault must not fail a
|
|
1192
|
+
// commit that already landed, so the helper never throws.
|
|
1193
|
+
if (proof !== undefined) {
|
|
1194
|
+
await this.persistProofIfContentMatches(blockId, actionId, rev, storage, proof, newBlock, latch);
|
|
1195
|
+
}
|
|
1196
|
+
|
|
1197
|
+
// Prune the now-superseded prior materialization (checkpoint retention). Runs LAST — after the
|
|
1198
|
+
// new rev's materialization + revision + transform + setLatest are all durable — so no crash
|
|
1199
|
+
// point can leave a rev unrecoverable: a crash BEFORE this leaves a redundant (harmless)
|
|
1200
|
+
// materialization the next commit's prune reclaims; a crash AFTER is fully consistent. The prune
|
|
1201
|
+
// only ever deletes a materialization reconstructible from the retained floor + transforms. Runs
|
|
1202
|
+
// under the block write latch already held here, so it serializes against concurrent commits.
|
|
1203
|
+
// NOTE: prune targets ONLY the immediate prior. A crash between setLatest and this call leaves that
|
|
1204
|
+
// one prior materialization un-pruned; since a later commit prunes ITS OWN prior (never the earlier
|
|
1205
|
+
// leaked rev), that copy is NOT auto-reclaimed — a bounded (≤1 block-copy per crash), harmless leak
|
|
1206
|
+
// (state stays consistent + reconstructible). If crash-before-prune leaks ever accumulate materially,
|
|
1207
|
+
// add a bounded look-back (prune non-retained mats in [rev-checkpointInterval, rev)) here, or a
|
|
1208
|
+
// periodic reconciliation sweep — do NOT reintroduce a per-read re-cache.
|
|
1209
|
+
if (latest !== undefined) {
|
|
1210
|
+
await storage.pruneSupersededMaterialization(latest, latch);
|
|
1211
|
+
}
|
|
1212
|
+
|
|
1213
|
+
// Report the affected collection for change-event routing. For a delete the
|
|
1214
|
+
// materialized block is undefined, so fall back to the prior block's header.
|
|
1215
|
+
// Either may be absent only for a malformed/headerless block — return
|
|
1216
|
+
// undefined so the caller skips it rather than emitting a bogus event.
|
|
1217
|
+
return newBlock?.header.collectionId ?? priorBlock?.header.collectionId;
|
|
1218
|
+
}
|
|
1219
|
+
|
|
1220
|
+
/**
|
|
1221
|
+
* Retain `proof` for a block this call found ALREADY committed at `(rev, actionId)` — the paths
|
|
1222
|
+
* that land (or find already landed) a revision without running {@link internalCommit}, and would
|
|
1223
|
+
* otherwise never retain a proof: the idempotent re-commit partition, the Crash-D3 `recover()`
|
|
1224
|
+
* partition, and {@link saveReplicatedBlock}'s monotonic no-op on a certified push. Strictly
|
|
1225
|
+
* additive: an existing proof is left alone, and the same digest-match rule as the fresh-commit
|
|
1226
|
+
* site decides retention.
|
|
1227
|
+
*
|
|
1228
|
+
* `rev`/`actionId` are passed separately rather than as a `CommitRequest` because the replica
|
|
1229
|
+
* caller has no commit request — it has the `(rev, actionId)` the push and the held revision
|
|
1230
|
+
* agree on.
|
|
1231
|
+
*
|
|
1232
|
+
* Callers must hold the block's write latch (`latch`). `getBlock` is local-only and can throw on
|
|
1233
|
+
* an unmaterializable or uncovered base — treated as "no local content", i.e. the proof is withheld.
|
|
1234
|
+
*/
|
|
1235
|
+
private async backFillProof(
|
|
1236
|
+
blockId: BlockId, storage: IBlockStorage, rev: number, actionId: ActionId, proof: BlockCommitProof | undefined,
|
|
1237
|
+
latch: BlockWriteLatch
|
|
1238
|
+
): Promise<void> {
|
|
1239
|
+
if (proof === undefined || await storage.getBlockProof(rev) !== undefined) {
|
|
1240
|
+
return;
|
|
1241
|
+
}
|
|
1242
|
+
let committedBlock: IBlock | undefined;
|
|
1243
|
+
try {
|
|
1244
|
+
committedBlock = (await storage.getBlock(rev))?.block;
|
|
1245
|
+
} catch {
|
|
1246
|
+
committedBlock = undefined;
|
|
1247
|
+
}
|
|
1248
|
+
await this.persistProofIfContentMatches(blockId, actionId, rev, storage, proof, committedBlock, latch);
|
|
1249
|
+
}
|
|
1250
|
+
|
|
1251
|
+
/**
|
|
1252
|
+
* The single retention rule for {@link BlockCommitProof}s, shared by the fresh-commit site
|
|
1253
|
+
* ({@link internalCommit}, after `setLatest`) and the already-landed back-fill
|
|
1254
|
+
* ({@link backFillProof}):
|
|
1255
|
+
*
|
|
1256
|
+
* > **A member persists the proof only when its own materialization matches the digest the
|
|
1257
|
+
* > commit operation declared for this block.**
|
|
1258
|
+
*
|
|
1259
|
+
* One rule covers every awkward case without a second flag: a DIVERGED member (committed onto a
|
|
1260
|
+
* lagging base) computes a different hash, stores no proof, and falls back to corroboration
|
|
1261
|
+
* exactly as today — the `commit:proof-digest-mismatch` log line is also the first signal this
|
|
1262
|
+
* system has ever had that a member diverged. A member that abstained at vote time still checks
|
|
1263
|
+
* here (by commit time it HAS materialized) and legitimately keeps the proof on agreement. A
|
|
1264
|
+
* tombstone (no `block`) and a commit with no `blockDigests` (pre-upgrade client) declare no
|
|
1265
|
+
* digest and store no proof (`commit:proof-undeclared`).
|
|
1266
|
+
*
|
|
1267
|
+
* Never throws: the commit this proof describes already durably landed, so a proof-persist
|
|
1268
|
+
* fault must not turn `commit()` into `success:false` for a landed commit — it is logged and
|
|
1269
|
+
* the proof simply is not retained (repair falls back to corroboration).
|
|
1270
|
+
*
|
|
1271
|
+
* NOTE: one commit of N blocks stores the SAME proof under each block's `(blockId, rev)` proofs-store key, and
|
|
1272
|
+
* the proof itself carries the commit op's N `blockIds`/`blockDigests` — so bytes retained per
|
|
1273
|
+
* commit grow with N². Measured base cost is ~4.6 KB for a 10-peer 2-block commit
|
|
1274
|
+
* (`test/commit-proof.spec.ts` "size"), and nothing today bounds `CommitRequest.blockIds`. Fine
|
|
1275
|
+
* at the handful-of-blocks batches the transactor produces now; if per-coordinator batches ever
|
|
1276
|
+
* grow large, store the proof once under its `messageHash` and key each revision to a pointer.
|
|
1277
|
+
*/
|
|
1278
|
+
private async persistProofIfContentMatches(
|
|
1279
|
+
blockId: BlockId,
|
|
1280
|
+
actionId: ActionId,
|
|
1281
|
+
rev: number,
|
|
1282
|
+
storage: IBlockStorage,
|
|
1283
|
+
proof: BlockCommitProof,
|
|
1284
|
+
block: IBlock | undefined,
|
|
1285
|
+
latch: BlockWriteLatch
|
|
1286
|
+
): Promise<void> {
|
|
1287
|
+
try {
|
|
1288
|
+
const declaredDigest = proofDeclaredDigest(proof, { blockId, rev, actionId });
|
|
1289
|
+
if (declaredDigest === undefined) {
|
|
1290
|
+
log('commit:proof-undeclared blockId=%s rev=%d actionId=%s', blockId, rev, actionId);
|
|
1291
|
+
return;
|
|
1292
|
+
}
|
|
1293
|
+
// A digest was declared but this node materialized nothing (tombstone / unmaterializable
|
|
1294
|
+
// read on the back-fill path): the local content provably is not the declared content.
|
|
1295
|
+
const localDigest = block === undefined ? undefined : await canonicalBlockHash(block);
|
|
1296
|
+
if (localDigest !== declaredDigest) {
|
|
1297
|
+
log('commit:proof-digest-mismatch blockId=%s rev=%d actionId=%s declared=%s local=%s',
|
|
1298
|
+
blockId, rev, actionId, declaredDigest, localDigest);
|
|
1299
|
+
return;
|
|
1300
|
+
}
|
|
1301
|
+
await storage.saveBlockProof(rev, proof, latch);
|
|
1302
|
+
} catch (err) {
|
|
1303
|
+
log('commit:proof-persist-failed blockId=%s rev=%d actionId=%s error=%s', blockId, rev, actionId,
|
|
1304
|
+
err instanceof Error ? err.message : String(err));
|
|
1305
|
+
}
|
|
1306
|
+
}
|
|
1307
|
+
|
|
1308
|
+
/**
|
|
1309
|
+
* The materialization this commit builds on: the block at `latest`, or `undefined` when the block
|
|
1310
|
+
* holds no committed revision yet (the normal insert case).
|
|
1311
|
+
*
|
|
1312
|
+
* `getBlock` THROWS when this node holds a `latest` it cannot materialize — a block already wedged
|
|
1313
|
+
* by a pre-fix commit, or by truncated history. That is the same divergence as having no base at
|
|
1314
|
+
* all, so it is translated into {@link MissingBaseRevisionError} rather than surfacing as an opaque
|
|
1315
|
+
* storage fault: the healing path can then repair the block instead of the fault resetting the
|
|
1316
|
+
* cluster stream, and a wedged node recovers on the next write touching the block.
|
|
1317
|
+
*
|
|
1318
|
+
* The catch is deliberately UNNARROWED — it also absorbs a transient fault (a raw-storage read
|
|
1319
|
+
* error, a `restoreCallback` timeout on a block whose `ranges` do not cover its own `latest`).
|
|
1320
|
+
* BlockStorage reports every one of these as a bare `Error`, so they cannot be told apart here,
|
|
1321
|
+
* and treating them as divergence is the safe default: this node genuinely cannot materialize the
|
|
1322
|
+
* base right now, and the cluster's policy is to heal rather than throw out of consensus. The
|
|
1323
|
+
* price is that a transient fault ALSO drops pending records — this block's (see
|
|
1324
|
+
* {@link refuseMissingBase}) AND, because {@link commit} keys its cleanup off the same error type,
|
|
1325
|
+
* every not-yet-reached block in the same batch — so those blocks converge by replication instead
|
|
1326
|
+
* of by a replay the retry could have done. That is a wider blast radius than the per-block
|
|
1327
|
+
* refusal alone, and it is why the discriminator must NOT be loosened beyond this error type.
|
|
1328
|
+
* Narrowing this would require typed faults out of BlockStorage; until then, prefer the tolerant
|
|
1329
|
+
* reading.
|
|
1330
|
+
*/
|
|
1331
|
+
private async readCommitBase(
|
|
1332
|
+
blockId: BlockId,
|
|
1333
|
+
actionId: ActionId,
|
|
1334
|
+
rev: number,
|
|
1335
|
+
storage: IBlockStorage,
|
|
1336
|
+
latest: ActionRev | undefined,
|
|
1337
|
+
latch: BlockWriteLatch
|
|
1338
|
+
): Promise<IBlock | undefined> {
|
|
1339
|
+
if (!latest) {
|
|
1340
|
+
return undefined;
|
|
1341
|
+
}
|
|
1342
|
+
// NOTE: this read is deliberately LOCAL-ONLY and does not heal. `getBlock` no longer restores
|
|
1343
|
+
// from a peer (that moved to the explicit `restoreRevision`, which `StorageRepo.get` calls), so a
|
|
1344
|
+
// base this node cannot materialize locally raises {@link MissingBaseRevisionError} here instead
|
|
1345
|
+
// of being fetched in line. The reason is the calling context, not the cost of a fetch: `commit`
|
|
1346
|
+
// holds the write latch of EVERY block in the batch across this call, and network I/O inside that
|
|
1347
|
+
// critical section makes one unreachable peer stall every writer of every block in the batch for
|
|
1348
|
+
// the length of a round trip. Healing is out-of-band instead — cohort reconcile supplies the
|
|
1349
|
+
// revision (`ClusterMember` → `saveReplicatedBlock`) and the action is retried, by which point
|
|
1350
|
+
// this read succeeds locally. Pinned by `test/storage-repo.spec.ts` "commit reads its base
|
|
1351
|
+
// locally", which wires a restore callback that would have answered and asserts it is never
|
|
1352
|
+
// called. Do not reintroduce a restore on this path; if a commit ever genuinely needs one, fetch
|
|
1353
|
+
// BEFORE taking the latches, not underneath them.
|
|
1354
|
+
//
|
|
1355
|
+
// NOTE: `latest.rev` is always inside `meta.ranges` today — every writer of `latest`
|
|
1356
|
+
// (`setLatest`, `saveForwardRevision`, `recover`) merges an open-ended range anchored at or
|
|
1357
|
+
// below the new latest in the same `saveMetadata` — so the RevisionNotCoveredError arm below
|
|
1358
|
+
// is unreachable from here and only truncated-history corruption lands in the catch. If a
|
|
1359
|
+
// future change can leave `latest` uncovered, the ordering in `get` becomes load-bearing: the
|
|
1360
|
+
// read-driven promotion runs BEFORE `readBlockHealing`, so a coverage gap under `latest` would
|
|
1361
|
+
// make `refuseMissingBase` delete the pending record moments before the healing read would
|
|
1362
|
+
// have restored it. Heal before refusing if that day comes.
|
|
1363
|
+
try {
|
|
1364
|
+
return (await storage.getBlock(latest.rev))?.block;
|
|
1365
|
+
} catch (err) {
|
|
1366
|
+
log('commit:unmaterializable-base blockId=%s baseRev=%d error=%s', blockId, latest.rev,
|
|
1367
|
+
err instanceof Error ? err.message : String(err));
|
|
1368
|
+
return await this.refuseMissingBase(blockId, actionId, rev, storage, latch,
|
|
1369
|
+
`local rev ${latest.rev} is not materializable here`);
|
|
1370
|
+
}
|
|
1371
|
+
}
|
|
1372
|
+
|
|
1373
|
+
/**
|
|
1374
|
+
* Refuse a commit this node cannot materialize. Always throws {@link MissingBaseRevisionError};
|
|
1375
|
+
* nothing durable has been written at this point, so the block is left exactly as it was minus the
|
|
1376
|
+
* pending record.
|
|
1377
|
+
*
|
|
1378
|
+
* The pending is dropped because it can never be promoted here: promotion needs a base this node
|
|
1379
|
+
* must obtain out-of-band, and once the healing path lands that revision `latest` is already >= rev,
|
|
1380
|
+
* so a commit retry partitions the block as already-done/stale and never revisits the pending.
|
|
1381
|
+
* Leaving it would also report a phantom conflicting action from {@link pend} for every later write.
|
|
1382
|
+
*/
|
|
1383
|
+
private async refuseMissingBase(
|
|
1384
|
+
blockId: BlockId,
|
|
1385
|
+
actionId: ActionId,
|
|
1386
|
+
rev: number,
|
|
1387
|
+
storage: IBlockStorage,
|
|
1388
|
+
latch: BlockWriteLatch,
|
|
1389
|
+
detail: string
|
|
1390
|
+
): Promise<never> {
|
|
1391
|
+
await storage.deletePendingTransaction(actionId, latch);
|
|
1392
|
+
log('commit:missing-base blockId=%s rev=%d actionId=%s detail=%s', blockId, rev, actionId, detail);
|
|
1393
|
+
throw new MissingBaseRevisionError(blockId, rev, detail);
|
|
1394
|
+
}
|
|
1395
|
+
}
|
|
1396
|
+
|
|
1397
|
+
/**
|
|
1398
|
+
* Converts list of missing actions per block into a list of missing actions across blocks.
|
|
1399
|
+
*
|
|
1400
|
+
* NOTE: relies on each (actionId, blockId) pair appearing at most once — one revision per action
|
|
1401
|
+
* per block. If a block ever records two revisions under the same actionId, concatTransform now
|
|
1402
|
+
* concatenates both revisions' ops into one array rather than dropping the earlier one — still
|
|
1403
|
+
* wrong, since ops from distinct revisions are not composable against a single base, but loud
|
|
1404
|
+
* rather than silent. Group by (actionId, rev) instead if that case becomes reachable.
|
|
1405
|
+
*/
|
|
1406
|
+
function perBlockActionTransformsToPerAction(missing: { blockId: BlockId; transforms: ActionTransform[]; }[]) {
|
|
1407
|
+
const missingFlat = missing.flatMap(({ blockId, transforms }) =>
|
|
1408
|
+
transforms.map(transform => ({ blockId, transform }))
|
|
1409
|
+
);
|
|
1410
|
+
const missingByActionId = groupBy(missingFlat, ({ transform }) => transform.actionId);
|
|
1411
|
+
return Object.entries(missingByActionId).map(([actionId, items]) =>
|
|
1412
|
+
items.reduce((acc, { blockId, transform }) => {
|
|
1413
|
+
acc.transforms = concatTransform(acc.transforms, blockId, transform.transform);
|
|
1414
|
+
return acc;
|
|
1415
|
+
}, {
|
|
1416
|
+
actionId: actionId as ActionId,
|
|
1417
|
+
rev: items[0]!.transform.rev, // Assumption: an action commits at one revision, so every block's entry for this actionId agrees. Distinct actionIds may still carry distinct revs.
|
|
1418
|
+
transforms: emptyTransforms()
|
|
1419
|
+
})
|
|
1420
|
+
);
|
|
1421
|
+
}
|