@optimystic/db-p2p 0.24.2 → 0.25.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/{readme.md → README.md} +688 -668
- package/dist/src/cluster/block-transfer-service.d.ts +133 -10
- package/dist/src/cluster/block-transfer-service.d.ts.map +1 -1
- package/dist/src/cluster/block-transfer-service.js +155 -9
- package/dist/src/cluster/block-transfer-service.js.map +1 -1
- package/dist/src/cluster/block-transfer.d.ts +67 -7
- package/dist/src/cluster/block-transfer.d.ts.map +1 -1
- package/dist/src/cluster/block-transfer.js +127 -24
- package/dist/src/cluster/block-transfer.js.map +1 -1
- package/dist/src/cluster/certified-claims.d.ts +170 -0
- package/dist/src/cluster/certified-claims.d.ts.map +1 -0
- package/dist/src/cluster/certified-claims.js +200 -0
- package/dist/src/cluster/certified-claims.js.map +1 -0
- package/dist/src/cluster/client-signature-verifier.d.ts +23 -0
- package/dist/src/cluster/client-signature-verifier.d.ts.map +1 -0
- package/dist/src/cluster/client-signature-verifier.js +33 -0
- package/dist/src/cluster/client-signature-verifier.js.map +1 -0
- package/dist/src/cluster/cluster-policy.d.ts +9 -3
- package/dist/src/cluster/cluster-policy.d.ts.map +1 -1
- package/dist/src/cluster/cluster-policy.js +111 -27
- package/dist/src/cluster/cluster-policy.js.map +1 -1
- package/dist/src/cluster/cluster-repo.d.ts +140 -11
- package/dist/src/cluster/cluster-repo.d.ts.map +1 -1
- package/dist/src/cluster/cluster-repo.js +485 -29
- package/dist/src/cluster/cluster-repo.js.map +1 -1
- package/dist/src/cluster/commit-proof.d.ts +134 -0
- package/dist/src/cluster/commit-proof.d.ts.map +1 -0
- package/dist/src/cluster/commit-proof.js +263 -0
- package/dist/src/cluster/commit-proof.js.map +1 -0
- package/dist/src/cluster/quorum-restore.d.ts +95 -8
- package/dist/src/cluster/quorum-restore.d.ts.map +1 -1
- package/dist/src/cluster/quorum-restore.js +113 -14
- package/dist/src/cluster/quorum-restore.js.map +1 -1
- package/dist/src/cluster/rebalance-monitor.d.ts +84 -0
- package/dist/src/cluster/rebalance-monitor.d.ts.map +1 -1
- package/dist/src/cluster/rebalance-monitor.js +207 -5
- package/dist/src/cluster/rebalance-monitor.js.map +1 -1
- package/dist/src/cluster/reconcile-block.d.ts +27 -7
- package/dist/src/cluster/reconcile-block.d.ts.map +1 -1
- package/dist/src/cluster/reconcile-block.js +179 -38
- package/dist/src/cluster/reconcile-block.js.map +1 -1
- package/dist/src/cluster/service.d.ts +13 -4
- package/dist/src/cluster/service.d.ts.map +1 -1
- package/dist/src/cluster/service.js +17 -18
- package/dist/src/cluster/service.js.map +1 -1
- package/dist/src/cluster/spread-on-churn.d.ts +8 -2
- package/dist/src/cluster/spread-on-churn.d.ts.map +1 -1
- package/dist/src/cluster/spread-on-churn.js +7 -7
- package/dist/src/cluster/spread-on-churn.js.map +1 -1
- package/dist/src/cohort-topic/host.d.ts.map +1 -1
- package/dist/src/cohort-topic/host.js +8 -7
- package/dist/src/cohort-topic/host.js.map +1 -1
- package/dist/src/cohort-topic/stream-util.d.ts +7 -6
- package/dist/src/cohort-topic/stream-util.d.ts.map +1 -1
- package/dist/src/cohort-topic/stream-util.js +14 -41
- package/dist/src/cohort-topic/stream-util.js.map +1 -1
- package/dist/src/dispute/cascade.d.ts +7 -8
- package/dist/src/dispute/cascade.d.ts.map +1 -1
- package/dist/src/dispute/cascade.js +24 -7
- package/dist/src/dispute/cascade.js.map +1 -1
- package/dist/src/dispute/dispute-service.d.ts +5 -0
- package/dist/src/dispute/dispute-service.d.ts.map +1 -1
- package/dist/src/dispute/dispute-service.js +13 -3
- package/dist/src/dispute/dispute-service.js.map +1 -1
- package/dist/src/dispute/invalidation.d.ts +33 -15
- package/dist/src/dispute/invalidation.d.ts.map +1 -1
- package/dist/src/dispute/invalidation.js +127 -34
- package/dist/src/dispute/invalidation.js.map +1 -1
- package/dist/src/dispute/service.d.ts.map +1 -1
- package/dist/src/dispute/service.js +2 -1
- package/dist/src/dispute/service.js.map +1 -1
- package/dist/src/index.d.ts +7 -0
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +7 -0
- package/dist/src/index.js.map +1 -1
- package/dist/src/libp2p-key-network.d.ts +14 -10
- package/dist/src/libp2p-key-network.d.ts.map +1 -1
- package/dist/src/libp2p-key-network.js +28 -58
- package/dist/src/libp2p-key-network.js.map +1 -1
- package/dist/src/libp2p-node-base.d.ts +38 -7
- package/dist/src/libp2p-node-base.d.ts.map +1 -1
- package/dist/src/libp2p-node-base.js +148 -63
- package/dist/src/libp2p-node-base.js.map +1 -1
- package/dist/src/network/open-protocol-stream.d.ts +56 -0
- package/dist/src/network/open-protocol-stream.d.ts.map +1 -0
- package/dist/src/network/open-protocol-stream.js +60 -0
- package/dist/src/network/open-protocol-stream.js.map +1 -0
- package/dist/src/network/register-protocol-handler.d.ts +66 -0
- package/dist/src/network/register-protocol-handler.d.ts.map +1 -0
- package/dist/src/network/register-protocol-handler.js +51 -0
- package/dist/src/network/register-protocol-handler.js.map +1 -0
- package/dist/src/peer-address-book.d.ts +60 -2
- package/dist/src/peer-address-book.d.ts.map +1 -1
- package/dist/src/peer-address-book.js +89 -1
- package/dist/src/peer-address-book.js.map +1 -1
- package/dist/src/pend-validation.d.ts +65 -0
- package/dist/src/pend-validation.d.ts.map +1 -0
- package/dist/src/pend-validation.js +69 -0
- package/dist/src/pend-validation.js.map +1 -0
- package/dist/src/reactivity/notify-transport.d.ts.map +1 -1
- package/dist/src/reactivity/notify-transport.js +2 -1
- package/dist/src/reactivity/notify-transport.js.map +1 -1
- package/dist/src/reactivity/push-state-gossip.d.ts.map +1 -1
- package/dist/src/reactivity/push-state-gossip.js +2 -1
- package/dist/src/reactivity/push-state-gossip.js.map +1 -1
- package/dist/src/repo/cluster-coordinator.d.ts +32 -6
- package/dist/src/repo/cluster-coordinator.d.ts.map +1 -1
- package/dist/src/repo/cluster-coordinator.js +44 -10
- package/dist/src/repo/cluster-coordinator.js.map +1 -1
- package/dist/src/repo/coordinator-repo.d.ts +181 -25
- package/dist/src/repo/coordinator-repo.d.ts.map +1 -1
- package/dist/src/repo/coordinator-repo.js +650 -62
- package/dist/src/repo/coordinator-repo.js.map +1 -1
- package/dist/src/repo/served-repo-proxy.d.ts +30 -0
- package/dist/src/repo/served-repo-proxy.d.ts.map +1 -0
- package/dist/src/repo/served-repo-proxy.js +50 -0
- package/dist/src/repo/served-repo-proxy.js.map +1 -0
- package/dist/src/repo/service.d.ts +16 -1
- package/dist/src/repo/service.d.ts.map +1 -1
- package/dist/src/repo/service.js +18 -16
- package/dist/src/repo/service.js.map +1 -1
- package/dist/src/reputation/types.d.ts +3 -1
- package/dist/src/reputation/types.d.ts.map +1 -1
- package/dist/src/reputation/types.js +3 -1
- package/dist/src/reputation/types.js.map +1 -1
- package/dist/src/rn.d.ts +24 -2
- package/dist/src/rn.d.ts.map +1 -1
- package/dist/src/rn.js +33 -4
- package/dist/src/rn.js.map +1 -1
- package/dist/src/storage/block-archive.d.ts +153 -0
- package/dist/src/storage/block-archive.d.ts.map +1 -0
- package/dist/src/storage/block-archive.js +194 -0
- package/dist/src/storage/block-archive.js.map +1 -0
- package/dist/src/storage/block-latch.d.ts +92 -0
- package/dist/src/storage/block-latch.d.ts.map +1 -0
- package/dist/src/storage/block-latch.js +132 -0
- package/dist/src/storage/block-latch.js.map +1 -0
- package/dist/src/storage/block-storage.d.ts +169 -12
- package/dist/src/storage/block-storage.d.ts.map +1 -1
- package/dist/src/storage/block-storage.js +415 -152
- package/dist/src/storage/block-storage.js.map +1 -1
- package/dist/src/storage/cached-raw-storage.d.ts +5 -1
- package/dist/src/storage/cached-raw-storage.d.ts.map +1 -1
- package/dist/src/storage/cached-raw-storage.js +16 -1
- package/dist/src/storage/cached-raw-storage.js.map +1 -1
- package/dist/src/storage/cached-store-driver.d.ts +14 -4
- package/dist/src/storage/cached-store-driver.d.ts.map +1 -1
- package/dist/src/storage/cached-store-driver.js +39 -5
- package/dist/src/storage/cached-store-driver.js.map +1 -1
- package/dist/src/storage/i-block-storage.d.ts +70 -19
- package/dist/src/storage/i-block-storage.d.ts.map +1 -1
- package/dist/src/storage/i-block-storage.js +21 -1
- package/dist/src/storage/i-block-storage.js.map +1 -1
- package/dist/src/storage/i-raw-storage.d.ts +38 -0
- package/dist/src/storage/i-raw-storage.d.ts.map +1 -1
- package/dist/src/storage/kv-raw-storage.d.ts +14 -8
- package/dist/src/storage/kv-raw-storage.d.ts.map +1 -1
- package/dist/src/storage/kv-raw-storage.js +29 -9
- package/dist/src/storage/kv-raw-storage.js.map +1 -1
- package/dist/src/storage/memory-store-driver.d.ts +4 -1
- package/dist/src/storage/memory-store-driver.d.ts.map +1 -1
- package/dist/src/storage/memory-store-driver.js +22 -2
- package/dist/src/storage/memory-store-driver.js.map +1 -1
- package/dist/src/storage/raw-store-codec.js +2 -2
- package/dist/src/storage/raw-store-driver.d.ts +44 -3
- package/dist/src/storage/raw-store-driver.d.ts.map +1 -1
- package/dist/src/storage/shared-cache-pool.d.ts +60 -3
- package/dist/src/storage/shared-cache-pool.d.ts.map +1 -1
- package/dist/src/storage/shared-cache-pool.js +68 -4
- package/dist/src/storage/shared-cache-pool.js.map +1 -1
- package/dist/src/storage/storage-repo.d.ts +204 -26
- package/dist/src/storage/storage-repo.d.ts.map +1 -1
- package/dist/src/storage/storage-repo.js +422 -119
- package/dist/src/storage/storage-repo.js.map +1 -1
- package/dist/src/storage/store-identity.d.ts +21 -0
- package/dist/src/storage/store-identity.d.ts.map +1 -0
- package/dist/src/storage/store-identity.js +33 -0
- package/dist/src/storage/store-identity.js.map +1 -0
- package/dist/src/storage/struct.d.ts +15 -0
- package/dist/src/storage/struct.d.ts.map +1 -1
- package/dist/src/storage/with-read-cache.d.ts +92 -0
- package/dist/src/storage/with-read-cache.d.ts.map +1 -0
- package/dist/src/storage/with-read-cache.js +132 -0
- package/dist/src/storage/with-read-cache.js.map +1 -0
- package/dist/src/sync/service.d.ts +13 -3
- package/dist/src/sync/service.d.ts.map +1 -1
- package/dist/src/sync/service.js +7 -30
- package/dist/src/sync/service.js.map +1 -1
- package/dist/src/testing/mesh-harness.d.ts +75 -2
- package/dist/src/testing/mesh-harness.d.ts.map +1 -1
- package/dist/src/testing/mesh-harness.js +193 -81
- package/dist/src/testing/mesh-harness.js.map +1 -1
- package/dist/src/testing/raw-storage-conformance.d.ts +8 -0
- package/dist/src/testing/raw-storage-conformance.d.ts.map +1 -1
- package/dist/src/testing/raw-storage-conformance.js +97 -2
- package/dist/src/testing/raw-storage-conformance.js.map +1 -1
- package/package.json +5 -6
- package/src/cluster/block-transfer-service.ts +528 -312
- package/src/cluster/block-transfer.ts +547 -424
- package/src/cluster/certified-claims.ts +319 -0
- package/src/cluster/client-signature-verifier.ts +34 -0
- package/src/cluster/cluster-policy.ts +294 -203
- package/src/cluster/cluster-repo.ts +2501 -2034
- package/src/cluster/commit-proof.ts +344 -0
- package/src/cluster/quorum-restore.ts +348 -223
- package/src/cluster/rebalance-monitor.ts +539 -254
- package/src/cluster/reconcile-block.ts +404 -203
- package/src/cluster/service.ts +23 -20
- package/src/cluster/spread-on-churn.ts +352 -346
- package/src/cohort-topic/host.ts +8 -7
- package/src/cohort-topic/stream-util.ts +14 -43
- package/src/dispute/cascade.ts +31 -15
- package/src/dispute/dispute-service.ts +706 -696
- package/src/dispute/invalidation.ts +162 -52
- package/src/dispute/service.ts +126 -125
- package/src/index.ts +7 -0
- package/src/libp2p-key-network.ts +1204 -1235
- package/src/libp2p-node-base.ts +185 -70
- package/src/network/open-protocol-stream.ts +87 -0
- package/src/network/register-protocol-handler.ts +85 -0
- package/src/peer-address-book.ts +96 -2
- package/src/pend-validation.ts +89 -0
- package/src/reactivity/notify-transport.ts +2 -1
- package/src/reactivity/push-state-gossip.ts +2 -1
- package/src/repo/cluster-coordinator.ts +65 -11
- package/src/repo/coordinator-repo.ts +729 -71
- package/src/repo/served-repo-proxy.ts +55 -0
- package/src/repo/service.ts +25 -16
- package/src/reputation/types.ts +3 -1
- package/src/rn.ts +34 -4
- package/src/storage/block-archive.ts +242 -0
- package/src/storage/block-latch.ts +144 -0
- package/src/storage/block-storage.ts +787 -498
- package/src/storage/cached-raw-storage.ts +21 -1
- package/src/storage/cached-store-driver.ts +898 -859
- package/src/storage/i-block-storage.ts +172 -116
- package/src/storage/i-raw-storage.ts +40 -0
- package/src/storage/kv-raw-storage.ts +34 -9
- package/src/storage/memory-store-driver.ts +26 -2
- package/src/storage/raw-store-codec.ts +42 -42
- package/src/storage/raw-store-driver.ts +45 -3
- package/src/storage/shared-cache-pool.ts +516 -452
- package/src/storage/storage-repo.ts +1421 -1014
- package/src/storage/store-identity.ts +41 -0
- package/src/storage/struct.ts +52 -34
- package/src/storage/with-read-cache.ts +182 -0
- package/src/sync/service.ts +178 -198
- package/src/testing/mesh-harness.ts +524 -344
- package/src/testing/raw-storage-conformance.ts +114 -2
|
@@ -1,2034 +1,2501 @@
|
|
|
1
|
-
import type { IRepo, ClusterRecord, ClusterPeers, Signature, RepoMessage, ITransactionValidator, ClusterConsensusConfig, CommitResult, BlockId, ActionId, ActionRev, CommitRequest, CommitCert, InvalidateRequest } from "@optimystic/db-core";
|
|
2
|
-
import type { ICluster } from "@optimystic/db-core";
|
|
3
|
-
import type { IPeerNetwork } from "@optimystic/db-core";
|
|
4
|
-
import { blockIdsForTransforms, DEFAULT_SUPER_MAJORITY_THRESHOLD } from "@optimystic/db-core";
|
|
5
|
-
import { computeClusterCommitHash, computeClusterMessageHash, computeClusterPromiseHash, membershipDigest, recordMembershipDigest, clampPriority, clusterVoteSigningPayload, clusterVoteVerificationPayload } from "@optimystic/db-core";
|
|
6
|
-
import { verifyInvalidationCertificate, type ArbitratorSetRecompute } from "../dispute/invalidation.js";
|
|
7
|
-
import { buildCommitCert, invalidationActionId } from "./commit-cert.js";
|
|
8
|
-
import { ClusterClient } from "./client.js";
|
|
9
|
-
import type { PeerId, PrivateKey } from "@libp2p/interface";
|
|
10
|
-
import { peerIdFromString } from "@libp2p/peer-id";
|
|
11
|
-
import { publicKeyFromRaw } from "@libp2p/crypto/keys";
|
|
12
|
-
import { peerIdBindsPublicKey } from "./peer-key-binding.js";
|
|
13
|
-
import { toString as uint8ArrayToString, fromString as uint8ArrayFromString } from 'uint8arrays';
|
|
14
|
-
import { createLogger } from '../logger.js'
|
|
15
|
-
import type { PartitionDetector } from "./partition-detector.js";
|
|
16
|
-
import type { FretService } from "p2p-fret";
|
|
17
|
-
import type { IPeerReputation } from "../reputation/types.js";
|
|
18
|
-
import { PenaltyReason } from "../reputation/types.js";
|
|
19
|
-
import type { ITransactionStateStore } from "./i-transaction-state-store.js";
|
|
20
|
-
import { isMissingBaseRevisionFailure } from "../storage/storage-repo.js";
|
|
21
|
-
import {
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
*
|
|
41
|
-
*
|
|
42
|
-
*
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
*
|
|
59
|
-
*
|
|
60
|
-
*
|
|
61
|
-
* peer
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
*
|
|
71
|
-
*
|
|
72
|
-
*
|
|
73
|
-
*
|
|
74
|
-
*
|
|
75
|
-
*
|
|
76
|
-
* @param
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
* {@link
|
|
84
|
-
*
|
|
85
|
-
*
|
|
86
|
-
*
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
*
|
|
94
|
-
*
|
|
95
|
-
*
|
|
96
|
-
*
|
|
97
|
-
*
|
|
98
|
-
*
|
|
99
|
-
*
|
|
100
|
-
*
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
*
|
|
108
|
-
*
|
|
109
|
-
*
|
|
110
|
-
*
|
|
111
|
-
*
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
*
|
|
119
|
-
*
|
|
120
|
-
*
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
/**
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
*
|
|
133
|
-
*
|
|
134
|
-
*
|
|
135
|
-
*
|
|
136
|
-
*
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
}
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
private
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
//
|
|
245
|
-
//
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
private
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
}
|
|
406
|
-
|
|
407
|
-
//
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
log('cluster-member:
|
|
501
|
-
messageHash: record.messageHash
|
|
502
|
-
});
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
//
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
}
|
|
702
|
-
|
|
703
|
-
}
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
}
|
|
746
|
-
|
|
747
|
-
/**
|
|
748
|
-
*
|
|
749
|
-
*
|
|
750
|
-
*
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
const
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
}
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
const
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
//
|
|
838
|
-
//
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
|
|
1122
|
-
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
|
|
1126
|
-
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
|
|
1149
|
-
|
|
1150
|
-
|
|
1151
|
-
|
|
1152
|
-
|
|
1153
|
-
|
|
1154
|
-
|
|
1155
|
-
|
|
1156
|
-
|
|
1157
|
-
|
|
1158
|
-
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
|
|
1165
|
-
|
|
1166
|
-
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
|
|
1170
|
-
|
|
1171
|
-
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
|
|
1180
|
-
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
-
|
|
1188
|
-
|
|
1189
|
-
|
|
1190
|
-
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
private async
|
|
1194
|
-
if (this.
|
|
1195
|
-
return
|
|
1196
|
-
}
|
|
1197
|
-
const
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
|
|
1203
|
-
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
-
|
|
1207
|
-
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
|
|
1214
|
-
|
|
1215
|
-
|
|
1216
|
-
|
|
1217
|
-
|
|
1218
|
-
|
|
1219
|
-
|
|
1220
|
-
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
|
|
1225
|
-
|
|
1226
|
-
|
|
1227
|
-
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
|
|
1241
|
-
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
|
|
1268
|
-
|
|
1269
|
-
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
-
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
|
|
1307
|
-
|
|
1308
|
-
|
|
1309
|
-
|
|
1310
|
-
|
|
1311
|
-
|
|
1312
|
-
|
|
1313
|
-
|
|
1314
|
-
|
|
1315
|
-
|
|
1316
|
-
|
|
1317
|
-
|
|
1318
|
-
|
|
1319
|
-
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
|
|
1337
|
-
|
|
1338
|
-
|
|
1339
|
-
|
|
1340
|
-
|
|
1341
|
-
|
|
1342
|
-
|
|
1343
|
-
|
|
1344
|
-
|
|
1345
|
-
|
|
1346
|
-
|
|
1347
|
-
|
|
1348
|
-
|
|
1349
|
-
|
|
1350
|
-
|
|
1351
|
-
|
|
1352
|
-
|
|
1353
|
-
|
|
1354
|
-
|
|
1355
|
-
|
|
1356
|
-
|
|
1357
|
-
|
|
1358
|
-
|
|
1359
|
-
|
|
1360
|
-
|
|
1361
|
-
|
|
1362
|
-
|
|
1363
|
-
|
|
1364
|
-
|
|
1365
|
-
|
|
1366
|
-
|
|
1367
|
-
|
|
1368
|
-
|
|
1369
|
-
|
|
1370
|
-
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
|
|
1375
|
-
|
|
1376
|
-
|
|
1377
|
-
|
|
1378
|
-
|
|
1379
|
-
|
|
1380
|
-
|
|
1381
|
-
|
|
1382
|
-
|
|
1383
|
-
|
|
1384
|
-
|
|
1385
|
-
|
|
1386
|
-
|
|
1387
|
-
|
|
1388
|
-
|
|
1389
|
-
|
|
1390
|
-
|
|
1391
|
-
|
|
1392
|
-
|
|
1393
|
-
|
|
1394
|
-
|
|
1395
|
-
|
|
1396
|
-
|
|
1397
|
-
|
|
1398
|
-
|
|
1399
|
-
|
|
1400
|
-
|
|
1401
|
-
|
|
1402
|
-
|
|
1403
|
-
|
|
1404
|
-
|
|
1405
|
-
|
|
1406
|
-
|
|
1407
|
-
|
|
1408
|
-
|
|
1409
|
-
|
|
1410
|
-
|
|
1411
|
-
|
|
1412
|
-
|
|
1413
|
-
|
|
1414
|
-
|
|
1415
|
-
|
|
1416
|
-
|
|
1417
|
-
|
|
1418
|
-
|
|
1419
|
-
|
|
1420
|
-
|
|
1421
|
-
|
|
1422
|
-
|
|
1423
|
-
|
|
1424
|
-
|
|
1425
|
-
|
|
1426
|
-
|
|
1427
|
-
|
|
1428
|
-
|
|
1429
|
-
|
|
1430
|
-
|
|
1431
|
-
|
|
1432
|
-
|
|
1433
|
-
|
|
1434
|
-
|
|
1435
|
-
|
|
1436
|
-
|
|
1437
|
-
|
|
1438
|
-
|
|
1439
|
-
|
|
1440
|
-
|
|
1441
|
-
|
|
1442
|
-
|
|
1443
|
-
|
|
1444
|
-
|
|
1445
|
-
|
|
1446
|
-
|
|
1447
|
-
|
|
1448
|
-
|
|
1449
|
-
|
|
1450
|
-
|
|
1451
|
-
|
|
1452
|
-
|
|
1453
|
-
|
|
1454
|
-
|
|
1455
|
-
|
|
1456
|
-
|
|
1457
|
-
|
|
1458
|
-
|
|
1459
|
-
|
|
1460
|
-
|
|
1461
|
-
|
|
1462
|
-
|
|
1463
|
-
|
|
1464
|
-
|
|
1465
|
-
|
|
1466
|
-
|
|
1467
|
-
|
|
1468
|
-
|
|
1469
|
-
|
|
1470
|
-
|
|
1471
|
-
|
|
1472
|
-
|
|
1473
|
-
|
|
1474
|
-
|
|
1475
|
-
|
|
1476
|
-
}
|
|
1477
|
-
|
|
1478
|
-
}
|
|
1479
|
-
|
|
1480
|
-
|
|
1481
|
-
|
|
1482
|
-
|
|
1483
|
-
|
|
1484
|
-
|
|
1485
|
-
|
|
1486
|
-
|
|
1487
|
-
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
|
|
1491
|
-
|
|
1492
|
-
|
|
1493
|
-
|
|
1494
|
-
|
|
1495
|
-
|
|
1496
|
-
|
|
1497
|
-
|
|
1498
|
-
|
|
1499
|
-
|
|
1500
|
-
|
|
1501
|
-
|
|
1502
|
-
|
|
1503
|
-
|
|
1504
|
-
|
|
1505
|
-
|
|
1506
|
-
|
|
1507
|
-
|
|
1508
|
-
|
|
1509
|
-
|
|
1510
|
-
}
|
|
1511
|
-
|
|
1512
|
-
|
|
1513
|
-
|
|
1514
|
-
|
|
1515
|
-
|
|
1516
|
-
|
|
1517
|
-
|
|
1518
|
-
|
|
1519
|
-
|
|
1520
|
-
|
|
1521
|
-
|
|
1522
|
-
|
|
1523
|
-
|
|
1524
|
-
|
|
1525
|
-
|
|
1526
|
-
|
|
1527
|
-
|
|
1528
|
-
|
|
1529
|
-
|
|
1530
|
-
|
|
1531
|
-
|
|
1532
|
-
|
|
1533
|
-
|
|
1534
|
-
|
|
1535
|
-
|
|
1536
|
-
|
|
1537
|
-
|
|
1538
|
-
|
|
1539
|
-
|
|
1540
|
-
|
|
1541
|
-
|
|
1542
|
-
|
|
1543
|
-
|
|
1544
|
-
|
|
1545
|
-
|
|
1546
|
-
|
|
1547
|
-
|
|
1548
|
-
|
|
1549
|
-
|
|
1550
|
-
|
|
1551
|
-
|
|
1552
|
-
|
|
1553
|
-
|
|
1554
|
-
|
|
1555
|
-
|
|
1556
|
-
|
|
1557
|
-
|
|
1558
|
-
|
|
1559
|
-
|
|
1560
|
-
|
|
1561
|
-
|
|
1562
|
-
|
|
1563
|
-
|
|
1564
|
-
|
|
1565
|
-
|
|
1566
|
-
|
|
1567
|
-
|
|
1568
|
-
|
|
1569
|
-
|
|
1570
|
-
|
|
1571
|
-
|
|
1572
|
-
|
|
1573
|
-
|
|
1574
|
-
|
|
1575
|
-
|
|
1576
|
-
|
|
1577
|
-
|
|
1578
|
-
|
|
1579
|
-
|
|
1580
|
-
|
|
1581
|
-
|
|
1582
|
-
|
|
1583
|
-
|
|
1584
|
-
|
|
1585
|
-
|
|
1586
|
-
|
|
1587
|
-
|
|
1588
|
-
|
|
1589
|
-
|
|
1590
|
-
|
|
1591
|
-
|
|
1592
|
-
|
|
1593
|
-
|
|
1594
|
-
|
|
1595
|
-
|
|
1596
|
-
|
|
1597
|
-
|
|
1598
|
-
|
|
1599
|
-
|
|
1600
|
-
|
|
1601
|
-
|
|
1602
|
-
|
|
1603
|
-
|
|
1604
|
-
|
|
1605
|
-
|
|
1606
|
-
|
|
1607
|
-
|
|
1608
|
-
|
|
1609
|
-
|
|
1610
|
-
|
|
1611
|
-
|
|
1612
|
-
|
|
1613
|
-
|
|
1614
|
-
|
|
1615
|
-
|
|
1616
|
-
|
|
1617
|
-
|
|
1618
|
-
|
|
1619
|
-
|
|
1620
|
-
|
|
1621
|
-
|
|
1622
|
-
|
|
1623
|
-
|
|
1624
|
-
|
|
1625
|
-
|
|
1626
|
-
|
|
1627
|
-
|
|
1628
|
-
|
|
1629
|
-
|
|
1630
|
-
|
|
1631
|
-
|
|
1632
|
-
|
|
1633
|
-
|
|
1634
|
-
|
|
1635
|
-
|
|
1636
|
-
|
|
1637
|
-
|
|
1638
|
-
|
|
1639
|
-
|
|
1640
|
-
|
|
1641
|
-
|
|
1642
|
-
|
|
1643
|
-
|
|
1644
|
-
|
|
1645
|
-
|
|
1646
|
-
|
|
1647
|
-
|
|
1648
|
-
|
|
1649
|
-
|
|
1650
|
-
|
|
1651
|
-
|
|
1652
|
-
|
|
1653
|
-
|
|
1654
|
-
|
|
1655
|
-
|
|
1656
|
-
|
|
1657
|
-
|
|
1658
|
-
|
|
1659
|
-
|
|
1660
|
-
|
|
1661
|
-
|
|
1662
|
-
|
|
1663
|
-
|
|
1664
|
-
|
|
1665
|
-
|
|
1666
|
-
|
|
1667
|
-
|
|
1668
|
-
|
|
1669
|
-
|
|
1670
|
-
|
|
1671
|
-
|
|
1672
|
-
|
|
1673
|
-
|
|
1674
|
-
|
|
1675
|
-
|
|
1676
|
-
}
|
|
1677
|
-
|
|
1678
|
-
|
|
1679
|
-
|
|
1680
|
-
|
|
1681
|
-
|
|
1682
|
-
|
|
1683
|
-
|
|
1684
|
-
|
|
1685
|
-
|
|
1686
|
-
|
|
1687
|
-
|
|
1688
|
-
|
|
1689
|
-
|
|
1690
|
-
|
|
1691
|
-
|
|
1692
|
-
|
|
1693
|
-
*
|
|
1694
|
-
*
|
|
1695
|
-
*
|
|
1696
|
-
*
|
|
1697
|
-
*
|
|
1698
|
-
*
|
|
1699
|
-
*
|
|
1700
|
-
*
|
|
1701
|
-
* (`
|
|
1702
|
-
*
|
|
1703
|
-
*
|
|
1704
|
-
*
|
|
1705
|
-
*
|
|
1706
|
-
*
|
|
1707
|
-
*
|
|
1708
|
-
*
|
|
1709
|
-
*
|
|
1710
|
-
*
|
|
1711
|
-
*
|
|
1712
|
-
*
|
|
1713
|
-
*
|
|
1714
|
-
*
|
|
1715
|
-
*
|
|
1716
|
-
*
|
|
1717
|
-
*
|
|
1718
|
-
*
|
|
1719
|
-
*
|
|
1720
|
-
*
|
|
1721
|
-
*
|
|
1722
|
-
*
|
|
1723
|
-
*
|
|
1724
|
-
*
|
|
1725
|
-
*
|
|
1726
|
-
*
|
|
1727
|
-
*
|
|
1728
|
-
*
|
|
1729
|
-
*
|
|
1730
|
-
*
|
|
1731
|
-
*
|
|
1732
|
-
*
|
|
1733
|
-
|
|
1734
|
-
|
|
1735
|
-
|
|
1736
|
-
|
|
1737
|
-
|
|
1738
|
-
|
|
1739
|
-
|
|
1740
|
-
|
|
1741
|
-
|
|
1742
|
-
|
|
1743
|
-
|
|
1744
|
-
|
|
1745
|
-
|
|
1746
|
-
|
|
1747
|
-
|
|
1748
|
-
|
|
1749
|
-
|
|
1750
|
-
|
|
1751
|
-
|
|
1752
|
-
|
|
1753
|
-
|
|
1754
|
-
|
|
1755
|
-
|
|
1756
|
-
|
|
1757
|
-
|
|
1758
|
-
|
|
1759
|
-
|
|
1760
|
-
|
|
1761
|
-
|
|
1762
|
-
|
|
1763
|
-
|
|
1764
|
-
|
|
1765
|
-
|
|
1766
|
-
|
|
1767
|
-
|
|
1768
|
-
|
|
1769
|
-
|
|
1770
|
-
|
|
1771
|
-
|
|
1772
|
-
|
|
1773
|
-
|
|
1774
|
-
|
|
1775
|
-
|
|
1776
|
-
|
|
1777
|
-
|
|
1778
|
-
|
|
1779
|
-
|
|
1780
|
-
|
|
1781
|
-
|
|
1782
|
-
|
|
1783
|
-
|
|
1784
|
-
|
|
1785
|
-
|
|
1786
|
-
|
|
1787
|
-
|
|
1788
|
-
|
|
1789
|
-
|
|
1790
|
-
|
|
1791
|
-
|
|
1792
|
-
|
|
1793
|
-
|
|
1794
|
-
|
|
1795
|
-
|
|
1796
|
-
|
|
1797
|
-
|
|
1798
|
-
|
|
1799
|
-
|
|
1800
|
-
|
|
1801
|
-
|
|
1802
|
-
|
|
1803
|
-
|
|
1804
|
-
|
|
1805
|
-
|
|
1806
|
-
|
|
1807
|
-
|
|
1808
|
-
|
|
1809
|
-
|
|
1810
|
-
|
|
1811
|
-
|
|
1812
|
-
|
|
1813
|
-
|
|
1814
|
-
|
|
1815
|
-
|
|
1816
|
-
|
|
1817
|
-
|
|
1818
|
-
}
|
|
1819
|
-
|
|
1820
|
-
|
|
1821
|
-
|
|
1822
|
-
|
|
1823
|
-
|
|
1824
|
-
|
|
1825
|
-
|
|
1826
|
-
|
|
1827
|
-
|
|
1828
|
-
|
|
1829
|
-
|
|
1830
|
-
//
|
|
1831
|
-
|
|
1832
|
-
|
|
1833
|
-
|
|
1834
|
-
|
|
1835
|
-
|
|
1836
|
-
|
|
1837
|
-
//
|
|
1838
|
-
//
|
|
1839
|
-
|
|
1840
|
-
|
|
1841
|
-
|
|
1842
|
-
|
|
1843
|
-
|
|
1844
|
-
|
|
1845
|
-
|
|
1846
|
-
|
|
1847
|
-
|
|
1848
|
-
|
|
1849
|
-
|
|
1850
|
-
|
|
1851
|
-
|
|
1852
|
-
|
|
1853
|
-
|
|
1854
|
-
|
|
1855
|
-
|
|
1856
|
-
|
|
1857
|
-
|
|
1858
|
-
|
|
1859
|
-
|
|
1860
|
-
|
|
1861
|
-
|
|
1862
|
-
|
|
1863
|
-
|
|
1864
|
-
|
|
1865
|
-
|
|
1866
|
-
|
|
1867
|
-
|
|
1868
|
-
|
|
1869
|
-
|
|
1870
|
-
|
|
1871
|
-
|
|
1872
|
-
|
|
1873
|
-
|
|
1874
|
-
|
|
1875
|
-
|
|
1876
|
-
|
|
1877
|
-
|
|
1878
|
-
|
|
1879
|
-
|
|
1880
|
-
|
|
1881
|
-
|
|
1882
|
-
|
|
1883
|
-
|
|
1884
|
-
|
|
1885
|
-
|
|
1886
|
-
|
|
1887
|
-
|
|
1888
|
-
|
|
1889
|
-
|
|
1890
|
-
|
|
1891
|
-
|
|
1892
|
-
|
|
1893
|
-
|
|
1894
|
-
|
|
1895
|
-
|
|
1896
|
-
|
|
1897
|
-
|
|
1898
|
-
|
|
1899
|
-
|
|
1900
|
-
|
|
1901
|
-
|
|
1902
|
-
|
|
1903
|
-
|
|
1904
|
-
|
|
1905
|
-
//
|
|
1906
|
-
|
|
1907
|
-
|
|
1908
|
-
|
|
1909
|
-
|
|
1910
|
-
|
|
1911
|
-
|
|
1912
|
-
|
|
1913
|
-
|
|
1914
|
-
|
|
1915
|
-
|
|
1916
|
-
|
|
1917
|
-
|
|
1918
|
-
|
|
1919
|
-
|
|
1920
|
-
|
|
1921
|
-
|
|
1922
|
-
|
|
1923
|
-
|
|
1924
|
-
|
|
1925
|
-
|
|
1926
|
-
|
|
1927
|
-
|
|
1928
|
-
|
|
1929
|
-
|
|
1930
|
-
|
|
1931
|
-
|
|
1932
|
-
|
|
1933
|
-
|
|
1934
|
-
|
|
1935
|
-
|
|
1936
|
-
|
|
1937
|
-
|
|
1938
|
-
|
|
1939
|
-
|
|
1940
|
-
|
|
1941
|
-
|
|
1942
|
-
|
|
1943
|
-
|
|
1944
|
-
|
|
1945
|
-
|
|
1946
|
-
|
|
1947
|
-
|
|
1948
|
-
|
|
1949
|
-
|
|
1950
|
-
|
|
1951
|
-
|
|
1952
|
-
|
|
1953
|
-
|
|
1954
|
-
|
|
1955
|
-
|
|
1956
|
-
|
|
1957
|
-
|
|
1958
|
-
|
|
1959
|
-
}
|
|
1960
|
-
|
|
1961
|
-
|
|
1962
|
-
|
|
1963
|
-
|
|
1964
|
-
|
|
1965
|
-
|
|
1966
|
-
|
|
1967
|
-
|
|
1968
|
-
|
|
1969
|
-
|
|
1970
|
-
|
|
1971
|
-
|
|
1972
|
-
|
|
1973
|
-
|
|
1974
|
-
|
|
1975
|
-
|
|
1976
|
-
|
|
1977
|
-
|
|
1978
|
-
|
|
1979
|
-
|
|
1980
|
-
|
|
1981
|
-
|
|
1982
|
-
|
|
1983
|
-
|
|
1984
|
-
|
|
1985
|
-
|
|
1986
|
-
|
|
1987
|
-
|
|
1988
|
-
|
|
1989
|
-
|
|
1990
|
-
|
|
1991
|
-
|
|
1992
|
-
|
|
1993
|
-
|
|
1994
|
-
|
|
1995
|
-
|
|
1996
|
-
|
|
1997
|
-
|
|
1998
|
-
|
|
1999
|
-
|
|
2000
|
-
|
|
2001
|
-
|
|
2002
|
-
|
|
2003
|
-
log('cluster-member:
|
|
2004
|
-
|
|
2005
|
-
|
|
2006
|
-
|
|
2007
|
-
|
|
2008
|
-
|
|
2009
|
-
|
|
2010
|
-
|
|
2011
|
-
|
|
2012
|
-
|
|
2013
|
-
|
|
2014
|
-
|
|
2015
|
-
|
|
2016
|
-
|
|
2017
|
-
|
|
2018
|
-
|
|
2019
|
-
|
|
2020
|
-
|
|
2021
|
-
|
|
2022
|
-
|
|
2023
|
-
|
|
2024
|
-
|
|
2025
|
-
|
|
2026
|
-
|
|
2027
|
-
|
|
2028
|
-
|
|
2029
|
-
|
|
2030
|
-
|
|
2031
|
-
|
|
2032
|
-
|
|
2033
|
-
|
|
2034
|
-
|
|
1
|
+
import type { IRepo, ClusterRecord, ClusterPeers, Signature, RepoMessage, ITransactionValidator, ClusterConsensusConfig, UnvalidatablePendPolicy, CommitResult, PendResult, BlockId, ActionId, ActionRev, CommitRequest, CommitCert, InvalidateRequest } from "@optimystic/db-core";
|
|
2
|
+
import type { ICluster } from "@optimystic/db-core";
|
|
3
|
+
import type { IPeerNetwork } from "@optimystic/db-core";
|
|
4
|
+
import { blockIdsForTransforms, isOwnRevision, DEFAULT_SUPER_MAJORITY_THRESHOLD } from "@optimystic/db-core";
|
|
5
|
+
import { computeClusterCommitHash, computeClusterMessageHash, computeClusterPromiseHash, membershipDigest, recordMembershipDigest, clampPriority, clusterVoteSigningPayload, clusterVoteVerificationPayload } from "@optimystic/db-core";
|
|
6
|
+
import { verifyInvalidationCertificate, type ArbitratorSetRecompute } from "../dispute/invalidation.js";
|
|
7
|
+
import { buildCommitCert, invalidationActionId } from "./commit-cert.js";
|
|
8
|
+
import { ClusterClient } from "./client.js";
|
|
9
|
+
import type { PeerId, PrivateKey } from "@libp2p/interface";
|
|
10
|
+
import { peerIdFromString } from "@libp2p/peer-id";
|
|
11
|
+
import { publicKeyFromRaw } from "@libp2p/crypto/keys";
|
|
12
|
+
import { peerIdBindsPublicKey } from "./peer-key-binding.js";
|
|
13
|
+
import { toString as uint8ArrayToString, fromString as uint8ArrayFromString } from 'uint8arrays';
|
|
14
|
+
import { createLogger } from '../logger.js'
|
|
15
|
+
import type { PartitionDetector } from "./partition-detector.js";
|
|
16
|
+
import type { FretService } from "p2p-fret";
|
|
17
|
+
import type { IPeerReputation } from "../reputation/types.js";
|
|
18
|
+
import { PenaltyReason } from "../reputation/types.js";
|
|
19
|
+
import type { ITransactionStateStore } from "./i-transaction-state-store.js";
|
|
20
|
+
import { isMissingBaseRevisionFailure, type CommitDigestPreview, type ICommitDigestPreviewer, type ICommitProofPersister, type IRevisionActionReader } from "../storage/storage-repo.js";
|
|
21
|
+
import { checkPendValidation } from "../pend-validation.js";
|
|
22
|
+
import { buildBlockCommitProof } from "./commit-proof.js";
|
|
23
|
+
import { RECONCILE_TIMEOUT_MS } from "./reconcile-block.js";
|
|
24
|
+
|
|
25
|
+
const log = createLogger('cluster-member')
|
|
26
|
+
|
|
27
|
+
/** State of a transaction in the cluster */
|
|
28
|
+
enum TransactionPhase {
|
|
29
|
+
Promising, // We have voted; still collecting promises from other peers
|
|
30
|
+
OurPromiseNeeded, // We need to provide our promise
|
|
31
|
+
OurConflictVoteNeeded, // We hold a conflicting race winner; we must answer with a conflict vote
|
|
32
|
+
OurCommitNeeded, // We need to provide our commit
|
|
33
|
+
Consensus, // Transaction has reached consensus
|
|
34
|
+
Rejected, // Transaction was rejected (validity judgement — enough reject votes)
|
|
35
|
+
ConflictSuperseded, // Terminal but retryable: conflict votes make super-majority unreachable
|
|
36
|
+
Propagating // Transaction is being propagated
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* A phase plus the data its handler needs. Only {@link TransactionPhase.OurConflictVoteNeeded}
|
|
41
|
+
* carries any — `conflictsWith`, the winning rival's messageHash — so the conflict-vote handler need
|
|
42
|
+
* not re-run conflict detection (whose race resolution has side effects) to learn what blocked it.
|
|
43
|
+
* Split by phase rather than an optional field so the handler reads it without an assertion, and so
|
|
44
|
+
* a future phase-with-data cannot silently inherit this one's payload.
|
|
45
|
+
*/
|
|
46
|
+
type PhaseResult =
|
|
47
|
+
| { phase: TransactionPhase.OurConflictVoteNeeded; conflictsWith: string }
|
|
48
|
+
| { phase: Exclude<TransactionPhase, TransactionPhase.OurConflictVoteNeeded> };
|
|
49
|
+
|
|
50
|
+
interface TransactionState {
|
|
51
|
+
record: ClusterRecord;
|
|
52
|
+
promiseTimeout?: NodeJS.Timeout;
|
|
53
|
+
resolutionTimeout?: NodeJS.Timeout;
|
|
54
|
+
lastUpdate: number;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* Result of verifying one vote signature. `penalize` distinguishes "identity was never proven"
|
|
59
|
+
* (no key / not Ed25519 / key not bound to the peer id / malformed input — reject, but do NOT
|
|
60
|
+
* report the named peer, whose id may have been attacker-chosen) from "the key IS the one the
|
|
61
|
+
* peer id names, yet the signature does not verify" (reject AND report — a genuine bad vote from
|
|
62
|
+
* that proven identity). Collapsing both into a bare `false` would let a coordinator get an honest
|
|
63
|
+
* peer penalized just by attaching a key it controls under that peer's id.
|
|
64
|
+
*/
|
|
65
|
+
type VerifyOutcome =
|
|
66
|
+
| { valid: true }
|
|
67
|
+
| { valid: false; penalize: boolean };
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* Actively reconciles a block this member committed without having seen the matching
|
|
71
|
+
* pend (cohort drift between the independent pend and commit cluster-transactions).
|
|
72
|
+
* Pulls the committed revision from a cohort peer that holds it and restores it into
|
|
73
|
+
* local storage. Injected so {@link ClusterMember} stays transport-agnostic — mirrors
|
|
74
|
+
* how `CoordinatorRepo` receives its `clusterLatestCallback`.
|
|
75
|
+
*
|
|
76
|
+
* @param blockId the under-replicated block to restore
|
|
77
|
+
* @param committed the committed `(actionId, rev)` agreed by consensus
|
|
78
|
+
* @param cohortPeerIds cohort members to pull from (self already excluded)
|
|
79
|
+
*/
|
|
80
|
+
export type ReconcileBlockCallback = (blockId: BlockId, committed: ActionRev, cohortPeerIds: string[]) => Promise<void>;
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* Sink for the {@link CommitCert} this member assembled at consensus, fired once per committed action
|
|
84
|
+
* **before** the commit is applied to local storage (so it is already retained when
|
|
85
|
+
* {@link StorageRepo.commit} emits the matching `CollectionChangeEvent` the reactivity bridge reads).
|
|
86
|
+
* The bytes are the cluster's own `approve` commit signatures, forwarded UNCHANGED — never re-signed.
|
|
87
|
+
* Optional; absent on nodes that do not originate reactivity notifications. A throwing sink is
|
|
88
|
+
* isolated + logged (it must never break consensus).
|
|
89
|
+
*/
|
|
90
|
+
export type CommitCertificateSink = (actionId: ActionId, cert: CommitCert) => void;
|
|
91
|
+
|
|
92
|
+
/**
|
|
93
|
+
* Applies a consensus-ordered {@link InvalidateRequest} to local storage — the deterministic
|
|
94
|
+
* reversal every member runs once consensus on the invalidation is reached. The implementation
|
|
95
|
+
* recomputes the per-block as-if-`T_inv`-absent content, writes the compensating revisions, and
|
|
96
|
+
* appends the durable invalidation log entry (see `applyInvalidation` in the dispute module). It
|
|
97
|
+
* is injected so {@link ClusterMember} stays storage/log-agnostic (mirrors {@link ReconcileBlockCallback}).
|
|
98
|
+
*
|
|
99
|
+
* The certificate is verified by {@link ClusterMember} *before* the sink is invoked, so a sink
|
|
100
|
+
* implementation may assume `request.resolution` is already a valid challenger-wins certificate.
|
|
101
|
+
* A throwing sink is logged and tolerated (never resets the cluster stream); the in-memory dedup
|
|
102
|
+
* marker is rolled back so a re-broadcast can retry.
|
|
103
|
+
*/
|
|
104
|
+
export type InvalidationApplySink = (request: InvalidateRequest) => Promise<void>;
|
|
105
|
+
|
|
106
|
+
/**
|
|
107
|
+
* Optional **layer-2** capability for invalidation-certificate verification: re-derives the
|
|
108
|
+
* legitimately-selected arbitrator set from this member's topology view and judges the carried set
|
|
109
|
+
* (see {@link ArbitratorSetRecompute}). Injected so {@link ClusterMember} stays network-agnostic; the
|
|
110
|
+
* composition root supplies it from FRET when available. When absent (or when the member cannot
|
|
111
|
+
* reconstruct the historical topology), {@link applyConsensusInvalidation} accepts a layer-1-valid
|
|
112
|
+
* certificate and logs that it applied an invalidation it could not fully anchor — the documented
|
|
113
|
+
* interim posture (see `tickets/plan/cohort-topic-membership-cert-trust-anchoring.md`).
|
|
114
|
+
*/
|
|
115
|
+
export type RecomputeArbitratorSetCapability = ArbitratorSetRecompute;
|
|
116
|
+
|
|
117
|
+
/**
|
|
118
|
+
* The member's own independently-derived view of a block's responsible cluster, plus FRET's confidence in
|
|
119
|
+
* the underlying network-size estimate (0..1). Returned by {@link DeriveExpectedClusterCallback} and
|
|
120
|
+
* consumed by {@link ClusterMember}'s membership admission gate: the member checks a coordinator-declared
|
|
121
|
+
* peer set against `peers` (its expected set `E`) and gates on `confidence` (low confidence ⇒ fail closed
|
|
122
|
+
* for any downsizing, the partition posture).
|
|
123
|
+
*/
|
|
124
|
+
export type ExpectedClusterView = {
|
|
125
|
+
/** The member's own derived responsible-peer set for the block (its view of the legitimate cluster). */
|
|
126
|
+
peers: ClusterPeers;
|
|
127
|
+
/** FRET's confidence in the current network-size estimate (0..1); ≤ threshold ⇒ treated as untrusted. */
|
|
128
|
+
confidence: number;
|
|
129
|
+
};
|
|
130
|
+
|
|
131
|
+
/**
|
|
132
|
+
* Independently derive this member's own view of a block's responsible cluster. Injected so
|
|
133
|
+
* {@link ClusterMember} stays transport-agnostic — the composition root supplies it from
|
|
134
|
+
* `IKeyNetwork.findCluster` + FRET (mirroring how the coordinator derives the cluster). Absent on nodes
|
|
135
|
+
* that cannot derive a view (no FRET, unit tests): with no derived view AND no asserted
|
|
136
|
+
* {@link ClusterConsensusConfig.assumedClusterSize} the gate preserves legacy approve behavior, but an
|
|
137
|
+
* asserted size still lets the gate fail closed on an unjustified downsize. See {@link ClusterMember}
|
|
138
|
+
* admission gate.
|
|
139
|
+
*/
|
|
140
|
+
export type DeriveExpectedClusterCallback = (blockId: BlockId) => Promise<ExpectedClusterView>;
|
|
141
|
+
|
|
142
|
+
/** Stable reject reason a member emits when a declared peer set fails the membership admission gate. */
|
|
143
|
+
export const MEMBERSHIP_NOT_ADMITTED = 'membership-not-admitted';
|
|
144
|
+
|
|
145
|
+
/**
|
|
146
|
+
* Stable reject reason a member emits when its own materialization of a commit's block disagrees
|
|
147
|
+
* with the content digest the transaction author declared (`CommitRequest.blockDigests`). Rides in
|
|
148
|
+
* the reject vote's `rejectReason`, which `clusterVoteSigningPayload` folds into the signed bytes —
|
|
149
|
+
* so the rejection itself is integrity-protected.
|
|
150
|
+
*/
|
|
151
|
+
export const CONTENT_DIGEST_MISMATCH = 'content-digest-mismatch';
|
|
152
|
+
|
|
153
|
+
/**
|
|
154
|
+
* The two stable reject reasons a validator-configured member emits from the shared
|
|
155
|
+
* {@link checkPendValidation}: `PEND_NOT_VALIDATABLE` for a pend carrying no `validation` payload
|
|
156
|
+
* under `ClusterConsensusConfig.unvalidatablePendPolicy: 'reject'`, and `VALIDATOR_FAULT` for a
|
|
157
|
+
* checker that threw. Defined in `pend-validation.ts` (which the storage tier runs too, so both
|
|
158
|
+
* tiers refuse with the same prefixes) and re-exported here next to its siblings above.
|
|
159
|
+
*/
|
|
160
|
+
export { PEND_NOT_VALIDATABLE, VALIDATOR_FAULT } from "../pend-validation.js";
|
|
161
|
+
|
|
162
|
+
interface ClusterMemberComponents {
|
|
163
|
+
storageRepo: IRepo;
|
|
164
|
+
peerNetwork: IPeerNetwork;
|
|
165
|
+
peerId: PeerId;
|
|
166
|
+
privateKey: PrivateKey;
|
|
167
|
+
protocolPrefix?: string;
|
|
168
|
+
// Reserved for partition-healing consumers (backlog ticket 6.5-partition-healing); not yet read by ClusterMember.
|
|
169
|
+
partitionDetector?: PartitionDetector;
|
|
170
|
+
fretService?: FretService;
|
|
171
|
+
validator?: ITransactionValidator;
|
|
172
|
+
reputation?: IPeerReputation;
|
|
173
|
+
consensusConfig?: ClusterConsensusConfig;
|
|
174
|
+
stateStore?: ITransactionStateStore;
|
|
175
|
+
/** Restores a block under-replicated by cohort drift; see {@link ReconcileBlockCallback}. */
|
|
176
|
+
reconcileBlock?: ReconcileBlockCallback;
|
|
177
|
+
/** Receives the consensus commit cert per committed action; see {@link CommitCertificateSink}. */
|
|
178
|
+
onCommitCertificate?: CommitCertificateSink;
|
|
179
|
+
/** Applies a consensus-ordered invalidation to local storage; see {@link InvalidationApplySink}. */
|
|
180
|
+
onInvalidate?: InvalidationApplySink;
|
|
181
|
+
/** Layer-2 arbitrator-set recompute for invalidation verification; see {@link RecomputeArbitratorSetCapability}. */
|
|
182
|
+
recomputeArbitratorSet?: RecomputeArbitratorSetCapability;
|
|
183
|
+
/** Member-side cluster derivation for the membership admission gate; see {@link DeriveExpectedClusterCallback}. */
|
|
184
|
+
deriveExpectedCluster?: DeriveExpectedClusterCallback;
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
export function clusterMember(components: ClusterMemberComponents): ClusterMember {
|
|
188
|
+
return new ClusterMember(
|
|
189
|
+
components.storageRepo,
|
|
190
|
+
components.peerNetwork,
|
|
191
|
+
components.peerId,
|
|
192
|
+
components.privateKey,
|
|
193
|
+
components.protocolPrefix,
|
|
194
|
+
components.partitionDetector,
|
|
195
|
+
components.fretService,
|
|
196
|
+
components.validator,
|
|
197
|
+
components.reputation,
|
|
198
|
+
components.consensusConfig,
|
|
199
|
+
components.stateStore,
|
|
200
|
+
components.reconcileBlock,
|
|
201
|
+
components.onCommitCertificate,
|
|
202
|
+
components.onInvalidate,
|
|
203
|
+
components.recomputeArbitratorSet,
|
|
204
|
+
components.deriveExpectedCluster
|
|
205
|
+
);
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
// How long to keep executed transaction records (10 minutes)
|
|
209
|
+
const ExecutedTransactionTtlMs = 10 * 60 * 1000;
|
|
210
|
+
|
|
211
|
+
// Upper bound on an awaited active reconciliation of a divergent commit. Bounds the
|
|
212
|
+
// consensus path so a slow/unreachable cohort peer can't stall the cluster stream;
|
|
213
|
+
// a timeout is logged and tolerated (never thrown — that would reset the stream).
|
|
214
|
+
// Shared with the read path's acquisition (see RECONCILE_TIMEOUT_MS) — same operation, same bound.
|
|
215
|
+
const ReconcileTimeoutMs = RECONCILE_TIMEOUT_MS;
|
|
216
|
+
|
|
217
|
+
/**
|
|
218
|
+
* True when a thrown storage error reports a missing pending action — i.e. this
|
|
219
|
+
* member reached commit-consensus without having seen the matching pend phase.
|
|
220
|
+
* That is recoverable local divergence (reconciled via sync / read-repair), not
|
|
221
|
+
* a transaction fault, so it must not reset the cluster stream.
|
|
222
|
+
*/
|
|
223
|
+
function isMissingPendingActionError(err: unknown): boolean {
|
|
224
|
+
return err instanceof Error && /pending action .+ not found/i.test(err.message);
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
/**
|
|
228
|
+
* Handles cluster-side operations, managing promises and commits for cluster updates
|
|
229
|
+
* and coordinating with the local storage repo.
|
|
230
|
+
*/
|
|
231
|
+
export class ClusterMember implements ICluster {
|
|
232
|
+
// Track active transactions by their message hash
|
|
233
|
+
private activeTransactions: Map<string, TransactionState> = new Map();
|
|
234
|
+
// Track executed consensus transactions to prevent duplicate execution (messageHash -> executedAt timestamp)
|
|
235
|
+
private executedTransactions: Map<string, number> = new Map();
|
|
236
|
+
// Local storage's verdict for a pend operation applied during consensus (messageHash -> PendResult).
|
|
237
|
+
// Retained so the coordinator can return storage's real answer instead of fabricating success —
|
|
238
|
+
// a pend that every member refused (rival pending action, or the revision already taken) must
|
|
239
|
+
// reach the writer as a conflict, not a win. Pruned alongside executedTransactions (same TTL).
|
|
240
|
+
private executedPendResults: Map<string, PendResult> = new Map();
|
|
241
|
+
// Local storage's verdict for a COMMIT operation applied during consensus (messageHash ->
|
|
242
|
+
// CommitResult). Retained so the coordinator can detect when the ahead-divergence tolerance in
|
|
243
|
+
// applyConsensusOperation swallowed a refusal whose real cause was a RIVAL action holding the
|
|
244
|
+
// requested revision — the commit-tier acknowledgement hole: a commit that assembled consensus
|
|
245
|
+
// inside every member's signed-but-not-yet-applied window is refused by every member's storage
|
|
246
|
+
// at apply, and without this verdict the coordinator fabricates a success no member durably
|
|
247
|
+
// stored. Pruned alongside executedTransactions (same TTL).
|
|
248
|
+
private executedCommitResults: Map<string, CommitResult> = new Map();
|
|
249
|
+
// Fast in-memory dedup for applied invalidations, keyed `${invalidatedActionId}:${disputeId}`.
|
|
250
|
+
// The durable source of truth is the invalidation log entry (Log.findInvalidation, re-checked
|
|
251
|
+
// inside the sink); this map only spares redundant work when the same invalidation reaches
|
|
252
|
+
// consensus twice (rebroadcast / sync) under different message hashes. (-> appliedAt timestamp)
|
|
253
|
+
private appliedInvalidations: Map<string, number> = new Map();
|
|
254
|
+
// Queue of transactions to clean up
|
|
255
|
+
private cleanupQueue: string[] = [];
|
|
256
|
+
// Serialize concurrent updates for the same transaction
|
|
257
|
+
private pendingUpdates: Map<string, Promise<ClusterRecord>> = new Map();
|
|
258
|
+
// Interval handles for periodic cleanup (stored so dispose() can clear them)
|
|
259
|
+
private readonly expirationInterval: NodeJS.Timeout;
|
|
260
|
+
private readonly cleanupInterval: NodeJS.Timeout;
|
|
261
|
+
|
|
262
|
+
/**
|
|
263
|
+
* Confidence floor at/below which FRET's network-size view is treated as untrustworthy for the
|
|
264
|
+
* membership gate. Above it the member trusts its derived view (confident path); at/below it the gate
|
|
265
|
+
* fails closed for downsizing. Matches the coordinator's `validateSmallCluster` confidence gate (> 0.5).
|
|
266
|
+
*/
|
|
267
|
+
private static readonly MembershipConfidenceThreshold = 0.5;
|
|
268
|
+
|
|
269
|
+
/**
|
|
270
|
+
* Effective super-majority threshold this member accepts as sufficient for a commit. Defaults to
|
|
271
|
+
* {@link DEFAULT_SUPER_MAJORITY_THRESHOLD} (0.75) when no config is supplied — the SAME default the
|
|
272
|
+
* coordinator uses, so the two can never silently disagree about whether a transaction is final.
|
|
273
|
+
* (Previously defaulted to 1.0/unanimity, which split the member from a coordinator committing at 0.75.)
|
|
274
|
+
*/
|
|
275
|
+
private readonly superMajorityThreshold: number;
|
|
276
|
+
// Membership admission gate parameters (see {@link admitMembership}). Read once from consensusConfig
|
|
277
|
+
// so the gate has stable thresholds independent of the (untrusted) values a record declares.
|
|
278
|
+
private readonly minAbsoluteClusterSize: number;
|
|
279
|
+
private readonly clusterSizeTolerance: number;
|
|
280
|
+
private readonly membershipAdmissionFraction: number;
|
|
281
|
+
/** Operator-asserted smallest genuine cohort size, or undefined when unknown. */
|
|
282
|
+
private readonly assumedClusterSize: number | undefined;
|
|
283
|
+
private readonly allowUnvalidatedSmallCluster: boolean;
|
|
284
|
+
/** What a validator-configured member does with a pend carrying no `validation` payload — see
|
|
285
|
+
* {@link ClusterConsensusConfig.unvalidatablePendPolicy}. Read once, like the gate parameters. */
|
|
286
|
+
private readonly unvalidatablePendPolicy: UnvalidatablePendPolicy;
|
|
287
|
+
|
|
288
|
+
constructor(
|
|
289
|
+
private readonly storageRepo: IRepo,
|
|
290
|
+
private readonly peerNetwork: IPeerNetwork,
|
|
291
|
+
private readonly peerId: PeerId,
|
|
292
|
+
private readonly privateKey: PrivateKey,
|
|
293
|
+
private readonly protocolPrefix?: string,
|
|
294
|
+
// Reserved for partition-healing (backlog ticket 6.5-partition-healing); held but not yet consumed.
|
|
295
|
+
_partitionDetector?: PartitionDetector,
|
|
296
|
+
private readonly fretService?: FretService,
|
|
297
|
+
private readonly validator?: ITransactionValidator,
|
|
298
|
+
private readonly reputation?: IPeerReputation,
|
|
299
|
+
consensusConfig?: ClusterConsensusConfig,
|
|
300
|
+
private readonly stateStore?: ITransactionStateStore,
|
|
301
|
+
private readonly reconcileBlock?: ReconcileBlockCallback,
|
|
302
|
+
private readonly onCommitCertificate?: CommitCertificateSink,
|
|
303
|
+
private readonly onInvalidate?: InvalidationApplySink,
|
|
304
|
+
private readonly recomputeArbitratorSet?: RecomputeArbitratorSetCapability,
|
|
305
|
+
private readonly deriveExpectedCluster?: DeriveExpectedClusterCallback
|
|
306
|
+
) {
|
|
307
|
+
this.superMajorityThreshold = consensusConfig?.superMajorityThreshold ?? DEFAULT_SUPER_MAJORITY_THRESHOLD;
|
|
308
|
+
this.minAbsoluteClusterSize = consensusConfig?.minAbsoluteClusterSize ?? 3;
|
|
309
|
+
this.clusterSizeTolerance = consensusConfig?.clusterSizeTolerance ?? 0.5;
|
|
310
|
+
this.membershipAdmissionFraction = consensusConfig?.membershipAdmissionFraction ?? 0.75;
|
|
311
|
+
this.assumedClusterSize = consensusConfig?.assumedClusterSize;
|
|
312
|
+
this.allowUnvalidatedSmallCluster = consensusConfig?.allowUnvalidatedSmallCluster ?? false;
|
|
313
|
+
this.unvalidatablePendPolicy = consensusConfig?.unvalidatablePendPolicy ?? 'accept';
|
|
314
|
+
// State the resolved gate parameters once, so an operator diagnosing a membership rejection can see
|
|
315
|
+
// what this node actually resolved. A fact, not a warning: `assumedClusterSize < clusterSize` is the
|
|
316
|
+
// normal default state, so warning on it would fire for every node and be ignored.
|
|
317
|
+
log('cluster-member:admission-config', {
|
|
318
|
+
assumedClusterSize: this.assumedClusterSize,
|
|
319
|
+
minAbsoluteClusterSize: this.minAbsoluteClusterSize,
|
|
320
|
+
membershipAdmissionFraction: this.membershipAdmissionFraction,
|
|
321
|
+
allowUnvalidatedSmallCluster: this.allowUnvalidatedSmallCluster
|
|
322
|
+
});
|
|
323
|
+
// Periodically clean up expired transactions (.unref() so tests/short-lived processes can exit)
|
|
324
|
+
this.expirationInterval = setInterval(() => this.queueExpiredTransactions(), 60000);
|
|
325
|
+
this.expirationInterval.unref();
|
|
326
|
+
// Process cleanup queue
|
|
327
|
+
this.cleanupInterval = setInterval(() => this.processCleanupQueue(), 1000);
|
|
328
|
+
this.cleanupInterval.unref();
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
/**
|
|
332
|
+
* The resolved super-majority threshold this member runs on. Exposed so the composition root can
|
|
333
|
+
* fail-fast if the member and the coordinator would run different thresholds (see the coupling
|
|
334
|
+
* assertion in `libp2p-node-base.ts`) — a mismatch is a latent phase-disagreement, caught at startup
|
|
335
|
+
* rather than mid-consensus.
|
|
336
|
+
*/
|
|
337
|
+
get effectiveSuperMajorityThreshold(): number {
|
|
338
|
+
return this.superMajorityThreshold;
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
/**
|
|
342
|
+
* Clears all interval and timeout handles and empties active state.
|
|
343
|
+
* Called during node shutdown to prevent leaked timers.
|
|
344
|
+
*/
|
|
345
|
+
dispose(): void {
|
|
346
|
+
clearInterval(this.expirationInterval);
|
|
347
|
+
clearInterval(this.cleanupInterval);
|
|
348
|
+
for (const [, state] of this.activeTransactions) {
|
|
349
|
+
if (state.promiseTimeout) clearTimeout(state.promiseTimeout);
|
|
350
|
+
if (state.resolutionTimeout) clearTimeout(state.resolutionTimeout);
|
|
351
|
+
}
|
|
352
|
+
this.activeTransactions.clear();
|
|
353
|
+
this.cleanupQueue.length = 0;
|
|
354
|
+
this.executedPendResults.clear();
|
|
355
|
+
this.executedCommitResults.clear();
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
/**
|
|
359
|
+
* Checks if a transaction's operations were already executed during consensus.
|
|
360
|
+
* Used by the coordinator to avoid duplicate execution in CoordinatorRepo.
|
|
361
|
+
*/
|
|
362
|
+
wasTransactionExecuted(messageHash: string): boolean {
|
|
363
|
+
return this.executedTransactions.has(messageHash);
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
/**
|
|
367
|
+
* Local storage's verdict for the pend operation this member applied at consensus for
|
|
368
|
+
* `messageHash`, when one was retained. The coordinator reads this so the answer a writer gets
|
|
369
|
+
* is the answer storage gave — the cluster path must not fabricate a success the single-node
|
|
370
|
+
* path (`CoordinatorRepo.pend`'s `peerCount <= 1` short-circuit) would never produce. Absent for
|
|
371
|
+
* transactions carrying no pend operation, for transactions applied before this member restarted
|
|
372
|
+
* (the map is in-memory only; the coordinator then falls back to its fabricated-success shape,
|
|
373
|
+
* with the promise-phase pending check in {@link validatePendOperations} narrowing that window),
|
|
374
|
+
* and after the executed-transaction TTL prunes it.
|
|
375
|
+
*/
|
|
376
|
+
getExecutedPendResult(messageHash: string): PendResult | undefined {
|
|
377
|
+
return this.executedPendResults.get(messageHash);
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
/**
|
|
381
|
+
* Commit-shaped sibling of {@link getExecutedPendResult}: local storage's verdict for a commit
|
|
382
|
+
* operation applied during consensus, when this member retained one. `CoordinatorRepo.commit`
|
|
383
|
+
* consults it after a locally-executed commit-consensus — a retained refusal whose cause a local
|
|
384
|
+
* re-read confirms as a rival holding the requested revision is returned to the writer as a
|
|
385
|
+
* retryable conflict instead of the fabricated success the ahead-divergence tolerance would
|
|
386
|
+
* otherwise imply. Same availability caveats as the pend accessor: in-memory only, absent for
|
|
387
|
+
* pre-restart applies, pruned on the executed-transaction TTL.
|
|
388
|
+
*/
|
|
389
|
+
getExecutedCommitResult(messageHash: string): CommitResult | undefined {
|
|
390
|
+
return this.executedCommitResults.get(messageHash);
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
/**
|
|
394
|
+
* Handles an incoming cluster update, managing the two-phase commit process
|
|
395
|
+
* and coordinating with the local storage repo
|
|
396
|
+
*/
|
|
397
|
+
async update(record: ClusterRecord): Promise<ClusterRecord> {
|
|
398
|
+
// Serialize concurrent updates for the same transaction
|
|
399
|
+
const existingUpdate = this.pendingUpdates.get(record.messageHash);
|
|
400
|
+
if (existingUpdate) {
|
|
401
|
+
log('cluster-member:concurrent-update-wait', { messageHash: record.messageHash });
|
|
402
|
+
await existingUpdate;
|
|
403
|
+
// After waiting, continue processing with the new incoming record
|
|
404
|
+
// to ensure proper merging of promises/commits from coordinator
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
// Create a promise for this update operation
|
|
408
|
+
const updatePromise = this.processUpdate(record);
|
|
409
|
+
this.pendingUpdates.set(record.messageHash, updatePromise);
|
|
410
|
+
|
|
411
|
+
try {
|
|
412
|
+
const result = await updatePromise;
|
|
413
|
+
return result;
|
|
414
|
+
} finally {
|
|
415
|
+
// Remove from pending updates after a short delay to allow concurrent calls to see it
|
|
416
|
+
setTimeout(() => {
|
|
417
|
+
this.pendingUpdates.delete(record.messageHash);
|
|
418
|
+
}, 100).unref();
|
|
419
|
+
}
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
private async processUpdate(record: ClusterRecord): Promise<ClusterRecord> {
|
|
423
|
+
const ourId = this.peerId.toString();
|
|
424
|
+
const inboundPhase = record.commits[ourId] ? 'commit' : record.promises[ourId] ? 'promise' : 'initial';
|
|
425
|
+
log('cluster-member:incoming', {
|
|
426
|
+
messageHash: record.messageHash,
|
|
427
|
+
phase: inboundPhase,
|
|
428
|
+
peerCount: Object.keys(record.peers).length,
|
|
429
|
+
promiseCount: Object.keys(record.promises).length,
|
|
430
|
+
commitCount: Object.keys(record.commits).length,
|
|
431
|
+
existingTransaction: this.activeTransactions.has(record.messageHash)
|
|
432
|
+
});
|
|
433
|
+
|
|
434
|
+
// Report network size hint to FRET if provided
|
|
435
|
+
if (this.fretService && record.networkSizeHint && record.networkSizeConfidence) {
|
|
436
|
+
try {
|
|
437
|
+
this.fretService.reportNetworkSize(
|
|
438
|
+
record.networkSizeHint,
|
|
439
|
+
record.networkSizeConfidence,
|
|
440
|
+
'cluster'
|
|
441
|
+
);
|
|
442
|
+
} catch (err) {
|
|
443
|
+
// Ignore errors reporting to FRET
|
|
444
|
+
}
|
|
445
|
+
}
|
|
446
|
+
|
|
447
|
+
// Validate the incoming record
|
|
448
|
+
await this.validateRecord(record);
|
|
449
|
+
|
|
450
|
+
const existingState = this.activeTransactions.get(record.messageHash);
|
|
451
|
+
let currentRecord = existingState?.record || record;
|
|
452
|
+
if (existingState) {
|
|
453
|
+
log('cluster-member:merge-start', {
|
|
454
|
+
messageHash: record.messageHash,
|
|
455
|
+
existingPromises: Object.keys(existingState.record.promises ?? {}),
|
|
456
|
+
existingCommits: Object.keys(existingState.record.commits ?? {}),
|
|
457
|
+
incomingPromises: Object.keys(record.promises ?? {}),
|
|
458
|
+
incomingCommits: Object.keys(record.commits ?? {})
|
|
459
|
+
});
|
|
460
|
+
}
|
|
461
|
+
|
|
462
|
+
// If we have an existing record, merge the signatures
|
|
463
|
+
if (existingState) {
|
|
464
|
+
currentRecord = await this.mergeRecords(existingState.record, record);
|
|
465
|
+
log('cluster-member:merge-complete', {
|
|
466
|
+
messageHash: record.messageHash,
|
|
467
|
+
mergedPromises: Object.keys(currentRecord.promises ?? {}),
|
|
468
|
+
mergedCommits: Object.keys(currentRecord.commits ?? {})
|
|
469
|
+
});
|
|
470
|
+
}
|
|
471
|
+
|
|
472
|
+
// Drive the phase machine to a FIXPOINT rather than handling one phase per delivery. Each
|
|
473
|
+
// vote this member adds can put the record straight into the next phase (our promise
|
|
474
|
+
// completes super-majority ⇒ our commit is due; our commit completes the majority ⇒
|
|
475
|
+
// consensus; our reject/conflict vote makes the record terminal), and any follow-on phase
|
|
476
|
+
// not re-handled here would silently wait for the coordinator's next delivery — e.g. a
|
|
477
|
+
// member whose promise the coordinator never collected receives the commit-phase record,
|
|
478
|
+
// adds its promise, and must then also commit in the SAME delivery. One loop replaces the
|
|
479
|
+
// hand-written per-branch re-checks that used to cover only the follow-ons their authors
|
|
480
|
+
// thought of. Only the three vote-adding phases continue; each strictly grows the record
|
|
481
|
+
// (adds a promise or commit key), so the loop terminates — the cap guards a
|
|
482
|
+
// phase-computation bug, not a real bound.
|
|
483
|
+
const MaxPhaseSteps = 8;
|
|
484
|
+
let shouldPersist = true;
|
|
485
|
+
phaseLoop: for (let step = 0; ; step++) {
|
|
486
|
+
if (step >= MaxPhaseSteps) {
|
|
487
|
+
log('cluster-member:phase-loop-overflow', { messageHash: record.messageHash, steps: step });
|
|
488
|
+
break;
|
|
489
|
+
}
|
|
490
|
+
const phaseResult = await this.getTransactionPhase(currentRecord);
|
|
491
|
+
log('cluster-member:phase', {
|
|
492
|
+
messageHash: record.messageHash,
|
|
493
|
+
phase: phaseResult.phase,
|
|
494
|
+
step,
|
|
495
|
+
promises: Object.keys(currentRecord.promises ?? {}),
|
|
496
|
+
commits: Object.keys(currentRecord.commits ?? {})
|
|
497
|
+
});
|
|
498
|
+
switch (phaseResult.phase) {
|
|
499
|
+
case TransactionPhase.OurPromiseNeeded:
|
|
500
|
+
log('cluster-member:action-promise', {
|
|
501
|
+
messageHash: record.messageHash
|
|
502
|
+
});
|
|
503
|
+
currentRecord = await this.handlePromiseNeeded(currentRecord);
|
|
504
|
+
log('cluster-member:action-promise-complete', {
|
|
505
|
+
messageHash: record.messageHash,
|
|
506
|
+
promises: Object.keys(currentRecord.promises ?? {})
|
|
507
|
+
});
|
|
508
|
+
// Our own vote can be terminal (a reject where maxAllowedRejections is 0) or complete
|
|
509
|
+
// the super-majority — recompute rather than guess which.
|
|
510
|
+
continue;
|
|
511
|
+
case TransactionPhase.OurConflictVoteNeeded:
|
|
512
|
+
currentRecord = await this.handleConflictVoteNeeded(currentRecord, phaseResult.conflictsWith);
|
|
513
|
+
// Never persist a record we conflict-voted: this member holds the WINNER, and
|
|
514
|
+
// persisting the loser would reserve the same blocks a second time — half of what
|
|
515
|
+
// made the silent-abstention failure self-sustaining.
|
|
516
|
+
shouldPersist = false;
|
|
517
|
+
continue;
|
|
518
|
+
case TransactionPhase.OurCommitNeeded:
|
|
519
|
+
log('cluster-member:action-commit', {
|
|
520
|
+
messageHash: record.messageHash
|
|
521
|
+
});
|
|
522
|
+
currentRecord = await this.handleCommitNeeded(currentRecord);
|
|
523
|
+
log('cluster-member:action-commit-complete', {
|
|
524
|
+
messageHash: record.messageHash,
|
|
525
|
+
commits: Object.keys(currentRecord.commits ?? {})
|
|
526
|
+
});
|
|
527
|
+
shouldPersist = false;
|
|
528
|
+
// Our commit may have completed the majority — recompute; Consensus executes below.
|
|
529
|
+
continue;
|
|
530
|
+
case TransactionPhase.Consensus:
|
|
531
|
+
log('cluster-member:action-consensus', {
|
|
532
|
+
messageHash: record.messageHash
|
|
533
|
+
});
|
|
534
|
+
await this.handleConsensus(currentRecord);
|
|
535
|
+
shouldPersist = false;
|
|
536
|
+
break phaseLoop;
|
|
537
|
+
case TransactionPhase.Rejected:
|
|
538
|
+
log('cluster-member:action-rejected', {
|
|
539
|
+
messageHash: record.messageHash
|
|
540
|
+
});
|
|
541
|
+
await this.handleRejection(currentRecord);
|
|
542
|
+
shouldPersist = false;
|
|
543
|
+
break phaseLoop;
|
|
544
|
+
case TransactionPhase.ConflictSuperseded:
|
|
545
|
+
// Enough conflict votes that super-majority is unreachable. NOT a rejection — the
|
|
546
|
+
// callers retry it as a fresh transaction — so it gets its own terminal phase and the
|
|
547
|
+
// record is cleared rather than held (holding a provably-dead loser would reserve its
|
|
548
|
+
// blocks against the very retry that is supposed to win).
|
|
549
|
+
log('cluster-member:action-conflict-superseded', {
|
|
550
|
+
messageHash: record.messageHash
|
|
551
|
+
});
|
|
552
|
+
shouldPersist = false;
|
|
553
|
+
break phaseLoop;
|
|
554
|
+
case TransactionPhase.Propagating:
|
|
555
|
+
// Transaction is complete and propagating - clean it up
|
|
556
|
+
log('cluster-member:phase-propagating', {
|
|
557
|
+
messageHash: record.messageHash
|
|
558
|
+
});
|
|
559
|
+
shouldPersist = false;
|
|
560
|
+
break phaseLoop;
|
|
561
|
+
case TransactionPhase.Promising:
|
|
562
|
+
// We have already voted (approve, reject, or conflict); the record is still
|
|
563
|
+
// collecting promises from the rest of the cohort. Nothing to add — retain the
|
|
564
|
+
// record only if our vote wasn't a conflict (`shouldPersist` already reflects that).
|
|
565
|
+
log('cluster-member:phase-promising-waiting', {
|
|
566
|
+
messageHash: record.messageHash
|
|
567
|
+
});
|
|
568
|
+
break phaseLoop;
|
|
569
|
+
}
|
|
570
|
+
}
|
|
571
|
+
|
|
572
|
+
if (shouldPersist) {
|
|
573
|
+
// Update transaction state
|
|
574
|
+
const timeouts = this.setupTimeouts(currentRecord);
|
|
575
|
+
this.activeTransactions.set(record.messageHash, {
|
|
576
|
+
record: currentRecord,
|
|
577
|
+
lastUpdate: Date.now(),
|
|
578
|
+
promiseTimeout: timeouts.promiseTimeout,
|
|
579
|
+
resolutionTimeout: timeouts.resolutionTimeout
|
|
580
|
+
});
|
|
581
|
+
this.persistParticipantState(record.messageHash, currentRecord);
|
|
582
|
+
log('cluster-member:state-persist', {
|
|
583
|
+
messageHash: record.messageHash,
|
|
584
|
+
storedPromises: Object.keys(currentRecord.promises ?? {}),
|
|
585
|
+
storedCommits: Object.keys(currentRecord.commits ?? {})
|
|
586
|
+
});
|
|
587
|
+
} else {
|
|
588
|
+
log('cluster-member:state-clear', {
|
|
589
|
+
messageHash: record.messageHash
|
|
590
|
+
});
|
|
591
|
+
this.clearTransaction(record.messageHash);
|
|
592
|
+
}
|
|
593
|
+
|
|
594
|
+
// Skip propagation - the coordinator manages distribution
|
|
595
|
+
// await this.propagateIfNeeded(currentRecord);
|
|
596
|
+
|
|
597
|
+
log('cluster-member:update-complete', {
|
|
598
|
+
messageHash: record.messageHash,
|
|
599
|
+
promiseCount: Object.keys(currentRecord.promises).length,
|
|
600
|
+
commitCount: Object.keys(currentRecord.commits).length
|
|
601
|
+
});
|
|
602
|
+
return currentRecord;
|
|
603
|
+
}
|
|
604
|
+
|
|
605
|
+
/**
|
|
606
|
+
* Merges two records, validating that non-signature fields match.
|
|
607
|
+
* Detects equivocation (same peer changing vote type) and applies penalties.
|
|
608
|
+
*/
|
|
609
|
+
private async mergeRecords(existing: ClusterRecord, incoming: ClusterRecord): Promise<ClusterRecord> {
|
|
610
|
+
log('cluster-member:merge-records', {
|
|
611
|
+
messageHash: existing.messageHash,
|
|
612
|
+
existingPromises: Object.keys(existing.promises ?? {}),
|
|
613
|
+
existingCommits: Object.keys(existing.commits ?? {}),
|
|
614
|
+
incomingPromises: Object.keys(incoming.promises ?? {}),
|
|
615
|
+
incomingCommits: Object.keys(incoming.commits ?? {})
|
|
616
|
+
});
|
|
617
|
+
// Verify that immutable fields match
|
|
618
|
+
if (existing.messageHash !== incoming.messageHash) {
|
|
619
|
+
throw new Error('Message hash mismatch');
|
|
620
|
+
}
|
|
621
|
+
if (ClusterMember.canonicalJson(existing.message) !== ClusterMember.canonicalJson(incoming.message)) {
|
|
622
|
+
throw new Error('Message content mismatch');
|
|
623
|
+
}
|
|
624
|
+
if (existing.membershipVersion === 2 || incoming.membershipVersion === 2) {
|
|
625
|
+
// v2: the sorted peer-id set (captured by membershipDigest) is bound into messageHash, so equal
|
|
626
|
+
// messageHash MUST imply equal membership on any honest path. A mismatch here — different digest
|
|
627
|
+
// or version at equal hash — is a protocol violation (a bug or a hash-collision attack), NOT an
|
|
628
|
+
// honest divergence: two honest members with different views now hold two DIFFERENT hashes, i.e.
|
|
629
|
+
// two competing transactions the race machinery resolves, not one contested record. Log loudly
|
|
630
|
+
// and reject; never silently adopt the incoming set. (validateRecord already proved each record's
|
|
631
|
+
// own digest matches its own peers, so multiaddr / pubkey churn within the SAME id set — which
|
|
632
|
+
// keeps the same digest and hash — does NOT trip this.)
|
|
633
|
+
if (existing.membershipVersion !== incoming.membershipVersion || existing.membershipDigest !== incoming.membershipDigest) {
|
|
634
|
+
log('cluster-member:peers-mismatch-invariant-violation', {
|
|
635
|
+
messageHash: existing.messageHash,
|
|
636
|
+
existingVersion: existing.membershipVersion,
|
|
637
|
+
incomingVersion: incoming.membershipVersion,
|
|
638
|
+
existingDigest: existing.membershipDigest,
|
|
639
|
+
incomingDigest: incoming.membershipDigest,
|
|
640
|
+
existingPeers: Object.keys(existing.peers ?? {}).sort(),
|
|
641
|
+
incomingPeers: Object.keys(incoming.peers ?? {}).sort()
|
|
642
|
+
});
|
|
643
|
+
throw new Error('Peers mismatch');
|
|
644
|
+
}
|
|
645
|
+
} else if (ClusterMember.canonicalJson(existing.peers) !== ClusterMember.canonicalJson(incoming.peers)) {
|
|
646
|
+
// v1 (legacy, membership unbound): full peer-object equality is the only available guard.
|
|
647
|
+
throw new Error('Peers mismatch');
|
|
648
|
+
}
|
|
649
|
+
|
|
650
|
+
// Merge signatures with equivocation detection
|
|
651
|
+
const mergedPromises = this.detectEquivocation(
|
|
652
|
+
existing.promises, incoming.promises, 'promise', existing.messageHash
|
|
653
|
+
);
|
|
654
|
+
const mergedCommits = this.detectEquivocation(
|
|
655
|
+
existing.commits, incoming.commits, 'commit', existing.messageHash
|
|
656
|
+
);
|
|
657
|
+
|
|
658
|
+
return {
|
|
659
|
+
...existing,
|
|
660
|
+
promises: mergedPromises,
|
|
661
|
+
commits: mergedCommits
|
|
662
|
+
};
|
|
663
|
+
}
|
|
664
|
+
|
|
665
|
+
/**
|
|
666
|
+
* Compares existing vs incoming signatures for the same peers.
|
|
667
|
+
* If a peer's vote type changed (approve↔reject), that's equivocation:
|
|
668
|
+
* report a penalty and keep the first-seen signature.
|
|
669
|
+
* New peers are accepted normally.
|
|
670
|
+
*/
|
|
671
|
+
private detectEquivocation(
|
|
672
|
+
existing: Record<string, Signature>,
|
|
673
|
+
incoming: Record<string, Signature>,
|
|
674
|
+
phase: 'promise' | 'commit',
|
|
675
|
+
messageHash: string
|
|
676
|
+
): Record<string, Signature> {
|
|
677
|
+
// NOTE: relies on validateSignatures() (via validateRecord in processUpdate) having already run on
|
|
678
|
+
// every record reaching here, so each peerId's signature is key-bound. Without that guarantee the
|
|
679
|
+
// Equivocation penalty below would act on self-asserted, unverified peer ids and could frame an
|
|
680
|
+
// honest peer. Do not call this on unvalidated signatures.
|
|
681
|
+
const merged = { ...existing };
|
|
682
|
+
|
|
683
|
+
for (const [peerId, incomingSig] of Object.entries(incoming)) {
|
|
684
|
+
const existingSig = existing[peerId];
|
|
685
|
+
if (existingSig) {
|
|
686
|
+
if (existingSig.type !== incomingSig.type) {
|
|
687
|
+
// Equivocation detected: peer changed their vote type
|
|
688
|
+
log('cluster-member:equivocation-detected', {
|
|
689
|
+
peerId,
|
|
690
|
+
phase,
|
|
691
|
+
messageHash,
|
|
692
|
+
existingType: existingSig.type,
|
|
693
|
+
incomingType: incomingSig.type
|
|
694
|
+
});
|
|
695
|
+
this.reputation?.reportPeer(
|
|
696
|
+
peerId,
|
|
697
|
+
PenaltyReason.Equivocation,
|
|
698
|
+
`${phase}:${messageHash}:${existingSig.type}->${incomingSig.type}`
|
|
699
|
+
);
|
|
700
|
+
// Keep first-seen signature — do not let the peer flip their vote
|
|
701
|
+
}
|
|
702
|
+
// Same type: keep existing (no-op, already in merged)
|
|
703
|
+
} else {
|
|
704
|
+
// New peer — accept normally
|
|
705
|
+
merged[peerId] = incomingSig;
|
|
706
|
+
}
|
|
707
|
+
}
|
|
708
|
+
|
|
709
|
+
return merged;
|
|
710
|
+
}
|
|
711
|
+
|
|
712
|
+
private async validateRecord(record: ClusterRecord): Promise<void> {
|
|
713
|
+
// Reject a record whose membership-binding version this code does not implement. The cluster
|
|
714
|
+
// consensus code is a single deployable unit (all cluster members upgrade together), so a version
|
|
715
|
+
// we don't understand is rejected rather than cross-version-consensus'd.
|
|
716
|
+
if (record.membershipVersion !== undefined && record.membershipVersion !== 1 && record.membershipVersion !== 2) {
|
|
717
|
+
throw new Error(`Unsupported membershipVersion: ${record.membershipVersion}`);
|
|
718
|
+
}
|
|
719
|
+
|
|
720
|
+
// v2: the declared membership digest must match the record's own peer set. A record whose declared
|
|
721
|
+
// digest doesn't match its peers is malformed (and its messageHash — computed over that digest —
|
|
722
|
+
// would not bind the real membership).
|
|
723
|
+
// NOTE: recomputes membershipDigest (one SHA256 over the sorted peer-id list) on every incoming v2
|
|
724
|
+
// record; if a hot cluster ever shows this as a cost, memoize per (messageHash → digest).
|
|
725
|
+
if (record.membershipVersion === 2) {
|
|
726
|
+
const expectedDigest = await membershipDigest(record.peers);
|
|
727
|
+
if (expectedDigest !== record.membershipDigest) {
|
|
728
|
+
throw new Error(`Membership digest mismatch: expected=${expectedDigest}, received=${record.membershipDigest ?? 'undefined'}`);
|
|
729
|
+
}
|
|
730
|
+
}
|
|
731
|
+
|
|
732
|
+
// Validate message hash matches the message content (v2 folds in the membership digest)
|
|
733
|
+
const expectedHash = await this.computeMessageHash(record);
|
|
734
|
+
if (expectedHash !== record.messageHash) {
|
|
735
|
+
throw new Error(`Message hash mismatch: expected=${expectedHash}, received=${record.messageHash}`);
|
|
736
|
+
}
|
|
737
|
+
|
|
738
|
+
// Validate signatures
|
|
739
|
+
await this.validateSignatures(record);
|
|
740
|
+
|
|
741
|
+
// Validate expiration
|
|
742
|
+
if (record.message.expiration && record.message.expiration < Date.now()) {
|
|
743
|
+
throw new Error('Transaction expired');
|
|
744
|
+
}
|
|
745
|
+
}
|
|
746
|
+
|
|
747
|
+
/**
|
|
748
|
+
* Compute message hash using the same algorithm as the coordinator. Version-dispatched: a v2 record
|
|
749
|
+
* folds its membership digest into the preimage, a v1 / unversioned record hashes byte-identically to
|
|
750
|
+
* before this change. Must match cluster-coordinator.ts createMessageHash().
|
|
751
|
+
*/
|
|
752
|
+
private async computeMessageHash(record: Pick<ClusterRecord, 'message' | 'membershipVersion' | 'membershipDigest'>): Promise<string> {
|
|
753
|
+
return computeClusterMessageHash(record.message, recordMembershipDigest(record));
|
|
754
|
+
}
|
|
755
|
+
|
|
756
|
+
private async validateSignatures(record: ClusterRecord): Promise<void> {
|
|
757
|
+
// Validate promise signatures. Reject on any failure, but only report an InvalidSignature
|
|
758
|
+
// penalty when the key was proven to belong to `peerId` (outcome.penalize) — otherwise the id
|
|
759
|
+
// is attacker-chosen and reporting it would let a coordinator frame an honest peer.
|
|
760
|
+
const promiseHash = await this.computePromiseHash(record);
|
|
761
|
+
for (const [peerId, signature] of Object.entries(record.promises)) {
|
|
762
|
+
const outcome = await this.verifySignature(record, peerId, promiseHash, signature);
|
|
763
|
+
if (!outcome.valid) {
|
|
764
|
+
if (outcome.penalize) {
|
|
765
|
+
this.reputation?.reportPeer(peerId, PenaltyReason.InvalidSignature, `promise:${record.messageHash}`);
|
|
766
|
+
}
|
|
767
|
+
throw new Error(`Invalid promise signature from ${peerId}`);
|
|
768
|
+
}
|
|
769
|
+
}
|
|
770
|
+
|
|
771
|
+
// Validate commit signatures
|
|
772
|
+
const commitHash = await this.computeCommitHash(record);
|
|
773
|
+
for (const [peerId, signature] of Object.entries(record.commits)) {
|
|
774
|
+
const outcome = await this.verifySignature(record, peerId, commitHash, signature);
|
|
775
|
+
if (!outcome.valid) {
|
|
776
|
+
if (outcome.penalize) {
|
|
777
|
+
this.reputation?.reportPeer(peerId, PenaltyReason.InvalidSignature, `commit:${record.messageHash}`);
|
|
778
|
+
}
|
|
779
|
+
throw new Error(`Invalid commit signature from ${peerId}`);
|
|
780
|
+
}
|
|
781
|
+
}
|
|
782
|
+
}
|
|
783
|
+
|
|
784
|
+
/**
|
|
785
|
+
* Deterministic JSON: sorts object keys so comparisons are order-independent.
|
|
786
|
+
* NOTE: this is a second copy of the canonicalization in db-core `membership.ts` (which feeds the hash
|
|
787
|
+
* preimages). It is used here only for equality checks (message-content / v1-peers compare), and those
|
|
788
|
+
* run *after* a messageHash-equality gate, so a drift between the two can't silently forge agreement —
|
|
789
|
+
* but keep them byte-identical. If a third caller appears, promote to a single exported helper.
|
|
790
|
+
*/
|
|
791
|
+
private static canonicalJson(value: unknown): string {
|
|
792
|
+
return JSON.stringify(value, (_, v) =>
|
|
793
|
+
v && typeof v === 'object' && !Array.isArray(v)
|
|
794
|
+
? Object.keys(v).sort().reduce((o: Record<string, unknown>, k) => { o[k] = v[k]; return o; }, {})
|
|
795
|
+
: v
|
|
796
|
+
);
|
|
797
|
+
}
|
|
798
|
+
|
|
799
|
+
private async computePromiseHash(record: ClusterRecord): Promise<string> {
|
|
800
|
+
return computeClusterPromiseHash(record.messageHash, record.message, recordMembershipDigest(record));
|
|
801
|
+
}
|
|
802
|
+
|
|
803
|
+
private async computeCommitHash(record: ClusterRecord): Promise<string> {
|
|
804
|
+
return computeClusterCommitHash(record.messageHash, record.message, record.promises, recordMembershipDigest(record));
|
|
805
|
+
}
|
|
806
|
+
|
|
807
|
+
private async signVote(hash: string, type: Signature['type'], extra?: string): Promise<string> {
|
|
808
|
+
const sigBytes = await this.privateKey.sign(clusterVoteSigningPayload(hash, type, extra));
|
|
809
|
+
return uint8ArrayToString(sigBytes, 'base64url');
|
|
810
|
+
}
|
|
811
|
+
|
|
812
|
+
/**
|
|
813
|
+
* Verify one vote signature and classify the outcome (see {@link VerifyOutcome}). Total on hostile
|
|
814
|
+
* input: a missing/empty key, a non-Ed25519 id, a key not bound to `peerId`, or malformed bytes all
|
|
815
|
+
* yield `{ valid:false, penalize:false }` (reject without penalizing an unproven identity) rather
|
|
816
|
+
* than throwing. Only after the key is proven to be the one `peerId` names does a failed
|
|
817
|
+
* cryptographic verify yield `{ valid:false, penalize:true }`.
|
|
818
|
+
*
|
|
819
|
+
* NOTE: the binding check (`peerIdBindsPublicKey`) proves the vote was signed by the key `peerId`
|
|
820
|
+
* names — it does NOT establish that `peerId` is legitimately in the cohort. A coordinator minting
|
|
821
|
+
* fresh keypairs and using each key's own derived id passes this for every one. Sybil/cohort
|
|
822
|
+
* membership is a separate layer (cohort-topic membership certificates), not solved here.
|
|
823
|
+
*/
|
|
824
|
+
private async verifySignature(record: ClusterRecord, peerId: string, hash: string, signature: Signature): Promise<VerifyOutcome> {
|
|
825
|
+
const peerInfo = record.peers[peerId];
|
|
826
|
+
if (!peerInfo?.publicKey?.length) {
|
|
827
|
+
// No key to check against — identity not proven. Reject without penalty.
|
|
828
|
+
return { valid: false, penalize: false };
|
|
829
|
+
}
|
|
830
|
+
let keyBytes: Uint8Array;
|
|
831
|
+
try {
|
|
832
|
+
// publicKey is base64url-encoded string (JSON-serialization safe)
|
|
833
|
+
keyBytes = uint8ArrayFromString(peerInfo.publicKey, 'base64url');
|
|
834
|
+
} catch {
|
|
835
|
+
return { valid: false, penalize: false };
|
|
836
|
+
}
|
|
837
|
+
// The key must be the one `peerId` provably names, else the vote could be attributed to any peer
|
|
838
|
+
// id while signed by a key the coordinator controls. Binding failure ⇒ identity unproven ⇒ no penalty.
|
|
839
|
+
if (!peerIdBindsPublicKey(peerId, keyBytes)) {
|
|
840
|
+
return { valid: false, penalize: false };
|
|
841
|
+
}
|
|
842
|
+
try {
|
|
843
|
+
const pubKey = publicKeyFromRaw(keyBytes);
|
|
844
|
+
const payload = clusterVoteVerificationPayload(hash, signature);
|
|
845
|
+
const sigBytes = uint8ArrayFromString(signature.signature, 'base64url');
|
|
846
|
+
const ok = await pubKey.verify(payload, sigBytes);
|
|
847
|
+
// Key is bound to peerId: a failed verify is a genuine bad vote from a proven identity → penalize.
|
|
848
|
+
// NOTE: residual — an Ed25519 peer's public key is derivable from its (public) id, so an attacker
|
|
849
|
+
// can attach a victim's REAL key with a garbage signature and still trip this InvalidSignature
|
|
850
|
+
// penalty on the victim. Binding narrows framing (the attacker must use the victim's own key, not
|
|
851
|
+
// an arbitrary one) but cannot eliminate it here: a single signature can't distinguish "victim
|
|
852
|
+
// signed badly" from "someone pasted the victim's public key + junk". Fully closing it needs an
|
|
853
|
+
// authenticated membership/channel layer (cohort-topic membership certs), out of scope for this fix.
|
|
854
|
+
return ok ? { valid: true } : { valid: false, penalize: true };
|
|
855
|
+
} catch {
|
|
856
|
+
// Malformed signature bytes / key decode failure: reject, but do not penalize on unparseable input.
|
|
857
|
+
return { valid: false, penalize: false };
|
|
858
|
+
}
|
|
859
|
+
}
|
|
860
|
+
|
|
861
|
+
private async getTransactionPhase(record: ClusterRecord): Promise<PhaseResult> {
|
|
862
|
+
const peerCount = Object.keys(record.peers).length;
|
|
863
|
+
const promiseCount = Object.keys(record.promises).length;
|
|
864
|
+
const ourId = this.peerId.toString();
|
|
865
|
+
|
|
866
|
+
const superMajority = Math.ceil(peerCount * this.superMajorityThreshold);
|
|
867
|
+
const maxAllowedRejections = peerCount - superMajority;
|
|
868
|
+
|
|
869
|
+
// Check for rejections — rejected if too many rejections to ever reach super-majority.
|
|
870
|
+
// ONLY `reject` votes count here: a `conflict` vote is "not now", never a validity
|
|
871
|
+
// judgement, so it must not push a record into the permanent `Rejected` phase.
|
|
872
|
+
const rejectedPromises = Object.values(record.promises).filter(s => s.type === 'reject');
|
|
873
|
+
const conflictPromises = Object.values(record.promises).filter(s => s.type === 'conflict');
|
|
874
|
+
const rejectedCommits = Object.values(record.commits).filter(s => s.type === 'reject');
|
|
875
|
+
if (rejectedPromises.length > maxAllowedRejections || this.hasMajority(rejectedCommits.length, peerCount)) {
|
|
876
|
+
return { phase: TransactionPhase.Rejected };
|
|
877
|
+
}
|
|
878
|
+
|
|
879
|
+
// Conflict votes don't judge validity, but enough of them still make super-majority
|
|
880
|
+
// unreachable — a distinct terminal outcome (retryable as a fresh transaction) so logs and
|
|
881
|
+
// reputation-adjacent paths keep meaning what they say.
|
|
882
|
+
if (conflictPromises.length > 0 && rejectedPromises.length + conflictPromises.length > maxAllowedRejections) {
|
|
883
|
+
return { phase: TransactionPhase.ConflictSuperseded };
|
|
884
|
+
}
|
|
885
|
+
|
|
886
|
+
// Check if we need to vote. A lost race is answered with a conflict vote, not silence:
|
|
887
|
+
// absence used to mean both "unreachable" and "refusing in favour of a rival", and the
|
|
888
|
+
// coordinator could not tell the two apart. Once our conflict vote is merged into
|
|
889
|
+
// `promises`, this branch is skipped forever — a conflict vote is terminal for this record;
|
|
890
|
+
// a retry must be a fresh transaction (new messageHash), which `CoordinatorRepo.pend`
|
|
891
|
+
// already mints per call.
|
|
892
|
+
if (!record.promises[ourId]) {
|
|
893
|
+
const conflict = this.findConflict(record);
|
|
894
|
+
if (conflict) {
|
|
895
|
+
return { phase: TransactionPhase.OurConflictVoteNeeded, conflictsWith: conflict.blockedBy };
|
|
896
|
+
}
|
|
897
|
+
return { phase: TransactionPhase.OurPromiseNeeded };
|
|
898
|
+
}
|
|
899
|
+
|
|
900
|
+
// Check if we have enough approved promises to proceed to commit. Deliberately blind to what
|
|
901
|
+
// OUR own vote was: the rest of the cohort reaching super-majority is the commit rule
|
|
902
|
+
// (Theorem 1 Case 2), so a member that rejected — or conflict-voted — still signs the commit
|
|
903
|
+
// the cohort decided on rather than stalling it. Only the rejection/superseded thresholds
|
|
904
|
+
// above can stop a record here, and both are checked first.
|
|
905
|
+
//
|
|
906
|
+
// NOTE: signing the commit drops this member's reservation on the record
|
|
907
|
+
// (`shouldPersist = false` in the caller), and the phase fixpoint means that can now happen on
|
|
908
|
+
// the FIRST delivery when the record already arrives at super-majority, rather than a
|
|
909
|
+
// round-trip later. The safety argument is quorum intersection (Theorem 9: no rival can
|
|
910
|
+
// assemble its own super-majority once this one has), NOT the reservation — the reservation
|
|
911
|
+
// only orders *concurrently-pending* rivals.
|
|
912
|
+
//
|
|
913
|
+
// The lost update this predicted WAS observed (a pend admitted between a rival's
|
|
914
|
+
// pend-consensus and commit-consensus, then refused by every member's storage at apply and
|
|
915
|
+
// still reported to the writer as a success). Holding the reservation until `handleConsensus`
|
|
916
|
+
// would not have closed it — the loser was approved before the winner's apply had even
|
|
917
|
+
// reached most members — so the cure went elsewhere: `validatePendOperations` now rejects a
|
|
918
|
+
// pend whose blocks are held by a different unresolved STORAGE pending record (the durable
|
|
919
|
+
// reservation that spans pend-apply → commit/cancel), and the coordinator returns storage's
|
|
920
|
+
// retained apply verdict instead of fabricating success (`getExecutedPendResult`). Residual:
|
|
921
|
+
// a member that has not yet applied the rival's pend abstains from that vote, and only the
|
|
922
|
+
// coordinating node's own verdict is threaded back — see the handoff notes on
|
|
923
|
+
// `CoordinatorRepo.pend`.
|
|
924
|
+
const approvedPromises = Object.values(record.promises).filter(s => s.type === 'approve');
|
|
925
|
+
if (approvedPromises.length >= superMajority && !record.commits[ourId]) {
|
|
926
|
+
return { phase: TransactionPhase.OurCommitNeeded };
|
|
927
|
+
}
|
|
928
|
+
|
|
929
|
+
// Check if still collecting promises
|
|
930
|
+
if (promiseCount < peerCount && approvedPromises.length < superMajority) {
|
|
931
|
+
return { phase: TransactionPhase.Promising };
|
|
932
|
+
}
|
|
933
|
+
|
|
934
|
+
// Check for consensus
|
|
935
|
+
const approvedCommits = Object.values(record.commits).filter(s => s.type === 'approve');
|
|
936
|
+
if (this.hasMajority(approvedCommits.length, peerCount)) {
|
|
937
|
+
return { phase: TransactionPhase.Consensus };
|
|
938
|
+
}
|
|
939
|
+
|
|
940
|
+
return { phase: TransactionPhase.Propagating };
|
|
941
|
+
}
|
|
942
|
+
|
|
943
|
+
private hasMajority(count: number, total: number): boolean {
|
|
944
|
+
return count > total / 2;
|
|
945
|
+
}
|
|
946
|
+
|
|
947
|
+
private async handlePromiseNeeded(record: ClusterRecord): Promise<ClusterRecord> {
|
|
948
|
+
// Membership admission gate runs BEFORE pend validation: a member independently checks the declared
|
|
949
|
+
// peer set is a legitimate cluster it belongs to, and refuses (reject vote) rather than rubber-stamping
|
|
950
|
+
// a set the coordinator chose (e.g. a self-shrunk minority-partition set). On admission failure we skip
|
|
951
|
+
// pend validation entirely and emit the membership rejection.
|
|
952
|
+
const validationResult = await this.evaluatePromise(record);
|
|
953
|
+
|
|
954
|
+
const promiseHash = await this.computePromiseHash(record);
|
|
955
|
+
const type = validationResult.valid ? 'approve' as const : 'reject' as const;
|
|
956
|
+
const rejectReason = validationResult.valid ? undefined : validationResult.reason;
|
|
957
|
+
const sig = await this.signVote(promiseHash, type, rejectReason);
|
|
958
|
+
|
|
959
|
+
const signature: Signature = validationResult.valid
|
|
960
|
+
? { type: 'approve', signature: sig }
|
|
961
|
+
: { type: 'reject', signature: sig, rejectReason };
|
|
962
|
+
|
|
963
|
+
if (!validationResult.valid) {
|
|
964
|
+
log('cluster-member:validation-rejected', {
|
|
965
|
+
messageHash: record.messageHash,
|
|
966
|
+
reason: validationResult.reason
|
|
967
|
+
});
|
|
968
|
+
}
|
|
969
|
+
|
|
970
|
+
return {
|
|
971
|
+
...record,
|
|
972
|
+
promises: {
|
|
973
|
+
...record.promises,
|
|
974
|
+
[this.peerId.toString()]: signature
|
|
975
|
+
}
|
|
976
|
+
};
|
|
977
|
+
}
|
|
978
|
+
|
|
979
|
+
/**
|
|
980
|
+
* Answer a record that lost the deterministic race to a transaction this member already holds
|
|
981
|
+
* (`docs/correctness.md` Theorems 1 & 9: the loser is TOLD it lost, not ignored — an unanswered
|
|
982
|
+
* loss is indistinguishable from an unreachable cohort at the coordinator). `conflictWith` — the
|
|
983
|
+
* winning rival's messageHash — is folded into the signed payload, so the claim is
|
|
984
|
+
* integrity-protected in transit and readable without parsing prose. NOT a validity judgement:
|
|
985
|
+
* {@link getTransactionPhase} never counts conflict votes toward the permanent-rejection
|
|
986
|
+
* threshold, and the coordinator surfaces them as a retryable loss, never a validator rejection.
|
|
987
|
+
*/
|
|
988
|
+
private async handleConflictVoteNeeded(record: ClusterRecord, conflictWith: string): Promise<ClusterRecord> {
|
|
989
|
+
log('cluster-member:action-conflict-vote', {
|
|
990
|
+
messageHash: record.messageHash,
|
|
991
|
+
conflictWith
|
|
992
|
+
});
|
|
993
|
+
const promiseHash = await this.computePromiseHash(record);
|
|
994
|
+
const sig = await this.signVote(promiseHash, 'conflict', conflictWith);
|
|
995
|
+
const signature: Signature = { type: 'conflict', signature: sig, conflictWith };
|
|
996
|
+
|
|
997
|
+
return {
|
|
998
|
+
...record,
|
|
999
|
+
promises: {
|
|
1000
|
+
...record.promises,
|
|
1001
|
+
[this.peerId.toString()]: signature
|
|
1002
|
+
}
|
|
1003
|
+
};
|
|
1004
|
+
}
|
|
1005
|
+
|
|
1006
|
+
/**
|
|
1007
|
+
* The full promise-phase decision for a record: admit the declared membership FIRST, then (only if
|
|
1008
|
+
* admitted) validate its pend operations, then its commit operations. Failing any yields a
|
|
1009
|
+
* `{ valid:false, reason }` the caller turns into a `reject` vote. Keeping the three separate keeps
|
|
1010
|
+
* the reason strings distinct — a `membership-not-admitted` reject is a different signal (feeds the
|
|
1011
|
+
* dispute path) than a stale-revision / custom-validator reject, which is different again from a
|
|
1012
|
+
* `content-digest-mismatch` (see {@link validateCommitOperations}). A record carries pend OR commit
|
|
1013
|
+
* operations, so in practice exactly one of the latter two has anything to inspect.
|
|
1014
|
+
*/
|
|
1015
|
+
private async evaluatePromise(record: ClusterRecord): Promise<{ valid: boolean; reason?: string }> {
|
|
1016
|
+
const admission = await this.admitMembership(record);
|
|
1017
|
+
if (!admission.admit) {
|
|
1018
|
+
return { valid: false, reason: admission.reason ?? MEMBERSHIP_NOT_ADMITTED };
|
|
1019
|
+
}
|
|
1020
|
+
const pendValidation = await this.validatePendOperations(record);
|
|
1021
|
+
if (!pendValidation.valid) {
|
|
1022
|
+
return pendValidation;
|
|
1023
|
+
}
|
|
1024
|
+
// Revision staleness before content digests: a commit whose revision a rival already took can
|
|
1025
|
+
// never win, and the sharper stale reject also skips the per-block digest previews (the digest
|
|
1026
|
+
// check would abstain on such a block anyway — its local base rev no longer matches the
|
|
1027
|
+
// declared one).
|
|
1028
|
+
const commitRevValidation = await this.validateCommitRevisions(record);
|
|
1029
|
+
if (!commitRevValidation.valid) {
|
|
1030
|
+
return commitRevValidation;
|
|
1031
|
+
}
|
|
1032
|
+
return await this.validateCommitOperations(record);
|
|
1033
|
+
}
|
|
1034
|
+
|
|
1035
|
+
/**
|
|
1036
|
+
* Membership admission gate. Decides whether the coordinator-declared peer set (`record.peers`, call it
|
|
1037
|
+
* `D`) is a *legitimate* cluster this member may vote inside, judged against the member's OWN
|
|
1038
|
+
* independently-derived view — not against anything the (untrusted) record declares about its size.
|
|
1039
|
+
* Evaluated on the promise path before the member signs an approve.
|
|
1040
|
+
*
|
|
1041
|
+
* The predicate admits `D` iff ALL hold:
|
|
1042
|
+
* 1. **Self-membership** — this member's id ∈ `D`; else this block is not its responsibility (and a
|
|
1043
|
+
* coordinator must not route a record to a non-member to pad approval counts).
|
|
1044
|
+
* 2. **Not a self-shrink below the floor** — with a confident derived view `E`, `|D| ≥ ⌈fraction·|E|⌉`
|
|
1045
|
+
* (and ≥ minAbsoluteClusterSize). `|E|` is the member's own confident cluster-size estimate `K_est`,
|
|
1046
|
+
* so a minority-partition set (small `D`) is rejected against the member's larger view.
|
|
1047
|
+
* 3. **Consistency with the derived view** — `|D △ E|` within `clusterSizeTolerance·|E|`; honest churn
|
|
1048
|
+
* of a peer or two is absorbed, a wholesale-disjoint or half-size set is not.
|
|
1049
|
+
*
|
|
1050
|
+
* **Fail-closed posture.** When the member cannot confidently derive `E` (no capability, low FRET
|
|
1051
|
+
* confidence — exactly what a partition induces), it must refuse any *downsizing* decision — but it
|
|
1052
|
+
* needs a size reference to judge "downsize" against, and it may NOT borrow `clusterSize` for that:
|
|
1053
|
+
* `clusterSize` is the replication factor (what a cohort should aim for), not a claim about how many
|
|
1054
|
+
* peers exist, so a small deployment configured with the default 10 would refuse every write. The
|
|
1055
|
+
* fallback yardstick is instead {@link ClusterConsensusConfig.assumedClusterSize} — the operator's own
|
|
1056
|
+
* assertion of the smallest cohort this deployment can genuinely field — run through the SAME
|
|
1057
|
+
* {@link admissionFloor} as the confident path, so the fallback can never be stricter than the measured
|
|
1058
|
+
* path (it was: it demanded the full configured size, with no fraction and no slack for churn or a peer
|
|
1059
|
+
* not yet discovered). With NEITHER a confident view NOR an asserted size the gate cannot judge a
|
|
1060
|
+
* downsize at all, so it preserves the legacy approve behavior (backward-compatible for nodes/tests with
|
|
1061
|
+
* no derivation wired). `allowUnvalidatedSmallCluster` is the explicit opt-in (single-node / local dev
|
|
1062
|
+
* knowingly below the safe floor), matching the coordinator's `validateSmallCluster` semantics.
|
|
1063
|
+
*/
|
|
1064
|
+
private async admitMembership(record: ClusterRecord): Promise<{ admit: boolean; reason?: string }> {
|
|
1065
|
+
const ourId = this.peerId.toString();
|
|
1066
|
+
const declared = Object.keys(record.peers ?? {});
|
|
1067
|
+
|
|
1068
|
+
// Predicate 1: self-membership. Always enforced (independent of any opt-in): a member does not vote
|
|
1069
|
+
// in a cluster it is not part of.
|
|
1070
|
+
if (!declared.includes(ourId)) {
|
|
1071
|
+
log('cluster-member:admission-reject', { messageHash: record.messageHash, reason: 'self-not-member', declaredSize: declared.length });
|
|
1072
|
+
return { admit: false, reason: `${MEMBERSHIP_NOT_ADMITTED}:self-not-member` };
|
|
1073
|
+
}
|
|
1074
|
+
|
|
1075
|
+
// Explicit opt-in: knowingly transact below the safe floor (single-node / local dev). Skips the
|
|
1076
|
+
// size/consistency gates but not self-membership above.
|
|
1077
|
+
if (this.allowUnvalidatedSmallCluster) {
|
|
1078
|
+
return { admit: true };
|
|
1079
|
+
}
|
|
1080
|
+
|
|
1081
|
+
const derived = await this.deriveExpectedClusterView(record);
|
|
1082
|
+
// An empty derived view (kEst === 0) carries no usable reference set: measured against it every
|
|
1083
|
+
// non-empty declared set is wholly "inconsistent" (maxDiff = ceil(tol·0) = 0), which would spuriously
|
|
1084
|
+
// reject a legitimate full cluster — a stricter, worse outcome than an absent view. Treat empty as
|
|
1085
|
+
// not-confident so it takes the fail-closed-or-legacy branch below instead. (Not normally reachable:
|
|
1086
|
+
// a responsible member's findCluster includes at least itself; this guards a transient empty read.)
|
|
1087
|
+
const derivedSize = derived !== undefined ? Object.keys(derived.peers ?? {}).length : 0;
|
|
1088
|
+
const confident = derived !== undefined
|
|
1089
|
+
&& derived.confidence > ClusterMember.MembershipConfidenceThreshold
|
|
1090
|
+
&& derivedSize > 0;
|
|
1091
|
+
|
|
1092
|
+
if (!confident) {
|
|
1093
|
+
// Fail closed for downsizing under low/absent confidence, measured against the operator's asserted
|
|
1094
|
+
// cohort size rather than the replication factor. With no asserted size the gate cannot tell a
|
|
1095
|
+
// downsize from a legitimately small cluster at all, so it preserves legacy approve behavior.
|
|
1096
|
+
if (this.assumedClusterSize === undefined) {
|
|
1097
|
+
return { admit: true };
|
|
1098
|
+
}
|
|
1099
|
+
const floor = this.admissionFloor(this.assumedClusterSize);
|
|
1100
|
+
if (declared.length >= floor) {
|
|
1101
|
+
return { admit: true };
|
|
1102
|
+
}
|
|
1103
|
+
log('cluster-member:admission-reject', {
|
|
1104
|
+
messageHash: record.messageHash,
|
|
1105
|
+
reason: 'low-confidence-downsize',
|
|
1106
|
+
declaredSize: declared.length,
|
|
1107
|
+
floor,
|
|
1108
|
+
assumedClusterSize: this.assumedClusterSize,
|
|
1109
|
+
confidence: derived?.confidence
|
|
1110
|
+
});
|
|
1111
|
+
// The numbers ride along in the reason: this rejection is caused by *local* configuration, and
|
|
1112
|
+
// without them a coordinator (or an operator reading a dispute record) has no hint which knob did it.
|
|
1113
|
+
// NOTE: two honest members with different local config now emit *different* reason strings for the
|
|
1114
|
+
// same record. Nothing compares reasons across peers today (`disputeEvidence.rejectReasons` is a
|
|
1115
|
+
// per-peer map, and the signed payload hashes the string each vote carries); if anything ever groups
|
|
1116
|
+
// or dedupes dispute reasons by string equality, group on the `membership-not-admitted:<variant>`
|
|
1117
|
+
// prefix, not the whole string.
|
|
1118
|
+
return {
|
|
1119
|
+
admit: false,
|
|
1120
|
+
reason: `${MEMBERSHIP_NOT_ADMITTED}:low-confidence-downsize (declared=${declared.length}, floor=${floor}, assumedClusterSize=${this.assumedClusterSize})`
|
|
1121
|
+
};
|
|
1122
|
+
}
|
|
1123
|
+
|
|
1124
|
+
const expected = Object.keys(derived!.peers ?? {});
|
|
1125
|
+
const kEst = expected.length;
|
|
1126
|
+
|
|
1127
|
+
// Predicate 2: floor derived from the member's OWN confident estimate.
|
|
1128
|
+
const floor = this.admissionFloor(kEst);
|
|
1129
|
+
if (declared.length < floor) {
|
|
1130
|
+
log('cluster-member:admission-reject', {
|
|
1131
|
+
messageHash: record.messageHash,
|
|
1132
|
+
reason: 'below-floor',
|
|
1133
|
+
declaredSize: declared.length,
|
|
1134
|
+
floor,
|
|
1135
|
+
kEst
|
|
1136
|
+
});
|
|
1137
|
+
return {
|
|
1138
|
+
admit: false,
|
|
1139
|
+
reason: `${MEMBERSHIP_NOT_ADMITTED}:below-floor (declared=${declared.length}, floor=${floor}, kEst=${kEst})`
|
|
1140
|
+
};
|
|
1141
|
+
}
|
|
1142
|
+
|
|
1143
|
+
// Predicate 3: consistency with the derived view within tolerance.
|
|
1144
|
+
const symmetricDiff = ClusterMember.symmetricDifferenceSize(declared, expected);
|
|
1145
|
+
const maxDiff = Math.ceil(this.clusterSizeTolerance * kEst);
|
|
1146
|
+
if (symmetricDiff > maxDiff) {
|
|
1147
|
+
log('cluster-member:admission-reject', {
|
|
1148
|
+
messageHash: record.messageHash,
|
|
1149
|
+
reason: 'inconsistent-with-derived-view',
|
|
1150
|
+
declaredSize: declared.length,
|
|
1151
|
+
kEst,
|
|
1152
|
+
symmetricDiff,
|
|
1153
|
+
maxDiff
|
|
1154
|
+
});
|
|
1155
|
+
return { admit: false, reason: `${MEMBERSHIP_NOT_ADMITTED}:inconsistent-with-derived-view` };
|
|
1156
|
+
}
|
|
1157
|
+
|
|
1158
|
+
return { admit: true };
|
|
1159
|
+
}
|
|
1160
|
+
|
|
1161
|
+
/**
|
|
1162
|
+
* The smallest declared peer set admissible against a cohort-size reference `k`, whether `k` is
|
|
1163
|
+
* measured (the confident path's `kEst`) or asserted (`assumedClusterSize`). One function so the
|
|
1164
|
+
* fallback can never be stricter than the measured path — which it was, demanding the full configured
|
|
1165
|
+
* size with no fraction and no slack. Clamped at `minAbsoluteClusterSize`, so a degenerate `k` of 0, 1
|
|
1166
|
+
* or negative yields the absolute floor rather than a floor that admits everything. A non-finite scaled
|
|
1167
|
+
* size (a `NaN` or `Infinity` config value) is likewise treated as no usable reference rather than
|
|
1168
|
+
* propagating: an unguarded `NaN` floor fails EVERY comparison, which would silently make the node
|
|
1169
|
+
* reject every unconfident write.
|
|
1170
|
+
*
|
|
1171
|
+
* NOTE: partition safety needs `2 · membershipAdmissionFraction · superMajorityThreshold > 1` — each
|
|
1172
|
+
* side of a split must recruit `fraction · threshold · K` distinct honest members, and two sides cannot
|
|
1173
|
+
* both find them in one K-peer cluster. At the shipped defaults (0.75 · 0.75 — both default to
|
|
1174
|
+
* `DEFAULT_SUPER_MAJORITY_THRESHOLD` / `membershipAdmissionFraction`'s own default) that product is
|
|
1175
|
+
* 1.125. If either default is ever lowered, re-check Theorem 2 in `docs/correctness.md` before shipping it.
|
|
1176
|
+
*/
|
|
1177
|
+
private admissionFloor(k: number): number {
|
|
1178
|
+
const scaled = Math.ceil(this.membershipAdmissionFraction * k);
|
|
1179
|
+
return Math.max(this.minAbsoluteClusterSize, Number.isFinite(scaled) ? scaled : 0);
|
|
1180
|
+
}
|
|
1181
|
+
|
|
1182
|
+
/**
|
|
1183
|
+
* Derive this member's own view of the record's block cluster via the injected capability, or
|
|
1184
|
+
* `undefined` when it cannot (no capability, no coordinating block id, a coordinating block not bound
|
|
1185
|
+
* to the record's own operations, or a derivation error — all of which the gate treats as "not
|
|
1186
|
+
* confident"). Derived from the record's coordinating block, the same key the coordinator used to
|
|
1187
|
+
* select the cluster.
|
|
1188
|
+
*
|
|
1189
|
+
* Read off `record.message`, NOT a top-level record field: `messageHash` covers the message only, so
|
|
1190
|
+
* only the in-message copy is tamper-evident to a relaying peer. (There is no top-level copy any more —
|
|
1191
|
+
* see {@link ClusterRecord.message}.)
|
|
1192
|
+
*/
|
|
1193
|
+
private async deriveExpectedClusterView(record: ClusterRecord): Promise<ExpectedClusterView | undefined> {
|
|
1194
|
+
if (!this.deriveExpectedCluster) {
|
|
1195
|
+
return undefined;
|
|
1196
|
+
}
|
|
1197
|
+
const blockId = record.message.coordinatingBlockIds?.[0];
|
|
1198
|
+
if (blockId === undefined) {
|
|
1199
|
+
return undefined;
|
|
1200
|
+
}
|
|
1201
|
+
// Hashing the field makes it tamper-evident to RELAYS, but the coordinator is the party this gate
|
|
1202
|
+
// exists to check and it picks the field before it computes the hash. Unbound, a Byzantine
|
|
1203
|
+
// coordinator declares a shrunken cohort `D` and names a coordinating block whose real cohort
|
|
1204
|
+
// resembles `D`: every member then derives that block's cohort, finds kEst = |D|, symmetric
|
|
1205
|
+
// difference 0, and admits — the gate fully defeated. Binding the coordinating block to a block the
|
|
1206
|
+
// record's OWN operations touch removes that free choice. `getAffectedBlockIds` is the same block
|
|
1207
|
+
// extraction conflict detection already runs on this message — one definition, so the set a
|
|
1208
|
+
// coordinating id must come from cannot drift from the set the record is judged to touch.
|
|
1209
|
+
if (!this.getAffectedBlockIds(record.message.operations).includes(blockId)) {
|
|
1210
|
+
log('cluster-member:coordinating-block-unbound', {
|
|
1211
|
+
messageHash: record.messageHash,
|
|
1212
|
+
coordinatingBlockId: blockId
|
|
1213
|
+
});
|
|
1214
|
+
// Fail closed into the branch that already exists rather than throwing: a hard reject would
|
|
1215
|
+
// change `validateRecord`'s failure surface, and "not confident" already refuses any downsize.
|
|
1216
|
+
return undefined;
|
|
1217
|
+
}
|
|
1218
|
+
// Which block a member derived its cohort view from is the single most useful fact when an
|
|
1219
|
+
// admission decision has to be explained after the fact — and the only externally visible sign
|
|
1220
|
+
// that the confident predicates ran at all rather than the fallback floor.
|
|
1221
|
+
log('cluster-member:derive-expected-cluster', { messageHash: record.messageHash, blockId });
|
|
1222
|
+
try {
|
|
1223
|
+
// NOTE: derives (findCluster) once per inbound record on the promise path — one routing lookup
|
|
1224
|
+
// per vote. If this shows up as hot, cache the derived view per (blockId, short TTL): it is a
|
|
1225
|
+
// pure read of current topology, so a few-seconds-stale view is safe for admission.
|
|
1226
|
+
return await this.deriveExpectedCluster(blockId as BlockId);
|
|
1227
|
+
} catch (err) {
|
|
1228
|
+
log('cluster-member:derive-expected-cluster-error', { messageHash: record.messageHash, error: (err as Error).message });
|
|
1229
|
+
return undefined;
|
|
1230
|
+
}
|
|
1231
|
+
}
|
|
1232
|
+
|
|
1233
|
+
/** |A △ B| over two id lists (order-independent set symmetric difference). */
|
|
1234
|
+
private static symmetricDifferenceSize(a: string[], b: string[]): number {
|
|
1235
|
+
const setA = new Set(a);
|
|
1236
|
+
const setB = new Set(b);
|
|
1237
|
+
let count = 0;
|
|
1238
|
+
for (const x of setA) if (!setB.has(x)) count++;
|
|
1239
|
+
for (const x of setB) if (!setA.has(x)) count++;
|
|
1240
|
+
return count;
|
|
1241
|
+
}
|
|
1242
|
+
|
|
1243
|
+
/**
|
|
1244
|
+
* Validates pend operations in a cluster record using the transaction validator.
|
|
1245
|
+
* Also checks for stale revisions, and for blocks held by a different unresolved pending
|
|
1246
|
+
* action, to prevent consensus on operations that storage would refuse at apply.
|
|
1247
|
+
* Returns success if no validator is configured (backwards compatibility).
|
|
1248
|
+
*/
|
|
1249
|
+
private async validatePendOperations(record: ClusterRecord): Promise<{ valid: boolean; reason?: string }> {
|
|
1250
|
+
// Find pend operations in the message
|
|
1251
|
+
for (const operation of record.message.operations) {
|
|
1252
|
+
if ('pend' in operation) {
|
|
1253
|
+
const pendRequest = operation.pend;
|
|
1254
|
+
const blockIds = blockIdsForTransforms(pendRequest.transforms);
|
|
1255
|
+
// One state read serves both checks below: `latest` for staleness, `pendings` for the
|
|
1256
|
+
// unresolved-rival check.
|
|
1257
|
+
const blockResults = await this.storageRepo.get({ blockIds });
|
|
1258
|
+
|
|
1259
|
+
// Check for stale revisions before allowing consensus
|
|
1260
|
+
if (pendRequest.rev !== undefined) {
|
|
1261
|
+
for (const blockId of blockIds) {
|
|
1262
|
+
const blockResult = blockResults[blockId];
|
|
1263
|
+
if (blockResult?.unavailable !== undefined) {
|
|
1264
|
+
// This member cannot establish the block's revision, so it cannot judge
|
|
1265
|
+
// staleness. Vote reject rather than approve on an answer it knows is a
|
|
1266
|
+
// guess — approving would let a stale pend reach consensus on the strength
|
|
1267
|
+
// of a member that could not check it. (Before StorageRepo caught
|
|
1268
|
+
// materialization faults per block, this read threw out of the promise
|
|
1269
|
+
// handler; rejecting keeps the fail-closed posture with a signed reason.)
|
|
1270
|
+
log('cluster-member:validation-block-unavailable', {
|
|
1271
|
+
messageHash: record.messageHash,
|
|
1272
|
+
blockId,
|
|
1273
|
+
reason: blockResult.unavailable
|
|
1274
|
+
});
|
|
1275
|
+
return { valid: false, reason: `block ${blockId} unavailable (${blockResult.unavailable}): cannot verify revision` };
|
|
1276
|
+
}
|
|
1277
|
+
const latest = blockResult?.state?.latest;
|
|
1278
|
+
if (latest !== undefined && latest.rev >= pendRequest.rev) {
|
|
1279
|
+
// Self is excluded so a redelivered pend for this same action stays
|
|
1280
|
+
// approvable — the same exclusion the pending-rival check below documents,
|
|
1281
|
+
// and the same rule storage applies (see {@link isOwnRevision}).
|
|
1282
|
+
if (isOwnRevision(latest, pendRequest.rev, pendRequest.actionId)) {
|
|
1283
|
+
continue;
|
|
1284
|
+
}
|
|
1285
|
+
log('cluster-member:validation-stale-revision', {
|
|
1286
|
+
messageHash: record.messageHash,
|
|
1287
|
+
blockId,
|
|
1288
|
+
requestedRev: pendRequest.rev,
|
|
1289
|
+
latestRev: latest.rev
|
|
1290
|
+
});
|
|
1291
|
+
// Deliberately prose-only: this reason is fed to computeSigningPayload, signed,
|
|
1292
|
+
// and carried as Signature.rejectReason, so adding a structured revision here
|
|
1293
|
+
// would change the signed byte layout and the Signature type — every peer would
|
|
1294
|
+
// have to agree on the new format or verification breaks across versions. This
|
|
1295
|
+
// is NOT a StaleFailure producer, so StaleFailure.staleAt does not apply; the
|
|
1296
|
+
// coordinator's own local re-read (CoordinatorRepo.classifyStaleRejection)
|
|
1297
|
+
// supplies that number when it can confirm the revision itself.
|
|
1298
|
+
return { valid: false, reason: `stale revision: block ${blockId} at rev ${latest.rev}, requested rev ${pendRequest.rev}` };
|
|
1299
|
+
}
|
|
1300
|
+
}
|
|
1301
|
+
}
|
|
1302
|
+
|
|
1303
|
+
// Reject a pend whose blocks are held by a DIFFERENT unresolved pending action. This is
|
|
1304
|
+
// the durable reservation the in-memory table (`findConflict` / `activeTransactions`)
|
|
1305
|
+
// cannot provide: that table clears the moment the rival's PEND record reaches
|
|
1306
|
+
// consensus, but the rival's storage pending record — written at pend-apply, removed at
|
|
1307
|
+
// commit or cancel — spans exactly the pend→commit window in which `latest.rev` has not
|
|
1308
|
+
// yet advanced. Storage's own pend would refuse this request at consensus-apply for the
|
|
1309
|
+
// same reason (`StorageRepo.pend`'s listPendingTransactions scan); voting reject here
|
|
1310
|
+
// moves that verdict into the phase where the cohort aggregates it, so the loser is
|
|
1311
|
+
// refused with a real answer instead of burning a consensus round it cannot win. A
|
|
1312
|
+
// member that has not yet applied the rival's pend has no record and simply abstains
|
|
1313
|
+
// from this reason; the coordinator returning the retained apply verdict
|
|
1314
|
+
// (getExecutedPendResult) catches that residual. Self is excluded so a redelivered pend
|
|
1315
|
+
// for this same action stays approvable. An unavailable block carries no `pendings` and
|
|
1316
|
+
// abstains (the rev branch above already fail-closes when a revision claim is at stake).
|
|
1317
|
+
// Reason stays plain prose: it is fed to computeSigningPayload and carried as
|
|
1318
|
+
// Signature.rejectReason, exactly like the stale-revision reason above.
|
|
1319
|
+
for (const blockId of blockIds) {
|
|
1320
|
+
const rivals = (blockResults[blockId]?.state?.pendings ?? []).filter(actionId => actionId !== pendRequest.actionId);
|
|
1321
|
+
if (rivals.length > 0) {
|
|
1322
|
+
log('cluster-member:validation-pending-conflict', {
|
|
1323
|
+
messageHash: record.messageHash,
|
|
1324
|
+
blockId,
|
|
1325
|
+
actionId: pendRequest.actionId,
|
|
1326
|
+
rivals
|
|
1327
|
+
});
|
|
1328
|
+
return { valid: false, reason: `pending conflict: block ${blockId} held by unresolved action(s) ${rivals.join(', ')}` };
|
|
1329
|
+
}
|
|
1330
|
+
}
|
|
1331
|
+
|
|
1332
|
+
// Re-check the transaction when a validator is configured. The unvalidatable-pend
|
|
1333
|
+
// policy and the throwing-validator catch live in the shared `checkPendValidation`,
|
|
1334
|
+
// which the storage tier runs too, so a member cannot vote approve on a shape its own
|
|
1335
|
+
// storage would refuse at apply. Its reasons are fed to computeSigningPayload and
|
|
1336
|
+
// carried as Signature.rejectReason, exactly like the stale-revision reason above, so
|
|
1337
|
+
// a fail-closed refusal here is signed evidence rather than a lost vote.
|
|
1338
|
+
const validator = this.validator;
|
|
1339
|
+
const validation = await checkPendValidation(
|
|
1340
|
+
pendRequest,
|
|
1341
|
+
validator && (({ transaction, operationsHash }) => validator.validate(transaction, operationsHash)),
|
|
1342
|
+
this.unvalidatablePendPolicy,
|
|
1343
|
+
event => event.kind === 'unvalidatable'
|
|
1344
|
+
// An operator can grep this line to see how much traffic goes unchecked.
|
|
1345
|
+
? log('cluster-member:pend-unvalidatable', {
|
|
1346
|
+
messageHash: record.messageHash,
|
|
1347
|
+
actionId: pendRequest.actionId,
|
|
1348
|
+
policy: event.policy
|
|
1349
|
+
})
|
|
1350
|
+
: log('cluster-member:validator-fault', {
|
|
1351
|
+
messageHash: record.messageHash,
|
|
1352
|
+
error: event.error
|
|
1353
|
+
})
|
|
1354
|
+
);
|
|
1355
|
+
if (!validation.valid) {
|
|
1356
|
+
return { valid: false, reason: validation.reason };
|
|
1357
|
+
}
|
|
1358
|
+
}
|
|
1359
|
+
}
|
|
1360
|
+
|
|
1361
|
+
return { valid: true };
|
|
1362
|
+
}
|
|
1363
|
+
|
|
1364
|
+
/**
|
|
1365
|
+
* Promise-round check that a commit record's requested revision is not already committed HERE
|
|
1366
|
+
* under a different action. This is the member-side arm that keeps a DEAD rival's re-broadcast
|
|
1367
|
+
* commit from assembling consensus: after a race winner commits and members clear its record
|
|
1368
|
+
* from the reservation table, a loser's re-driven commit meets no conflict votes — without this
|
|
1369
|
+
* check every caught-up member would abstain (the content-digest check below abstains for
|
|
1370
|
+
* update-only transforms whose base moved) and the loser could reach commit-consensus for a
|
|
1371
|
+
* write no member will ever durably store (its apply is refused stale and tolerated as 'ahead'
|
|
1372
|
+
* divergence — see `applyConsensusOperation`).
|
|
1373
|
+
*
|
|
1374
|
+
* Same "must run on the promise round" rule as {@link validateCommitOperations}: the
|
|
1375
|
+
* commit-round vote is deliberately blind, so promise votes are the only ones that carry
|
|
1376
|
+
* "I checked this". The four-way rule, per committed block:
|
|
1377
|
+
* - no local `latest`, or `latest.rev < commit.rev` → abstain (approve). Preserves the
|
|
1378
|
+
* lagging-member tolerance (`coordinator-repo-commit-divergence.spec.ts`): a member behind
|
|
1379
|
+
* the commit cannot judge it.
|
|
1380
|
+
* - `latest.rev === commit.rev` with the SAME action → abstain (approve). Idempotent
|
|
1381
|
+
* redelivery of an already-durable commit; rejecting would make the writer rebase and
|
|
1382
|
+
* re-append an action that already landed — a duplicate entry. (Storage's `alreadyDone`
|
|
1383
|
+
* partition returns success for this shape at apply.)
|
|
1384
|
+
* - `latest.rev === commit.rev` with a DIFFERENT action → reject: a rival took the revision.
|
|
1385
|
+
* - `latest.rev > commit.rev` → consult the {@link IRevisionActionReader} capability for who
|
|
1386
|
+
* holds `commit.rev`: a different action → reject; the same action → abstain (already
|
|
1387
|
+
* durable, history simply moved on); no record / capability absent / read fault → abstain.
|
|
1388
|
+
*
|
|
1389
|
+
* Never throws out of the vote path: any read fault is an abstain (mirroring the digest check's
|
|
1390
|
+
* preview-error arm), because a member that fails to vote at all is worse than one that
|
|
1391
|
+
* abstains. Reason stays plain prose — it is fed to computeSigningPayload and carried as
|
|
1392
|
+
* Signature.rejectReason, exactly like the stale-revision pend reject.
|
|
1393
|
+
*
|
|
1394
|
+
* Residual (see the commit-tier handoff): a member that signed the winner's commit but has not
|
|
1395
|
+
* yet APPLIED it sits in a window where it holds neither the winner's record (reservation
|
|
1396
|
+
* dropped at commit-sign, `shouldPersist = false`) nor the winner's revision (storage still
|
|
1397
|
+
* behind) — it abstains here. A capability-less member, or one with history truncated below
|
|
1398
|
+
* `latest`, abstains at `latest.rev > commit.rev` too. A rival's commit can therefore still
|
|
1399
|
+
* pass the promise round if EVERY member is simultaneously in one of those states — on a fast
|
|
1400
|
+
* cohort that window is the COMMON case, not the corner. The backstop is downstream of
|
|
1401
|
+
* consensus: every member's apply then refuses the rival as stale, the coordinating node's own
|
|
1402
|
+
* member retains that refusal (`getExecutedCommitResult`), and `CoordinatorRepo.commit`
|
|
1403
|
+
* confirms the rival against local storage and answers the writer with a retryable conflict
|
|
1404
|
+
* instead of a fabricated success. `ConflictRaceLostError` conversion and classified
|
|
1405
|
+
* rejections close the re-drive route the same way.
|
|
1406
|
+
*/
|
|
1407
|
+
private async validateCommitRevisions(record: ClusterRecord): Promise<{ valid: boolean; reason?: string }> {
|
|
1408
|
+
for (const operation of record.message.operations) {
|
|
1409
|
+
if (!('commit' in operation)) {
|
|
1410
|
+
continue;
|
|
1411
|
+
}
|
|
1412
|
+
const commit = operation.commit;
|
|
1413
|
+
let blockResults: Awaited<ReturnType<IRepo['get']>>;
|
|
1414
|
+
try {
|
|
1415
|
+
// The member's raw storage repo (no cluster recursion) — the same seam
|
|
1416
|
+
// validatePendOperations reads on every pend vote.
|
|
1417
|
+
blockResults = await this.storageRepo.get({ blockIds: commit.blockIds });
|
|
1418
|
+
} catch (err) {
|
|
1419
|
+
log('cluster-member:commit-staleness-read-error', {
|
|
1420
|
+
messageHash: record.messageHash,
|
|
1421
|
+
actionId: commit.actionId,
|
|
1422
|
+
error: err instanceof Error ? err.message : String(err)
|
|
1423
|
+
});
|
|
1424
|
+
continue; // a local read fault is an abstain, never an escape out of the vote path
|
|
1425
|
+
}
|
|
1426
|
+
for (const blockId of commit.blockIds) {
|
|
1427
|
+
const latest = blockResults[blockId]?.state?.latest;
|
|
1428
|
+
if (!latest || latest.rev < commit.rev) {
|
|
1429
|
+
continue; // behind (or block never seen): cannot judge — abstain
|
|
1430
|
+
}
|
|
1431
|
+
if (isOwnRevision(latest, commit.rev, commit.actionId)) {
|
|
1432
|
+
continue; // idempotent redelivery of an already-durable commit — MUST NOT reject
|
|
1433
|
+
}
|
|
1434
|
+
if (latest.rev === commit.rev) {
|
|
1435
|
+
log('cluster-member:validation-stale-commit', {
|
|
1436
|
+
messageHash: record.messageHash,
|
|
1437
|
+
blockId,
|
|
1438
|
+
actionId: commit.actionId,
|
|
1439
|
+
rev: commit.rev,
|
|
1440
|
+
committedBy: latest.actionId
|
|
1441
|
+
});
|
|
1442
|
+
return { valid: false, reason: `stale commit: block ${blockId} rev ${commit.rev} committed by a different action` };
|
|
1443
|
+
}
|
|
1444
|
+
// latest.rev > commit.rev: latest can no longer name who took commit.rev — ask the
|
|
1445
|
+
// revision index. Structural probe, same pattern as previewCommitDigest below: a repo
|
|
1446
|
+
// without the capability abstains.
|
|
1447
|
+
const reader = this.storageRepo as IRepo & Partial<IRevisionActionReader>;
|
|
1448
|
+
if (typeof reader.getRevisionAction !== 'function') {
|
|
1449
|
+
continue;
|
|
1450
|
+
}
|
|
1451
|
+
let takenBy: ActionId | undefined;
|
|
1452
|
+
try {
|
|
1453
|
+
takenBy = await reader.getRevisionAction(blockId, commit.rev);
|
|
1454
|
+
} catch (err) {
|
|
1455
|
+
log('cluster-member:commit-staleness-revision-read-error', {
|
|
1456
|
+
messageHash: record.messageHash,
|
|
1457
|
+
blockId,
|
|
1458
|
+
rev: commit.rev,
|
|
1459
|
+
error: err instanceof Error ? err.message : String(err)
|
|
1460
|
+
});
|
|
1461
|
+
continue; // read fault → abstain
|
|
1462
|
+
}
|
|
1463
|
+
if (takenBy !== undefined && takenBy !== commit.actionId) {
|
|
1464
|
+
log('cluster-member:validation-stale-commit', {
|
|
1465
|
+
messageHash: record.messageHash,
|
|
1466
|
+
blockId,
|
|
1467
|
+
actionId: commit.actionId,
|
|
1468
|
+
rev: commit.rev,
|
|
1469
|
+
committedBy: takenBy,
|
|
1470
|
+
latestRev: latest.rev
|
|
1471
|
+
});
|
|
1472
|
+
return { valid: false, reason: `stale commit: block ${blockId} rev ${commit.rev} committed by a different action` };
|
|
1473
|
+
}
|
|
1474
|
+
// takenBy === commit.actionId (already durable, history moved on) or undefined
|
|
1475
|
+
// (truncated history — unknown): abstain either way.
|
|
1476
|
+
}
|
|
1477
|
+
}
|
|
1478
|
+
return { valid: true };
|
|
1479
|
+
}
|
|
1480
|
+
|
|
1481
|
+
/**
|
|
1482
|
+
* Promise-round check of a commit record's declared content digests
|
|
1483
|
+
* (`CommitRequest.blockDigests`) against what this member's OWN pended copy of each transform
|
|
1484
|
+
* would materialize (`StorageRepo.previewCommitDigest`). This is what makes a promise approval on
|
|
1485
|
+
* a commit record MEAN something about content: before this hook, the promise round validated
|
|
1486
|
+
* nothing for commits (`validatePendOperations` only inspects pend operations).
|
|
1487
|
+
*
|
|
1488
|
+
* MUST run on the promise round, not the commit round: the commit-round vote is cast deliberately
|
|
1489
|
+
* blind — `getTransactionPhase` signs the commit whenever promise approvals reach super-majority,
|
|
1490
|
+
* regardless of this member's own promise vote — so promise approvals are the only votes that
|
|
1491
|
+
* carry "I checked this". Do not move it.
|
|
1492
|
+
*
|
|
1493
|
+
* Checkable/abstain rule, keyed on the member's OWN pended transform (the payload the client
|
|
1494
|
+
* authored, delivered at pend — a hostile declarer cannot force or dodge a check by mis-declaring
|
|
1495
|
+
* `baseRev`):
|
|
1496
|
+
* - transform carries an `insert` → base-independent, ALWAYS check (declared `baseRev` ignored);
|
|
1497
|
+
* - `updates` only → check iff this member's local base rev equals the declared `baseRev`
|
|
1498
|
+
* (StorageRepo.commit accepts any `latest.rev < request.rev`, so a lagging member applying an
|
|
1499
|
+
* update-only transform to an older base legitimately materializes different bytes);
|
|
1500
|
+
* - `delete` only / no base / unmaterializable base → materializes nothing to compare, abstain;
|
|
1501
|
+
* - no pending transform for the action (this member never saw the pend) → abstain.
|
|
1502
|
+
* "Abstain" = contribute no content attestation: approve exactly as before this check existed.
|
|
1503
|
+
*
|
|
1504
|
+
* Residual: a false digest survives only when the declarer lies AND enough of the cohort is
|
|
1505
|
+
* simultaneously unable to check (lagging on update-only blocks, missed pends) that no honest
|
|
1506
|
+
* checker is left — any single caught-up honest member rejects. Strictly stronger than before,
|
|
1507
|
+
* when commit signatures bound no content at all. Verifying/persisting a durable content proof is
|
|
1508
|
+
* later work (persist-block-commit-proof).
|
|
1509
|
+
*/
|
|
1510
|
+
private async validateCommitOperations(record: ClusterRecord): Promise<{ valid: boolean; reason?: string }> {
|
|
1511
|
+
// Capability probe: `storageRepo` is typed IRepo, and only a repo that owns the local
|
|
1512
|
+
// materialization can preview one. A repo without the capability abstains everywhere (also
|
|
1513
|
+
// keeps mock-repo harnesses and non-storage compositions on the legacy approve path).
|
|
1514
|
+
// NOTE: probing structurally means a decorating/caching repo later inserted at this seam
|
|
1515
|
+
// silently disables the whole check with no signal. `ICommitDigestPreviewer` exists so such a
|
|
1516
|
+
// decorator has a named contract to forward; if a non-forwarding wrapper is ever wired here,
|
|
1517
|
+
// promote this to a typed component field rather than widening the probe.
|
|
1518
|
+
const repo = this.storageRepo as IRepo & Partial<ICommitDigestPreviewer>;
|
|
1519
|
+
if (typeof repo.previewCommitDigest !== 'function') {
|
|
1520
|
+
return { valid: true };
|
|
1521
|
+
}
|
|
1522
|
+
|
|
1523
|
+
for (const operation of record.message.operations) {
|
|
1524
|
+
if (!('commit' in operation)) {
|
|
1525
|
+
continue;
|
|
1526
|
+
}
|
|
1527
|
+
const commit = operation.commit;
|
|
1528
|
+
// An upgraded member receiving a commit with no declarations abstains everywhere.
|
|
1529
|
+
if (!commit.blockDigests) {
|
|
1530
|
+
continue;
|
|
1531
|
+
}
|
|
1532
|
+
// One Set per commit operation: the surplus-entry filter below is a membership test per
|
|
1533
|
+
// declared id, and `blockIds` is a per-coordinator batch that can be wide.
|
|
1534
|
+
const committedIds = new Set<string>(commit.blockIds);
|
|
1535
|
+
// NOTE: previews run one block at a time, so a commit declaring N blocks adds N sequential
|
|
1536
|
+
// preview round-trips (each 1-3 block-storage reads plus a structuredClone of the base and
|
|
1537
|
+
// the transform) to this member's promise-round latency. Unmeasured, and sequencing buys the
|
|
1538
|
+
// short-circuit on the first mismatch. If wide commits ever show up as promise latency,
|
|
1539
|
+
// fan the previews out with Promise.all and reduce the results, rather than sampling a
|
|
1540
|
+
// subset of the declared ids — a skipped id is an unchecked id.
|
|
1541
|
+
for (const [blockId, declared] of Object.entries(commit.blockDigests)) {
|
|
1542
|
+
// Surplus (or hostile) entry for a block this commit does not even cover: ignore it —
|
|
1543
|
+
// never throw out of the vote path, and never reject on content nobody is committing.
|
|
1544
|
+
if (!committedIds.has(blockId)) {
|
|
1545
|
+
continue;
|
|
1546
|
+
}
|
|
1547
|
+
// `blockDigests` is untrusted wire data with no ingress schema behind it, so the entry
|
|
1548
|
+
// need not be the shape the type promises. A malformed entry is treated as an omitted
|
|
1549
|
+
// one (abstain) rather than a mismatch: rejecting on it would let a garbled request
|
|
1550
|
+
// look like forged content, and reading through it would throw a TypeError out of the
|
|
1551
|
+
// vote path — this member would then fail to vote at all instead of voting reject.
|
|
1552
|
+
if (typeof declared?.digest !== 'string') {
|
|
1553
|
+
continue;
|
|
1554
|
+
}
|
|
1555
|
+
let preview: CommitDigestPreview | undefined;
|
|
1556
|
+
try {
|
|
1557
|
+
preview = await repo.previewCommitDigest(blockId as BlockId, commit.actionId, commit.rev);
|
|
1558
|
+
} catch (err) {
|
|
1559
|
+
log('cluster-member:content-digest-preview-error', {
|
|
1560
|
+
messageHash: record.messageHash,
|
|
1561
|
+
blockId,
|
|
1562
|
+
error: err instanceof Error ? err.message : String(err)
|
|
1563
|
+
});
|
|
1564
|
+
continue; // a local preview fault is an abstain, never a content judgement
|
|
1565
|
+
}
|
|
1566
|
+
// No pend seen here, or the transform materializes nothing to compare (tombstone,
|
|
1567
|
+
// updates with no base, unmaterializable base) → abstain.
|
|
1568
|
+
if (preview === undefined || preview.digest === undefined) {
|
|
1569
|
+
continue;
|
|
1570
|
+
}
|
|
1571
|
+
// `typeof === 'number'` rather than `!== undefined` for the same untrusted-shape reason
|
|
1572
|
+
// as the digest guard above: a non-numeric declared baseRev can never equal a local
|
|
1573
|
+
// one, so it degrades to an abstain instead of comparing junk.
|
|
1574
|
+
const checkable = preview.baseIndependent
|
|
1575
|
+
|| (typeof declared.baseRev === 'number' && preview.baseRev === declared.baseRev);
|
|
1576
|
+
if (!checkable) {
|
|
1577
|
+
continue;
|
|
1578
|
+
}
|
|
1579
|
+
if (preview.digest !== declared.digest) {
|
|
1580
|
+
log('cluster-member:content-digest-mismatch', {
|
|
1581
|
+
messageHash: record.messageHash,
|
|
1582
|
+
blockId,
|
|
1583
|
+
actionId: commit.actionId,
|
|
1584
|
+
rev: commit.rev,
|
|
1585
|
+
declaredDigest: declared.digest,
|
|
1586
|
+
declaredBaseRev: declared.baseRev,
|
|
1587
|
+
previewDigest: preview.digest,
|
|
1588
|
+
previewBaseRev: preview.baseRev,
|
|
1589
|
+
baseIndependent: preview.baseIndependent
|
|
1590
|
+
});
|
|
1591
|
+
// One vote per record: a single mismatching block rejects the whole record.
|
|
1592
|
+
return { valid: false, reason: CONTENT_DIGEST_MISMATCH };
|
|
1593
|
+
}
|
|
1594
|
+
}
|
|
1595
|
+
}
|
|
1596
|
+
|
|
1597
|
+
return { valid: true };
|
|
1598
|
+
}
|
|
1599
|
+
|
|
1600
|
+
private async handleCommitNeeded(record: ClusterRecord): Promise<ClusterRecord> {
|
|
1601
|
+
if (this.hasLocalCommit(record)) {
|
|
1602
|
+
return record;
|
|
1603
|
+
}
|
|
1604
|
+
const commitHash = await this.computeCommitHash(record);
|
|
1605
|
+
const sig = await this.signVote(commitHash, 'approve');
|
|
1606
|
+
const signature: Signature = {
|
|
1607
|
+
type: 'approve',
|
|
1608
|
+
signature: sig
|
|
1609
|
+
};
|
|
1610
|
+
|
|
1611
|
+
return {
|
|
1612
|
+
...record,
|
|
1613
|
+
commits: {
|
|
1614
|
+
...record.commits,
|
|
1615
|
+
[this.peerId.toString()]: signature
|
|
1616
|
+
}
|
|
1617
|
+
};
|
|
1618
|
+
}
|
|
1619
|
+
|
|
1620
|
+
/**
|
|
1621
|
+
* Executes operations after consensus is reached.
|
|
1622
|
+
*
|
|
1623
|
+
* @warning This method executes on ALL cluster peers, not just the coordinator.
|
|
1624
|
+
* Each peer independently applies the operations to its local storage.
|
|
1625
|
+
*
|
|
1626
|
+
* @pitfall **Check-then-act race** - The in-memory guard must be checked AND set
|
|
1627
|
+
* atomically (before any `await`) to prevent duplicate execution; JavaScript's
|
|
1628
|
+
* single-threaded nature makes that synchronous check-and-set atomic. The durable
|
|
1629
|
+
* marker, by contrast, is persisted only *after* apply succeeds — writing it eagerly
|
|
1630
|
+
* would leave a stuck marker on a caught fault or a crash mid-apply, silently dropping
|
|
1631
|
+
* the transaction on this member on redelivery.
|
|
1632
|
+
*
|
|
1633
|
+
* @pitfall **Independent node storage** - Each node has its own storage. After consensus,
|
|
1634
|
+
* each node applies operations locally. Nodes must fetch missing blocks from cluster
|
|
1635
|
+
* peers via `restoreCallback` if they don't have prior revisions.
|
|
1636
|
+
*
|
|
1637
|
+
* @see docs/internals.md "Check-Then-Act Race in Consensus" and "Independent Node Storage" pitfalls
|
|
1638
|
+
*/
|
|
1639
|
+
private async handleConsensus(record: ClusterRecord): Promise<void> {
|
|
1640
|
+
// Check persistent store first for post-recovery dedup (in-memory map is cleared on restart).
|
|
1641
|
+
// wasTransactionExecutedAsync also checks the in-memory map as a fast path.
|
|
1642
|
+
if (await this.wasTransactionExecutedAsync(record.messageHash)) {
|
|
1643
|
+
log('cluster-member:consensus-already-executed', { messageHash: record.messageHash });
|
|
1644
|
+
return;
|
|
1645
|
+
}
|
|
1646
|
+
// Check-and-set ATOMICALLY to prevent race condition where multiple calls
|
|
1647
|
+
// pass the async check before any completes. Since JavaScript is single-threaded,
|
|
1648
|
+
// this synchronous check-and-set is atomic before any await.
|
|
1649
|
+
if (this.executedTransactions.has(record.messageHash)) {
|
|
1650
|
+
log('cluster-member:consensus-already-executed', { messageHash: record.messageHash });
|
|
1651
|
+
return;
|
|
1652
|
+
}
|
|
1653
|
+
// Set the in-memory guard IMMEDIATELY, before any async operations: its synchronous
|
|
1654
|
+
// check-and-set (line above's `has` + this `set`) is what prevents the concurrent
|
|
1655
|
+
// apply-window race where two handleConsensus calls for the same hash both pass the
|
|
1656
|
+
// async check. The durable marker is deliberately NOT written here — see below.
|
|
1657
|
+
const executedAt = Date.now();
|
|
1658
|
+
this.executedTransactions.set(record.messageHash, executedAt);
|
|
1659
|
+
|
|
1660
|
+
try {
|
|
1661
|
+
for (const operation of record.message.operations) {
|
|
1662
|
+
await this.applyConsensusOperation(record, operation);
|
|
1663
|
+
}
|
|
1664
|
+
} catch (err) {
|
|
1665
|
+
// A genuinely unexpected fault (e.g. storage I/O) — roll back the in-memory
|
|
1666
|
+
// marker so a corrected retry can re-run, and propagate so the caller learns
|
|
1667
|
+
// the real cause. The durable marker was never written (it lands only after
|
|
1668
|
+
// apply succeeds, below), so there is nothing to roll back. Recoverable local
|
|
1669
|
+
// divergence is absorbed inside applyConsensusOperation and never reaches here.
|
|
1670
|
+
// A retained pend verdict rolls back with the marker: it belongs to an apply
|
|
1671
|
+
// that is now considered not-executed, and a re-run will retain a fresh one.
|
|
1672
|
+
this.executedTransactions.delete(record.messageHash);
|
|
1673
|
+
this.executedPendResults.delete(record.messageHash);
|
|
1674
|
+
this.executedCommitResults.delete(record.messageHash);
|
|
1675
|
+
throw err;
|
|
1676
|
+
}
|
|
1677
|
+
|
|
1678
|
+
// Persist the durable marker only now that apply has actually succeeded. Writing it
|
|
1679
|
+
// eagerly (before the loop) would leave a stuck marker on a caught fault OR a crash
|
|
1680
|
+
// mid-apply, and on redelivery handleConsensus short-circuits at the async
|
|
1681
|
+
// wasTransactionExecuted check — silently dropping the transaction on this member
|
|
1682
|
+
// forever. The durable marker exists only for post-restart dedup (the in-memory map
|
|
1683
|
+
// is empty after restart), and the narrow window between "apply succeeded" and
|
|
1684
|
+
// "durable write landed" is safe to re-run on restart: re-applying an
|
|
1685
|
+
// already-applied consensus transaction is idempotent (the "ahead" divergence path
|
|
1686
|
+
// in applyConsensusOperation tolerates it as a no-op), so it converges rather than
|
|
1687
|
+
// dropping. Fire-and-forget: a persist failure must not fail the apply that succeeded.
|
|
1688
|
+
this.stateStore?.markExecuted(record.messageHash, executedAt)
|
|
1689
|
+
.catch(err => log('cluster-member:persist-executed-error', { messageHash: record.messageHash, error: (err as Error).message }));
|
|
1690
|
+
}
|
|
1691
|
+
|
|
1692
|
+
/**
|
|
1693
|
+
* Applies one consensus-approved operation to local storage.
|
|
1694
|
+
*
|
|
1695
|
+
* By the time we reach here the cluster has *already* reached consensus, so the
|
|
1696
|
+
* operation is authoritative cluster-wide. A failure applying it to THIS member's
|
|
1697
|
+
* local store therefore does not mean the operation is invalid — it means our
|
|
1698
|
+
* local state has diverged from the agreed history:
|
|
1699
|
+
*
|
|
1700
|
+
* - **ahead**: we already hold a newer revision, so a stale pend/commit is a
|
|
1701
|
+
* no-op for us (`StorageRepo.commit` returns `success:false` with `missing`);
|
|
1702
|
+
* - **behind**: we missed the prior `pend` cluster-transaction (cohort drift
|
|
1703
|
+
* between the independent pend and commit phases, or transient unreachability),
|
|
1704
|
+
* so we lack the pending action — `StorageRepo.commit` *throws* "Pending
|
|
1705
|
+
* action … not found";
|
|
1706
|
+
* - **behind (no base)**: we DID see the pend, but we never saw the revision that
|
|
1707
|
+
* created the block, so the transform has nothing to apply to —
|
|
1708
|
+
* `StorageRepo.commit` returns `success:false` with a `missing-base-revision`
|
|
1709
|
+
* reason rather than recording a revision it could not materialize.
|
|
1710
|
+
*
|
|
1711
|
+
* For both **behind** cases we hold no usable revision of the committed blocks, so we
|
|
1712
|
+
* actively reconcile: pull the committed revision from a cohort peer that holds it
|
|
1713
|
+
* (`reconcileBlock`) and restore it locally. Lazy read-repair on a later read cannot
|
|
1714
|
+
* recover it on its own when cohort drift has left the block under-replicated (no
|
|
1715
|
+
* reachable peer the reader sees holds the newer rev), so reconciling here is what
|
|
1716
|
+
* keeps cross-cohort transactions converging. For the **ahead** case we already hold
|
|
1717
|
+
* ≥ the committed rev, so we tolerate the no-op without reconciling downward.
|
|
1718
|
+
*
|
|
1719
|
+
* Whatever happens, we must NOT reset the stream: throwing here would reset the
|
|
1720
|
+
* cluster stream the coordinator is awaiting and surface as a spurious
|
|
1721
|
+
* `StreamResetError`, sinking an otherwise-successful transaction. So divergence —
|
|
1722
|
+
* and any reconciliation failure — is tolerated (logged for observability). A
|
|
1723
|
+
* genuinely *invalid* pend can never get this far: it is rejected during the promise
|
|
1724
|
+
* phase (`validatePendOperations`, which validates pend ops only — commits carry no
|
|
1725
|
+
* promise-phase validation).
|
|
1726
|
+
*
|
|
1727
|
+
* The propagate-vs-tolerate split keys off the failure's *nature* via `CommitResult`,
|
|
1728
|
+
* not throw-vs-return: a missing pend (thrown "not found") or a stale/ahead commit
|
|
1729
|
+
* (`success:false` with `missing`) is divergence and tolerated, whereas a genuine
|
|
1730
|
+
* mid-commit `internalCommit` fault (`success:false` with a bare `reason`, no
|
|
1731
|
+
* `missing`) is propagated so {@link handleConsensus} rolls back the executed marker
|
|
1732
|
+
* and rethrows — exactly like an unexpected thrown fault.
|
|
1733
|
+
*/
|
|
1734
|
+
private async applyConsensusOperation(record: ClusterRecord, operation: RepoMessage['operations'][number]): Promise<void> {
|
|
1735
|
+
const messageHash = record.messageHash;
|
|
1736
|
+
if ('get' in operation) {
|
|
1737
|
+
await this.storageRepo.get(operation.get);
|
|
1738
|
+
return;
|
|
1739
|
+
}
|
|
1740
|
+
if ('cancel' in operation) {
|
|
1741
|
+
await this.storageRepo.cancel(operation.cancel.actionRef);
|
|
1742
|
+
return;
|
|
1743
|
+
}
|
|
1744
|
+
if ('pend' in operation) {
|
|
1745
|
+
const result = await this.storageRepo.pend(operation.pend);
|
|
1746
|
+
// Retain the verdict either way so the coordinator can hand the writer storage's real
|
|
1747
|
+
// answer (see getExecutedPendResult). A refusal here is NOT local divergence the way a
|
|
1748
|
+
// commit refusal is: pend-consensus confers no durability — a refusal carrying `pending`
|
|
1749
|
+
// (a rival's unresolved action holds the blocks) or `missing` (the requested revision is
|
|
1750
|
+
// already committed) is the optimistic-concurrency verdict, and swallowing it acknowledged
|
|
1751
|
+
// writes that no member stored.
|
|
1752
|
+
this.executedPendResults.set(messageHash, result);
|
|
1753
|
+
if (!result.success) {
|
|
1754
|
+
log('cluster-member:consensus-pend-diverged', {
|
|
1755
|
+
messageHash,
|
|
1756
|
+
actionId: operation.pend.actionId,
|
|
1757
|
+
reason: result.reason,
|
|
1758
|
+
hasMissing: !!result.missing?.length,
|
|
1759
|
+
hasPending: !!result.pending?.length
|
|
1760
|
+
});
|
|
1761
|
+
}
|
|
1762
|
+
return;
|
|
1763
|
+
}
|
|
1764
|
+
if ('commit' in operation) {
|
|
1765
|
+
const commit = operation.commit;
|
|
1766
|
+
// Capture the consensus commit cert BEFORE applying to storage: StorageRepo.commit emits the
|
|
1767
|
+
// CollectionChangeEvent synchronously at the end of the call below, and the reactivity bridge
|
|
1768
|
+
// resolves the cert from that event — so it must already be retained when commit() returns.
|
|
1769
|
+
//
|
|
1770
|
+
// The commit-vote signed preimage is computed HERE, before the synchronous capture+commit
|
|
1771
|
+
// sequence below: it is the exact bytes each approving signer endorsed
|
|
1772
|
+
// (`utf8(commitHash + ":approve")`), reproduced from this already-validated `record` (its
|
|
1773
|
+
// commit signatures were verified against the same `computeCommitHash`). Reactivity sets a
|
|
1774
|
+
// notification's `digest` from it so a subscriber's threshold-verify over `digest` succeeds.
|
|
1775
|
+
// The one `await` happens before `captureCommitCert` runs, so the cert is still retained
|
|
1776
|
+
// synchronously before `commit()` emits its change event (do not move this past the commit).
|
|
1777
|
+
// Gated on the sink: with no reactivity wired the preimage has no consumer, so a sink-less
|
|
1778
|
+
// node pays neither the extra `sha256` nor the extra microtask — the true zero-cost default.
|
|
1779
|
+
if (this.onCommitCertificate) {
|
|
1780
|
+
const commitSignedPayload = clusterVoteSigningPayload(await this.computeCommitHash(record), 'approve');
|
|
1781
|
+
this.captureCommitCert(record, commit.actionId, commitSignedPayload);
|
|
1782
|
+
}
|
|
1783
|
+
// Project the consensus record into a durable BlockCommitProof and hand it down the commit
|
|
1784
|
+
// path — StorageRepo persists it only where the local materialization matches the declared
|
|
1785
|
+
// digest (see persistProofIfContentMatches). A cheap projection (no hashing/signing).
|
|
1786
|
+
// `undefined` for a v1 / unversioned record: its hashes bind no peer set, so it is never
|
|
1787
|
+
// certifiable — logged so an operator can see why a cohort retains no proofs. The cast is the
|
|
1788
|
+
// named ICommitProofPersister contract; a plain IRepo mock ignores the extra argument.
|
|
1789
|
+
const proof = buildBlockCommitProof(record);
|
|
1790
|
+
if (proof === undefined) {
|
|
1791
|
+
log('cluster-member:commit-proof-skipped', {
|
|
1792
|
+
messageHash,
|
|
1793
|
+
actionId: commit.actionId,
|
|
1794
|
+
membershipVersion: record.membershipVersion
|
|
1795
|
+
});
|
|
1796
|
+
}
|
|
1797
|
+
let result: CommitResult;
|
|
1798
|
+
try {
|
|
1799
|
+
result = await (this.storageRepo as IRepo & ICommitProofPersister).commit(commit, undefined, proof);
|
|
1800
|
+
} catch (err) {
|
|
1801
|
+
// `StorageRepo.commit` throws (rather than returning success:false) when
|
|
1802
|
+
// the pending action is missing — the canonical "behind" signal: this
|
|
1803
|
+
// member reached commit-consensus without the matching pend (cohort drift).
|
|
1804
|
+
if (isMissingPendingActionError(err)) {
|
|
1805
|
+
log('cluster-member:consensus-commit-diverged', {
|
|
1806
|
+
messageHash,
|
|
1807
|
+
actionId: commit.actionId,
|
|
1808
|
+
divergence: 'behind',
|
|
1809
|
+
reason: (err as Error).message
|
|
1810
|
+
});
|
|
1811
|
+
// We hold no revision of these blocks; pull the committed revision from a
|
|
1812
|
+
// cohort peer so the block is no longer under-replicated. Best-effort:
|
|
1813
|
+
// failures are logged inside, never thrown (a throw would reset the stream).
|
|
1814
|
+
await this.reconcileDivergentCommit(record, commit);
|
|
1815
|
+
return;
|
|
1816
|
+
}
|
|
1817
|
+
throw err;
|
|
1818
|
+
}
|
|
1819
|
+
// Retain the verdict either way (see getExecutedCommitResult): a success confirms local
|
|
1820
|
+
// durability, and an ahead-shaped refusal is the only evidence the coordinator has that
|
|
1821
|
+
// the tolerance below swallowed a rival's win at the requested revision. The
|
|
1822
|
+
// missing-pending throw path above retains nothing — no CommitResult exists there, and
|
|
1823
|
+
// the coordinator's fabricated-success fallback plus cohort reconcile is the right shape
|
|
1824
|
+
// for a member that is genuinely behind.
|
|
1825
|
+
this.executedCommitResults.set(messageHash, result);
|
|
1826
|
+
if (!result.success) {
|
|
1827
|
+
// success:false is a StaleFailure. `missing` ⇒ ahead/stale divergence
|
|
1828
|
+
// (we already hold ≥ this rev): tolerate, do NOT reconcile downward. A
|
|
1829
|
+
// missing-base reason ⇒ behind divergence, reconcile (below). Any other bare
|
|
1830
|
+
// `reason` with no `missing` ⇒ a genuine internalCommit fault: propagate so
|
|
1831
|
+
// handleConsensus rolls back the executed marker and rethrows.
|
|
1832
|
+
//
|
|
1833
|
+
// NOTE: this 'ahead' tolerance is what turns a rival's commit that somehow reaches
|
|
1834
|
+
// consensus into a reported success no member durably stored (consensus without
|
|
1835
|
+
// durability — the commit-tier acknowledgement hole). It must stay: a member
|
|
1836
|
+
// genuinely ahead of a redelivered/lagging commit is the common, correct case. The
|
|
1837
|
+
// guards live UPSTREAM: `validateCommitRevisions` rejects the rival at the promise
|
|
1838
|
+
// round, `CoordinatorRepo.commit` returns lost races as retryable conflicts instead
|
|
1839
|
+
// of re-driving them, and the verdict retained just above (getExecutedCommitResult)
|
|
1840
|
+
// lets the coordinating node convert its OWN member's rival-confirmed refusal into a
|
|
1841
|
+
// conflict answer — that last guard is what closes the signed-but-not-yet-applied
|
|
1842
|
+
// window, where two commits for one revision both assemble consensus because signing
|
|
1843
|
+
// drops each member's reservation before applying advances its storage. If
|
|
1844
|
+
// consensus-without-durability is ever observed again, look at those guards' abstain
|
|
1845
|
+
// residuals (non-coordinating members' verdicts are not threaded back; capability-less
|
|
1846
|
+
// or history-truncated storage abstains), not at this branch.
|
|
1847
|
+
if (result.missing?.length) {
|
|
1848
|
+
log('cluster-member:consensus-commit-diverged', {
|
|
1849
|
+
messageHash,
|
|
1850
|
+
actionId: commit.actionId,
|
|
1851
|
+
divergence: 'ahead',
|
|
1852
|
+
reason: result.reason,
|
|
1853
|
+
hasMissing: true
|
|
1854
|
+
});
|
|
1855
|
+
return;
|
|
1856
|
+
}
|
|
1857
|
+
// This member holds no materializable base for one of the blocks, so
|
|
1858
|
+
// `StorageRepo.commit` REFUSED rather than record a revision it could never serve.
|
|
1859
|
+
// Same "behind" divergence as a missing pend — and the same cure: pull the committed
|
|
1860
|
+
// revision from a cohort peer. Reconciling here (after commit released its per-block
|
|
1861
|
+
// latches) is what makes refusing safe; fetching inside the commit path would deadlock
|
|
1862
|
+
// against the latch `saveReplicatedBlock` needs to persist what it fetched.
|
|
1863
|
+
if (isMissingBaseRevisionFailure(result)) {
|
|
1864
|
+
log('cluster-member:consensus-commit-diverged', {
|
|
1865
|
+
messageHash,
|
|
1866
|
+
actionId: commit.actionId,
|
|
1867
|
+
divergence: 'behind',
|
|
1868
|
+
reason: result.reason
|
|
1869
|
+
});
|
|
1870
|
+
await this.reconcileDivergentCommit(record, commit);
|
|
1871
|
+
return;
|
|
1872
|
+
}
|
|
1873
|
+
throw new Error(`Consensus commit for action ${commit.actionId} failed: ${result.reason ?? 'unknown reason'}`);
|
|
1874
|
+
}
|
|
1875
|
+
return;
|
|
1876
|
+
}
|
|
1877
|
+
if ('invalidate' in operation) {
|
|
1878
|
+
await this.applyConsensusInvalidation(record, operation.invalidate);
|
|
1879
|
+
return;
|
|
1880
|
+
}
|
|
1881
|
+
}
|
|
1882
|
+
|
|
1883
|
+
/**
|
|
1884
|
+
* Applies a consensus-ordered invalidation on this member: dedup → certificate verification →
|
|
1885
|
+
* capture the invalidation's commit cert (for reactivity reuse) → delegate the compensating write +
|
|
1886
|
+
* durable log append to the injected {@link InvalidationApplySink}.
|
|
1887
|
+
*
|
|
1888
|
+
* Like every other branch of {@link applyConsensusOperation}, a failure here is tolerated rather
|
|
1889
|
+
* than thrown — a throw would reset the cluster stream. A forged/sub-threshold certificate is
|
|
1890
|
+
* rejected (and never reaches the sink); a sink fault is logged and its dedup marker rolled back
|
|
1891
|
+
* so a re-broadcast can retry. The durable, authoritative dedup is the invalidation log entry the
|
|
1892
|
+
* sink consults; the in-memory map is only a fast path.
|
|
1893
|
+
*/
|
|
1894
|
+
private async applyConsensusInvalidation(record: ClusterRecord, request: InvalidateRequest): Promise<void> {
|
|
1895
|
+
const messageHash = record.messageHash;
|
|
1896
|
+
const dedupKey = `${request.invalidatedActionId}:${request.resolution.disputeId}`;
|
|
1897
|
+
if (this.appliedInvalidations.has(dedupKey)) {
|
|
1898
|
+
log('cluster-member:consensus-invalidate-duplicate', { messageHash, dedupKey });
|
|
1899
|
+
return;
|
|
1900
|
+
}
|
|
1901
|
+
|
|
1902
|
+
// Certificate verification BEFORE apply — never trust the originator's say-so. Verify the proof
|
|
1903
|
+
// against THIS request's own target: the votes are bound to the transaction the dispute resolved,
|
|
1904
|
+
// so a genuine proof carried in a request that points at a different (innocent) action/blocks fails
|
|
1905
|
+
// here (the network-facing replay boundary, #2). The proof is also bound to the legitimately-selected
|
|
1906
|
+
// arbitrator set (#1): membership + the challenger's set signature gate it (layer 1), and when a
|
|
1907
|
+
// recompute capability is wired we re-derive the eligible set from our topology and reject a forged
|
|
1908
|
+
// one (layer 2). When we cannot reconstruct the historical topology, we accept on layer 1 and LOG
|
|
1909
|
+
// that the invalidation was applied without full anchoring — the documented interim posture.
|
|
1910
|
+
const certified = await verifyInvalidationCertificate(
|
|
1911
|
+
request.resolution,
|
|
1912
|
+
{ invalidatedActionId: request.invalidatedActionId, blockIds: request.blockIds },
|
|
1913
|
+
{
|
|
1914
|
+
recomputeArbitratorSet: this.recomputeArbitratorSet,
|
|
1915
|
+
onUnanchored: (info) => log('cluster-member:consensus-invalidate-unanchored', {
|
|
1916
|
+
messageHash,
|
|
1917
|
+
invalidatedActionId: request.invalidatedActionId,
|
|
1918
|
+
disputeId: info.disputeId,
|
|
1919
|
+
reason: info.reason,
|
|
1920
|
+
arbitratorSetSize: info.arbitratorSet.length
|
|
1921
|
+
})
|
|
1922
|
+
}
|
|
1923
|
+
);
|
|
1924
|
+
if (!certified) {
|
|
1925
|
+
log('cluster-member:consensus-invalidate-reject-certificate', {
|
|
1926
|
+
messageHash,
|
|
1927
|
+
invalidatedActionId: request.invalidatedActionId,
|
|
1928
|
+
disputeId: request.resolution.disputeId,
|
|
1929
|
+
outcome: request.resolution.outcome
|
|
1930
|
+
});
|
|
1931
|
+
return;
|
|
1932
|
+
}
|
|
1933
|
+
|
|
1934
|
+
// Capture the invalidation's own commit cert — the threshold signature the cohort produced for
|
|
1935
|
+
// THIS consensus-ordered `invalidate` op (over `computeCommitHash(record)`), the reversal analogue
|
|
1936
|
+
// of the commit-cert capture in {@link applyConsensusOperation}. Reactivity reuses it bit-for-bit
|
|
1937
|
+
// as the invalidation notification's `sig` (never re-signed). Keyed on the deterministic
|
|
1938
|
+
// {@link invalidationActionId} the invalidation's change event also carries, so the bridge's
|
|
1939
|
+
// cert extractor resolves it. Gated on the sink — a node with no reactivity wired pays nothing.
|
|
1940
|
+
if (this.onCommitCertificate) {
|
|
1941
|
+
const invSignedPayload = clusterVoteSigningPayload(await this.computeCommitHash(record), 'approve');
|
|
1942
|
+
this.captureCommitCert(record, invalidationActionId(request.invalidatedActionId, request.resolution.disputeId), invSignedPayload);
|
|
1943
|
+
}
|
|
1944
|
+
|
|
1945
|
+
if (!this.onInvalidate) {
|
|
1946
|
+
log('cluster-member:consensus-invalidate-no-sink', { messageHash, dedupKey });
|
|
1947
|
+
return;
|
|
1948
|
+
}
|
|
1949
|
+
|
|
1950
|
+
this.appliedInvalidations.set(dedupKey, Date.now());
|
|
1951
|
+
try {
|
|
1952
|
+
await this.onInvalidate(request);
|
|
1953
|
+
log('cluster-member:consensus-invalidate-applied', {
|
|
1954
|
+
messageHash,
|
|
1955
|
+
invalidatedActionId: request.invalidatedActionId,
|
|
1956
|
+
disputeId: request.resolution.disputeId,
|
|
1957
|
+
blockCount: request.blockIds.length
|
|
1958
|
+
});
|
|
1959
|
+
} catch (err) {
|
|
1960
|
+
// Tolerate (don't reset the stream); roll back the marker so a re-broadcast retries.
|
|
1961
|
+
this.appliedInvalidations.delete(dedupKey);
|
|
1962
|
+
log('cluster-member:consensus-invalidate-sink-error', { messageHash, dedupKey, error: (err as Error).message });
|
|
1963
|
+
}
|
|
1964
|
+
}
|
|
1965
|
+
|
|
1966
|
+
/**
|
|
1967
|
+
* Record the consensus commit cert for `actionId` into the injected {@link CommitCertificateSink},
|
|
1968
|
+
* if one is wired. The cert is built from the agreed `record.commits` (the per-member `approve`
|
|
1969
|
+
* commit signatures), forwarded UNCHANGED for reactivity to reuse. `signedPayload` is the exact
|
|
1970
|
+
* commit-vote preimage those signatures were produced over (`utf8(commitHash + ":approve")`),
|
|
1971
|
+
* computed by the caller from the same `record` *before* the synchronous commit — reactivity sets a
|
|
1972
|
+
* notification's `digest` from it so a real threshold-verify over `digest` succeeds. Stays
|
|
1973
|
+
* **synchronous** (no `await` here) so the cert is retained before `StorageRepo.commit` emits its
|
|
1974
|
+
* change event. No-op (zero cost) when no sink is configured; a throwing sink is isolated + logged so
|
|
1975
|
+
* it can never break consensus.
|
|
1976
|
+
*/
|
|
1977
|
+
private captureCommitCert(record: ClusterRecord, actionId: ActionId, signedPayload: Uint8Array): void {
|
|
1978
|
+
if (!this.onCommitCertificate) {
|
|
1979
|
+
return;
|
|
1980
|
+
}
|
|
1981
|
+
const minSigs = Math.ceil(Object.keys(record.peers).length * this.superMajorityThreshold);
|
|
1982
|
+
try {
|
|
1983
|
+
this.onCommitCertificate(actionId, buildCommitCert(record, minSigs, signedPayload));
|
|
1984
|
+
} catch (err) {
|
|
1985
|
+
log('cluster-member:commit-cert-sink-error', { actionId, error: (err as Error).message });
|
|
1986
|
+
}
|
|
1987
|
+
}
|
|
1988
|
+
|
|
1989
|
+
/**
|
|
1990
|
+
* After tolerating a "behind" commit divergence, pull the committed revision of
|
|
1991
|
+
* each block from a cohort peer that holds it and restore it locally. Best-effort:
|
|
1992
|
+
* a missing callback, an empty cohort, or a per-block failure/timeout is logged and
|
|
1993
|
+
* tolerated — never thrown, since a throw out of consensus execution resets the
|
|
1994
|
+
* cluster stream.
|
|
1995
|
+
*/
|
|
1996
|
+
private async reconcileDivergentCommit(record: ClusterRecord, commit: CommitRequest): Promise<void> {
|
|
1997
|
+
if (!this.reconcileBlock) {
|
|
1998
|
+
log('cluster-member:consensus-commit-reconcile-skip', { messageHash: record.messageHash, reason: 'no-callback' });
|
|
1999
|
+
return;
|
|
2000
|
+
}
|
|
2001
|
+
const cohortPeerIds = Object.keys(record.peers).filter(id => id !== this.peerId.toString());
|
|
2002
|
+
if (cohortPeerIds.length === 0) {
|
|
2003
|
+
log('cluster-member:consensus-commit-reconcile-skip', { messageHash: record.messageHash, reason: 'no-cohort-peers' });
|
|
2004
|
+
return;
|
|
2005
|
+
}
|
|
2006
|
+
const committed: ActionRev = { actionId: commit.actionId, rev: commit.rev };
|
|
2007
|
+
await Promise.all(
|
|
2008
|
+
commit.blockIds.map(blockId => this.reconcileOneBlock(record.messageHash, blockId, committed, cohortPeerIds))
|
|
2009
|
+
);
|
|
2010
|
+
}
|
|
2011
|
+
|
|
2012
|
+
/** Reconcile a single block, bounding the awaited callback and swallowing failures. */
|
|
2013
|
+
private async reconcileOneBlock(messageHash: string, blockId: BlockId, committed: ActionRev, cohortPeerIds: string[]): Promise<void> {
|
|
2014
|
+
try {
|
|
2015
|
+
await this.withReconcileTimeout(this.reconcileBlock!(blockId, committed, cohortPeerIds), blockId);
|
|
2016
|
+
// "attempted", not "reconciled": the callback returns void, and a quorum decline is a
|
|
2017
|
+
// normal, non-throwing outcome — so reaching here means the pass ran to completion, NOT
|
|
2018
|
+
// that anything was restored. `reconcile:restored` (reconcile-block.ts) is the line that
|
|
2019
|
+
// says the bytes actually landed; `reconcile:no-rev-quorum` / `reconcile:no-content-quorum`
|
|
2020
|
+
// say they did not.
|
|
2021
|
+
log('cluster-member:consensus-commit-reconcile-attempted', { messageHash, blockId, rev: committed.rev });
|
|
2022
|
+
} catch (err) {
|
|
2023
|
+
log('cluster-member:consensus-commit-reconcile-failed', {
|
|
2024
|
+
messageHash,
|
|
2025
|
+
blockId,
|
|
2026
|
+
rev: committed.rev,
|
|
2027
|
+
error: (err as Error).message
|
|
2028
|
+
});
|
|
2029
|
+
}
|
|
2030
|
+
}
|
|
2031
|
+
|
|
2032
|
+
/** Bound an awaited reconcile so a slow/unreachable cohort peer can't stall consensus. */
|
|
2033
|
+
private withReconcileTimeout<T>(promise: Promise<T>, blockId: BlockId): Promise<T> {
|
|
2034
|
+
let timer: NodeJS.Timeout | undefined;
|
|
2035
|
+
const timeout = new Promise<never>((_, reject) => {
|
|
2036
|
+
timer = setTimeout(
|
|
2037
|
+
() => reject(new Error(`reconcile for block ${blockId} timed out after ${ReconcileTimeoutMs}ms`)),
|
|
2038
|
+
ReconcileTimeoutMs
|
|
2039
|
+
);
|
|
2040
|
+
timer.unref();
|
|
2041
|
+
});
|
|
2042
|
+
return Promise.race([promise, timeout]).finally(() => {
|
|
2043
|
+
if (timer) clearTimeout(timer);
|
|
2044
|
+
});
|
|
2045
|
+
}
|
|
2046
|
+
|
|
2047
|
+
private async handleRejection(_record: ClusterRecord): Promise<void> {
|
|
2048
|
+
// Clean up any resources - will be cleared by shouldPersist = false in the main flow
|
|
2049
|
+
}
|
|
2050
|
+
|
|
2051
|
+
private setupTimeouts(record: ClusterRecord): { promiseTimeout?: NodeJS.Timeout; resolutionTimeout?: NodeJS.Timeout } {
|
|
2052
|
+
if (!record.message.expiration) {
|
|
2053
|
+
return {};
|
|
2054
|
+
}
|
|
2055
|
+
|
|
2056
|
+
return {
|
|
2057
|
+
promiseTimeout: setTimeout(
|
|
2058
|
+
() => this.handleExpiration(record.messageHash),
|
|
2059
|
+
record.message.expiration - Date.now()
|
|
2060
|
+
).unref(),
|
|
2061
|
+
resolutionTimeout: setTimeout(
|
|
2062
|
+
() => this.resolveWithPeers(record.messageHash),
|
|
2063
|
+
record.message.expiration + 5000 - Date.now()
|
|
2064
|
+
).unref()
|
|
2065
|
+
};
|
|
2066
|
+
}
|
|
2067
|
+
|
|
2068
|
+
/**
|
|
2069
|
+
* Scan this member's reservation table (`activeTransactions`) for a held transaction that
|
|
2070
|
+
* conflicts with `record` AND wins the deterministic race against it. Returns the winner's
|
|
2071
|
+
* identity — `{ blockedBy: messageHash }` — rather than a bare boolean, because that identity is
|
|
2072
|
+
* exactly what the resulting conflict vote must name (`Signature.conflictWith`); the old boolean
|
|
2073
|
+
* lost it. `undefined` means no blocking conflict. Side-effectful on the way through: stale
|
|
2074
|
+
* entries are swept, and a held transaction that LOSES the race to `record` is cleared.
|
|
2075
|
+
*/
|
|
2076
|
+
private findConflict(record: ClusterRecord): { blockedBy: string } | undefined {
|
|
2077
|
+
const now = Date.now();
|
|
2078
|
+
const staleThresholdMs = 2000; // 2 seconds - allow more time for distributed consensus
|
|
2079
|
+
|
|
2080
|
+
const incomingBlockIds = this.getAffectedBlockIds(record.message.operations);
|
|
2081
|
+
log('cluster-member:findConflict-check', {
|
|
2082
|
+
messageHash: record.messageHash,
|
|
2083
|
+
activeCount: this.activeTransactions.size,
|
|
2084
|
+
incomingBlockIds
|
|
2085
|
+
});
|
|
2086
|
+
|
|
2087
|
+
for (const [existingHash, state] of Array.from(this.activeTransactions.entries())) {
|
|
2088
|
+
if (existingHash === record.messageHash) {
|
|
2089
|
+
continue;
|
|
2090
|
+
}
|
|
2091
|
+
|
|
2092
|
+
const existingBlockIds = this.getAffectedBlockIds(state.record.message.operations);
|
|
2093
|
+
log('cluster-member:findConflict-compare', {
|
|
2094
|
+
existing: existingHash,
|
|
2095
|
+
incoming: record.messageHash,
|
|
2096
|
+
existingBlockIds,
|
|
2097
|
+
incomingBlockIds
|
|
2098
|
+
});
|
|
2099
|
+
|
|
2100
|
+
// Clean up stale transactions that have been around too long
|
|
2101
|
+
if (now - state.lastUpdate > staleThresholdMs) {
|
|
2102
|
+
log('cluster-member:stale-cleanup', {
|
|
2103
|
+
messageHash: existingHash,
|
|
2104
|
+
age: now - state.lastUpdate
|
|
2105
|
+
});
|
|
2106
|
+
this.clearTransaction(existingHash);
|
|
2107
|
+
continue;
|
|
2108
|
+
}
|
|
2109
|
+
|
|
2110
|
+
if (this.operationsConflict(state.record.message.operations, record.message.operations)) {
|
|
2111
|
+
// Use race resolution to determine winner
|
|
2112
|
+
const resolution = this.resolveRace(state.record, record);
|
|
2113
|
+
|
|
2114
|
+
if (resolution === 'keep-existing') {
|
|
2115
|
+
log('cluster-member:race-keep-existing', {
|
|
2116
|
+
existing: existingHash,
|
|
2117
|
+
incoming: record.messageHash
|
|
2118
|
+
});
|
|
2119
|
+
return { blockedBy: existingHash }; // Reject incoming, naming the winner
|
|
2120
|
+
} else {
|
|
2121
|
+
// Accept incoming, abort existing
|
|
2122
|
+
log('cluster-member:race-accept-incoming', {
|
|
2123
|
+
existing: existingHash,
|
|
2124
|
+
incoming: record.messageHash
|
|
2125
|
+
});
|
|
2126
|
+
this.clearTransaction(existingHash);
|
|
2127
|
+
continue; // Check other conflicts
|
|
2128
|
+
}
|
|
2129
|
+
}
|
|
2130
|
+
}
|
|
2131
|
+
|
|
2132
|
+
return undefined; // No blocking conflicts
|
|
2133
|
+
}
|
|
2134
|
+
|
|
2135
|
+
/** Number of *approve* promise votes on a record — the count the commit rule uses. */
|
|
2136
|
+
private static approvalCount(record: ClusterRecord): number {
|
|
2137
|
+
return Object.values(record.promises).filter(s => s.type === 'approve').length;
|
|
2138
|
+
}
|
|
2139
|
+
|
|
2140
|
+
/**
|
|
2141
|
+
* Resolve a race between two conflicting transactions. Total and deterministic, so every honest
|
|
2142
|
+
* member computes the identical winner (the Theorem 1 Case-2 premise). Order:
|
|
2143
|
+
* 1. more *approve* promise signatures wins (progress monotonicity — see safety note below);
|
|
2144
|
+
* 2. equal approval counts → higher aged priority wins (fairness — see {@link recordPriority});
|
|
2145
|
+
* 3. still tied → higher message hash wins.
|
|
2146
|
+
*
|
|
2147
|
+
* The count is APPROVALS, not `promises` keys. `promises` is the vote map — a reject occupies a key
|
|
2148
|
+
* there exactly as an approve does — so counting keys would treat a rejection as progress, letting a
|
|
2149
|
+
* record that can never commit outrank (and therefore block, via {@link findConflict}) a fresh rival
|
|
2150
|
+
* for the whole staleness window. Approvals is also the count the invariant below actually needs:
|
|
2151
|
+
* the commit rule is `approvedPromises >= superMajority`, which never looks at rejections.
|
|
2152
|
+
*
|
|
2153
|
+
* Approval count is FIRST so this comparison never displaces a transaction that is further along.
|
|
2154
|
+
* That restores the pre-priority safety invariant: a member commits purely on promise supermajority
|
|
2155
|
+
* (`handleCommitNeeded` signs whenever `approvedPromises >= superMajority`; the commit path has NO
|
|
2156
|
+
* conflict re-check), so `resolveRace` is the ONLY arbiter among concurrently-pending conflicts.
|
|
2157
|
+
* With approvals-first, once transaction X holds a promise supermajority every conflicting rival Y has
|
|
2158
|
+
* strictly fewer approvals — Y can only match X's count by getting the intersecting quorum member to
|
|
2159
|
+
* approve it, but that member already holds X at supermajority and `resolveRace(X, Y)` returns
|
|
2160
|
+
* `keep-existing` on X's higher count, so it never does. By quorum intersection any Y-supermajority
|
|
2161
|
+
* overlaps X's in ≥1 honest member, and that member rejects Y. One winner (docs/correctness.md
|
|
2162
|
+
* Theorem 9). Priority-first would break this: it could displace an already-quorum-reached X for a
|
|
2163
|
+
* higher-priority Y with fewer approvals, letting BOTH commit (split brain) — the regression fixed by
|
|
2164
|
+
* ticket occ-priority-first-breaks-promise-monotonicity.
|
|
2165
|
+
*
|
|
2166
|
+
* Priority is now a tie-break that runs only at EQUAL approval counts, which is exactly the
|
|
2167
|
+
* concurrent-starvation case aging targets (two fresh rivals, 0 promises each, otherwise coin-flipping
|
|
2168
|
+
* on the hash). Priority still breaks those ties deterministically, so aging still solves the stated
|
|
2169
|
+
* fairness problem in its common case. It only orders two *concurrently-pending* conflicts; it does NOT
|
|
2170
|
+
* defer a fresh pend for an absent aged transaction (that residual — sequential sub-window starvation —
|
|
2171
|
+
* is the deferred feat-occ-priority-reservation).
|
|
2172
|
+
*
|
|
2173
|
+
* NOTE: residual-fairness tripwire. Under approvals-first an aged transaction can still lose to a fresh
|
|
2174
|
+
* rival that has *legitimately* gathered even one more approval — that is not the pure-coin-flip
|
|
2175
|
+
* starvation aging targets (equal counts, priority wins), it is the monotonicity behaviour we WANT (a
|
|
2176
|
+
* more-progressed rival is never displaced). If deeper fairness against a genuinely-more-progressed
|
|
2177
|
+
* rival is ever needed, it belongs to feat-occ-priority-reservation (reserve/defer at pend time), NOT
|
|
2178
|
+
* to this race tie-break.
|
|
2179
|
+
*
|
|
2180
|
+
* NOTE: Byzantine self-assert is a fairness DoS, not a safety hole. A coordinator can stamp
|
|
2181
|
+
* priority == MaxPriority on every transaction; recordPriority clamps to the cap so it cannot
|
|
2182
|
+
* exceed it, and priority never influences validity/operationsHash/stale-read checks — and now sits
|
|
2183
|
+
* below the approval count, so it can only break equal-count ties it might have ~50% won anyway,
|
|
2184
|
+
* degrading to at-worst-status-quo fairness (the same graceful-degradation class as spam under
|
|
2185
|
+
* honest-majority). Binding priority to provable age is out of scope (feat-occ-priority-reservation).
|
|
2186
|
+
*
|
|
2187
|
+
* NOTE: keep priority a self-contained additive message field + this one comparison key so it
|
|
2188
|
+
* composes with — does not block — a future HLC/crdt-sync redesign of this same path
|
|
2189
|
+
* (design-hot-log-tail-sharding-guidance).
|
|
2190
|
+
*/
|
|
2191
|
+
private resolveRace(existing: ClusterRecord, incoming: ClusterRecord): 'keep-existing' | 'accept-incoming' {
|
|
2192
|
+
// 1. Transaction with more APPROVALS wins — never displace a more-progressed rival (safety, see
|
|
2193
|
+
// above). Counting `promises` keys instead would count reject votes as progress: a record holding
|
|
2194
|
+
// one rejection would outrank an untouched rival and reserve its blocks for the whole staleness
|
|
2195
|
+
// window, and the commit rule this ordering protects (`approvedPromises >= superMajority`) never
|
|
2196
|
+
// looks at rejections anyway.
|
|
2197
|
+
const existingCount = ClusterMember.approvalCount(existing);
|
|
2198
|
+
const incomingCount = ClusterMember.approvalCount(incoming);
|
|
2199
|
+
if (existingCount !== incomingCount) {
|
|
2200
|
+
return existingCount > incomingCount ? 'keep-existing' : 'accept-incoming';
|
|
2201
|
+
}
|
|
2202
|
+
|
|
2203
|
+
// 2. Equal approval counts → higher aged priority wins (fairness tie-break).
|
|
2204
|
+
const existingPriority = this.recordPriority(existing);
|
|
2205
|
+
const incomingPriority = this.recordPriority(incoming);
|
|
2206
|
+
if (existingPriority !== incomingPriority) {
|
|
2207
|
+
return existingPriority > incomingPriority ? 'keep-existing' : 'accept-incoming';
|
|
2208
|
+
}
|
|
2209
|
+
|
|
2210
|
+
// 3. Tie-breaker: higher message hash wins (deterministic).
|
|
2211
|
+
return existing.messageHash > incoming.messageHash ? 'keep-existing' : 'accept-incoming';
|
|
2212
|
+
}
|
|
2213
|
+
|
|
2214
|
+
/**
|
|
2215
|
+
* Aged advisory priority carried by a record's pend operation, clamped to [0, MaxPriority].
|
|
2216
|
+
* The multi-collection path carries it on `pend.validation.transaction.priority`; the single-collection
|
|
2217
|
+
* (`Collection.sync`) path carries it as top-level `pend.priority`; a record with neither — a
|
|
2218
|
+
* legacy/unversioned coordinator's transaction, or a non-pend operation — is priority 0
|
|
2219
|
+
* (backward compatible: such transactions simply never age). Both carriers live inside the signed
|
|
2220
|
+
* `message`, so priority is integrity-protected in transit; clamping here bounds a self-asserted
|
|
2221
|
+
* out-of-range value to the cap.
|
|
2222
|
+
*
|
|
2223
|
+
* NOTE: `message` is fixed for a transaction's whole lifecycle (promises/commits accrue in the
|
|
2224
|
+
* separate `promises`/`commits` maps, never in `message`), so a transaction keeps its rank through
|
|
2225
|
+
* the commit phase — there is no "priority drops to 0 at commit" asymmetry. resolveRace is only
|
|
2226
|
+
* consulted at the promise decision (findConflict), i.e. between two still-open conflicting
|
|
2227
|
+
* transactions, which is exactly the concurrent-contention case priority is meant to order.
|
|
2228
|
+
*/
|
|
2229
|
+
private recordPriority(record: ClusterRecord): number {
|
|
2230
|
+
for (const op of record.message.operations) {
|
|
2231
|
+
if ('pend' in op) {
|
|
2232
|
+
// Every hop optional: `validation` arrives off the wire inside a signed message whose
|
|
2233
|
+
// hash binds its bytes, not its shape, so a malformed pair must yield priority 0 (what
|
|
2234
|
+
// clampPriority already does for a missing or Byzantine number) rather than throw out
|
|
2235
|
+
// of the vote path — the lost vote this fail-closed pass exists to prevent.
|
|
2236
|
+
return clampPriority(op.pend.validation?.transaction?.priority ?? op.pend.priority);
|
|
2237
|
+
}
|
|
2238
|
+
}
|
|
2239
|
+
return 0;
|
|
2240
|
+
}
|
|
2241
|
+
|
|
2242
|
+
private operationsConflict(ops1: RepoMessage['operations'], ops2: RepoMessage['operations']): boolean {
|
|
2243
|
+
// Check if one is a commit for the same action as a pend - these don't conflict
|
|
2244
|
+
const actionId1 = this.getActionId(ops1);
|
|
2245
|
+
const actionId2 = this.getActionId(ops2);
|
|
2246
|
+
if (actionId1 && actionId2 && actionId1 === actionId2) {
|
|
2247
|
+
// Same action - commit is resolving the pend, not conflicting
|
|
2248
|
+
return false;
|
|
2249
|
+
}
|
|
2250
|
+
|
|
2251
|
+
const blocks1 = new Set(this.getAffectedBlockIds(ops1));
|
|
2252
|
+
const blocks2 = new Set(this.getAffectedBlockIds(ops2));
|
|
2253
|
+
|
|
2254
|
+
for (const block of Array.from(blocks1)) {
|
|
2255
|
+
if (blocks2.has(block)) {
|
|
2256
|
+
log('cluster-member:conflict-detected', {
|
|
2257
|
+
blocks1: Array.from(blocks1),
|
|
2258
|
+
blocks2: Array.from(blocks2),
|
|
2259
|
+
conflictingBlock: block
|
|
2260
|
+
});
|
|
2261
|
+
return true;
|
|
2262
|
+
}
|
|
2263
|
+
}
|
|
2264
|
+
|
|
2265
|
+
return false;
|
|
2266
|
+
}
|
|
2267
|
+
|
|
2268
|
+
private getActionId(operations: RepoMessage['operations']): string | undefined {
|
|
2269
|
+
for (const operation of operations) {
|
|
2270
|
+
if ('pend' in operation) {
|
|
2271
|
+
return operation.pend.actionId;
|
|
2272
|
+
} else if ('commit' in operation) {
|
|
2273
|
+
return operation.commit.actionId;
|
|
2274
|
+
} else if ('cancel' in operation) {
|
|
2275
|
+
return operation.cancel.actionRef.actionId;
|
|
2276
|
+
}
|
|
2277
|
+
}
|
|
2278
|
+
return undefined;
|
|
2279
|
+
}
|
|
2280
|
+
|
|
2281
|
+
/**
|
|
2282
|
+
* Every block id the message's own operations name. Two consumers, deliberately sharing one
|
|
2283
|
+
* definition: conflict detection (which writes must serialize against each other) and the membership
|
|
2284
|
+
* admission gate's binding check (the set a legitimate `coordinatingBlockIds[0]` must come from —
|
|
2285
|
+
* {@link ClusterMember.deriveExpectedClusterView}). If the two ever disagreed, a coordinator could
|
|
2286
|
+
* name a block the record is not judged to touch.
|
|
2287
|
+
*/
|
|
2288
|
+
private getAffectedBlockIds(operations: RepoMessage['operations']): string[] {
|
|
2289
|
+
const blockIds = new Set<string>();
|
|
2290
|
+
|
|
2291
|
+
for (const operation of operations) {
|
|
2292
|
+
if ('get' in operation) {
|
|
2293
|
+
operation.get.blockIds.forEach(id => blockIds.add(id));
|
|
2294
|
+
} else if ('pend' in operation) {
|
|
2295
|
+
// Use blockIdsForTransforms to correctly extract block IDs from Transforms structure
|
|
2296
|
+
blockIdsForTransforms(operation.pend.transforms).forEach(id => blockIds.add(id));
|
|
2297
|
+
} else if ('commit' in operation) {
|
|
2298
|
+
operation.commit.blockIds.forEach(id => blockIds.add(id));
|
|
2299
|
+
} else if ('cancel' in operation) {
|
|
2300
|
+
operation.cancel.actionRef.blockIds.forEach(id => blockIds.add(id));
|
|
2301
|
+
} else if ('invalidate' in operation) {
|
|
2302
|
+
// The invalidation writes compensating revisions to these blocks; surfacing them lets
|
|
2303
|
+
// conflict detection serialize a concurrent commit racing the invalidation on a block.
|
|
2304
|
+
operation.invalidate.blockIds.forEach(id => blockIds.add(id));
|
|
2305
|
+
}
|
|
2306
|
+
}
|
|
2307
|
+
|
|
2308
|
+
return Array.from(blockIds);
|
|
2309
|
+
}
|
|
2310
|
+
|
|
2311
|
+
private async propagateIfNeeded(record: ClusterRecord): Promise<void> {
|
|
2312
|
+
const promises = [];
|
|
2313
|
+
for (const peerId of Object.keys(record.peers)) {
|
|
2314
|
+
if (peerId === this.peerId.toString()) continue;
|
|
2315
|
+
|
|
2316
|
+
try {
|
|
2317
|
+
const client = ClusterClient.create(peerIdFromString(peerId), this.peerNetwork, this.protocolPrefix);
|
|
2318
|
+
promises.push(client.update(record));
|
|
2319
|
+
} catch (error) {
|
|
2320
|
+
log('ERROR: Failed to propagate to peer %s: %o', peerId, error);
|
|
2321
|
+
}
|
|
2322
|
+
}
|
|
2323
|
+
await Promise.allSettled(promises);
|
|
2324
|
+
}
|
|
2325
|
+
|
|
2326
|
+
private async handleExpiration(messageHash: string): Promise<void> {
|
|
2327
|
+
const state = this.activeTransactions.get(messageHash);
|
|
2328
|
+
if (!state) return;
|
|
2329
|
+
|
|
2330
|
+
if (!state.record.promises[this.peerId.toString()]) {
|
|
2331
|
+
const rejectReason = 'Transaction expired';
|
|
2332
|
+
const promiseHash = await this.computePromiseHash(state.record);
|
|
2333
|
+
const sig = await this.signVote(promiseHash, 'reject', rejectReason);
|
|
2334
|
+
const signature: Signature = {
|
|
2335
|
+
type: 'reject',
|
|
2336
|
+
signature: sig,
|
|
2337
|
+
rejectReason
|
|
2338
|
+
};
|
|
2339
|
+
|
|
2340
|
+
const updatedRecord = {
|
|
2341
|
+
...state.record,
|
|
2342
|
+
promises: {
|
|
2343
|
+
...state.record.promises,
|
|
2344
|
+
[this.peerId.toString()]: signature
|
|
2345
|
+
}
|
|
2346
|
+
};
|
|
2347
|
+
|
|
2348
|
+
this.activeTransactions.set(messageHash, {
|
|
2349
|
+
...state,
|
|
2350
|
+
record: updatedRecord
|
|
2351
|
+
});
|
|
2352
|
+
|
|
2353
|
+
await this.propagateIfNeeded(updatedRecord);
|
|
2354
|
+
}
|
|
2355
|
+
}
|
|
2356
|
+
|
|
2357
|
+
private async resolveWithPeers(messageHash: string): Promise<void> {
|
|
2358
|
+
// This method is disabled - the coordinator handles all retry logic
|
|
2359
|
+
// Keeping the skeleton in case we need peer-initiated recovery in the future
|
|
2360
|
+
log('cluster-member:resolve-skipped', { messageHash, reason: 'coordinator-handles-retry' });
|
|
2361
|
+
}
|
|
2362
|
+
|
|
2363
|
+
private queueExpiredTransactions(): void {
|
|
2364
|
+
const now = Date.now();
|
|
2365
|
+
for (const [messageHash, state] of Array.from(this.activeTransactions.entries())) {
|
|
2366
|
+
if (state.record.message.expiration && state.record.message.expiration < now) {
|
|
2367
|
+
this.cleanupQueue.push(messageHash);
|
|
2368
|
+
}
|
|
2369
|
+
}
|
|
2370
|
+
// Also clean up old executed transaction records
|
|
2371
|
+
const expirationThreshold = now - ExecutedTransactionTtlMs;
|
|
2372
|
+
for (const [messageHash, executedAt] of Array.from(this.executedTransactions.entries())) {
|
|
2373
|
+
if (executedAt < expirationThreshold) {
|
|
2374
|
+
this.executedTransactions.delete(messageHash);
|
|
2375
|
+
this.executedPendResults.delete(messageHash);
|
|
2376
|
+
this.executedCommitResults.delete(messageHash);
|
|
2377
|
+
}
|
|
2378
|
+
}
|
|
2379
|
+
// Prune old applied-invalidation dedup markers on the same TTL.
|
|
2380
|
+
for (const [dedupKey, appliedAt] of Array.from(this.appliedInvalidations.entries())) {
|
|
2381
|
+
if (appliedAt < expirationThreshold) {
|
|
2382
|
+
this.appliedInvalidations.delete(dedupKey);
|
|
2383
|
+
}
|
|
2384
|
+
}
|
|
2385
|
+
this.stateStore?.pruneExecuted(expirationThreshold)
|
|
2386
|
+
.catch(err => log('cluster-member:prune-executed-error', { error: (err as Error).message }));
|
|
2387
|
+
}
|
|
2388
|
+
|
|
2389
|
+
private async processCleanupQueue(): Promise<void> {
|
|
2390
|
+
while (this.cleanupQueue.length > 0) {
|
|
2391
|
+
const messageHash = this.cleanupQueue.shift();
|
|
2392
|
+
if (!messageHash) continue;
|
|
2393
|
+
|
|
2394
|
+
const state = this.activeTransactions.get(messageHash);
|
|
2395
|
+
if (!state) continue;
|
|
2396
|
+
|
|
2397
|
+
// NOTE: an expired entry already in a terminal phase is deliberately left alone here —
|
|
2398
|
+
// `processUpdate` clears those on the update that made them terminal, so reaching this point
|
|
2399
|
+
// in one means that update never arrived. It is not stranded: `findConflict`'s 2 s staleness
|
|
2400
|
+
// sweep drops it on the next conflicting arrival. But on a member that then goes idle the
|
|
2401
|
+
// entry lingers until traffic returns. If member memory ever shows entries outliving their
|
|
2402
|
+
// expiration, delete unconditionally here instead of exempting the terminal phases.
|
|
2403
|
+
const { phase } = await this.getTransactionPhase(state.record);
|
|
2404
|
+
if (phase !== TransactionPhase.Consensus && phase !== TransactionPhase.Rejected) {
|
|
2405
|
+
this.activeTransactions.delete(messageHash);
|
|
2406
|
+
}
|
|
2407
|
+
}
|
|
2408
|
+
}
|
|
2409
|
+
|
|
2410
|
+
private hasLocalCommit(record: ClusterRecord): boolean {
|
|
2411
|
+
const ourId = this.peerId.toString();
|
|
2412
|
+
return Boolean(record.commits[ourId]);
|
|
2413
|
+
}
|
|
2414
|
+
|
|
2415
|
+
private clearTransaction(messageHash: string): void {
|
|
2416
|
+
const state = this.activeTransactions.get(messageHash);
|
|
2417
|
+
if (!state) {
|
|
2418
|
+
log('cluster-member:clear-miss', { messageHash });
|
|
2419
|
+
return;
|
|
2420
|
+
}
|
|
2421
|
+
if (state.promiseTimeout) {
|
|
2422
|
+
clearTimeout(state.promiseTimeout);
|
|
2423
|
+
}
|
|
2424
|
+
if (state.resolutionTimeout) {
|
|
2425
|
+
clearTimeout(state.resolutionTimeout);
|
|
2426
|
+
}
|
|
2427
|
+
this.activeTransactions.delete(messageHash);
|
|
2428
|
+
this.stateStore?.deleteParticipantState(messageHash)
|
|
2429
|
+
.catch(err => log('cluster-member:persist-delete-error', { messageHash, error: (err as Error).message }));
|
|
2430
|
+
log('cluster-member:clear-done', {
|
|
2431
|
+
messageHash,
|
|
2432
|
+
remaining: Array.from(this.activeTransactions.keys())
|
|
2433
|
+
});
|
|
2434
|
+
}
|
|
2435
|
+
|
|
2436
|
+
/** Fire-and-forget persist — errors are logged, never thrown. */
|
|
2437
|
+
private persistParticipantState(messageHash: string, record: ClusterRecord): void {
|
|
2438
|
+
if (!this.stateStore) return;
|
|
2439
|
+
this.stateStore.saveParticipantState(messageHash, {
|
|
2440
|
+
messageHash,
|
|
2441
|
+
record,
|
|
2442
|
+
lastUpdate: Date.now()
|
|
2443
|
+
}).catch(err => log('cluster-member:persist-error', { messageHash, error: (err as Error).message }));
|
|
2444
|
+
}
|
|
2445
|
+
|
|
2446
|
+
/**
|
|
2447
|
+
* Recover member transactions from persistent store after a restart.
|
|
2448
|
+
* Called during node startup, before accepting new requests.
|
|
2449
|
+
*/
|
|
2450
|
+
async recoverTransactions(): Promise<void> {
|
|
2451
|
+
if (!this.stateStore) return;
|
|
2452
|
+
const now = Date.now();
|
|
2453
|
+
|
|
2454
|
+
// 1. Prune expired executed entries from persistent store
|
|
2455
|
+
await this.stateStore.pruneExecuted(now - ExecutedTransactionTtlMs);
|
|
2456
|
+
// Note: executed transactions are checked via wasTransactionExecutedAsync() at runtime,
|
|
2457
|
+
// which falls back to the persistent store when the in-memory map misses.
|
|
2458
|
+
|
|
2459
|
+
// 2. Restore active participant states
|
|
2460
|
+
const participantStates = await this.stateStore.getAllParticipantStates();
|
|
2461
|
+
for (const state of participantStates) {
|
|
2462
|
+
const { messageHash } = state;
|
|
2463
|
+
// Expired — clean up
|
|
2464
|
+
if (state.record.message.expiration && state.record.message.expiration < now) {
|
|
2465
|
+
log('cluster-member:recovery-expired', { messageHash });
|
|
2466
|
+
await this.stateStore.deleteParticipantState(messageHash);
|
|
2467
|
+
continue;
|
|
2468
|
+
}
|
|
2469
|
+
// Restore into activeTransactions with fresh timeouts
|
|
2470
|
+
log('cluster-member:recovery-restore', { messageHash });
|
|
2471
|
+
const timeouts = this.setupTimeouts(state.record);
|
|
2472
|
+
this.activeTransactions.set(messageHash, {
|
|
2473
|
+
record: state.record,
|
|
2474
|
+
lastUpdate: state.lastUpdate,
|
|
2475
|
+
promiseTimeout: timeouts.promiseTimeout,
|
|
2476
|
+
resolutionTimeout: timeouts.resolutionTimeout
|
|
2477
|
+
});
|
|
2478
|
+
}
|
|
2479
|
+
|
|
2480
|
+
log('cluster-member:recovery-complete', {
|
|
2481
|
+
restoredActive: this.activeTransactions.size,
|
|
2482
|
+
restoredExecuted: this.executedTransactions.size
|
|
2483
|
+
});
|
|
2484
|
+
}
|
|
2485
|
+
|
|
2486
|
+
/**
|
|
2487
|
+
* Checks if a transaction's operations were already executed during consensus.
|
|
2488
|
+
* Falls back to the persistent store when the in-memory map misses.
|
|
2489
|
+
*/
|
|
2490
|
+
async wasTransactionExecutedAsync(messageHash: string): Promise<boolean> {
|
|
2491
|
+
if (this.executedTransactions.has(messageHash)) return true;
|
|
2492
|
+
if (!this.stateStore) return false;
|
|
2493
|
+
const persisted = await this.stateStore.wasExecuted(messageHash);
|
|
2494
|
+
if (persisted) {
|
|
2495
|
+
// Re-populate in-memory map for future synchronous checks
|
|
2496
|
+
this.executedTransactions.set(messageHash, Date.now());
|
|
2497
|
+
}
|
|
2498
|
+
return persisted;
|
|
2499
|
+
}
|
|
2500
|
+
}
|
|
2501
|
+
|