@ocap/resolver 1.31.9 → 1.32.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/esm/_virtual/rolldown_runtime.mjs +7 -0
- package/esm/api.d.mts +33 -5
- package/esm/api.mjs +939 -53
- package/esm/dashboard.d.mts +199 -0
- package/esm/dashboard.mjs +2481 -0
- package/esm/explorer.d.mts +29 -0
- package/esm/explorer.mjs +227 -0
- package/esm/follower.d.mts +294 -0
- package/esm/follower.mjs +2452 -0
- package/esm/hooks.mjs +1 -1
- package/esm/hotstuff-runtime.d.mts +271 -0
- package/esm/hotstuff-runtime.mjs +2668 -0
- package/esm/index.d.mts +79 -4
- package/esm/index.mjs +911 -174
- package/esm/node-preferences.d.mts +203 -0
- package/esm/node-preferences.mjs +1021 -0
- package/esm/replayer.d.mts +114 -0
- package/esm/replayer.mjs +430 -0
- package/esm/token-distribution.mjs +2 -2
- package/esm/token-flow.mjs +1 -1
- package/esm/types.d.mts +25 -2
- package/lib/api.cjs +941 -52
- package/lib/api.d.cts +33 -5
- package/lib/dashboard.cjs +2491 -0
- package/lib/dashboard.d.cts +199 -0
- package/lib/explorer.cjs +227 -0
- package/lib/explorer.d.cts +29 -0
- package/lib/follower.cjs +2458 -0
- package/lib/follower.d.cts +294 -0
- package/lib/hooks.cjs +1 -1
- package/lib/hotstuff-runtime.cjs +2671 -0
- package/lib/hotstuff-runtime.d.cts +271 -0
- package/lib/index.cjs +946 -174
- package/lib/index.d.cts +79 -4
- package/lib/node-preferences.cjs +1043 -0
- package/lib/node-preferences.d.cts +203 -0
- package/lib/replayer.cjs +432 -0
- package/lib/replayer.d.cts +114 -0
- package/lib/token-distribution.cjs +2 -2
- package/lib/token-flow.cjs +1 -1
- package/lib/types.d.cts +25 -2
- package/package.json +20 -14
|
@@ -0,0 +1,2671 @@
|
|
|
1
|
+
const require_rolldown_runtime = require('./_virtual/rolldown_runtime.cjs');
|
|
2
|
+
const require_replayer = require('./replayer.cjs');
|
|
3
|
+
let _arcblock_ledger_core = require("@arcblock/ledger-core");
|
|
4
|
+
let node_crypto = require("node:crypto");
|
|
5
|
+
let _ocap_util_lib_error = require("@ocap/util/lib/error");
|
|
6
|
+
let node_http = require("node:http");
|
|
7
|
+
node_http = require_rolldown_runtime.__toESM(node_http);
|
|
8
|
+
|
|
9
|
+
//#region src/hotstuff-runtime.ts
|
|
10
|
+
/**
|
|
11
|
+
* Split-Plane WireGuard Consensus Listener & HotStuff-Lite 3f+1 BFT HA Runtime (#1712)
|
|
12
|
+
*
|
|
13
|
+
* Implements:
|
|
14
|
+
* 1. Dedicated private consensus `node:http` server bound STRICTLY to `CONSENSUS_BIND_HOST`
|
|
15
|
+
* (loopback or private WireGuard IP — never `0.0.0.0` / `::` / public IP, and never mounted on
|
|
16
|
+
* the public Express RPC server).
|
|
17
|
+
* 2. Mutual Ed25519 request + response authentication (`X-Ocap-Consensus-*` headers) against the
|
|
18
|
+
* `HOTSTUFF_VALIDATORS` roster with `±60s` clock-skew bounds, bounded nonce replay cache,
|
|
19
|
+
* `redirect: 'error'`, and strict pre-`JSON.parse` byte caps (`2 MiB` data / `256 KiB` control).
|
|
20
|
+
* 3. 1-RTT `Instant Settle` HotStuff-Lite execution (`executeWithConsensusLock` + `IConsensusCommitter`)
|
|
21
|
+
* across rotating views (`validators[view % N]`), replica SQLite `SAVEPOINT` dry-run voting
|
|
22
|
+
* (`LedgerEntryReplayer.dryRunVerifyEntries`), Pacemaker heartbeats + `TimeoutCertificate` failover,
|
|
23
|
+
* uncommitted `tc.lockedCheckpoint` recovery, quorum-lease isolation (`1-vs-3` partition), and
|
|
24
|
+
* stale-leader automatic demotion + `LeaderRouter` write forwarding.
|
|
25
|
+
*/
|
|
26
|
+
const MAX_DATA_ROUTE_BYTES = 2 * 1024 * 1024;
|
|
27
|
+
const MAX_CONTROL_ROUTE_BYTES = 256 * 1024;
|
|
28
|
+
const MAX_CLOCK_SKEW_MS = 6e4;
|
|
29
|
+
const REPLAY_TTL_MS = 12e4;
|
|
30
|
+
const MAX_REPLAY_CACHE_ENTRIES = 8192;
|
|
31
|
+
const DEFAULT_PROPOSAL_TIMEOUT_MS = 2500;
|
|
32
|
+
const DEFAULT_HTTP_TIMEOUT_MS = 3e3;
|
|
33
|
+
const DEFAULT_HEARTBEAT_INTERVAL_MS = 400;
|
|
34
|
+
const DEFAULT_VIEW_TIMEOUT_MS = 1500;
|
|
35
|
+
const MAX_VIEW_TIMEOUT_BACKOFF_MULTIPLIER = 8;
|
|
36
|
+
const DEFAULT_STARTUP_QUORUM_GRACE_MS = 2e3;
|
|
37
|
+
const NEXT_LEADER_COMMIT_WAIT_MS = 300;
|
|
38
|
+
const MAX_CATCHUP_SEGMENTS_PER_REQUEST = 50;
|
|
39
|
+
const MAX_IN_MEMORY_UNCOMMITTED_PROPOSALS = 8;
|
|
40
|
+
const DEFAULT_MAX_EMPTY_VIEW_MS = 12e3;
|
|
41
|
+
var ConsensusAbortError = class extends _ocap_util_lib_error.CustomError {
|
|
42
|
+
constructor(code, message) {
|
|
43
|
+
super(code, message);
|
|
44
|
+
this.isConsensusAbort = true;
|
|
45
|
+
this.code = code;
|
|
46
|
+
this.name = "ConsensusAbortError";
|
|
47
|
+
}
|
|
48
|
+
};
|
|
49
|
+
function sha256HexBytes(bytes) {
|
|
50
|
+
return `0x${(0, node_crypto.createHash)("sha256").update(bytes).digest("hex")}`;
|
|
51
|
+
}
|
|
52
|
+
function sha256Utf8Bytes(input) {
|
|
53
|
+
return new Uint8Array((0, node_crypto.createHash)("sha256").update(input, "utf8").digest());
|
|
54
|
+
}
|
|
55
|
+
function normalizeHexCompare(hex) {
|
|
56
|
+
return String(hex ?? "").trim().replace(/^0x/i, "").toLowerCase();
|
|
57
|
+
}
|
|
58
|
+
function buildCanonicalHttpAuthDigest(params) {
|
|
59
|
+
const bodyHash = sha256HexBytes(params.bodyBytes);
|
|
60
|
+
return sha256Utf8Bytes(JSON.stringify([
|
|
61
|
+
"arcblock:consensus-http:v1",
|
|
62
|
+
params.kind,
|
|
63
|
+
params.chainId,
|
|
64
|
+
params.methodOrStatus.toUpperCase(),
|
|
65
|
+
params.pathWithQuery,
|
|
66
|
+
params.senderDid,
|
|
67
|
+
String(params.timestampMs),
|
|
68
|
+
params.nonce,
|
|
69
|
+
bodyHash
|
|
70
|
+
]));
|
|
71
|
+
}
|
|
72
|
+
async function readIncomingBytesBounded(req, maxBytes) {
|
|
73
|
+
const rawLen = req.headers["content-length"];
|
|
74
|
+
if (typeof rawLen === "string" && rawLen.trim().length > 0) {
|
|
75
|
+
const declared = Number(rawLen);
|
|
76
|
+
if (Number.isFinite(declared) && declared > maxBytes) return {
|
|
77
|
+
ok: false,
|
|
78
|
+
status: 413,
|
|
79
|
+
code: "PAYLOAD_TOO_LARGE",
|
|
80
|
+
error: `Consensus request Content-Length (${declared}) exceeds ${maxBytes} bytes`
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
return new Promise((resolve) => {
|
|
84
|
+
const chunks = [];
|
|
85
|
+
let total = 0;
|
|
86
|
+
let settled = false;
|
|
87
|
+
const finish = (res) => {
|
|
88
|
+
if (settled) return;
|
|
89
|
+
settled = true;
|
|
90
|
+
resolve(res);
|
|
91
|
+
};
|
|
92
|
+
req.on("data", (chunk) => {
|
|
93
|
+
if (settled) return;
|
|
94
|
+
const buf = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk, "utf8");
|
|
95
|
+
total += buf.byteLength;
|
|
96
|
+
if (total > maxBytes) {
|
|
97
|
+
req.pause();
|
|
98
|
+
finish({
|
|
99
|
+
ok: false,
|
|
100
|
+
status: 413,
|
|
101
|
+
code: "PAYLOAD_TOO_LARGE",
|
|
102
|
+
error: `Consensus request body exceeds ${maxBytes}-byte limit`
|
|
103
|
+
});
|
|
104
|
+
return;
|
|
105
|
+
}
|
|
106
|
+
chunks.push(buf);
|
|
107
|
+
});
|
|
108
|
+
req.on("end", () => {
|
|
109
|
+
if (settled) return;
|
|
110
|
+
finish({
|
|
111
|
+
ok: true,
|
|
112
|
+
bytes: new Uint8Array(Buffer.concat(chunks, total))
|
|
113
|
+
});
|
|
114
|
+
});
|
|
115
|
+
req.on("error", () => {
|
|
116
|
+
finish({
|
|
117
|
+
ok: false,
|
|
118
|
+
status: 400,
|
|
119
|
+
code: "STREAM_ERROR",
|
|
120
|
+
error: "Failed to read consensus request body"
|
|
121
|
+
});
|
|
122
|
+
});
|
|
123
|
+
});
|
|
124
|
+
}
|
|
125
|
+
async function readFetchResponseBytesBounded(response, maxBytes) {
|
|
126
|
+
const contentLength = response.headers?.get?.("content-length");
|
|
127
|
+
if (contentLength) {
|
|
128
|
+
const declared = Number(contentLength);
|
|
129
|
+
if (Number.isFinite(declared) && declared > maxBytes) {
|
|
130
|
+
await response.body?.cancel().catch(() => void 0);
|
|
131
|
+
throw new _ocap_util_lib_error.CustomError("RESPONSE_TOO_LARGE", `Consensus response (${declared}) exceeds ${maxBytes} bytes`);
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
if (!response.body || typeof response.body.getReader !== "function") {
|
|
135
|
+
const buf = new Uint8Array(await response.arrayBuffer());
|
|
136
|
+
if (buf.byteLength > maxBytes) throw new _ocap_util_lib_error.CustomError("RESPONSE_TOO_LARGE", `Consensus response (${buf.byteLength}) exceeds ${maxBytes} bytes`);
|
|
137
|
+
return buf;
|
|
138
|
+
}
|
|
139
|
+
const reader = response.body.getReader();
|
|
140
|
+
const chunks = [];
|
|
141
|
+
let total = 0;
|
|
142
|
+
try {
|
|
143
|
+
while (true) {
|
|
144
|
+
const { done, value } = await reader.read();
|
|
145
|
+
if (done) break;
|
|
146
|
+
if (!value || value.byteLength === 0) continue;
|
|
147
|
+
total += value.byteLength;
|
|
148
|
+
if (total > maxBytes) {
|
|
149
|
+
await reader.cancel().catch(() => void 0);
|
|
150
|
+
throw new _ocap_util_lib_error.CustomError("RESPONSE_TOO_LARGE", `Consensus streaming response exceeded ${maxBytes} bytes`);
|
|
151
|
+
}
|
|
152
|
+
chunks.push(value);
|
|
153
|
+
}
|
|
154
|
+
} finally {
|
|
155
|
+
try {
|
|
156
|
+
reader.releaseLock();
|
|
157
|
+
} catch {}
|
|
158
|
+
}
|
|
159
|
+
const out = new Uint8Array(total);
|
|
160
|
+
let offset = 0;
|
|
161
|
+
for (const c of chunks) {
|
|
162
|
+
out.set(c, offset);
|
|
163
|
+
offset += c.byteLength;
|
|
164
|
+
}
|
|
165
|
+
return out;
|
|
166
|
+
}
|
|
167
|
+
var InMemoryHotStuffStateStore = class {
|
|
168
|
+
constructor() {
|
|
169
|
+
this.lock = null;
|
|
170
|
+
this.currentView = 0;
|
|
171
|
+
this.lockedQC = null;
|
|
172
|
+
this.highQC = null;
|
|
173
|
+
this.lastTC = null;
|
|
174
|
+
this.uncommitted = /* @__PURE__ */ new Map();
|
|
175
|
+
}
|
|
176
|
+
async getLock() {
|
|
177
|
+
return this.lock;
|
|
178
|
+
}
|
|
179
|
+
async setLock(record) {
|
|
180
|
+
this.lock = { ...record };
|
|
181
|
+
this.currentView = Math.max(this.currentView, record.lastVotedView, record.lastPreparedView ?? 0, record.lastCommitVotedView ?? 0, record.lastTimedOutView ?? 0);
|
|
182
|
+
}
|
|
183
|
+
async loadRuntimeState(chainId, voterDid) {
|
|
184
|
+
return {
|
|
185
|
+
chainId,
|
|
186
|
+
voterDid,
|
|
187
|
+
currentView: this.currentView,
|
|
188
|
+
lockRecord: this.lock,
|
|
189
|
+
lockedQC: this.lockedQC,
|
|
190
|
+
highQC: this.highQC,
|
|
191
|
+
lastTC: this.lastTC
|
|
192
|
+
};
|
|
193
|
+
}
|
|
194
|
+
async saveRuntimeState(params) {
|
|
195
|
+
if (params.lockedQC !== void 0) this.lockedQC = (0, _arcblock_ledger_core.selectHighestQC)([this.lockedQC, params.lockedQC]);
|
|
196
|
+
if (params.highQC !== void 0) this.highQC = (0, _arcblock_ledger_core.selectHighestQC)([this.highQC, params.highQC]);
|
|
197
|
+
if (params.lastTC && (!this.lastTC || params.lastTC.viewNumber > this.lastTC.viewNumber || params.lastTC.viewNumber === this.lastTC.viewNumber && params.lastTC.signatures.length > this.lastTC.signatures.length)) this.lastTC = params.lastTC;
|
|
198
|
+
this.currentView = Math.max(this.currentView, params.currentView ?? 0, this.highQC ? this.highQC.viewNumber + 1 : 0, this.lastTC ? this.lastTC.viewNumber + 1 : 0);
|
|
199
|
+
const pruneUpTo = params.pruneUncommittedUpToHeight ?? this.highQC?.height ?? 0;
|
|
200
|
+
if (pruneUpTo > 0) {
|
|
201
|
+
for (const [k, v] of this.uncommitted.entries()) if (v.height <= pruneUpTo) this.uncommitted.delete(k);
|
|
202
|
+
if (this.lock?.lockedPrepareQC && this.lock.lockedPrepareQC.height <= pruneUpTo) {
|
|
203
|
+
const pruned = { ...this.lock };
|
|
204
|
+
delete pruned.lockedPrepareQC;
|
|
205
|
+
delete pruned.lockedPrepareCheckpoint;
|
|
206
|
+
this.lock = pruned;
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
async saveUncommittedProposalEntries(params) {
|
|
211
|
+
if (this.highQC && params.height <= this.highQC.height) return;
|
|
212
|
+
const cleanKey = normalizeHexCompare(params.checkpointHash);
|
|
213
|
+
if (!cleanKey) return;
|
|
214
|
+
this.uncommitted.set(cleanKey, {
|
|
215
|
+
height: params.height,
|
|
216
|
+
entries: params.entries,
|
|
217
|
+
...params.checkpoint ? { checkpoint: params.checkpoint } : {},
|
|
218
|
+
updatedAt: params.updatedAt ?? Date.now()
|
|
219
|
+
});
|
|
220
|
+
if (this.uncommitted.size > MAX_IN_MEMORY_UNCOMMITTED_PROPOSALS) {
|
|
221
|
+
const pinned = /* @__PURE__ */ new Set();
|
|
222
|
+
for (const raw of [
|
|
223
|
+
this.lock?.lockedPrepareQC?.checkpointHash,
|
|
224
|
+
this.lock?.lockedPrepareCheckpoint?.hash,
|
|
225
|
+
this.lock?.lastPreparedCheckpointHash,
|
|
226
|
+
this.lock?.lastCommitVotedCheckpointHash,
|
|
227
|
+
this.lock?.lastVotedCheckpointHash,
|
|
228
|
+
this.lastTC?.lockedCheckpoint?.hash,
|
|
229
|
+
this.lastTC?.highestPrepareCheckpoint?.hash,
|
|
230
|
+
this.lastTC?.highestPrepareQC?.checkpointHash
|
|
231
|
+
]) {
|
|
232
|
+
const norm = normalizeHexCompare(raw);
|
|
233
|
+
if (norm) pinned.add(norm);
|
|
234
|
+
}
|
|
235
|
+
const evictable = Array.from(this.uncommitted.entries()).filter(([k]) => !pinned.has(k)).sort((a, b) => a[1].updatedAt - b[1].updatedAt);
|
|
236
|
+
while (this.uncommitted.size > MAX_IN_MEMORY_UNCOMMITTED_PROPOSALS && evictable.length > 0) {
|
|
237
|
+
const drop = evictable.shift();
|
|
238
|
+
if (drop) this.uncommitted.delete(drop[0]);
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
async getUncommittedProposalEntries(_chainId, _voterDid, checkpointHash) {
|
|
243
|
+
return this.uncommitted.get(normalizeHexCompare(checkpointHash))?.entries ?? null;
|
|
244
|
+
}
|
|
245
|
+
async getUncommittedProposalCheckpoint(_chainId, _voterDid, checkpointHash) {
|
|
246
|
+
return this.uncommitted.get(normalizeHexCompare(checkpointHash))?.checkpoint ?? null;
|
|
247
|
+
}
|
|
248
|
+
};
|
|
249
|
+
var HotStuffNodeRuntime = class {
|
|
250
|
+
constructor(options) {
|
|
251
|
+
this.role = "validator";
|
|
252
|
+
this.server = null;
|
|
253
|
+
this.boundPort = 0;
|
|
254
|
+
this.heartbeatTimer = null;
|
|
255
|
+
this.viewTimer = null;
|
|
256
|
+
this.mutexTail = Promise.resolve();
|
|
257
|
+
this.inConsensusLock = false;
|
|
258
|
+
this.startedAtMs = 0;
|
|
259
|
+
this.stopped = false;
|
|
260
|
+
this.halted = false;
|
|
261
|
+
this.haltReason = null;
|
|
262
|
+
this.currentView = 0;
|
|
263
|
+
this.lastSelfVotedView = -1;
|
|
264
|
+
this.lastSelfPreparedView = -1;
|
|
265
|
+
this.currentViewEnteredAtMs = Date.now();
|
|
266
|
+
this.pendingDemandForView = -1;
|
|
267
|
+
this.consecutiveViewTimeouts = 0;
|
|
268
|
+
this.highQC = null;
|
|
269
|
+
this.lockedQC = null;
|
|
270
|
+
this.lastTC = null;
|
|
271
|
+
this.quorumLeaseValid = true;
|
|
272
|
+
this.quorumContactObserved = false;
|
|
273
|
+
this.unleadableView = -1;
|
|
274
|
+
this.activeProposalSlot = null;
|
|
275
|
+
this.pendingBroadcastSegments = [];
|
|
276
|
+
this.peerLastSeenAtMs = /* @__PURE__ */ new Map();
|
|
277
|
+
this.replayCache = /* @__PURE__ */ new Map();
|
|
278
|
+
this.collectedTimeoutsByView = /* @__PURE__ */ new Map();
|
|
279
|
+
this.demandSendersByView = /* @__PURE__ */ new Map();
|
|
280
|
+
this.viewAdvanceWaiters = /* @__PURE__ */ new Set();
|
|
281
|
+
this.resolver = options.resolver;
|
|
282
|
+
this.writePolicy = options.writePolicy ?? "forward";
|
|
283
|
+
this.chainId = options.resolver.config.chainId || options.resolver.ledger.chainId;
|
|
284
|
+
this.replayer = new require_replayer.LedgerEntryReplayer(options.resolver);
|
|
285
|
+
this.consensusBindHost = (0, _arcblock_ledger_core.assertValidConsensusBindHost)(options.consensusBindHost, { allowPublicBind: options.allowPublicBind });
|
|
286
|
+
if (!Number.isSafeInteger(options.consensusPort) || options.consensusPort < 0 || options.consensusPort > 65535) throw new _ocap_util_lib_error.CustomError("INVALID_CONSENSUS_PORT", `Invalid consensusPort: ${String(options.consensusPort)}`);
|
|
287
|
+
this.configuredConsensusPort = options.consensusPort;
|
|
288
|
+
if (!Array.isArray(options.validators) || options.validators.length < 4) throw new _ocap_util_lib_error.CustomError("INVALID_HOTSTUFF_VALIDATORS", "HotStuffNodeRuntime requires at least 4 validators (3f+1 BFT)");
|
|
289
|
+
this.validatorMap = /* @__PURE__ */ new Map();
|
|
290
|
+
const normalizedValidators = [];
|
|
291
|
+
for (const v of options.validators) {
|
|
292
|
+
const normalizedConsensusUrl = (0, _arcblock_ledger_core.assertValidConsensusPeerUrl)(v.consensusUrl, { allowPrivateHttp: options.allowPrivateHttpPeers !== false });
|
|
293
|
+
const normalizedRpcUrl = (0, _arcblock_ledger_core.normalizeHttpBaseUrl)(v.rpcUrl, `validator(${v.did}).rpcUrl`);
|
|
294
|
+
const entry = {
|
|
295
|
+
did: v.did.trim(),
|
|
296
|
+
publicKey: v.publicKey.trim(),
|
|
297
|
+
rpcUrl: normalizedRpcUrl,
|
|
298
|
+
consensusUrl: normalizedConsensusUrl
|
|
299
|
+
};
|
|
300
|
+
this.validatorMap.set(entry.did, entry);
|
|
301
|
+
normalizedValidators.push(entry);
|
|
302
|
+
}
|
|
303
|
+
this.validators = normalizedValidators;
|
|
304
|
+
this.selfDid = options.validatorDid.trim();
|
|
305
|
+
if (!this.validatorMap.get(this.selfDid)) throw new _ocap_util_lib_error.CustomError("INVALID_HOTSTUFF_VALIDATORS", `Local validator DID ${this.selfDid} is not present in HOTSTUFF_VALIDATORS roster`);
|
|
306
|
+
this.peerValidators = this.validators.filter((v) => v.did !== this.selfDid);
|
|
307
|
+
this.clusterConfig = (0, _arcblock_ledger_core.toHotStuffClusterConfig)(this.chainId, this.validators, options.quorumThreshold);
|
|
308
|
+
this.validatorWallet = options.validatorWallet;
|
|
309
|
+
this.signDigest = (0, _arcblock_ledger_core.createCheckpointSigner)(options.validatorWallet, this.selfDid);
|
|
310
|
+
const rawAnchor = (0, _arcblock_ledger_core.createHotStuffTrustAnchor)(this.clusterConfig);
|
|
311
|
+
this.trustAnchorVerify = async (data, signature, signerDid) => Boolean(rawAnchor.verifier(data, signature, signerDid));
|
|
312
|
+
this.stateStore = options.stateStore ?? new InMemoryHotStuffStateStore();
|
|
313
|
+
this.proposalTimeoutMs = typeof options.proposalTimeoutMs === "number" && options.proposalTimeoutMs >= 100 ? options.proposalTimeoutMs : DEFAULT_PROPOSAL_TIMEOUT_MS;
|
|
314
|
+
this.httpTimeoutMs = typeof options.httpTimeoutMs === "number" && options.httpTimeoutMs >= 100 ? options.httpTimeoutMs : DEFAULT_HTTP_TIMEOUT_MS;
|
|
315
|
+
this.heartbeatIntervalMs = typeof options.heartbeatIntervalMs === "number" && options.heartbeatIntervalMs >= 40 ? options.heartbeatIntervalMs : DEFAULT_HEARTBEAT_INTERVAL_MS;
|
|
316
|
+
this.baseViewTimeoutMs = typeof options.viewTimeoutMs === "number" && options.viewTimeoutMs >= 100 ? options.viewTimeoutMs : DEFAULT_VIEW_TIMEOUT_MS;
|
|
317
|
+
this.maxEmptyViewMs = typeof options.maxEmptyViewMs === "number" && options.maxEmptyViewMs >= 200 ? options.maxEmptyViewMs : Math.max(DEFAULT_MAX_EMPTY_VIEW_MS, this.baseViewTimeoutMs * 6);
|
|
318
|
+
this.quorumLeaseTimeoutMs = typeof options.quorumLeaseTimeoutMs === "number" && options.quorumLeaseTimeoutMs >= 150 ? options.quorumLeaseTimeoutMs : Math.max(this.heartbeatIntervalMs * 4, this.baseViewTimeoutMs * 2);
|
|
319
|
+
this.startupQuorumGraceMs = typeof options.startupQuorumGraceMs === "number" && options.startupQuorumGraceMs >= 0 ? options.startupQuorumGraceMs : DEFAULT_STARTUP_QUORUM_GRACE_MS;
|
|
320
|
+
this.fetchImpl = options.fetchImpl ?? globalThis.fetch;
|
|
321
|
+
this.onOutboundConsensusRequest = options.onOutboundConsensusRequest;
|
|
322
|
+
this.onLeaderReplicaVoteReceived = options.onLeaderReplicaVoteReceived;
|
|
323
|
+
this.onLeaderReplicaPrecommitReceived = options.onLeaderReplicaPrecommitReceived;
|
|
324
|
+
this.leaderRouter = options.leaderRouter ?? new _arcblock_ledger_core.LeaderRouter({
|
|
325
|
+
chainId: this.chainId,
|
|
326
|
+
peerUrls: this.peerValidators.map((v) => v.rpcUrl),
|
|
327
|
+
hotstuffClusterConfig: this.clusterConfig,
|
|
328
|
+
requireSignedForwardReceipt: true,
|
|
329
|
+
fetchImpl: this.fetchImpl,
|
|
330
|
+
probeTimeoutMs: Math.min(2e3, this.httpTimeoutMs),
|
|
331
|
+
timeoutMs: Math.max(5e3, this.proposalTimeoutMs * 3),
|
|
332
|
+
failoverRetryTimeoutMs: Math.max(6e3, this.baseViewTimeoutMs * 4)
|
|
333
|
+
});
|
|
334
|
+
}
|
|
335
|
+
getConsensusMode() {
|
|
336
|
+
return "hotstuff";
|
|
337
|
+
}
|
|
338
|
+
getConsensusView() {
|
|
339
|
+
return this.currentView;
|
|
340
|
+
}
|
|
341
|
+
getHighQC() {
|
|
342
|
+
return this.highQC;
|
|
343
|
+
}
|
|
344
|
+
getLockedQC() {
|
|
345
|
+
return this.lockedQC;
|
|
346
|
+
}
|
|
347
|
+
getLastTC() {
|
|
348
|
+
return this.lastTC;
|
|
349
|
+
}
|
|
350
|
+
getListeningPort() {
|
|
351
|
+
return this.boundPort;
|
|
352
|
+
}
|
|
353
|
+
getConsensusBaseUrl() {
|
|
354
|
+
return `http://${this.consensusBindHost.includes(":") ? `[${this.consensusBindHost}]` : this.consensusBindHost}:${this.boundPort || this.configuredConsensusPort}`;
|
|
355
|
+
}
|
|
356
|
+
getConsensusCommitter() {
|
|
357
|
+
return this;
|
|
358
|
+
}
|
|
359
|
+
isHalted() {
|
|
360
|
+
return this.halted;
|
|
361
|
+
}
|
|
362
|
+
getHaltReason() {
|
|
363
|
+
return this.haltReason;
|
|
364
|
+
}
|
|
365
|
+
updateRuntimeConfig(patch) {
|
|
366
|
+
if (patch.peerUrls !== void 0) throw new _ocap_util_lib_error.CustomError("IMMUTABLE_CLUSTER_CONFIG", "Validator peerUrls cannot be mutated via runtime preferences in HotStuff mode");
|
|
367
|
+
if (patch.writePolicy !== void 0) {
|
|
368
|
+
if (patch.writePolicy !== "reject" && patch.writePolicy !== "forward") throw new _ocap_util_lib_error.CustomError("INVALID_CONFIG", `Invalid writePolicy "${String(patch.writePolicy)}"`);
|
|
369
|
+
this.writePolicy = patch.writePolicy;
|
|
370
|
+
}
|
|
371
|
+
}
|
|
372
|
+
getRuntimeDiagnostics(options) {
|
|
373
|
+
const includePrivate = options?.includePrivateTopology === true;
|
|
374
|
+
const nowMs = Date.now();
|
|
375
|
+
const quorumLeaseValid = this.refreshQuorumLease(nowMs);
|
|
376
|
+
const currentLeader = (0, _arcblock_ledger_core.getLeaderForView)(this.clusterConfig, this.currentView);
|
|
377
|
+
const livePeerCount = this.computeLivePeerCount(nowMs);
|
|
378
|
+
const validators = this.validators.slice(0, 64).map((v, idx) => {
|
|
379
|
+
const isSelf = v.did === this.selfDid;
|
|
380
|
+
const seenAt = this.peerLastSeenAtMs.get(v.did);
|
|
381
|
+
const reachable = isSelf ? !this.halted && !this.stopped : Boolean(seenAt && nowMs - seenAt <= this.quorumLeaseTimeoutMs);
|
|
382
|
+
return {
|
|
383
|
+
index: idx,
|
|
384
|
+
did: v.did,
|
|
385
|
+
publicKey: v.publicKey,
|
|
386
|
+
isSelf,
|
|
387
|
+
isCurrentViewLeader: v.did === currentLeader.did,
|
|
388
|
+
reachable,
|
|
389
|
+
lastSeenAgoMs: isSelf ? 0 : seenAt !== void 0 ? Math.max(0, nowMs - seenAt) : null,
|
|
390
|
+
...includePrivate ? {
|
|
391
|
+
rpcUrl: v.rpcUrl,
|
|
392
|
+
consensusUrl: v.consensusUrl
|
|
393
|
+
} : {}
|
|
394
|
+
};
|
|
395
|
+
});
|
|
396
|
+
return {
|
|
397
|
+
mode: "hotstuff",
|
|
398
|
+
consensusMode: "hotstuff",
|
|
399
|
+
selfDid: this.selfDid,
|
|
400
|
+
writePolicy: this.writePolicy,
|
|
401
|
+
currentView: this.currentView,
|
|
402
|
+
currentLeaderDid: currentLeader.did,
|
|
403
|
+
isCurrentViewLeader: currentLeader.did === this.selfDid,
|
|
404
|
+
writableLocally: this.shouldHandleWriteLocally(),
|
|
405
|
+
quorumThreshold: this.clusterConfig.quorumThreshold ?? 3,
|
|
406
|
+
totalValidators: this.validators.length,
|
|
407
|
+
livePeerCount,
|
|
408
|
+
quorumLeaseValid,
|
|
409
|
+
quorumLeaseTimeoutMs: this.quorumLeaseTimeoutMs,
|
|
410
|
+
consecutiveViewTimeouts: this.consecutiveViewTimeouts,
|
|
411
|
+
...includePrivate ? {
|
|
412
|
+
consensusBindHost: this.consensusBindHost,
|
|
413
|
+
consensusPort: this.boundPort || this.configuredConsensusPort
|
|
414
|
+
} : {},
|
|
415
|
+
highQC: this.highQC ? {
|
|
416
|
+
viewNumber: this.highQC.viewNumber,
|
|
417
|
+
height: this.highQC.height,
|
|
418
|
+
sequence: this.highQC.sequence,
|
|
419
|
+
proposer: this.highQC.proposer,
|
|
420
|
+
signerCount: this.highQC.signatures.length,
|
|
421
|
+
checkpointHash: this.highQC.checkpointHash
|
|
422
|
+
} : null,
|
|
423
|
+
lockedQC: this.lockedQC ? {
|
|
424
|
+
viewNumber: this.lockedQC.viewNumber,
|
|
425
|
+
height: this.lockedQC.height,
|
|
426
|
+
sequence: this.lockedQC.sequence,
|
|
427
|
+
proposer: this.lockedQC.proposer,
|
|
428
|
+
signerCount: this.lockedQC.signatures.length,
|
|
429
|
+
checkpointHash: this.lockedQC.checkpointHash
|
|
430
|
+
} : null,
|
|
431
|
+
lastTC: this.lastTC ? {
|
|
432
|
+
viewNumber: this.lastTC.viewNumber,
|
|
433
|
+
signerCount: this.lastTC.signatures.length,
|
|
434
|
+
highestQCView: this.lastTC.highestQC?.viewNumber ?? null
|
|
435
|
+
} : null,
|
|
436
|
+
validators
|
|
437
|
+
};
|
|
438
|
+
}
|
|
439
|
+
async triggerSyncOrProbe() {
|
|
440
|
+
if (this.halted) throw new _ocap_util_lib_error.CustomError("NODE_HALTED", `Validator node is halted: ${this.haltReason || "verification error"}`);
|
|
441
|
+
const promoted = await this.leaderRouter.probeAndPromoteLeader();
|
|
442
|
+
return {
|
|
443
|
+
mode: "hotstuff",
|
|
444
|
+
currentView: this.currentView,
|
|
445
|
+
currentLeaderDid: (0, _arcblock_ledger_core.getLeaderForView)(this.clusterConfig, this.currentView).did,
|
|
446
|
+
livePeerCount: this.computeLivePeerCount(),
|
|
447
|
+
quorumLeaseValid: this.refreshQuorumLease(),
|
|
448
|
+
hasReachablePeerLeader: Boolean(promoted)
|
|
449
|
+
};
|
|
450
|
+
}
|
|
451
|
+
haltFailClosed(reason) {
|
|
452
|
+
this.halted = true;
|
|
453
|
+
this.haltReason = reason;
|
|
454
|
+
this.clearTimers();
|
|
455
|
+
this.resolver.logger.error("CRITICAL: HotStuffNodeRuntime halted fail-closed", {
|
|
456
|
+
chainId: this.chainId,
|
|
457
|
+
validatorDid: this.selfDid,
|
|
458
|
+
haltReason: reason
|
|
459
|
+
});
|
|
460
|
+
}
|
|
461
|
+
computeLivePeerCount(nowMs = Date.now()) {
|
|
462
|
+
let live = 0;
|
|
463
|
+
for (const peer of this.peerValidators) {
|
|
464
|
+
const seenAt = this.peerLastSeenAtMs.get(peer.did);
|
|
465
|
+
if (seenAt !== void 0 && nowMs - seenAt <= this.quorumLeaseTimeoutMs) live++;
|
|
466
|
+
}
|
|
467
|
+
return live;
|
|
468
|
+
}
|
|
469
|
+
refreshQuorumLease(nowMs = Date.now()) {
|
|
470
|
+
const threshold = this.clusterConfig.quorumThreshold ?? 3;
|
|
471
|
+
if (this.computeLivePeerCount(nowMs) + 1 >= threshold) {
|
|
472
|
+
this.quorumContactObserved = true;
|
|
473
|
+
this.quorumLeaseValid = true;
|
|
474
|
+
return true;
|
|
475
|
+
}
|
|
476
|
+
if (!this.quorumContactObserved && this.startedAtMs > 0 && nowMs - this.startedAtMs <= this.startupQuorumGraceMs && this.currentView === 0 && this.highQC === null) return this.quorumLeaseValid;
|
|
477
|
+
this.quorumLeaseValid = false;
|
|
478
|
+
return false;
|
|
479
|
+
}
|
|
480
|
+
recordAuthenticatedPeerContact(peerDid, nowMs = Date.now()) {
|
|
481
|
+
if (peerDid && peerDid !== this.selfDid && this.validatorMap.has(peerDid)) {
|
|
482
|
+
this.peerLastSeenAtMs.set(peerDid, nowMs);
|
|
483
|
+
this.refreshQuorumLease(nowMs);
|
|
484
|
+
}
|
|
485
|
+
}
|
|
486
|
+
hasSpentTurnInView(viewNumber) {
|
|
487
|
+
return Math.max(this.lastSelfVotedView, this.lastSelfPreparedView) >= viewNumber;
|
|
488
|
+
}
|
|
489
|
+
advanceCurrentViewTo(nextView) {
|
|
490
|
+
if (nextView <= this.currentView) return;
|
|
491
|
+
this.currentView = nextView;
|
|
492
|
+
this.currentViewEnteredAtMs = Date.now();
|
|
493
|
+
if (this.pendingDemandForView < nextView) this.pendingDemandForView = -1;
|
|
494
|
+
for (const v of Array.from(this.demandSendersByView.keys())) if (v < nextView) this.demandSendersByView.delete(v);
|
|
495
|
+
for (const waiter of Array.from(this.viewAdvanceWaiters)) if (nextView > waiter.stalledView) {
|
|
496
|
+
this.viewAdvanceWaiters.delete(waiter);
|
|
497
|
+
waiter.resolve();
|
|
498
|
+
}
|
|
499
|
+
}
|
|
500
|
+
shouldSuppressHeartbeatKeepalive(lock, viewNumber, nowMs = Date.now()) {
|
|
501
|
+
const highQcHeight = this.highQC?.height ?? 0;
|
|
502
|
+
if ((lock?.lastTimedOutView ?? -1) >= viewNumber) return true;
|
|
503
|
+
if ((lock?.lastPreparedHeight ?? 0) > highQcHeight && (lock?.lastPreparedView ?? -1) >= viewNumber) return true;
|
|
504
|
+
if ((lock?.lastVotedHeight ?? 0) > highQcHeight && (lock?.lastVotedView ?? -1) >= viewNumber) return true;
|
|
505
|
+
if (this.lastTC && this.lastTC.viewNumber === viewNumber - 1 && this.lastTC.lockedCheckpoint && this.lastTC.lockedCheckpoint.height === highQcHeight + 1) return true;
|
|
506
|
+
if (this.pendingDemandForView === viewNumber) return true;
|
|
507
|
+
if (nowMs - this.currentViewEnteredAtMs >= this.maxEmptyViewMs) return true;
|
|
508
|
+
return false;
|
|
509
|
+
}
|
|
510
|
+
shouldHandleWriteLocally() {
|
|
511
|
+
if (this.halted || this.stopped) return false;
|
|
512
|
+
if (!this.refreshQuorumLease()) return false;
|
|
513
|
+
if (this.hasSpentTurnInView(this.currentView) || this.unleadableView === this.currentView) return false;
|
|
514
|
+
return (0, _arcblock_ledger_core.getLeaderForView)(this.clusterConfig, this.currentView).did === this.selfDid;
|
|
515
|
+
}
|
|
516
|
+
async isSynced() {
|
|
517
|
+
if (this.halted || this.stopped) return false;
|
|
518
|
+
const localHeight = (await this.resolver.ledger.getCheckpoint().catch(() => null))?.height ?? 0;
|
|
519
|
+
return (this.highQC?.height ?? 0) - localHeight <= 2;
|
|
520
|
+
}
|
|
521
|
+
async getFollowerStatus() {
|
|
522
|
+
const [localSequence, latestCp, synced] = await Promise.all([
|
|
523
|
+
this.resolver.ledger.getLatestSequence(),
|
|
524
|
+
this.resolver.ledger.getCheckpoint(),
|
|
525
|
+
this.isSynced()
|
|
526
|
+
]);
|
|
527
|
+
const localStateCommitHash = typeof this.resolver.statedb.getCurrentCommitHash === "function" ? await this.resolver.statedb.getCurrentCommitHash() : "UNKNOWN";
|
|
528
|
+
const currentLeader = (0, _arcblock_ledger_core.getLeaderForView)(this.clusterConfig, this.currentView);
|
|
529
|
+
const upstreamSeq = this.highQC ? Math.max(localSequence, this.highQC.sequence) : localSequence;
|
|
530
|
+
return {
|
|
531
|
+
chainId: this.chainId,
|
|
532
|
+
nodeRole: "validator",
|
|
533
|
+
writePolicy: this.writePolicy,
|
|
534
|
+
syncMode: "primary",
|
|
535
|
+
synced,
|
|
536
|
+
halted: this.halted,
|
|
537
|
+
haltReason: this.haltReason,
|
|
538
|
+
localSequence,
|
|
539
|
+
upstreamSequence: upstreamSeq,
|
|
540
|
+
syncLagEntries: Math.max(0, upstreamSeq - localSequence),
|
|
541
|
+
verifiedCheckpointHeight: latestCp?.height ?? 0,
|
|
542
|
+
verifiedCheckpointHash: latestCp?.hash ?? null,
|
|
543
|
+
localStateCommitHash,
|
|
544
|
+
activeLeaderUrl: currentLeader.rpcUrl,
|
|
545
|
+
lastUpstreamSuccessAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
546
|
+
};
|
|
547
|
+
}
|
|
548
|
+
async triggerClientDemandViewChange(stalledView) {
|
|
549
|
+
if (this.halted || this.stopped) return;
|
|
550
|
+
if (this.currentView === stalledView) {
|
|
551
|
+
this.pendingDemandForView = Math.max(this.pendingDemandForView, stalledView);
|
|
552
|
+
const t = setTimeout(() => {
|
|
553
|
+
this.onViewTimerExpired(stalledView).catch(() => void 0);
|
|
554
|
+
}, 0);
|
|
555
|
+
if (typeof t.unref === "function") t.unref();
|
|
556
|
+
await Promise.allSettled(this.peerValidators.map((peer) => this.sendConsensusJson(peer, "POST", "/consensus/demand", { viewNumber: stalledView }, Math.min(1e3, this.httpTimeoutMs), MAX_CONTROL_ROUTE_BYTES)));
|
|
557
|
+
}
|
|
558
|
+
if (this.currentView > stalledView) return;
|
|
559
|
+
const waitMs = Math.min(4500, Math.max(400, this.baseViewTimeoutMs * 3));
|
|
560
|
+
await new Promise((resolve) => {
|
|
561
|
+
const waiter = {
|
|
562
|
+
stalledView,
|
|
563
|
+
resolve: () => {}
|
|
564
|
+
};
|
|
565
|
+
const timer = setTimeout(() => {
|
|
566
|
+
this.viewAdvanceWaiters.delete(waiter);
|
|
567
|
+
resolve();
|
|
568
|
+
}, waitMs);
|
|
569
|
+
if (typeof timer.unref === "function") timer.unref();
|
|
570
|
+
waiter.resolve = () => {
|
|
571
|
+
clearTimeout(timer);
|
|
572
|
+
resolve();
|
|
573
|
+
};
|
|
574
|
+
this.viewAdvanceWaiters.add(waiter);
|
|
575
|
+
});
|
|
576
|
+
}
|
|
577
|
+
async forwardSendTx(args, ctx) {
|
|
578
|
+
if (this.halted) throw new _ocap_util_lib_error.CustomError("NODE_HALTED", `Validator node is halted fail-closed: ${this.haltReason || "verification error"}`);
|
|
579
|
+
try {
|
|
580
|
+
return await this.leaderRouter.forwardSendTx(args, ctx);
|
|
581
|
+
} catch (firstErr) {
|
|
582
|
+
if (this.halted || this.stopped || !this.refreshQuorumLease()) throw firstErr;
|
|
583
|
+
const stalledView = this.currentView;
|
|
584
|
+
await this.triggerClientDemandViewChange(stalledView);
|
|
585
|
+
if (this.shouldHandleWriteLocally() && typeof this.resolver.sendTx === "function") return this.resolver.sendTx(args, ctx);
|
|
586
|
+
return this.leaderRouter.forwardSendTx(args, ctx);
|
|
587
|
+
}
|
|
588
|
+
}
|
|
589
|
+
runExclusive(fn) {
|
|
590
|
+
const prev = this.mutexTail;
|
|
591
|
+
let release;
|
|
592
|
+
const gate = new Promise((resolve) => {
|
|
593
|
+
release = resolve;
|
|
594
|
+
});
|
|
595
|
+
this.mutexTail = prev.then(() => gate);
|
|
596
|
+
return prev.then(async () => {
|
|
597
|
+
this.inConsensusLock = true;
|
|
598
|
+
try {
|
|
599
|
+
return await fn();
|
|
600
|
+
} finally {
|
|
601
|
+
this.inConsensusLock = false;
|
|
602
|
+
release();
|
|
603
|
+
}
|
|
604
|
+
});
|
|
605
|
+
}
|
|
606
|
+
/**
|
|
607
|
+
* Serialized local leader execution wrapper (#1712):
|
|
608
|
+
* 1. Acquires `runExclusive` so `SqliteStateDB` never overlaps dry-run/replay transactions.
|
|
609
|
+
* 2. Catches up finalized segments if behind `highQC`, and recovers any pending `lastTC.lockedCheckpoint`
|
|
610
|
+
* BEFORE `executor.execute` opens `statedb.runAsLambda`.
|
|
611
|
+
* 3. Asserts active view leadership + valid quorum lease.
|
|
612
|
+
* 4. Runs `fn()` (`executor.execute`), then broadcasts `/consensus/commit` after releasing `runExclusive`.
|
|
613
|
+
*/
|
|
614
|
+
async executeWithConsensusLock(fn) {
|
|
615
|
+
let segmentsToBroadcast = [];
|
|
616
|
+
let result;
|
|
617
|
+
try {
|
|
618
|
+
result = await this.runExclusive(async () => {
|
|
619
|
+
this.pendingBroadcastSegments = [];
|
|
620
|
+
this.activeProposalSlot = null;
|
|
621
|
+
await this.ensureReadyToLeadLocked();
|
|
622
|
+
try {
|
|
623
|
+
return await fn();
|
|
624
|
+
} finally {
|
|
625
|
+
if (this.pendingBroadcastSegments.length > 0) segmentsToBroadcast = [...this.pendingBroadcastSegments];
|
|
626
|
+
this.activeProposalSlot = null;
|
|
627
|
+
this.pendingBroadcastSegments = [];
|
|
628
|
+
}
|
|
629
|
+
});
|
|
630
|
+
} catch (err) {
|
|
631
|
+
if (segmentsToBroadcast.length > 0) await this.broadcastCommittedSegments(segmentsToBroadcast).catch(() => void 0);
|
|
632
|
+
throw err;
|
|
633
|
+
}
|
|
634
|
+
if (segmentsToBroadcast.length > 0) await this.broadcastCommittedSegments(segmentsToBroadcast).catch(() => void 0);
|
|
635
|
+
return result;
|
|
636
|
+
}
|
|
637
|
+
async ensureReadyToLeadLocked() {
|
|
638
|
+
if (this.halted || this.stopped) throw new ConsensusAbortError("CONSENSUS_ABORTED", "Validator runtime is halted or stopped");
|
|
639
|
+
const localHeight = (await this.resolver.ledger.getCheckpoint())?.height ?? 0;
|
|
640
|
+
if (this.highQC && localHeight < this.highQC.height) await this.catchUpToHeightLocked(this.highQC.height);
|
|
641
|
+
if (!this.refreshQuorumLease()) throw new ConsensusAbortError("CONSENSUS_QUORUM_LOST", `Validator ${this.selfDid} lacks active BFT quorum lease`);
|
|
642
|
+
let leader = (0, _arcblock_ledger_core.getLeaderForView)(this.clusterConfig, this.currentView);
|
|
643
|
+
if (leader.did !== this.selfDid) throw new ConsensusAbortError("CONSENSUS_NOT_LEADER", `Validator ${this.selfDid} is not leader for view ${this.currentView} (leader=${leader.did})`);
|
|
644
|
+
if (this.hasSpentTurnInView(this.currentView) || this.unleadableView === this.currentView) throw new ConsensusAbortError("CONSENSUS_VIEW_CHANGING", `Validator ${this.selfDid} cannot lead view ${this.currentView}; awaiting view change`);
|
|
645
|
+
const expectedNextHeight = (this.highQC?.height ?? 0) + 1;
|
|
646
|
+
if (this.lastTC && this.lastTC.viewNumber === this.currentView - 1 && this.lastTC.lockedCheckpoint && this.lastTC.lockedCheckpoint.height === expectedNextHeight) {
|
|
647
|
+
let recoveredSegment = null;
|
|
648
|
+
try {
|
|
649
|
+
recoveredSegment = await this.recoverLockedCheckpointLocked(this.lastTC);
|
|
650
|
+
} catch (err) {
|
|
651
|
+
this.markCurrentViewUnleadableAndScheduleTimeout();
|
|
652
|
+
throw err;
|
|
653
|
+
}
|
|
654
|
+
if (recoveredSegment) this.pendingBroadcastSegments.push(recoveredSegment);
|
|
655
|
+
else if (((await this.resolver.ledger.getCheckpoint())?.height ?? 0) < expectedNextHeight) {
|
|
656
|
+
this.markCurrentViewUnleadableAndScheduleTimeout();
|
|
657
|
+
throw new ConsensusAbortError("CONSENSUS_VIEW_CHANGING", `Validator ${this.selfDid} could not recover lockedCheckpoint at height ${expectedNextHeight} in view ${this.currentView}; rotating view`);
|
|
658
|
+
}
|
|
659
|
+
leader = (0, _arcblock_ledger_core.getLeaderForView)(this.clusterConfig, this.currentView);
|
|
660
|
+
if (leader.did !== this.selfDid) {
|
|
661
|
+
const toFlush = [...this.pendingBroadcastSegments];
|
|
662
|
+
this.pendingBroadcastSegments = [];
|
|
663
|
+
this.broadcastCommittedSegments(toFlush).catch(() => void 0);
|
|
664
|
+
throw new ConsensusAbortError("CONSENSUS_NOT_LEADER", `Recovered lockedCheckpoint at view ${this.currentView - 1}; leadership rotated to ${leader.did} for view ${this.currentView}`);
|
|
665
|
+
}
|
|
666
|
+
if (this.hasSpentTurnInView(this.currentView)) throw new ConsensusAbortError("CONSENSUS_VIEW_CHANGING", `Validator ${this.selfDid} already voted in view ${this.currentView}; awaiting view change`);
|
|
667
|
+
}
|
|
668
|
+
}
|
|
669
|
+
async prepareProposalSlot() {
|
|
670
|
+
if (!this.inConsensusLock) throw new ConsensusAbortError("CONSENSUS_ABORTED", "prepareProposalSlot must execute inside executeWithConsensusLock");
|
|
671
|
+
if (this.halted || this.stopped) throw new ConsensusAbortError("CONSENSUS_ABORTED", "Validator runtime is halted or stopped");
|
|
672
|
+
if (!this.refreshQuorumLease()) throw new ConsensusAbortError("CONSENSUS_QUORUM_LOST", "Validator lost BFT quorum lease");
|
|
673
|
+
if ((0, _arcblock_ledger_core.getLeaderForView)(this.clusterConfig, this.currentView).did !== this.selfDid) throw new ConsensusAbortError("CONSENSUS_NOT_LEADER", `Validator ${this.selfDid} is not leader for view ${this.currentView}`);
|
|
674
|
+
if (this.hasSpentTurnInView(this.currentView)) throw new ConsensusAbortError("CONSENSUS_VIEW_CHANGING", `Validator ${this.selfDid} already voted in view ${this.currentView}; awaiting view change`);
|
|
675
|
+
const latestCp = await this.resolver.ledger.getCheckpoint();
|
|
676
|
+
const sequence = (latestCp?.sequence ?? 0) + 1;
|
|
677
|
+
const timestampMs = Math.max(Date.now(), (latestCp?.timestamp ?? 0) + 1);
|
|
678
|
+
this.activeProposalSlot = {
|
|
679
|
+
sequence,
|
|
680
|
+
timestampMs,
|
|
681
|
+
viewNumber: this.currentView
|
|
682
|
+
};
|
|
683
|
+
return {
|
|
684
|
+
sequence,
|
|
685
|
+
timestampMs
|
|
686
|
+
};
|
|
687
|
+
}
|
|
688
|
+
async commitStagedEntry(params) {
|
|
689
|
+
if (!this.inConsensusLock || !this.activeProposalSlot) throw new ConsensusAbortError("CONSENSUS_ABORTED", "Missing active consensus proposal slot");
|
|
690
|
+
if (this.halted || this.stopped) throw new ConsensusAbortError("CONSENSUS_ABORTED", "Validator runtime is halted or stopped");
|
|
691
|
+
const slot = this.activeProposalSlot;
|
|
692
|
+
if (this.currentView !== slot.viewNumber) throw new ConsensusAbortError("CONSENSUS_VIEW_CHANGING", `Consensus view advanced during execution (${slot.viewNumber} -> ${this.currentView})`);
|
|
693
|
+
if ((0, _arcblock_ledger_core.getLeaderForView)(this.clusterConfig, this.currentView).did !== this.selfDid) throw new ConsensusAbortError("CONSENSUS_NOT_LEADER", `Validator ${this.selfDid} is no longer leader for view ${this.currentView}`);
|
|
694
|
+
const latestCp = await this.resolver.ledger.getCheckpoint();
|
|
695
|
+
const prevSequence = latestCp?.sequence ?? 0;
|
|
696
|
+
if (slot.sequence !== prevSequence + 1 || params.sequence !== slot.sequence) throw new ConsensusAbortError("CONSENSUS_ABORTED", `Proposal slot sequence (${slot.sequence}) no longer matches ledger tip + 1 (${prevSequence + 1})`);
|
|
697
|
+
const txInput = params.txInput;
|
|
698
|
+
const v2Cutoff = this.resolver.ledger.getEntryHashV2From?.() ?? 1;
|
|
699
|
+
const hashVersion = slot.sequence >= v2Cutoff ? 2 : 1;
|
|
700
|
+
const entryHash = (0, _arcblock_ledger_core.computeEntryHash)(slot.sequence, txInput.txRaw, params.stateCommitHash, hashVersion);
|
|
701
|
+
const entry = {
|
|
702
|
+
sequence: slot.sequence,
|
|
703
|
+
timestamp: slot.timestampMs,
|
|
704
|
+
hash: txInput.txHash.toUpperCase(),
|
|
705
|
+
from: txInput.from,
|
|
706
|
+
nonce: Number(txInput.nonce),
|
|
707
|
+
pk: txInput.pk,
|
|
708
|
+
signature: txInput.signature,
|
|
709
|
+
txRaw: txInput.txRaw,
|
|
710
|
+
tx: txInput.tx,
|
|
711
|
+
txType: txInput.txType,
|
|
712
|
+
...txInput.txExtra !== void 0 ? { txExtra: txInput.txExtra } : {},
|
|
713
|
+
chainId: this.chainId,
|
|
714
|
+
entryHash,
|
|
715
|
+
stateCommitHash: params.stateCommitHash,
|
|
716
|
+
finalized: true
|
|
717
|
+
};
|
|
718
|
+
const tree = new _arcblock_ledger_core.IncrementalMerkleTree();
|
|
719
|
+
tree.append(entryHash);
|
|
720
|
+
const txRoot = tree.getHexRoot();
|
|
721
|
+
const unsignedCp = {
|
|
722
|
+
height: (latestCp?.height ?? 0) + 1,
|
|
723
|
+
sequence: slot.sequence,
|
|
724
|
+
prevSequence,
|
|
725
|
+
txCount: 1,
|
|
726
|
+
timestamp: slot.timestampMs,
|
|
727
|
+
txRoot,
|
|
728
|
+
stateRoot: params.stateCommitHash,
|
|
729
|
+
prevCheckpoint: latestCp?.hash ?? null,
|
|
730
|
+
proposer: this.selfDid
|
|
731
|
+
};
|
|
732
|
+
const cpHash = (0, _arcblock_ledger_core.computeCheckpointHash)(unsignedCp);
|
|
733
|
+
const cpSigRaw = await this.signDigest(cpHash);
|
|
734
|
+
const cpSignature = cpSigRaw.startsWith("0x") ? cpSigRaw : `0x${cpSigRaw}`;
|
|
735
|
+
const baseCheckpoint = {
|
|
736
|
+
...unsignedCp,
|
|
737
|
+
hash: cpHash,
|
|
738
|
+
signature: cpSignature
|
|
739
|
+
};
|
|
740
|
+
const segmentContentHash = (0, _arcblock_ledger_core.computeSegmentContentHash)({
|
|
741
|
+
chainId: this.chainId,
|
|
742
|
+
checkpoint: baseCheckpoint,
|
|
743
|
+
entries: [entry]
|
|
744
|
+
});
|
|
745
|
+
const segCommitDigest = (0, _arcblock_ledger_core.computeSegmentCommitmentDigest)(cpHash, segmentContentHash);
|
|
746
|
+
const segCommitSigRaw = await this.signDigest(segCommitDigest);
|
|
747
|
+
const checkpoint = {
|
|
748
|
+
...baseCheckpoint,
|
|
749
|
+
segmentContentHash,
|
|
750
|
+
segmentCommitmentSignature: segCommitSigRaw.startsWith("0x") ? segCommitSigRaw : `0x${segCommitSigRaw}`
|
|
751
|
+
};
|
|
752
|
+
const effectiveHighQC = (0, _arcblock_ledger_core.selectHighestQC)([this.highQC, this.lastTC?.winningHighQC]);
|
|
753
|
+
const needsTC = Boolean(this.lastTC) && this.lastTC.viewNumber === this.currentView - 1 && (effectiveHighQC ? this.currentView > effectiveHighQC.viewNumber + 1 : this.currentView > 0);
|
|
754
|
+
let proposal;
|
|
755
|
+
try {
|
|
756
|
+
proposal = await (0, _arcblock_ledger_core.createHotStuffProposal)({
|
|
757
|
+
config: this.clusterConfig,
|
|
758
|
+
viewNumber: this.currentView,
|
|
759
|
+
checkpoint,
|
|
760
|
+
leaderWallet: this.validatorWallet,
|
|
761
|
+
leaderDid: this.selfDid,
|
|
762
|
+
highQC: effectiveHighQC,
|
|
763
|
+
timeoutCert: needsTC ? this.lastTC : null,
|
|
764
|
+
timestamp: slot.timestampMs
|
|
765
|
+
});
|
|
766
|
+
} catch (err) {
|
|
767
|
+
if (err?.isConsensusAbort === true) throw err;
|
|
768
|
+
throw new ConsensusAbortError("CONSENSUS_ABORTED", err instanceof Error ? err.message : String(err));
|
|
769
|
+
}
|
|
770
|
+
const preVoteCheck = await (0, _arcblock_ledger_core.canVoterPrepareProposal)({
|
|
771
|
+
config: this.clusterConfig,
|
|
772
|
+
proposal,
|
|
773
|
+
voterDid: this.selfDid,
|
|
774
|
+
lockedQC: this.lockedQC,
|
|
775
|
+
voterStateStore: this.stateStore
|
|
776
|
+
});
|
|
777
|
+
if (!preVoteCheck.ok) throw new ConsensusAbortError("CONSENSUS_ABORTED", preVoteCheck.reason || "Leader voter lock rejected proposal");
|
|
778
|
+
await this.stateStore.saveUncommittedProposalEntries?.({
|
|
779
|
+
chainId: this.chainId,
|
|
780
|
+
voterDid: this.selfDid,
|
|
781
|
+
checkpointHash: checkpoint.hash,
|
|
782
|
+
height: checkpoint.height,
|
|
783
|
+
entries: [entry],
|
|
784
|
+
checkpoint,
|
|
785
|
+
updatedAt: slot.timestampMs
|
|
786
|
+
});
|
|
787
|
+
this.lastSelfPreparedView = Math.max(this.lastSelfPreparedView, proposal.viewNumber);
|
|
788
|
+
let qc;
|
|
789
|
+
try {
|
|
790
|
+
qc = await this.collectVotesAndFormQC(proposal, [entry], params.stateCommitHash, slot.timestampMs);
|
|
791
|
+
} catch (err) {
|
|
792
|
+
if (this.currentView === proposal.viewNumber) {
|
|
793
|
+
const stalledView = proposal.viewNumber;
|
|
794
|
+
const t = setTimeout(() => {
|
|
795
|
+
this.onViewTimerExpired(stalledView).catch(() => void 0);
|
|
796
|
+
}, 0);
|
|
797
|
+
if (typeof t.unref === "function") t.unref();
|
|
798
|
+
}
|
|
799
|
+
if (err?.isConsensusAbort === true) throw err;
|
|
800
|
+
throw new ConsensusAbortError("CONSENSUS_ABORTED", err instanceof Error ? err.message : String(err));
|
|
801
|
+
}
|
|
802
|
+
const sealedCheckpoint = {
|
|
803
|
+
...checkpoint,
|
|
804
|
+
qc
|
|
805
|
+
};
|
|
806
|
+
const finalizedSegment = {
|
|
807
|
+
chainId: this.chainId,
|
|
808
|
+
checkpoint: sealedCheckpoint,
|
|
809
|
+
entries: [entry]
|
|
810
|
+
};
|
|
811
|
+
await this.resolver.ledger.importSegment(finalizedSegment, {
|
|
812
|
+
hotstuffClusterConfig: this.clusterConfig,
|
|
813
|
+
verifyStateRoot: true
|
|
814
|
+
});
|
|
815
|
+
this.lockedQC = (0, _arcblock_ledger_core.selectHighestQC)([this.lockedQC, effectiveHighQC]);
|
|
816
|
+
this.highQC = (0, _arcblock_ledger_core.selectHighestQC)([this.highQC, qc]);
|
|
817
|
+
this.advanceCurrentViewTo(qc.viewNumber + 1);
|
|
818
|
+
this.currentViewEnteredAtMs = Date.now();
|
|
819
|
+
this.pendingDemandForView = -1;
|
|
820
|
+
this.consecutiveViewTimeouts = 0;
|
|
821
|
+
this.leaderRouter.advanceMinVerifiedCheckpointHeight(sealedCheckpoint.height);
|
|
822
|
+
try {
|
|
823
|
+
await this.stateStore.saveRuntimeState?.({
|
|
824
|
+
chainId: this.chainId,
|
|
825
|
+
voterDid: this.selfDid,
|
|
826
|
+
currentView: this.currentView,
|
|
827
|
+
lockedQC: this.lockedQC,
|
|
828
|
+
highQC: this.highQC,
|
|
829
|
+
lastTC: this.lastTC,
|
|
830
|
+
pruneUncommittedUpToHeight: sealedCheckpoint.height
|
|
831
|
+
});
|
|
832
|
+
} catch (saveErr) {
|
|
833
|
+
this.resolver.logger.warn("Failed to persist post-commit HotStuff runtime metadata; continuing commit", {
|
|
834
|
+
chainId: this.chainId,
|
|
835
|
+
height: sealedCheckpoint.height,
|
|
836
|
+
error: saveErr instanceof Error ? saveErr.message : String(saveErr)
|
|
837
|
+
});
|
|
838
|
+
}
|
|
839
|
+
this.scheduleViewTimeout();
|
|
840
|
+
this.pendingBroadcastSegments.push(finalizedSegment);
|
|
841
|
+
return entry;
|
|
842
|
+
}
|
|
843
|
+
async collectVotesAndFormQC(proposal, entries, expectedStateRoot, createdAtMs) {
|
|
844
|
+
const threshold = this.clusterConfig.quorumThreshold ?? 3;
|
|
845
|
+
const peerPrepareVotes = [];
|
|
846
|
+
const seenPreparePeerDids = /* @__PURE__ */ new Set();
|
|
847
|
+
let reachablePeers = 0;
|
|
848
|
+
await Promise.all(this.peerValidators.map(async (peer) => {
|
|
849
|
+
try {
|
|
850
|
+
const res = await this.sendConsensusJson(peer, "POST", "/consensus/propose", {
|
|
851
|
+
proposal,
|
|
852
|
+
entries
|
|
853
|
+
}, this.proposalTimeoutMs, MAX_CONTROL_ROUTE_BYTES);
|
|
854
|
+
if (res.authenticated) reachablePeers++;
|
|
855
|
+
if (res.body?.highQC) this.tryAbsorbRemoteQC(res.body.highQC);
|
|
856
|
+
if (res.body?.lastTC) this.tryAbsorbRemoteTC(res.body.lastTC);
|
|
857
|
+
if (res.status === 200 && res.body?.prepareVote) {
|
|
858
|
+
const pv = res.body.prepareVote;
|
|
859
|
+
if (pv.voterDid === peer.did && pv.viewNumber === proposal.viewNumber && normalizeHexCompare(pv.checkpointHash) === normalizeHexCompare(proposal.checkpoint.hash) && !seenPreparePeerDids.has(peer.did) && (0, _arcblock_ledger_core.verifyHotStuffPrepareVote)(pv, this.clusterConfig).ok) {
|
|
860
|
+
seenPreparePeerDids.add(peer.did);
|
|
861
|
+
peerPrepareVotes.push(pv);
|
|
862
|
+
if (this.onLeaderReplicaVoteReceived) await this.onLeaderReplicaVoteReceived({
|
|
863
|
+
leaderDid: this.selfDid,
|
|
864
|
+
voterDid: peer.did,
|
|
865
|
+
viewNumber: proposal.viewNumber,
|
|
866
|
+
height: proposal.checkpoint.height,
|
|
867
|
+
voteCountIncludingSelf: peerPrepareVotes.length + 1
|
|
868
|
+
});
|
|
869
|
+
}
|
|
870
|
+
}
|
|
871
|
+
} catch {}
|
|
872
|
+
}));
|
|
873
|
+
if (reachablePeers + 1 >= threshold) {
|
|
874
|
+
this.quorumContactObserved = true;
|
|
875
|
+
this.quorumLeaseValid = true;
|
|
876
|
+
} else this.quorumLeaseValid = false;
|
|
877
|
+
if (this.currentView > proposal.viewNumber) throw new ConsensusAbortError("CONSENSUS_VIEW_CHANGING", `Stepped down during proposal view ${proposal.viewNumber} -> ${this.currentView}`);
|
|
878
|
+
if (peerPrepareVotes.length + 1 < threshold) throw new ConsensusAbortError(reachablePeers + 1 < threshold ? "CONSENSUS_QUORUM_LOST" : "CONSENSUS_ABORTED", `Failed to reach ${threshold} BFT PrepareVotes for height ${proposal.checkpoint.height} in view ${proposal.viewNumber} (got ${peerPrepareVotes.length + 1}, reachable=${reachablePeers + 1})`);
|
|
879
|
+
let selfPrepareVote;
|
|
880
|
+
try {
|
|
881
|
+
selfPrepareVote = await (0, _arcblock_ledger_core.verifyProposalAndCreatePrepareVote)({
|
|
882
|
+
config: this.clusterConfig,
|
|
883
|
+
proposal,
|
|
884
|
+
entries,
|
|
885
|
+
voterWallet: this.validatorWallet,
|
|
886
|
+
voterDid: this.selfDid,
|
|
887
|
+
lockedQC: this.lockedQC,
|
|
888
|
+
voterStateStore: this.stateStore,
|
|
889
|
+
expectedStateRoot,
|
|
890
|
+
requireSegmentCommitment: true,
|
|
891
|
+
timestamp: createdAtMs
|
|
892
|
+
});
|
|
893
|
+
} catch (err) {
|
|
894
|
+
if (err?.isConsensusAbort === true) throw err;
|
|
895
|
+
throw new ConsensusAbortError("CONSENSUS_ABORTED", err instanceof Error ? err.message : String(err));
|
|
896
|
+
}
|
|
897
|
+
const prepareQC = (0, _arcblock_ledger_core.aggregatePrepareQuorumCertificate)({
|
|
898
|
+
config: this.clusterConfig,
|
|
899
|
+
proposal,
|
|
900
|
+
votes: [selfPrepareVote, ...peerPrepareVotes],
|
|
901
|
+
createdAt: createdAtMs
|
|
902
|
+
});
|
|
903
|
+
const effectiveParentQC = (0, _arcblock_ledger_core.selectHighestQC)([proposal.highQC, proposal.timeoutCert?.winningHighQC]);
|
|
904
|
+
const peerCommitVotes = [];
|
|
905
|
+
const seenCommitPeerDids = /* @__PURE__ */ new Set();
|
|
906
|
+
let precommitReachablePeers = 0;
|
|
907
|
+
await Promise.all(this.peerValidators.map(async (peer) => {
|
|
908
|
+
try {
|
|
909
|
+
let res = await this.sendConsensusJson(peer, "POST", "/consensus/precommit", {
|
|
910
|
+
viewNumber: proposal.viewNumber,
|
|
911
|
+
prepareQC,
|
|
912
|
+
checkpoint: proposal.checkpoint,
|
|
913
|
+
parentHighQC: effectiveParentQC
|
|
914
|
+
}, this.proposalTimeoutMs, MAX_CONTROL_ROUTE_BYTES);
|
|
915
|
+
if (res.status === 409 && res.body?.code === "MISSING_UNCOMMITTED_ENTRIES") res = await this.sendConsensusJson(peer, "POST", "/consensus/precommit", {
|
|
916
|
+
viewNumber: proposal.viewNumber,
|
|
917
|
+
prepareQC,
|
|
918
|
+
checkpoint: proposal.checkpoint,
|
|
919
|
+
parentHighQC: effectiveParentQC,
|
|
920
|
+
entries
|
|
921
|
+
}, this.proposalTimeoutMs, MAX_CONTROL_ROUTE_BYTES);
|
|
922
|
+
if (res.authenticated) precommitReachablePeers++;
|
|
923
|
+
if (res.body?.highQC) this.tryAbsorbRemoteQC(res.body.highQC);
|
|
924
|
+
if (res.body?.lastTC) this.tryAbsorbRemoteTC(res.body.lastTC);
|
|
925
|
+
if (res.status === 200 && res.body?.vote) {
|
|
926
|
+
const cv = res.body.vote;
|
|
927
|
+
if (cv.voterDid === peer.did && cv.viewNumber === proposal.viewNumber && normalizeHexCompare(cv.checkpointHash) === normalizeHexCompare(proposal.checkpoint.hash) && !seenCommitPeerDids.has(peer.did) && (0, _arcblock_ledger_core.verifyHotStuffVote)(cv, this.clusterConfig).ok) {
|
|
928
|
+
seenCommitPeerDids.add(peer.did);
|
|
929
|
+
peerCommitVotes.push(cv);
|
|
930
|
+
if (this.onLeaderReplicaPrecommitReceived) await this.onLeaderReplicaPrecommitReceived({
|
|
931
|
+
leaderDid: this.selfDid,
|
|
932
|
+
voterDid: peer.did,
|
|
933
|
+
viewNumber: proposal.viewNumber,
|
|
934
|
+
height: proposal.checkpoint.height,
|
|
935
|
+
commitVoteCountIncludingSelf: peerCommitVotes.length + 1
|
|
936
|
+
});
|
|
937
|
+
}
|
|
938
|
+
}
|
|
939
|
+
} catch {}
|
|
940
|
+
}));
|
|
941
|
+
if (precommitReachablePeers + 1 < threshold) this.quorumLeaseValid = false;
|
|
942
|
+
if (this.currentView > proposal.viewNumber) throw new ConsensusAbortError("CONSENSUS_VIEW_CHANGING", `Stepped down during precommit view ${proposal.viewNumber} -> ${this.currentView}`);
|
|
943
|
+
if (peerCommitVotes.length + 1 < threshold) throw new ConsensusAbortError(precommitReachablePeers + 1 < threshold ? "CONSENSUS_QUORUM_LOST" : "CONSENSUS_ABORTED", `Failed to reach ${threshold} BFT CommitVotes for height ${proposal.checkpoint.height} in view ${proposal.viewNumber} (got ${peerCommitVotes.length + 1}, reachable=${precommitReachablePeers + 1})`);
|
|
944
|
+
const commitTimestampMs = Math.max(createdAtMs, Date.now());
|
|
945
|
+
let selfCommitVote;
|
|
946
|
+
try {
|
|
947
|
+
selfCommitVote = await (0, _arcblock_ledger_core.verifyPrepareQCAndCreateCommitVote)({
|
|
948
|
+
config: this.clusterConfig,
|
|
949
|
+
prepareQC,
|
|
950
|
+
checkpoint: proposal.checkpoint,
|
|
951
|
+
voterWallet: this.validatorWallet,
|
|
952
|
+
voterDid: this.selfDid,
|
|
953
|
+
parentHighQC: effectiveParentQC,
|
|
954
|
+
voterStateStore: this.stateStore,
|
|
955
|
+
timestamp: commitTimestampMs
|
|
956
|
+
});
|
|
957
|
+
} catch (err) {
|
|
958
|
+
if (err?.isConsensusAbort === true) throw err;
|
|
959
|
+
throw new ConsensusAbortError("CONSENSUS_ABORTED", err instanceof Error ? err.message : String(err));
|
|
960
|
+
}
|
|
961
|
+
this.lastSelfVotedView = Math.max(this.lastSelfVotedView, selfCommitVote.viewNumber);
|
|
962
|
+
return (0, _arcblock_ledger_core.aggregateQuorumCertificate)({
|
|
963
|
+
config: this.clusterConfig,
|
|
964
|
+
proposal,
|
|
965
|
+
votes: [selfCommitVote, ...peerCommitVotes],
|
|
966
|
+
createdAt: commitTimestampMs
|
|
967
|
+
});
|
|
968
|
+
}
|
|
969
|
+
async hasValidCheckpointSegmentCommitment(cp, expectedSegContentHash) {
|
|
970
|
+
if (!cp || !cp.segmentContentHash || !cp.segmentCommitmentSignature) return false;
|
|
971
|
+
if (normalizeHexCompare(cp.segmentContentHash) !== normalizeHexCompare(expectedSegContentHash)) return false;
|
|
972
|
+
const cpProposerMember = this.clusterConfig.validators.find((v) => v.did === cp.proposer);
|
|
973
|
+
if (!cpProposerMember) return false;
|
|
974
|
+
return (0, _arcblock_ledger_core.verifySegmentCommitmentSignature)(cp, expectedSegContentHash, cpProposerMember.publicKey, cpProposerMember.did);
|
|
975
|
+
}
|
|
976
|
+
async recoverLockedCheckpointLocked(tc) {
|
|
977
|
+
let lockedCp = tc.lockedCheckpoint;
|
|
978
|
+
if (!lockedCp) return null;
|
|
979
|
+
const localCp = await this.resolver.ledger.getCheckpoint();
|
|
980
|
+
if (localCp && localCp.height >= lockedCp.height) return null;
|
|
981
|
+
let entries = await this.stateStore.getUncommittedProposalEntries?.(this.chainId, this.selfDid, lockedCp.hash);
|
|
982
|
+
const savedCp = await this.stateStore.getUncommittedProposalCheckpoint?.(this.chainId, this.selfDid, lockedCp.hash);
|
|
983
|
+
let expectedSegContentHash = entries && entries.length === lockedCp.txCount ? (0, _arcblock_ledger_core.computeSegmentContentHash)({
|
|
984
|
+
chainId: this.chainId,
|
|
985
|
+
checkpoint: lockedCp,
|
|
986
|
+
entries
|
|
987
|
+
}) : "";
|
|
988
|
+
let hasValidCommitment = expectedSegContentHash ? await this.hasValidCheckpointSegmentCommitment(lockedCp, expectedSegContentHash) : false;
|
|
989
|
+
if (!hasValidCommitment && expectedSegContentHash && savedCp && normalizeHexCompare(savedCp.hash) === normalizeHexCompare(lockedCp.hash) && await this.hasValidCheckpointSegmentCommitment(savedCp, expectedSegContentHash)) {
|
|
990
|
+
lockedCp = {
|
|
991
|
+
...lockedCp,
|
|
992
|
+
segmentContentHash: savedCp.segmentContentHash,
|
|
993
|
+
segmentCommitmentSignature: savedCp.segmentCommitmentSignature
|
|
994
|
+
};
|
|
995
|
+
hasValidCommitment = true;
|
|
996
|
+
}
|
|
997
|
+
if (!entries || entries.length !== lockedCp.txCount || !hasValidCommitment) for (const peer of this.peerValidators) try {
|
|
998
|
+
const res = await this.sendConsensusJson(peer, "GET", `/consensus/uncommitted?checkpointHash=${encodeURIComponent(lockedCp.hash)}`, void 0, this.httpTimeoutMs, MAX_DATA_ROUTE_BYTES);
|
|
999
|
+
if (!res.authenticated || res.status !== 200) continue;
|
|
1000
|
+
if (!entries || entries.length !== lockedCp.txCount) {
|
|
1001
|
+
if (!Array.isArray(res.body?.entries) || res.body.entries.length !== lockedCp.txCount) continue;
|
|
1002
|
+
const dryRun$1 = await this.replayer.dryRunVerifyEntries(res.body.entries, {
|
|
1003
|
+
expectedPrevSequence: lockedCp.prevSequence,
|
|
1004
|
+
prevStateRoot: localCp?.stateRoot ?? null
|
|
1005
|
+
}).catch(() => null);
|
|
1006
|
+
if (!dryRun$1 || normalizeHexCompare(dryRun$1.derivedStateRoot) !== normalizeHexCompare(lockedCp.stateRoot)) continue;
|
|
1007
|
+
const tree = new _arcblock_ledger_core.IncrementalMerkleTree();
|
|
1008
|
+
for (const h of dryRun$1.entryHashes) tree.append(h);
|
|
1009
|
+
if (normalizeHexCompare(tree.getHexRoot()) !== normalizeHexCompare(lockedCp.txRoot)) continue;
|
|
1010
|
+
entries = res.body.entries;
|
|
1011
|
+
expectedSegContentHash = (0, _arcblock_ledger_core.computeSegmentContentHash)({
|
|
1012
|
+
chainId: this.chainId,
|
|
1013
|
+
checkpoint: lockedCp,
|
|
1014
|
+
entries
|
|
1015
|
+
});
|
|
1016
|
+
hasValidCommitment = await this.hasValidCheckpointSegmentCommitment(lockedCp, expectedSegContentHash);
|
|
1017
|
+
if (!hasValidCommitment && savedCp && normalizeHexCompare(savedCp.hash) === normalizeHexCompare(lockedCp.hash) && await this.hasValidCheckpointSegmentCommitment(savedCp, expectedSegContentHash)) {
|
|
1018
|
+
lockedCp = {
|
|
1019
|
+
...lockedCp,
|
|
1020
|
+
segmentContentHash: savedCp.segmentContentHash,
|
|
1021
|
+
segmentCommitmentSignature: savedCp.segmentCommitmentSignature
|
|
1022
|
+
};
|
|
1023
|
+
hasValidCommitment = true;
|
|
1024
|
+
}
|
|
1025
|
+
}
|
|
1026
|
+
if (!hasValidCommitment && expectedSegContentHash && res.body?.checkpoint && normalizeHexCompare(res.body.checkpoint.hash) === normalizeHexCompare(lockedCp.hash) && await this.hasValidCheckpointSegmentCommitment(res.body.checkpoint, expectedSegContentHash)) {
|
|
1027
|
+
lockedCp = {
|
|
1028
|
+
...lockedCp,
|
|
1029
|
+
segmentContentHash: res.body.checkpoint.segmentContentHash,
|
|
1030
|
+
segmentCommitmentSignature: res.body.checkpoint.segmentCommitmentSignature
|
|
1031
|
+
};
|
|
1032
|
+
hasValidCommitment = true;
|
|
1033
|
+
}
|
|
1034
|
+
if (entries && entries.length === lockedCp.txCount && hasValidCommitment) break;
|
|
1035
|
+
} catch {}
|
|
1036
|
+
if (!entries || entries.length !== lockedCp.txCount) throw new ConsensusAbortError("CONSENSUS_ABORTED", `Unable to fetch uncommitted entries for lockedCheckpoint ${lockedCp.hash} at height ${lockedCp.height}`);
|
|
1037
|
+
if (!hasValidCommitment) throw new ConsensusAbortError("CONSENSUS_ABORTED", `Unable to recover verified segmentCommitmentSignature for lockedCheckpoint ${lockedCp.hash} at height ${lockedCp.height}`);
|
|
1038
|
+
const dryRun = await this.replayer.dryRunVerifyEntries(entries, {
|
|
1039
|
+
expectedPrevSequence: lockedCp.prevSequence,
|
|
1040
|
+
prevStateRoot: localCp?.stateRoot ?? null
|
|
1041
|
+
});
|
|
1042
|
+
if (normalizeHexCompare(dryRun.derivedStateRoot) !== normalizeHexCompare(lockedCp.stateRoot)) throw new ConsensusAbortError("CONSENSUS_ABORTED", `Recovered lockedCheckpoint stateRoot mismatch at height ${lockedCp.height}`);
|
|
1043
|
+
await this.stateStore.saveUncommittedProposalEntries?.({
|
|
1044
|
+
chainId: this.chainId,
|
|
1045
|
+
voterDid: this.selfDid,
|
|
1046
|
+
checkpointHash: lockedCp.hash,
|
|
1047
|
+
height: lockedCp.height,
|
|
1048
|
+
entries,
|
|
1049
|
+
checkpoint: lockedCp
|
|
1050
|
+
});
|
|
1051
|
+
const effectiveHighQC = (0, _arcblock_ledger_core.selectHighestQC)([this.highQC, tc.winningHighQC]);
|
|
1052
|
+
const nowMs = Date.now();
|
|
1053
|
+
let proposal;
|
|
1054
|
+
try {
|
|
1055
|
+
proposal = await (0, _arcblock_ledger_core.createHotStuffProposal)({
|
|
1056
|
+
config: this.clusterConfig,
|
|
1057
|
+
viewNumber: this.currentView,
|
|
1058
|
+
checkpoint: lockedCp,
|
|
1059
|
+
leaderWallet: this.validatorWallet,
|
|
1060
|
+
leaderDid: this.selfDid,
|
|
1061
|
+
highQC: effectiveHighQC,
|
|
1062
|
+
timeoutCert: tc,
|
|
1063
|
+
timestamp: nowMs
|
|
1064
|
+
});
|
|
1065
|
+
} catch (err) {
|
|
1066
|
+
if (err?.isConsensusAbort === true) throw err;
|
|
1067
|
+
throw new ConsensusAbortError("CONSENSUS_ABORTED", err instanceof Error ? err.message : String(err));
|
|
1068
|
+
}
|
|
1069
|
+
const preVoteCheck = await (0, _arcblock_ledger_core.canVoterPrepareProposal)({
|
|
1070
|
+
config: this.clusterConfig,
|
|
1071
|
+
proposal,
|
|
1072
|
+
voterDid: this.selfDid,
|
|
1073
|
+
lockedQC: this.lockedQC,
|
|
1074
|
+
voterStateStore: this.stateStore
|
|
1075
|
+
});
|
|
1076
|
+
if (!preVoteCheck.ok) throw new ConsensusAbortError("CONSENSUS_ABORTED", preVoteCheck.reason || "Leader voter lock rejected recovery proposal");
|
|
1077
|
+
this.lastSelfPreparedView = Math.max(this.lastSelfPreparedView, proposal.viewNumber);
|
|
1078
|
+
let qc;
|
|
1079
|
+
try {
|
|
1080
|
+
qc = await this.collectVotesAndFormQC(proposal, entries, dryRun.derivedStateRoot ?? void 0, nowMs);
|
|
1081
|
+
} catch (err) {
|
|
1082
|
+
if (this.currentView === proposal.viewNumber) {
|
|
1083
|
+
const stalledView = proposal.viewNumber;
|
|
1084
|
+
const t = setTimeout(() => {
|
|
1085
|
+
this.onViewTimerExpired(stalledView).catch(() => void 0);
|
|
1086
|
+
}, 0);
|
|
1087
|
+
if (typeof t.unref === "function") t.unref();
|
|
1088
|
+
}
|
|
1089
|
+
if (err?.isConsensusAbort === true) throw err;
|
|
1090
|
+
throw new ConsensusAbortError("CONSENSUS_ABORTED", err instanceof Error ? err.message : String(err));
|
|
1091
|
+
}
|
|
1092
|
+
const sealedCheckpoint = {
|
|
1093
|
+
...lockedCp,
|
|
1094
|
+
qc
|
|
1095
|
+
};
|
|
1096
|
+
const finalizedSegment = {
|
|
1097
|
+
chainId: this.chainId,
|
|
1098
|
+
checkpoint: sealedCheckpoint,
|
|
1099
|
+
entries
|
|
1100
|
+
};
|
|
1101
|
+
if (typeof this.resolver.ledger.verifySegmentForImport === "function") await this.resolver.ledger.verifySegmentForImport(finalizedSegment, {
|
|
1102
|
+
hotstuffClusterConfig: this.clusterConfig,
|
|
1103
|
+
verifyStateRoot: true,
|
|
1104
|
+
allowAheadLocalSequence: true
|
|
1105
|
+
});
|
|
1106
|
+
await this.resolver.ledger.importSegment(finalizedSegment, {
|
|
1107
|
+
hotstuffClusterConfig: this.clusterConfig,
|
|
1108
|
+
verifyStateRoot: true
|
|
1109
|
+
});
|
|
1110
|
+
for (const entry of entries) await this.replayer.replayEntry(entry, { appendToLedger: false });
|
|
1111
|
+
this.lockedQC = (0, _arcblock_ledger_core.selectHighestQC)([this.lockedQC, effectiveHighQC]);
|
|
1112
|
+
this.highQC = (0, _arcblock_ledger_core.selectHighestQC)([this.highQC, qc]);
|
|
1113
|
+
this.advanceCurrentViewTo(qc.viewNumber + 1);
|
|
1114
|
+
this.currentViewEnteredAtMs = Date.now();
|
|
1115
|
+
this.pendingDemandForView = -1;
|
|
1116
|
+
this.consecutiveViewTimeouts = 0;
|
|
1117
|
+
this.leaderRouter.advanceMinVerifiedCheckpointHeight(sealedCheckpoint.height);
|
|
1118
|
+
try {
|
|
1119
|
+
await this.stateStore.saveRuntimeState?.({
|
|
1120
|
+
chainId: this.chainId,
|
|
1121
|
+
voterDid: this.selfDid,
|
|
1122
|
+
currentView: this.currentView,
|
|
1123
|
+
lockedQC: this.lockedQC,
|
|
1124
|
+
highQC: this.highQC,
|
|
1125
|
+
lastTC: this.lastTC,
|
|
1126
|
+
pruneUncommittedUpToHeight: sealedCheckpoint.height
|
|
1127
|
+
});
|
|
1128
|
+
} catch {}
|
|
1129
|
+
this.scheduleViewTimeout();
|
|
1130
|
+
return finalizedSegment;
|
|
1131
|
+
}
|
|
1132
|
+
async broadcastCommittedSegments(segments) {
|
|
1133
|
+
for (const segment of segments) {
|
|
1134
|
+
const nextView = (segment.checkpoint.qc?.viewNumber ?? this.currentView - 1) + 1;
|
|
1135
|
+
const nextLeader = (0, _arcblock_ledger_core.getLeaderForView)(this.clusterConfig, nextView);
|
|
1136
|
+
const tasks = this.peerValidators.map(async (peer) => {
|
|
1137
|
+
try {
|
|
1138
|
+
await this.sendConsensusJson(peer, "POST", "/consensus/commit", { segment }, this.httpTimeoutMs, MAX_CONTROL_ROUTE_BYTES);
|
|
1139
|
+
} catch {}
|
|
1140
|
+
});
|
|
1141
|
+
const nextLeaderIdx = this.peerValidators.findIndex((p) => p.did === nextLeader.did);
|
|
1142
|
+
if (nextLeaderIdx >= 0) await Promise.race([tasks[nextLeaderIdx], new Promise((resolve) => {
|
|
1143
|
+
const t = setTimeout(resolve, NEXT_LEADER_COMMIT_WAIT_MS);
|
|
1144
|
+
if (typeof t.unref === "function") t.unref();
|
|
1145
|
+
})]);
|
|
1146
|
+
Promise.allSettled(tasks);
|
|
1147
|
+
}
|
|
1148
|
+
}
|
|
1149
|
+
tryAbsorbRemoteQC(candidateQC) {
|
|
1150
|
+
if (!candidateQC) return false;
|
|
1151
|
+
if (!(0, _arcblock_ledger_core.verifyQuorumCertificate)(candidateQC, this.clusterConfig).ok) return false;
|
|
1152
|
+
const prevView = this.currentView;
|
|
1153
|
+
const prevHeight = this.highQC?.height ?? 0;
|
|
1154
|
+
this.highQC = (0, _arcblock_ledger_core.selectHighestQC)([this.highQC, candidateQC]);
|
|
1155
|
+
const heightAdvanced = (this.highQC?.height ?? 0) > prevHeight;
|
|
1156
|
+
if (this.highQC && this.highQC.viewNumber + 1 > this.currentView) {
|
|
1157
|
+
this.advanceCurrentViewTo(this.highQC.viewNumber + 1);
|
|
1158
|
+
this.consecutiveViewTimeouts = 0;
|
|
1159
|
+
} else if (heightAdvanced) {
|
|
1160
|
+
this.currentViewEnteredAtMs = Date.now();
|
|
1161
|
+
this.pendingDemandForView = -1;
|
|
1162
|
+
this.consecutiveViewTimeouts = 0;
|
|
1163
|
+
}
|
|
1164
|
+
return this.currentView > prevView || heightAdvanced;
|
|
1165
|
+
}
|
|
1166
|
+
isBetterTimeoutCertificate(nextTC) {
|
|
1167
|
+
if (!this.lastTC) return true;
|
|
1168
|
+
if (nextTC.viewNumber > this.lastTC.viewNumber) return true;
|
|
1169
|
+
if (nextTC.viewNumber === this.lastTC.viewNumber && nextTC.signatures.length > this.lastTC.signatures.length) return true;
|
|
1170
|
+
return false;
|
|
1171
|
+
}
|
|
1172
|
+
markCurrentViewUnleadableAndScheduleTimeout() {
|
|
1173
|
+
const stalledView = this.currentView;
|
|
1174
|
+
this.unleadableView = stalledView;
|
|
1175
|
+
const t = setTimeout(() => {
|
|
1176
|
+
this.onViewTimerExpired(stalledView).catch(() => void 0);
|
|
1177
|
+
}, 0);
|
|
1178
|
+
if (typeof t.unref === "function") t.unref();
|
|
1179
|
+
}
|
|
1180
|
+
tryAbsorbRemoteTC(candidateTC) {
|
|
1181
|
+
if (!candidateTC) return false;
|
|
1182
|
+
if (!(0, _arcblock_ledger_core.verifyTimeoutCertificate)(candidateTC, this.clusterConfig).ok) return false;
|
|
1183
|
+
if (candidateTC.winningHighQC) this.tryAbsorbRemoteQC(candidateTC.winningHighQC);
|
|
1184
|
+
const prevView = this.currentView;
|
|
1185
|
+
let upgradedSameViewTC = false;
|
|
1186
|
+
if (this.isBetterTimeoutCertificate(candidateTC)) {
|
|
1187
|
+
upgradedSameViewTC = Boolean(this.lastTC && candidateTC.viewNumber === this.lastTC.viewNumber);
|
|
1188
|
+
this.lastTC = candidateTC;
|
|
1189
|
+
if (upgradedSameViewTC && this.unleadableView === candidateTC.viewNumber + 1) this.unleadableView = -1;
|
|
1190
|
+
}
|
|
1191
|
+
if (candidateTC.viewNumber + 1 > this.currentView) {
|
|
1192
|
+
this.advanceCurrentViewTo(candidateTC.viewNumber + 1);
|
|
1193
|
+
this.consecutiveViewTimeouts = 0;
|
|
1194
|
+
}
|
|
1195
|
+
return this.currentView > prevView || upgradedSameViewTC;
|
|
1196
|
+
}
|
|
1197
|
+
async catchUpToHeightLocked(targetHeight) {
|
|
1198
|
+
let localCp = await this.resolver.ledger.getCheckpoint();
|
|
1199
|
+
let currentHeight = localCp?.height ?? 0;
|
|
1200
|
+
if (currentHeight >= targetHeight || this.peerValidators.length === 0) return;
|
|
1201
|
+
let rotationOffset = Math.max(0, this.currentView) % this.peerValidators.length;
|
|
1202
|
+
while (!this.halted && !this.stopped && currentHeight < targetHeight) {
|
|
1203
|
+
const fromHeight = currentHeight + 1;
|
|
1204
|
+
const toHeight = Math.min(targetHeight, fromHeight + MAX_CATCHUP_SEGMENTS_PER_REQUEST - 1);
|
|
1205
|
+
let importedInBatch = 0;
|
|
1206
|
+
for (let i = 0; i < this.peerValidators.length; i++) {
|
|
1207
|
+
const peer = this.peerValidators[(rotationOffset + i) % this.peerValidators.length];
|
|
1208
|
+
let candidateSegments = null;
|
|
1209
|
+
try {
|
|
1210
|
+
const res = await this.sendConsensusJson(peer, "GET", `/consensus/entries?fromHeight=${fromHeight}&toHeight=${toHeight}`, void 0, this.httpTimeoutMs, MAX_DATA_ROUTE_BYTES);
|
|
1211
|
+
if (res.authenticated && res.status === 200 && Array.isArray(res.body?.segments) && res.body.segments.length > 0) candidateSegments = res.body.segments;
|
|
1212
|
+
} catch {}
|
|
1213
|
+
if (!candidateSegments) continue;
|
|
1214
|
+
let importedFromPeer = 0;
|
|
1215
|
+
for (const seg of candidateSegments) {
|
|
1216
|
+
if (!seg || seg.chainId !== this.chainId || !seg.checkpoint || seg.checkpoint.height !== currentHeight + 1 || !Array.isArray(seg.entries)) break;
|
|
1217
|
+
if (!(0, _arcblock_ledger_core.verifyCheckpointQuorumCertificate)(seg.checkpoint, this.clusterConfig).ok) break;
|
|
1218
|
+
try {
|
|
1219
|
+
if (typeof this.resolver.ledger.verifySegmentForImport === "function") await this.resolver.ledger.verifySegmentForImport(seg, {
|
|
1220
|
+
hotstuffClusterConfig: this.clusterConfig,
|
|
1221
|
+
verifyStateRoot: true,
|
|
1222
|
+
allowAheadLocalSequence: true
|
|
1223
|
+
});
|
|
1224
|
+
if (seg.entries.length > 0 && typeof this.resolver.statedb.runWithinDryRunSession === "function") {
|
|
1225
|
+
if (normalizeHexCompare((await this.replayer.dryRunVerifyEntries(seg.entries, {
|
|
1226
|
+
expectedPrevSequence: localCp?.sequence ?? 0,
|
|
1227
|
+
prevStateRoot: localCp?.stateRoot ?? null
|
|
1228
|
+
})).derivedStateRoot) !== normalizeHexCompare(seg.checkpoint.stateRoot)) break;
|
|
1229
|
+
}
|
|
1230
|
+
} catch {
|
|
1231
|
+
break;
|
|
1232
|
+
}
|
|
1233
|
+
await this.resolver.ledger.importSegment(seg, {
|
|
1234
|
+
hotstuffClusterConfig: this.clusterConfig,
|
|
1235
|
+
verifyStateRoot: true
|
|
1236
|
+
});
|
|
1237
|
+
for (const entry of seg.entries) await this.replayer.replayEntry(entry, { appendToLedger: false });
|
|
1238
|
+
this.tryAbsorbRemoteQC(seg.checkpoint.qc);
|
|
1239
|
+
currentHeight = seg.checkpoint.height;
|
|
1240
|
+
localCp = seg.checkpoint;
|
|
1241
|
+
this.leaderRouter.advanceMinVerifiedCheckpointHeight(currentHeight);
|
|
1242
|
+
importedFromPeer++;
|
|
1243
|
+
importedInBatch++;
|
|
1244
|
+
}
|
|
1245
|
+
if (importedFromPeer > 0) {
|
|
1246
|
+
rotationOffset = (rotationOffset + i) % this.peerValidators.length;
|
|
1247
|
+
break;
|
|
1248
|
+
}
|
|
1249
|
+
}
|
|
1250
|
+
if (importedInBatch === 0) break;
|
|
1251
|
+
}
|
|
1252
|
+
if (localCp && localCp.height > 0) try {
|
|
1253
|
+
await this.stateStore.saveRuntimeState?.({
|
|
1254
|
+
chainId: this.chainId,
|
|
1255
|
+
voterDid: this.selfDid,
|
|
1256
|
+
currentView: this.currentView,
|
|
1257
|
+
lockedQC: this.lockedQC,
|
|
1258
|
+
highQC: this.highQC,
|
|
1259
|
+
lastTC: this.lastTC,
|
|
1260
|
+
pruneUncommittedUpToHeight: localCp.height
|
|
1261
|
+
});
|
|
1262
|
+
} catch {}
|
|
1263
|
+
}
|
|
1264
|
+
async reconcileCrashWindowLocked() {
|
|
1265
|
+
const latestSeq = await this.resolver.ledger.getLatestSequence();
|
|
1266
|
+
if (latestSeq <= 0) return;
|
|
1267
|
+
const latestEntry = await this.resolver.ledger.getBySequence(latestSeq);
|
|
1268
|
+
if (!latestEntry || latestEntry.finalized !== true) return;
|
|
1269
|
+
const currentCommit = typeof this.resolver.statedb.getCurrentCommitHash === "function" ? await this.resolver.statedb.getCurrentCommitHash() : null;
|
|
1270
|
+
if (!currentCommit || currentCommit === "UNKNOWN") return;
|
|
1271
|
+
const normCurrentCommit = normalizeHexCompare(currentCommit);
|
|
1272
|
+
const isGenesisState = currentCommit === "GENESIS" || !normCurrentCommit;
|
|
1273
|
+
if (!isGenesisState && latestEntry.stateCommitHash && normCurrentCommit === normalizeHexCompare(latestEntry.stateCommitHash)) return;
|
|
1274
|
+
const unappliedReversed = [];
|
|
1275
|
+
let matchedBase = false;
|
|
1276
|
+
for (let seq = latestSeq; seq >= 1; seq--) {
|
|
1277
|
+
const entry = await this.resolver.ledger.getBySequence(seq);
|
|
1278
|
+
if (!entry || entry.finalized !== true) break;
|
|
1279
|
+
const normEntryCommit = normalizeHexCompare(entry.stateCommitHash);
|
|
1280
|
+
if (!isGenesisState && normEntryCommit && normEntryCommit === normCurrentCommit) {
|
|
1281
|
+
matchedBase = true;
|
|
1282
|
+
break;
|
|
1283
|
+
}
|
|
1284
|
+
unappliedReversed.push(entry);
|
|
1285
|
+
if (seq === 1 && isGenesisState) matchedBase = true;
|
|
1286
|
+
}
|
|
1287
|
+
const unappliedOrdered = [...unappliedReversed].reverse();
|
|
1288
|
+
if (!matchedBase && unappliedOrdered.length > 0 && unappliedOrdered[0].sequence === 1 && typeof this.resolver.statedb.runWithinDryRunSession === "function") {
|
|
1289
|
+
const dryRun = await this.replayer.dryRunVerifyEntries(unappliedOrdered, {
|
|
1290
|
+
expectedPrevSequence: 0,
|
|
1291
|
+
prevStateRoot: currentCommit
|
|
1292
|
+
}).catch(() => null);
|
|
1293
|
+
if (dryRun && latestEntry.stateCommitHash && normalizeHexCompare(dryRun.derivedStateRoot) === normalizeHexCompare(latestEntry.stateCommitHash)) matchedBase = true;
|
|
1294
|
+
}
|
|
1295
|
+
if (!matchedBase) throw new _ocap_util_lib_error.CustomError("STATE_COMMIT_MISMATCH", `Validator crash reconciliation could not align statedb commit (${currentCommit}) with finalized ledger tip (seq=${latestSeq})`);
|
|
1296
|
+
for (const entry of unappliedOrdered) await this.replayer.replayEntry(entry, { appendToLedger: false });
|
|
1297
|
+
}
|
|
1298
|
+
async signHttpEnvelope(params) {
|
|
1299
|
+
const digest = buildCanonicalHttpAuthDigest({
|
|
1300
|
+
kind: params.kind,
|
|
1301
|
+
chainId: this.chainId,
|
|
1302
|
+
methodOrStatus: params.methodOrStatus,
|
|
1303
|
+
pathWithQuery: params.pathWithQuery,
|
|
1304
|
+
senderDid: this.selfDid,
|
|
1305
|
+
timestampMs: params.timestampMs,
|
|
1306
|
+
nonce: params.nonce,
|
|
1307
|
+
bodyBytes: params.bodyBytes
|
|
1308
|
+
});
|
|
1309
|
+
const sig = await this.signDigest(digest);
|
|
1310
|
+
return sig.startsWith("0x") ? sig : `0x${sig}`;
|
|
1311
|
+
}
|
|
1312
|
+
isReplayKeyActive(key, nowMs) {
|
|
1313
|
+
const existingExpiry = this.replayCache.get(key);
|
|
1314
|
+
return existingExpiry !== void 0 && existingExpiry > nowMs;
|
|
1315
|
+
}
|
|
1316
|
+
pruneAndCheckReplayKey(key, nowMs) {
|
|
1317
|
+
if (this.isReplayKeyActive(key, nowMs)) return false;
|
|
1318
|
+
if (this.replayCache.size >= MAX_REPLAY_CACHE_ENTRIES) for (const [k, exp] of this.replayCache.entries()) {
|
|
1319
|
+
if (exp <= nowMs || this.replayCache.size >= MAX_REPLAY_CACHE_ENTRIES) this.replayCache.delete(k);
|
|
1320
|
+
if (this.replayCache.size < MAX_REPLAY_CACHE_ENTRIES - 64) break;
|
|
1321
|
+
}
|
|
1322
|
+
this.replayCache.set(key, nowMs + REPLAY_TTL_MS);
|
|
1323
|
+
return true;
|
|
1324
|
+
}
|
|
1325
|
+
precheckIncomingHttpAuthHeaders(req) {
|
|
1326
|
+
const chainHeader = String(req.headers["x-ocap-consensus-chain-id"] ?? "").trim();
|
|
1327
|
+
const senderDid = String(req.headers["x-ocap-consensus-sender"] ?? "").trim();
|
|
1328
|
+
const tsRaw = String(req.headers["x-ocap-consensus-timestamp"] ?? "").trim();
|
|
1329
|
+
const nonce = String(req.headers["x-ocap-consensus-nonce"] ?? "").trim();
|
|
1330
|
+
const signature = String(req.headers["x-ocap-consensus-signature"] ?? "").trim();
|
|
1331
|
+
if (!chainHeader || !senderDid || !tsRaw || !nonce || !signature) return {
|
|
1332
|
+
ok: false,
|
|
1333
|
+
status: 401,
|
|
1334
|
+
code: "CONSENSUS_AUTH_REQUIRED",
|
|
1335
|
+
error: "Missing mandatory X-Ocap-Consensus-* authentication headers"
|
|
1336
|
+
};
|
|
1337
|
+
if (chainHeader !== this.chainId) return {
|
|
1338
|
+
ok: false,
|
|
1339
|
+
status: 403,
|
|
1340
|
+
code: "CHAIN_ID_MISMATCH",
|
|
1341
|
+
error: `Consensus chainId mismatch: expected ${this.chainId}, got ${chainHeader}`
|
|
1342
|
+
};
|
|
1343
|
+
if (!this.validatorMap.has(senderDid)) return {
|
|
1344
|
+
ok: false,
|
|
1345
|
+
status: 403,
|
|
1346
|
+
code: "UNKNOWN_VALIDATOR",
|
|
1347
|
+
error: `Sender ${senderDid} is not in HOTSTUFF_VALIDATORS allowlist`
|
|
1348
|
+
};
|
|
1349
|
+
if (senderDid === this.selfDid) return {
|
|
1350
|
+
ok: false,
|
|
1351
|
+
status: 403,
|
|
1352
|
+
code: "SELF_SENDER_REJECTED",
|
|
1353
|
+
error: `Loopback/self sender ${senderDid} is not permitted on peer consensus HTTP listener`
|
|
1354
|
+
};
|
|
1355
|
+
if (!/^[0-9a-f]{16,64}$/i.test(nonce)) return {
|
|
1356
|
+
ok: false,
|
|
1357
|
+
status: 401,
|
|
1358
|
+
code: "INVALID_NONCE",
|
|
1359
|
+
error: "Invalid X-Ocap-Consensus-Nonce format"
|
|
1360
|
+
};
|
|
1361
|
+
const timestampMs = Number(tsRaw);
|
|
1362
|
+
const nowMs = Date.now();
|
|
1363
|
+
if (!Number.isSafeInteger(timestampMs) || Math.abs(nowMs - timestampMs) > MAX_CLOCK_SKEW_MS) return {
|
|
1364
|
+
ok: false,
|
|
1365
|
+
status: 401,
|
|
1366
|
+
code: "CLOCK_SKEW_EXCEEDED",
|
|
1367
|
+
error: "Consensus request timestamp outside ±60s clock skew window"
|
|
1368
|
+
};
|
|
1369
|
+
const replayKey = `req:${senderDid}:${timestampMs}:${nonce}`;
|
|
1370
|
+
if (this.isReplayKeyActive(replayKey, nowMs)) return {
|
|
1371
|
+
ok: false,
|
|
1372
|
+
status: 401,
|
|
1373
|
+
code: "REPLAY_DETECTED",
|
|
1374
|
+
error: "Replayed consensus HTTP request rejected"
|
|
1375
|
+
};
|
|
1376
|
+
return {
|
|
1377
|
+
ok: true,
|
|
1378
|
+
senderDid,
|
|
1379
|
+
timestampMs,
|
|
1380
|
+
nonce,
|
|
1381
|
+
signature
|
|
1382
|
+
};
|
|
1383
|
+
}
|
|
1384
|
+
async verifyIncomingHttpAuth(req, pathWithQuery, bodyBytes, prechecked) {
|
|
1385
|
+
const headerCheck = prechecked ?? this.precheckIncomingHttpAuthHeaders(req);
|
|
1386
|
+
if ("ok" in headerCheck && headerCheck.ok === false) return headerCheck;
|
|
1387
|
+
const { senderDid, timestampMs, nonce, signature } = headerCheck;
|
|
1388
|
+
const nowMs = Date.now();
|
|
1389
|
+
if (Math.abs(nowMs - timestampMs) > MAX_CLOCK_SKEW_MS) return {
|
|
1390
|
+
ok: false,
|
|
1391
|
+
status: 401,
|
|
1392
|
+
code: "CLOCK_SKEW_EXCEEDED",
|
|
1393
|
+
error: "Consensus request timestamp outside ±60s clock skew window"
|
|
1394
|
+
};
|
|
1395
|
+
const digest = buildCanonicalHttpAuthDigest({
|
|
1396
|
+
kind: "req",
|
|
1397
|
+
chainId: this.chainId,
|
|
1398
|
+
methodOrStatus: req.method || "GET",
|
|
1399
|
+
pathWithQuery,
|
|
1400
|
+
senderDid,
|
|
1401
|
+
timestampMs,
|
|
1402
|
+
nonce,
|
|
1403
|
+
bodyBytes
|
|
1404
|
+
});
|
|
1405
|
+
if (!await this.trustAnchorVerify(digest, signature, senderDid)) return {
|
|
1406
|
+
ok: false,
|
|
1407
|
+
status: 401,
|
|
1408
|
+
code: "INVALID_CONSENSUS_SIGNATURE",
|
|
1409
|
+
error: `Invalid consensus Ed25519 signature from ${senderDid}`
|
|
1410
|
+
};
|
|
1411
|
+
const replayKey = `req:${senderDid}:${timestampMs}:${nonce}`;
|
|
1412
|
+
if (!this.pruneAndCheckReplayKey(replayKey, nowMs)) return {
|
|
1413
|
+
ok: false,
|
|
1414
|
+
status: 401,
|
|
1415
|
+
code: "REPLAY_DETECTED",
|
|
1416
|
+
error: "Replayed consensus HTTP request rejected"
|
|
1417
|
+
};
|
|
1418
|
+
this.recordAuthenticatedPeerContact(senderDid, nowMs);
|
|
1419
|
+
return {
|
|
1420
|
+
ok: true,
|
|
1421
|
+
senderDid
|
|
1422
|
+
};
|
|
1423
|
+
}
|
|
1424
|
+
async sendAuthenticatedJsonResponse(res, pathWithQuery, statusCode, payload) {
|
|
1425
|
+
const json = JSON.stringify(payload);
|
|
1426
|
+
const bodyBytes = new Uint8Array(Buffer.from(json, "utf8"));
|
|
1427
|
+
const timestampMs = Date.now();
|
|
1428
|
+
const nonce = (0, node_crypto.randomBytes)(16).toString("hex");
|
|
1429
|
+
const signature = await this.signHttpEnvelope({
|
|
1430
|
+
kind: "res",
|
|
1431
|
+
methodOrStatus: String(statusCode),
|
|
1432
|
+
pathWithQuery,
|
|
1433
|
+
timestampMs,
|
|
1434
|
+
nonce,
|
|
1435
|
+
bodyBytes
|
|
1436
|
+
});
|
|
1437
|
+
res.statusCode = statusCode;
|
|
1438
|
+
res.setHeader("Content-Type", "application/json; charset=utf-8");
|
|
1439
|
+
res.setHeader("X-Content-Type-Options", "nosniff");
|
|
1440
|
+
res.setHeader("Cache-Control", "no-store");
|
|
1441
|
+
res.setHeader("X-Ocap-Consensus-Chain-Id", this.chainId);
|
|
1442
|
+
res.setHeader("X-Ocap-Consensus-Sender", this.selfDid);
|
|
1443
|
+
res.setHeader("X-Ocap-Consensus-Timestamp", String(timestampMs));
|
|
1444
|
+
res.setHeader("X-Ocap-Consensus-Nonce", nonce);
|
|
1445
|
+
res.setHeader("X-Ocap-Consensus-Signature", signature);
|
|
1446
|
+
res.end(Buffer.from(bodyBytes));
|
|
1447
|
+
}
|
|
1448
|
+
async sendConsensusJson(peer, method, pathWithQuery, bodyObj, timeoutMs, maxResponseBytes) {
|
|
1449
|
+
const url = `${peer.consensusUrl}${pathWithQuery}`;
|
|
1450
|
+
if (this.onOutboundConsensusRequest) await this.onOutboundConsensusRequest({
|
|
1451
|
+
fromDid: this.selfDid,
|
|
1452
|
+
toDid: peer.did,
|
|
1453
|
+
method,
|
|
1454
|
+
pathWithQuery,
|
|
1455
|
+
url
|
|
1456
|
+
});
|
|
1457
|
+
const bodyText = bodyObj !== void 0 ? JSON.stringify(bodyObj) : "";
|
|
1458
|
+
const bodyBytes = bodyText ? new Uint8Array(Buffer.from(bodyText, "utf8")) : new Uint8Array(0);
|
|
1459
|
+
const timestampMs = Date.now();
|
|
1460
|
+
const nonce = (0, node_crypto.randomBytes)(16).toString("hex");
|
|
1461
|
+
const signature = await this.signHttpEnvelope({
|
|
1462
|
+
kind: "req",
|
|
1463
|
+
methodOrStatus: method,
|
|
1464
|
+
pathWithQuery,
|
|
1465
|
+
timestampMs,
|
|
1466
|
+
nonce,
|
|
1467
|
+
bodyBytes
|
|
1468
|
+
});
|
|
1469
|
+
const response = await this.fetchImpl(url, {
|
|
1470
|
+
method,
|
|
1471
|
+
headers: {
|
|
1472
|
+
accept: "application/json",
|
|
1473
|
+
...bodyText ? { "content-type": "application/json" } : {},
|
|
1474
|
+
"x-ocap-consensus-chain-id": this.chainId,
|
|
1475
|
+
"x-ocap-consensus-sender": this.selfDid,
|
|
1476
|
+
"x-ocap-consensus-timestamp": String(timestampMs),
|
|
1477
|
+
"x-ocap-consensus-nonce": nonce,
|
|
1478
|
+
"x-ocap-consensus-signature": signature
|
|
1479
|
+
},
|
|
1480
|
+
...bodyText ? { body: bodyText } : {},
|
|
1481
|
+
redirect: "error",
|
|
1482
|
+
signal: AbortSignal.timeout(timeoutMs)
|
|
1483
|
+
});
|
|
1484
|
+
const respBytes = await readFetchResponseBytesBounded(response, maxResponseBytes);
|
|
1485
|
+
let authenticated = false;
|
|
1486
|
+
const respSender = String(response.headers.get("x-ocap-consensus-sender") ?? "").trim();
|
|
1487
|
+
const respChain = String(response.headers.get("x-ocap-consensus-chain-id") ?? "").trim();
|
|
1488
|
+
const respTs = Number(response.headers.get("x-ocap-consensus-timestamp") ?? "");
|
|
1489
|
+
const respNonce = String(response.headers.get("x-ocap-consensus-nonce") ?? "").trim();
|
|
1490
|
+
const respSig = String(response.headers.get("x-ocap-consensus-signature") ?? "").trim();
|
|
1491
|
+
const nowMs = Date.now();
|
|
1492
|
+
if (respSender === peer.did && respChain === this.chainId && Number.isSafeInteger(respTs) && Math.abs(nowMs - respTs) <= MAX_CLOCK_SKEW_MS && /^[0-9a-f]{16,64}$/i.test(respNonce) && respSig) {
|
|
1493
|
+
const respDigest = buildCanonicalHttpAuthDigest({
|
|
1494
|
+
kind: "res",
|
|
1495
|
+
chainId: this.chainId,
|
|
1496
|
+
methodOrStatus: String(response.status),
|
|
1497
|
+
pathWithQuery,
|
|
1498
|
+
senderDid: peer.did,
|
|
1499
|
+
timestampMs: respTs,
|
|
1500
|
+
nonce: respNonce,
|
|
1501
|
+
bodyBytes: respBytes
|
|
1502
|
+
});
|
|
1503
|
+
if (await this.trustAnchorVerify(respDigest, respSig, peer.did)) {
|
|
1504
|
+
const replayKey = `res:${peer.did}:${respTs}:${respNonce}`;
|
|
1505
|
+
if (this.pruneAndCheckReplayKey(replayKey, nowMs)) {
|
|
1506
|
+
authenticated = true;
|
|
1507
|
+
this.recordAuthenticatedPeerContact(peer.did, nowMs);
|
|
1508
|
+
}
|
|
1509
|
+
}
|
|
1510
|
+
}
|
|
1511
|
+
const decoded = respBytes.byteLength > 0 ? new TextDecoder().decode(respBytes) : "";
|
|
1512
|
+
const parsed = decoded ? JSON.parse(decoded) : {};
|
|
1513
|
+
return {
|
|
1514
|
+
status: response.status,
|
|
1515
|
+
body: parsed,
|
|
1516
|
+
authenticated
|
|
1517
|
+
};
|
|
1518
|
+
}
|
|
1519
|
+
async handleConsensusHttpRequest(req, res) {
|
|
1520
|
+
const rawUrl = req.url || "/";
|
|
1521
|
+
let parsedUrl;
|
|
1522
|
+
try {
|
|
1523
|
+
parsedUrl = new URL(rawUrl, "http://localhost");
|
|
1524
|
+
} catch {
|
|
1525
|
+
res.statusCode = 400;
|
|
1526
|
+
res.end(JSON.stringify({
|
|
1527
|
+
code: "INVALID_URL",
|
|
1528
|
+
error: "Malformed URL"
|
|
1529
|
+
}));
|
|
1530
|
+
return;
|
|
1531
|
+
}
|
|
1532
|
+
const pathname = parsedUrl.pathname;
|
|
1533
|
+
const pathWithQuery = `${pathname}${parsedUrl.search}`;
|
|
1534
|
+
const isDataRoute = pathname === "/consensus/propose" || pathname === "/consensus/precommit" || pathname === "/consensus/commit" || pathname === "/consensus/entries" || pathname === "/consensus/uncommitted";
|
|
1535
|
+
if (!isDataRoute && !(pathname === "/consensus/timeout" || pathname === "/consensus/new-view" || pathname === "/consensus/heartbeat" || pathname === "/consensus/demand" || pathname === "/consensus/status")) {
|
|
1536
|
+
res.statusCode = 404;
|
|
1537
|
+
res.setHeader("Content-Type", "application/json; charset=utf-8");
|
|
1538
|
+
res.end(JSON.stringify({
|
|
1539
|
+
code: "NOT_FOUND",
|
|
1540
|
+
error: `Unknown consensus endpoint: ${pathname}`
|
|
1541
|
+
}));
|
|
1542
|
+
return;
|
|
1543
|
+
}
|
|
1544
|
+
const prechecked = this.precheckIncomingHttpAuthHeaders(req);
|
|
1545
|
+
if (!prechecked.ok) {
|
|
1546
|
+
req.resume();
|
|
1547
|
+
res.statusCode = prechecked.status;
|
|
1548
|
+
res.setHeader("Content-Type", "application/json; charset=utf-8");
|
|
1549
|
+
res.end(JSON.stringify({
|
|
1550
|
+
code: prechecked.code,
|
|
1551
|
+
error: prechecked.error
|
|
1552
|
+
}));
|
|
1553
|
+
return;
|
|
1554
|
+
}
|
|
1555
|
+
const bodyRead = await readIncomingBytesBounded(req, isDataRoute ? MAX_DATA_ROUTE_BYTES : MAX_CONTROL_ROUTE_BYTES);
|
|
1556
|
+
if (!bodyRead.ok) {
|
|
1557
|
+
res.statusCode = bodyRead.status;
|
|
1558
|
+
res.setHeader("Content-Type", "application/json; charset=utf-8");
|
|
1559
|
+
res.end(JSON.stringify({
|
|
1560
|
+
code: bodyRead.code,
|
|
1561
|
+
error: bodyRead.error
|
|
1562
|
+
}));
|
|
1563
|
+
return;
|
|
1564
|
+
}
|
|
1565
|
+
const auth = await this.verifyIncomingHttpAuth(req, pathWithQuery, bodyRead.bytes, prechecked);
|
|
1566
|
+
if (!auth.ok) {
|
|
1567
|
+
res.statusCode = auth.status;
|
|
1568
|
+
res.setHeader("Content-Type", "application/json; charset=utf-8");
|
|
1569
|
+
res.end(JSON.stringify({
|
|
1570
|
+
code: auth.code,
|
|
1571
|
+
error: auth.error
|
|
1572
|
+
}));
|
|
1573
|
+
return;
|
|
1574
|
+
}
|
|
1575
|
+
if (this.halted) {
|
|
1576
|
+
await this.sendAuthenticatedJsonResponse(res, pathWithQuery, 503, {
|
|
1577
|
+
code: "NODE_HALTED",
|
|
1578
|
+
error: this.haltReason || "Validator halted fail-closed"
|
|
1579
|
+
});
|
|
1580
|
+
return;
|
|
1581
|
+
}
|
|
1582
|
+
let bodyJson = {};
|
|
1583
|
+
if (bodyRead.bytes.byteLength > 0) try {
|
|
1584
|
+
bodyJson = JSON.parse(new TextDecoder().decode(bodyRead.bytes));
|
|
1585
|
+
} catch {
|
|
1586
|
+
await this.sendAuthenticatedJsonResponse(res, pathWithQuery, 400, {
|
|
1587
|
+
code: "INVALID_JSON",
|
|
1588
|
+
error: "Malformed JSON body"
|
|
1589
|
+
});
|
|
1590
|
+
return;
|
|
1591
|
+
}
|
|
1592
|
+
try {
|
|
1593
|
+
if (req.method === "GET" && pathname === "/consensus/status") {
|
|
1594
|
+
const status = await this.buildConsensusStatus();
|
|
1595
|
+
await this.sendAuthenticatedJsonResponse(res, pathWithQuery, 200, status);
|
|
1596
|
+
return;
|
|
1597
|
+
}
|
|
1598
|
+
if (req.method === "GET" && pathname === "/consensus/entries") {
|
|
1599
|
+
const fromHeight = Number(parsedUrl.searchParams.get("fromHeight") ?? "0");
|
|
1600
|
+
const toHeight = Number(parsedUrl.searchParams.get("toHeight") ?? String(fromHeight));
|
|
1601
|
+
if (!Number.isSafeInteger(fromHeight) || !Number.isSafeInteger(toHeight) || fromHeight < 1 || toHeight < fromHeight) {
|
|
1602
|
+
await this.sendAuthenticatedJsonResponse(res, pathWithQuery, 400, {
|
|
1603
|
+
code: "INVALID_PARAMS",
|
|
1604
|
+
error: "Invalid fromHeight / toHeight"
|
|
1605
|
+
});
|
|
1606
|
+
return;
|
|
1607
|
+
}
|
|
1608
|
+
const cappedTo = Math.min(toHeight, fromHeight + MAX_CATCHUP_SEGMENTS_PER_REQUEST - 1);
|
|
1609
|
+
const segments = await this.readFinalizedSegmentsRange(fromHeight, cappedTo);
|
|
1610
|
+
await this.sendAuthenticatedJsonResponse(res, pathWithQuery, 200, { segments });
|
|
1611
|
+
return;
|
|
1612
|
+
}
|
|
1613
|
+
if (req.method === "GET" && pathname === "/consensus/uncommitted") {
|
|
1614
|
+
const checkpointHash = String(parsedUrl.searchParams.get("checkpointHash") ?? "").trim();
|
|
1615
|
+
const [entries, checkpoint] = checkpointHash ? await Promise.all([this.stateStore.getUncommittedProposalEntries?.(this.chainId, this.selfDid, checkpointHash), this.stateStore.getUncommittedProposalCheckpoint?.(this.chainId, this.selfDid, checkpointHash)]) : [null, null];
|
|
1616
|
+
if (!entries) {
|
|
1617
|
+
await this.sendAuthenticatedJsonResponse(res, pathWithQuery, 404, {
|
|
1618
|
+
code: "NOT_FOUND",
|
|
1619
|
+
error: "Uncommitted entries not found"
|
|
1620
|
+
});
|
|
1621
|
+
return;
|
|
1622
|
+
}
|
|
1623
|
+
await this.sendAuthenticatedJsonResponse(res, pathWithQuery, 200, {
|
|
1624
|
+
entries,
|
|
1625
|
+
...checkpoint ? { checkpoint } : {}
|
|
1626
|
+
});
|
|
1627
|
+
return;
|
|
1628
|
+
}
|
|
1629
|
+
if (req.method === "POST" && pathname === "/consensus/heartbeat") {
|
|
1630
|
+
const result = await this.handleHeartbeat(auth.senderDid, bodyJson);
|
|
1631
|
+
await this.sendAuthenticatedJsonResponse(res, pathWithQuery, 200, result);
|
|
1632
|
+
return;
|
|
1633
|
+
}
|
|
1634
|
+
if (req.method === "POST" && pathname === "/consensus/propose") {
|
|
1635
|
+
const result = await this.handlePropose(auth.senderDid, bodyJson);
|
|
1636
|
+
await this.sendAuthenticatedJsonResponse(res, pathWithQuery, result.status, result.body);
|
|
1637
|
+
return;
|
|
1638
|
+
}
|
|
1639
|
+
if (req.method === "POST" && pathname === "/consensus/precommit") {
|
|
1640
|
+
const result = await this.handlePrecommit(auth.senderDid, bodyJson);
|
|
1641
|
+
await this.sendAuthenticatedJsonResponse(res, pathWithQuery, result.status, result.body);
|
|
1642
|
+
return;
|
|
1643
|
+
}
|
|
1644
|
+
if (req.method === "POST" && pathname === "/consensus/demand") {
|
|
1645
|
+
const result = await this.handleDemand(auth.senderDid, bodyJson);
|
|
1646
|
+
await this.sendAuthenticatedJsonResponse(res, pathWithQuery, result.status, result.body);
|
|
1647
|
+
return;
|
|
1648
|
+
}
|
|
1649
|
+
if (req.method === "POST" && pathname === "/consensus/commit") {
|
|
1650
|
+
const result = await this.handleCommit(bodyJson);
|
|
1651
|
+
await this.sendAuthenticatedJsonResponse(res, pathWithQuery, result.status, result.body);
|
|
1652
|
+
return;
|
|
1653
|
+
}
|
|
1654
|
+
if (req.method === "POST" && pathname === "/consensus/timeout") {
|
|
1655
|
+
const result = await this.handleTimeoutMessage(bodyJson);
|
|
1656
|
+
await this.sendAuthenticatedJsonResponse(res, pathWithQuery, result.status, result.body);
|
|
1657
|
+
return;
|
|
1658
|
+
}
|
|
1659
|
+
if (req.method === "POST" && pathname === "/consensus/new-view") {
|
|
1660
|
+
const result = await this.handleNewView(bodyJson);
|
|
1661
|
+
await this.sendAuthenticatedJsonResponse(res, pathWithQuery, result.status, result.body);
|
|
1662
|
+
return;
|
|
1663
|
+
}
|
|
1664
|
+
await this.sendAuthenticatedJsonResponse(res, pathWithQuery, 405, {
|
|
1665
|
+
code: "METHOD_NOT_ALLOWED",
|
|
1666
|
+
error: `Unsupported method ${String(req.method)} for ${pathname}`
|
|
1667
|
+
});
|
|
1668
|
+
} catch (err) {
|
|
1669
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
1670
|
+
await this.sendAuthenticatedJsonResponse(res, pathWithQuery, 500, {
|
|
1671
|
+
code: err?.code || "CONSENSUS_INTERNAL_ERROR",
|
|
1672
|
+
error: message
|
|
1673
|
+
});
|
|
1674
|
+
}
|
|
1675
|
+
}
|
|
1676
|
+
async buildConsensusStatus() {
|
|
1677
|
+
const [latestSequence, latestCp] = await Promise.all([this.resolver.ledger.getLatestSequence(), this.resolver.ledger.getCheckpoint()]);
|
|
1678
|
+
return {
|
|
1679
|
+
chainId: this.chainId,
|
|
1680
|
+
validatorDid: this.selfDid,
|
|
1681
|
+
currentView: this.currentView,
|
|
1682
|
+
latestSequence,
|
|
1683
|
+
checkpointHeight: latestCp?.height ?? 0,
|
|
1684
|
+
highQC: this.highQC,
|
|
1685
|
+
lockedQC: this.lockedQC,
|
|
1686
|
+
lastTC: this.lastTC,
|
|
1687
|
+
quorumLeaseValid: this.refreshQuorumLease()
|
|
1688
|
+
};
|
|
1689
|
+
}
|
|
1690
|
+
async readFinalizedSegmentsRange(fromHeight, toHeight) {
|
|
1691
|
+
const out = [];
|
|
1692
|
+
for (let h = fromHeight; h <= toHeight; h++) {
|
|
1693
|
+
const cp = await this.resolver.ledger.getCheckpoint(h);
|
|
1694
|
+
if (!cp || !cp.qc) break;
|
|
1695
|
+
const entries = cp.sequence > cp.prevSequence ? await this.resolver.ledger.getRange(cp.prevSequence + 1, cp.sequence) : [];
|
|
1696
|
+
if (entries.length !== cp.txCount) break;
|
|
1697
|
+
out.push({
|
|
1698
|
+
chainId: this.chainId,
|
|
1699
|
+
checkpoint: cp,
|
|
1700
|
+
entries
|
|
1701
|
+
});
|
|
1702
|
+
}
|
|
1703
|
+
return out;
|
|
1704
|
+
}
|
|
1705
|
+
async handleHeartbeat(senderDid, body) {
|
|
1706
|
+
const hbHighQC = body.highQC ?? null;
|
|
1707
|
+
const hbLastTC = body.lastTC ?? null;
|
|
1708
|
+
const hbView = typeof body.viewNumber === "number" ? body.viewNumber : -1;
|
|
1709
|
+
const advancedFromQC = this.tryAbsorbRemoteQC(hbHighQC);
|
|
1710
|
+
const advancedFromTC = this.tryAbsorbRemoteTC(hbLastTC);
|
|
1711
|
+
if (advancedFromQC || advancedFromTC) this.runExclusive(async () => {
|
|
1712
|
+
if (this.highQC) await this.catchUpToHeightLocked(this.highQC.height);
|
|
1713
|
+
}).catch(() => void 0);
|
|
1714
|
+
if (senderDid === (0, _arcblock_ledger_core.getLeaderForView)(this.clusterConfig, this.currentView).did && hbView === this.currentView) {
|
|
1715
|
+
const lock = await Promise.resolve(this.stateStore.getLock(this.chainId, this.selfDid)).catch(() => null);
|
|
1716
|
+
if (!this.shouldSuppressHeartbeatKeepalive(lock, this.currentView)) this.scheduleViewTimeout();
|
|
1717
|
+
}
|
|
1718
|
+
return {
|
|
1719
|
+
ok: true,
|
|
1720
|
+
currentView: this.currentView,
|
|
1721
|
+
highQC: this.highQC,
|
|
1722
|
+
lastTC: this.lastTC
|
|
1723
|
+
};
|
|
1724
|
+
}
|
|
1725
|
+
async handleDemand(senderDid, body) {
|
|
1726
|
+
const demandView = Number(body.viewNumber);
|
|
1727
|
+
if (!Number.isSafeInteger(demandView) || demandView < 0) return {
|
|
1728
|
+
status: 400,
|
|
1729
|
+
body: {
|
|
1730
|
+
code: "INVALID_DEMAND_VIEW",
|
|
1731
|
+
error: "Invalid viewNumber in /consensus/demand"
|
|
1732
|
+
}
|
|
1733
|
+
};
|
|
1734
|
+
if (demandView !== this.currentView) return {
|
|
1735
|
+
status: 200,
|
|
1736
|
+
body: {
|
|
1737
|
+
ok: true,
|
|
1738
|
+
ignored: true,
|
|
1739
|
+
currentView: this.currentView,
|
|
1740
|
+
highQC: this.highQC
|
|
1741
|
+
}
|
|
1742
|
+
};
|
|
1743
|
+
let senders = this.demandSendersByView.get(demandView);
|
|
1744
|
+
if (!senders) {
|
|
1745
|
+
senders = /* @__PURE__ */ new Set();
|
|
1746
|
+
this.demandSendersByView.set(demandView, senders);
|
|
1747
|
+
}
|
|
1748
|
+
if (!senders.has(senderDid)) {
|
|
1749
|
+
senders.add(senderDid);
|
|
1750
|
+
this.pendingDemandForView = demandView;
|
|
1751
|
+
if (!this.stopped && !this.halted) {
|
|
1752
|
+
if (this.viewTimer) {
|
|
1753
|
+
clearTimeout(this.viewTimer);
|
|
1754
|
+
this.viewTimer = null;
|
|
1755
|
+
}
|
|
1756
|
+
const scheduledView = this.currentView;
|
|
1757
|
+
const clampMs = Math.min(this.baseViewTimeoutMs, 400);
|
|
1758
|
+
this.viewTimer = setTimeout(() => {
|
|
1759
|
+
this.onViewTimerExpired(scheduledView).catch(() => void 0);
|
|
1760
|
+
}, clampMs);
|
|
1761
|
+
if (typeof this.viewTimer.unref === "function") this.viewTimer.unref();
|
|
1762
|
+
}
|
|
1763
|
+
}
|
|
1764
|
+
return {
|
|
1765
|
+
status: 200,
|
|
1766
|
+
body: {
|
|
1767
|
+
ok: true,
|
|
1768
|
+
currentView: this.currentView,
|
|
1769
|
+
highQC: this.highQC
|
|
1770
|
+
}
|
|
1771
|
+
};
|
|
1772
|
+
}
|
|
1773
|
+
async handlePropose(senderDid, body) {
|
|
1774
|
+
return this.runExclusive(async () => {
|
|
1775
|
+
const proposal = body.proposal;
|
|
1776
|
+
const entries = body.entries;
|
|
1777
|
+
if (!proposal || typeof proposal !== "object" || !proposal.checkpoint || !Array.isArray(entries)) return {
|
|
1778
|
+
status: 400,
|
|
1779
|
+
body: {
|
|
1780
|
+
code: "INVALID_PROPOSAL",
|
|
1781
|
+
error: "Missing proposal or entries array"
|
|
1782
|
+
}
|
|
1783
|
+
};
|
|
1784
|
+
if (proposal.proposerDid !== senderDid) return {
|
|
1785
|
+
status: 403,
|
|
1786
|
+
body: {
|
|
1787
|
+
code: "PROPOSER_SENDER_MISMATCH",
|
|
1788
|
+
error: "Authenticated sender does not match proposal.proposerDid"
|
|
1789
|
+
}
|
|
1790
|
+
};
|
|
1791
|
+
if (proposal.highQC) this.tryAbsorbRemoteQC(proposal.highQC);
|
|
1792
|
+
if (proposal.timeoutCert) this.tryAbsorbRemoteTC(proposal.timeoutCert);
|
|
1793
|
+
if (proposal.viewNumber < this.currentView) return {
|
|
1794
|
+
status: 409,
|
|
1795
|
+
body: {
|
|
1796
|
+
code: "VIEW_REGRESSION",
|
|
1797
|
+
error: `Proposal view (${proposal.viewNumber}) < local currentView (${this.currentView})`,
|
|
1798
|
+
currentView: this.currentView,
|
|
1799
|
+
highQC: this.highQC,
|
|
1800
|
+
lastTC: this.lastTC
|
|
1801
|
+
}
|
|
1802
|
+
};
|
|
1803
|
+
const effectiveParentQC = (0, _arcblock_ledger_core.selectHighestQC)([proposal.highQC, proposal.timeoutCert?.winningHighQC]);
|
|
1804
|
+
const parentHeight = effectiveParentQC?.height ?? 0;
|
|
1805
|
+
let localCp = await this.resolver.ledger.getCheckpoint();
|
|
1806
|
+
if ((localCp?.height ?? 0) < parentHeight) {
|
|
1807
|
+
await this.catchUpToHeightLocked(parentHeight);
|
|
1808
|
+
localCp = await this.resolver.ledger.getCheckpoint();
|
|
1809
|
+
}
|
|
1810
|
+
const localHeight = localCp?.height ?? 0;
|
|
1811
|
+
if (localHeight >= proposal.checkpoint.height) return {
|
|
1812
|
+
status: 409,
|
|
1813
|
+
body: {
|
|
1814
|
+
code: "HEIGHT_ALREADY_COMMITTED",
|
|
1815
|
+
error: `Height ${proposal.checkpoint.height} is already finalized locally`,
|
|
1816
|
+
currentView: this.currentView,
|
|
1817
|
+
highQC: this.highQC,
|
|
1818
|
+
lastTC: this.lastTC
|
|
1819
|
+
}
|
|
1820
|
+
};
|
|
1821
|
+
if (localHeight !== proposal.checkpoint.height - 1) return {
|
|
1822
|
+
status: 409,
|
|
1823
|
+
body: {
|
|
1824
|
+
code: "PARENT_HEIGHT_MISSING",
|
|
1825
|
+
error: `Local height (${localHeight}) is behind parent height (${proposal.checkpoint.height - 1})`,
|
|
1826
|
+
currentView: this.currentView,
|
|
1827
|
+
highQC: this.highQC,
|
|
1828
|
+
lastTC: this.lastTC
|
|
1829
|
+
}
|
|
1830
|
+
};
|
|
1831
|
+
const expectedViewLeader = (0, _arcblock_ledger_core.getLeaderForView)(this.clusterConfig, proposal.viewNumber);
|
|
1832
|
+
if (proposal.proposerDid !== expectedViewLeader.did) return {
|
|
1833
|
+
status: 403,
|
|
1834
|
+
body: {
|
|
1835
|
+
code: "NOT_VIEW_LEADER",
|
|
1836
|
+
error: `Validator ${proposal.proposerDid} is not leader (${expectedViewLeader.did}) for view ${proposal.viewNumber}`
|
|
1837
|
+
}
|
|
1838
|
+
};
|
|
1839
|
+
const segContentHash = (0, _arcblock_ledger_core.computeSegmentContentHash)({
|
|
1840
|
+
chainId: this.chainId,
|
|
1841
|
+
checkpoint: proposal.checkpoint,
|
|
1842
|
+
entries
|
|
1843
|
+
});
|
|
1844
|
+
if (!proposal.checkpoint.segmentContentHash || normalizeHexCompare(proposal.checkpoint.segmentContentHash) !== normalizeHexCompare(segContentHash)) return {
|
|
1845
|
+
status: 400,
|
|
1846
|
+
body: {
|
|
1847
|
+
code: "SEGMENT_CONTENT_HASH_MISMATCH",
|
|
1848
|
+
error: "Proposal checkpoint is missing or mismatched segmentContentHash"
|
|
1849
|
+
}
|
|
1850
|
+
};
|
|
1851
|
+
if (!await this.hasValidCheckpointSegmentCommitment(proposal.checkpoint, segContentHash)) return {
|
|
1852
|
+
status: 400,
|
|
1853
|
+
body: {
|
|
1854
|
+
code: "INVALID_SEGMENT_COMMITMENT_SIGNATURE",
|
|
1855
|
+
error: "Proposal checkpoint is missing a valid proposer segmentCommitmentSignature"
|
|
1856
|
+
}
|
|
1857
|
+
};
|
|
1858
|
+
const preparePreCheck = await (0, _arcblock_ledger_core.canVoterPrepareProposal)({
|
|
1859
|
+
config: this.clusterConfig,
|
|
1860
|
+
proposal,
|
|
1861
|
+
voterDid: this.selfDid,
|
|
1862
|
+
lockedQC: this.lockedQC,
|
|
1863
|
+
voterStateStore: this.stateStore
|
|
1864
|
+
});
|
|
1865
|
+
if (!preparePreCheck.ok) return {
|
|
1866
|
+
status: 409,
|
|
1867
|
+
body: {
|
|
1868
|
+
code: "VOTER_PREPARE_REJECTED",
|
|
1869
|
+
error: preparePreCheck.reason || "Proposal rejected by voter safety lock",
|
|
1870
|
+
currentView: this.currentView,
|
|
1871
|
+
highQC: this.highQC,
|
|
1872
|
+
lastTC: this.lastTC
|
|
1873
|
+
}
|
|
1874
|
+
};
|
|
1875
|
+
let dryRun;
|
|
1876
|
+
try {
|
|
1877
|
+
dryRun = await this.replayer.dryRunVerifyEntries(entries, {
|
|
1878
|
+
expectedPrevSequence: proposal.checkpoint.prevSequence,
|
|
1879
|
+
prevStateRoot: localCp?.stateRoot ?? null
|
|
1880
|
+
});
|
|
1881
|
+
} catch (err) {
|
|
1882
|
+
return {
|
|
1883
|
+
status: 400,
|
|
1884
|
+
body: {
|
|
1885
|
+
code: "PROPOSAL_DRY_RUN_FAILED",
|
|
1886
|
+
error: err instanceof Error ? err.message : String(err)
|
|
1887
|
+
}
|
|
1888
|
+
};
|
|
1889
|
+
}
|
|
1890
|
+
if (dryRun.derivedStateRoot && normalizeHexCompare(dryRun.derivedStateRoot) !== normalizeHexCompare(proposal.checkpoint.stateRoot)) return {
|
|
1891
|
+
status: 400,
|
|
1892
|
+
body: {
|
|
1893
|
+
code: "STATE_ROOT_MISMATCH",
|
|
1894
|
+
error: `Proposal stateRoot ${proposal.checkpoint.stateRoot} != derived ${dryRun.derivedStateRoot}`
|
|
1895
|
+
}
|
|
1896
|
+
};
|
|
1897
|
+
let prepareVote;
|
|
1898
|
+
try {
|
|
1899
|
+
prepareVote = await (0, _arcblock_ledger_core.verifyProposalAndCreatePrepareVote)({
|
|
1900
|
+
config: this.clusterConfig,
|
|
1901
|
+
proposal,
|
|
1902
|
+
entries,
|
|
1903
|
+
voterWallet: this.validatorWallet,
|
|
1904
|
+
voterDid: this.selfDid,
|
|
1905
|
+
lockedQC: this.lockedQC,
|
|
1906
|
+
voterStateStore: this.stateStore,
|
|
1907
|
+
expectedStateRoot: dryRun.derivedStateRoot ?? void 0,
|
|
1908
|
+
requireSegmentCommitment: true
|
|
1909
|
+
});
|
|
1910
|
+
} catch (err) {
|
|
1911
|
+
return {
|
|
1912
|
+
status: 409,
|
|
1913
|
+
body: {
|
|
1914
|
+
code: "VOTER_PREPARE_REJECTED",
|
|
1915
|
+
error: err instanceof Error ? err.message : String(err),
|
|
1916
|
+
currentView: this.currentView,
|
|
1917
|
+
highQC: this.highQC,
|
|
1918
|
+
lastTC: this.lastTC
|
|
1919
|
+
}
|
|
1920
|
+
};
|
|
1921
|
+
}
|
|
1922
|
+
await this.stateStore.saveUncommittedProposalEntries?.({
|
|
1923
|
+
chainId: this.chainId,
|
|
1924
|
+
voterDid: this.selfDid,
|
|
1925
|
+
checkpointHash: proposal.checkpoint.hash,
|
|
1926
|
+
height: proposal.checkpoint.height,
|
|
1927
|
+
entries,
|
|
1928
|
+
checkpoint: proposal.checkpoint
|
|
1929
|
+
});
|
|
1930
|
+
this.lastSelfPreparedView = Math.max(this.lastSelfPreparedView, prepareVote.viewNumber);
|
|
1931
|
+
this.lockedQC = (0, _arcblock_ledger_core.selectHighestQC)([this.lockedQC, effectiveParentQC]);
|
|
1932
|
+
this.advanceCurrentViewTo(proposal.viewNumber);
|
|
1933
|
+
try {
|
|
1934
|
+
await this.stateStore.saveRuntimeState?.({
|
|
1935
|
+
chainId: this.chainId,
|
|
1936
|
+
voterDid: this.selfDid,
|
|
1937
|
+
currentView: this.currentView,
|
|
1938
|
+
lockedQC: this.lockedQC,
|
|
1939
|
+
highQC: this.highQC,
|
|
1940
|
+
lastTC: this.lastTC
|
|
1941
|
+
});
|
|
1942
|
+
} catch {}
|
|
1943
|
+
this.scheduleViewTimeout();
|
|
1944
|
+
return {
|
|
1945
|
+
status: 200,
|
|
1946
|
+
body: {
|
|
1947
|
+
prepareVote,
|
|
1948
|
+
currentView: this.currentView,
|
|
1949
|
+
highQC: this.highQC
|
|
1950
|
+
}
|
|
1951
|
+
};
|
|
1952
|
+
});
|
|
1953
|
+
}
|
|
1954
|
+
async handlePrecommit(senderDid, body) {
|
|
1955
|
+
return this.runExclusive(async () => {
|
|
1956
|
+
const prepareQC = body.prepareQC;
|
|
1957
|
+
const checkpoint = body.checkpoint;
|
|
1958
|
+
const parentHighQC = body.parentHighQC ?? null;
|
|
1959
|
+
const inlineEntries = Array.isArray(body.entries) ? body.entries : void 0;
|
|
1960
|
+
if (!prepareQC || typeof prepareQC !== "object" || !checkpoint || typeof checkpoint !== "object") return {
|
|
1961
|
+
status: 400,
|
|
1962
|
+
body: {
|
|
1963
|
+
code: "INVALID_PRECOMMIT",
|
|
1964
|
+
error: "Missing prepareQC or checkpoint in /consensus/precommit"
|
|
1965
|
+
}
|
|
1966
|
+
};
|
|
1967
|
+
if (prepareQC.leaderDid !== senderDid) return {
|
|
1968
|
+
status: 403,
|
|
1969
|
+
body: {
|
|
1970
|
+
code: "LEADER_SENDER_MISMATCH",
|
|
1971
|
+
error: "Authenticated sender does not match prepareQC.leaderDid"
|
|
1972
|
+
}
|
|
1973
|
+
};
|
|
1974
|
+
const expectedViewLeader = (0, _arcblock_ledger_core.getLeaderForView)(this.clusterConfig, prepareQC.viewNumber);
|
|
1975
|
+
if (expectedViewLeader.did !== senderDid) return {
|
|
1976
|
+
status: 403,
|
|
1977
|
+
body: {
|
|
1978
|
+
code: "NOT_VIEW_LEADER",
|
|
1979
|
+
error: `Sender ${senderDid} is not leader (${expectedViewLeader.did}) for view ${prepareQC.viewNumber}`
|
|
1980
|
+
}
|
|
1981
|
+
};
|
|
1982
|
+
if (prepareQC.height !== checkpoint.height || normalizeHexCompare(prepareQC.checkpointHash) !== normalizeHexCompare(checkpoint.hash)) return {
|
|
1983
|
+
status: 400,
|
|
1984
|
+
body: {
|
|
1985
|
+
code: "PREPARE_QC_CHECKPOINT_MISMATCH",
|
|
1986
|
+
error: "PrepareQC does not match checkpoint"
|
|
1987
|
+
}
|
|
1988
|
+
};
|
|
1989
|
+
const pqcCheck = (0, _arcblock_ledger_core.verifyPrepareQuorumCertificate)(prepareQC, this.clusterConfig);
|
|
1990
|
+
if (!pqcCheck.ok) return {
|
|
1991
|
+
status: 400,
|
|
1992
|
+
body: {
|
|
1993
|
+
code: "INVALID_PREPARE_QC",
|
|
1994
|
+
error: pqcCheck.reason || "Invalid PrepareQuorumCertificate"
|
|
1995
|
+
}
|
|
1996
|
+
};
|
|
1997
|
+
if (parentHighQC) this.tryAbsorbRemoteQC(parentHighQC);
|
|
1998
|
+
if (prepareQC.viewNumber < this.currentView) return {
|
|
1999
|
+
status: 409,
|
|
2000
|
+
body: {
|
|
2001
|
+
code: "VIEW_REGRESSION",
|
|
2002
|
+
error: `PrepareQC view (${prepareQC.viewNumber}) < local currentView (${this.currentView})`,
|
|
2003
|
+
currentView: this.currentView,
|
|
2004
|
+
highQC: this.highQC,
|
|
2005
|
+
lastTC: this.lastTC
|
|
2006
|
+
}
|
|
2007
|
+
};
|
|
2008
|
+
let localCp = await this.resolver.ledger.getCheckpoint();
|
|
2009
|
+
const targetParentHeight = checkpoint.height - 1;
|
|
2010
|
+
if ((localCp?.height ?? 0) < targetParentHeight && (this.highQC?.height ?? 0) >= targetParentHeight) {
|
|
2011
|
+
await this.catchUpToHeightLocked(targetParentHeight);
|
|
2012
|
+
localCp = await this.resolver.ledger.getCheckpoint();
|
|
2013
|
+
}
|
|
2014
|
+
const localHeight = localCp?.height ?? 0;
|
|
2015
|
+
if (localHeight >= checkpoint.height) return {
|
|
2016
|
+
status: 409,
|
|
2017
|
+
body: {
|
|
2018
|
+
code: "HEIGHT_ALREADY_COMMITTED",
|
|
2019
|
+
error: `Height ${checkpoint.height} is already finalized locally`,
|
|
2020
|
+
currentView: this.currentView,
|
|
2021
|
+
highQC: this.highQC,
|
|
2022
|
+
lastTC: this.lastTC
|
|
2023
|
+
}
|
|
2024
|
+
};
|
|
2025
|
+
if (localHeight !== targetParentHeight) return {
|
|
2026
|
+
status: 409,
|
|
2027
|
+
body: {
|
|
2028
|
+
code: "PARENT_HEIGHT_MISSING",
|
|
2029
|
+
error: `Local height (${localHeight}) is behind parent height (${targetParentHeight})`,
|
|
2030
|
+
currentView: this.currentView,
|
|
2031
|
+
highQC: this.highQC,
|
|
2032
|
+
lastTC: this.lastTC
|
|
2033
|
+
}
|
|
2034
|
+
};
|
|
2035
|
+
let stagedEntries = await this.stateStore.getUncommittedProposalEntries?.(this.chainId, this.selfDid, checkpoint.hash);
|
|
2036
|
+
if (!stagedEntries || stagedEntries.length !== checkpoint.txCount) {
|
|
2037
|
+
if (!inlineEntries || inlineEntries.length !== checkpoint.txCount) return {
|
|
2038
|
+
status: 409,
|
|
2039
|
+
body: {
|
|
2040
|
+
code: "MISSING_UNCOMMITTED_ENTRIES",
|
|
2041
|
+
error: "Replica lacks staged uncommitted entries for PrepareQC; retry with inline entries",
|
|
2042
|
+
currentView: this.currentView,
|
|
2043
|
+
highQC: this.highQC
|
|
2044
|
+
}
|
|
2045
|
+
};
|
|
2046
|
+
const inlineSegContentHash = (0, _arcblock_ledger_core.computeSegmentContentHash)({
|
|
2047
|
+
chainId: this.chainId,
|
|
2048
|
+
checkpoint,
|
|
2049
|
+
entries: inlineEntries
|
|
2050
|
+
});
|
|
2051
|
+
if (!await this.hasValidCheckpointSegmentCommitment(checkpoint, inlineSegContentHash)) return {
|
|
2052
|
+
status: 400,
|
|
2053
|
+
body: {
|
|
2054
|
+
code: "INVALID_SEGMENT_COMMITMENT_SIGNATURE",
|
|
2055
|
+
error: "Inline precommit entries failed segmentContentHash / segmentCommitmentSignature verification"
|
|
2056
|
+
}
|
|
2057
|
+
};
|
|
2058
|
+
let dryRun;
|
|
2059
|
+
try {
|
|
2060
|
+
dryRun = await this.replayer.dryRunVerifyEntries(inlineEntries, {
|
|
2061
|
+
expectedPrevSequence: checkpoint.prevSequence,
|
|
2062
|
+
prevStateRoot: localCp?.stateRoot ?? null
|
|
2063
|
+
});
|
|
2064
|
+
} catch (err) {
|
|
2065
|
+
return {
|
|
2066
|
+
status: 400,
|
|
2067
|
+
body: {
|
|
2068
|
+
code: "PRECOMMIT_DRY_RUN_FAILED",
|
|
2069
|
+
error: err instanceof Error ? err.message : String(err)
|
|
2070
|
+
}
|
|
2071
|
+
};
|
|
2072
|
+
}
|
|
2073
|
+
const tree = new _arcblock_ledger_core.IncrementalMerkleTree();
|
|
2074
|
+
for (const h of dryRun.entryHashes) tree.append(h);
|
|
2075
|
+
if (normalizeHexCompare(dryRun.derivedStateRoot) !== normalizeHexCompare(checkpoint.stateRoot) || normalizeHexCompare(tree.getHexRoot()) !== normalizeHexCompare(checkpoint.txRoot)) return {
|
|
2076
|
+
status: 400,
|
|
2077
|
+
body: {
|
|
2078
|
+
code: "STATE_ROOT_MISMATCH",
|
|
2079
|
+
error: "Inline precommit entries failed txRoot/stateRoot check"
|
|
2080
|
+
}
|
|
2081
|
+
};
|
|
2082
|
+
await this.stateStore.saveUncommittedProposalEntries?.({
|
|
2083
|
+
chainId: this.chainId,
|
|
2084
|
+
voterDid: this.selfDid,
|
|
2085
|
+
checkpointHash: checkpoint.hash,
|
|
2086
|
+
height: checkpoint.height,
|
|
2087
|
+
entries: inlineEntries,
|
|
2088
|
+
checkpoint
|
|
2089
|
+
});
|
|
2090
|
+
stagedEntries = inlineEntries;
|
|
2091
|
+
} else {
|
|
2092
|
+
const expectedSegContentHash = (0, _arcblock_ledger_core.computeSegmentContentHash)({
|
|
2093
|
+
chainId: this.chainId,
|
|
2094
|
+
checkpoint,
|
|
2095
|
+
entries: stagedEntries
|
|
2096
|
+
});
|
|
2097
|
+
if (!await this.hasValidCheckpointSegmentCommitment(checkpoint, expectedSegContentHash)) return {
|
|
2098
|
+
status: 400,
|
|
2099
|
+
body: {
|
|
2100
|
+
code: "INVALID_SEGMENT_COMMITMENT_SIGNATURE",
|
|
2101
|
+
error: "Precommit checkpoint failed segmentContentHash / segmentCommitmentSignature verification"
|
|
2102
|
+
}
|
|
2103
|
+
};
|
|
2104
|
+
}
|
|
2105
|
+
let vote;
|
|
2106
|
+
try {
|
|
2107
|
+
vote = await (0, _arcblock_ledger_core.verifyPrepareQCAndCreateCommitVote)({
|
|
2108
|
+
config: this.clusterConfig,
|
|
2109
|
+
prepareQC,
|
|
2110
|
+
checkpoint,
|
|
2111
|
+
voterWallet: this.validatorWallet,
|
|
2112
|
+
voterDid: this.selfDid,
|
|
2113
|
+
parentHighQC,
|
|
2114
|
+
voterStateStore: this.stateStore
|
|
2115
|
+
});
|
|
2116
|
+
} catch (err) {
|
|
2117
|
+
return {
|
|
2118
|
+
status: 409,
|
|
2119
|
+
body: {
|
|
2120
|
+
code: "VOTER_COMMIT_REJECTED",
|
|
2121
|
+
error: err instanceof Error ? err.message : String(err),
|
|
2122
|
+
currentView: this.currentView,
|
|
2123
|
+
highQC: this.highQC,
|
|
2124
|
+
lastTC: this.lastTC
|
|
2125
|
+
}
|
|
2126
|
+
};
|
|
2127
|
+
}
|
|
2128
|
+
this.lastSelfVotedView = Math.max(this.lastSelfVotedView, vote.viewNumber);
|
|
2129
|
+
this.lockedQC = (0, _arcblock_ledger_core.selectHighestQC)([this.lockedQC, parentHighQC]);
|
|
2130
|
+
this.advanceCurrentViewTo(prepareQC.viewNumber);
|
|
2131
|
+
try {
|
|
2132
|
+
await this.stateStore.saveRuntimeState?.({
|
|
2133
|
+
chainId: this.chainId,
|
|
2134
|
+
voterDid: this.selfDid,
|
|
2135
|
+
currentView: this.currentView,
|
|
2136
|
+
lockedQC: this.lockedQC,
|
|
2137
|
+
highQC: this.highQC,
|
|
2138
|
+
lastTC: this.lastTC
|
|
2139
|
+
});
|
|
2140
|
+
} catch {}
|
|
2141
|
+
this.scheduleViewTimeout();
|
|
2142
|
+
return {
|
|
2143
|
+
status: 200,
|
|
2144
|
+
body: {
|
|
2145
|
+
vote,
|
|
2146
|
+
currentView: this.currentView,
|
|
2147
|
+
highQC: this.highQC
|
|
2148
|
+
}
|
|
2149
|
+
};
|
|
2150
|
+
});
|
|
2151
|
+
}
|
|
2152
|
+
async handleCommit(body) {
|
|
2153
|
+
return this.runExclusive(async () => {
|
|
2154
|
+
const segment = body.segment;
|
|
2155
|
+
if (!segment || segment.chainId !== this.chainId || !segment.checkpoint?.qc || !Array.isArray(segment.entries)) return {
|
|
2156
|
+
status: 400,
|
|
2157
|
+
body: {
|
|
2158
|
+
code: "INVALID_COMMIT_SEGMENT",
|
|
2159
|
+
error: "Malformed committed segment"
|
|
2160
|
+
}
|
|
2161
|
+
};
|
|
2162
|
+
if (!(0, _arcblock_ledger_core.verifyCheckpointQuorumCertificate)(segment.checkpoint, this.clusterConfig).ok) return {
|
|
2163
|
+
status: 400,
|
|
2164
|
+
body: {
|
|
2165
|
+
code: "INVALID_CHECKPOINT_QC",
|
|
2166
|
+
error: "Invalid 2f+1 QuorumCertificate on commit"
|
|
2167
|
+
}
|
|
2168
|
+
};
|
|
2169
|
+
let localCp = await this.resolver.ledger.getCheckpoint();
|
|
2170
|
+
const targetHeight = segment.checkpoint.height;
|
|
2171
|
+
if (localCp && localCp.height >= targetHeight) {
|
|
2172
|
+
const existingAtTarget = await this.resolver.ledger.getCheckpoint(targetHeight);
|
|
2173
|
+
if (existingAtTarget && normalizeHexCompare(existingAtTarget.hash) !== normalizeHexCompare(segment.checkpoint.hash)) {
|
|
2174
|
+
this.haltFailClosed(`BFT safety violation at height ${targetHeight}: local ${existingAtTarget.hash} vs QC ${segment.checkpoint.hash}`);
|
|
2175
|
+
return {
|
|
2176
|
+
status: 503,
|
|
2177
|
+
body: {
|
|
2178
|
+
code: "EQUIVOCATION_HALT",
|
|
2179
|
+
error: this.haltReason
|
|
2180
|
+
}
|
|
2181
|
+
};
|
|
2182
|
+
}
|
|
2183
|
+
this.tryAbsorbRemoteQC(segment.checkpoint.qc);
|
|
2184
|
+
this.scheduleViewTimeout();
|
|
2185
|
+
return {
|
|
2186
|
+
status: 200,
|
|
2187
|
+
body: {
|
|
2188
|
+
ok: true,
|
|
2189
|
+
alreadyCommitted: true,
|
|
2190
|
+
currentView: this.currentView
|
|
2191
|
+
}
|
|
2192
|
+
};
|
|
2193
|
+
}
|
|
2194
|
+
if ((localCp?.height ?? 0) < targetHeight - 1) {
|
|
2195
|
+
await this.catchUpToHeightLocked(targetHeight - 1);
|
|
2196
|
+
localCp = await this.resolver.ledger.getCheckpoint();
|
|
2197
|
+
}
|
|
2198
|
+
if ((localCp?.height ?? 0) !== targetHeight - 1) return {
|
|
2199
|
+
status: 409,
|
|
2200
|
+
body: {
|
|
2201
|
+
code: "MISSING_PRIOR_HEIGHTS",
|
|
2202
|
+
error: `Cannot commit height ${targetHeight} while at local height ${localCp?.height ?? 0}`
|
|
2203
|
+
}
|
|
2204
|
+
};
|
|
2205
|
+
try {
|
|
2206
|
+
await this.resolver.ledger.verifySegmentForImport(segment, {
|
|
2207
|
+
hotstuffClusterConfig: this.clusterConfig,
|
|
2208
|
+
verifyStateRoot: true,
|
|
2209
|
+
allowAheadLocalSequence: true
|
|
2210
|
+
});
|
|
2211
|
+
} catch (err) {
|
|
2212
|
+
return {
|
|
2213
|
+
status: 400,
|
|
2214
|
+
body: {
|
|
2215
|
+
code: "INVALID_COMMIT_SEGMENT",
|
|
2216
|
+
error: err instanceof Error ? err.message : String(err)
|
|
2217
|
+
}
|
|
2218
|
+
};
|
|
2219
|
+
}
|
|
2220
|
+
let dryRun;
|
|
2221
|
+
try {
|
|
2222
|
+
dryRun = await this.replayer.dryRunVerifyEntries(segment.entries, {
|
|
2223
|
+
expectedPrevSequence: localCp?.sequence ?? 0,
|
|
2224
|
+
prevStateRoot: localCp?.stateRoot ?? null
|
|
2225
|
+
});
|
|
2226
|
+
} catch (err) {
|
|
2227
|
+
return {
|
|
2228
|
+
status: 400,
|
|
2229
|
+
body: {
|
|
2230
|
+
code: "COMMIT_DRY_RUN_FAILED",
|
|
2231
|
+
error: err instanceof Error ? err.message : String(err)
|
|
2232
|
+
}
|
|
2233
|
+
};
|
|
2234
|
+
}
|
|
2235
|
+
if (dryRun.derivedStateRoot && normalizeHexCompare(dryRun.derivedStateRoot) !== normalizeHexCompare(segment.checkpoint.stateRoot)) return {
|
|
2236
|
+
status: 400,
|
|
2237
|
+
body: {
|
|
2238
|
+
code: "STATE_ROOT_MISMATCH",
|
|
2239
|
+
error: "Segment stateRoot mismatch"
|
|
2240
|
+
}
|
|
2241
|
+
};
|
|
2242
|
+
await this.resolver.ledger.importSegment(segment, {
|
|
2243
|
+
hotstuffClusterConfig: this.clusterConfig,
|
|
2244
|
+
verifyStateRoot: true
|
|
2245
|
+
});
|
|
2246
|
+
for (const entry of segment.entries) await this.replayer.replayEntry(entry, { appendToLedger: false });
|
|
2247
|
+
this.lockedQC = (0, _arcblock_ledger_core.selectHighestQC)([this.lockedQC, this.highQC]);
|
|
2248
|
+
this.highQC = (0, _arcblock_ledger_core.selectHighestQC)([this.highQC, segment.checkpoint.qc]);
|
|
2249
|
+
this.advanceCurrentViewTo(segment.checkpoint.qc.viewNumber + 1);
|
|
2250
|
+
this.currentViewEnteredAtMs = Date.now();
|
|
2251
|
+
this.pendingDemandForView = -1;
|
|
2252
|
+
this.consecutiveViewTimeouts = 0;
|
|
2253
|
+
this.leaderRouter.advanceMinVerifiedCheckpointHeight(targetHeight);
|
|
2254
|
+
try {
|
|
2255
|
+
await this.stateStore.saveRuntimeState?.({
|
|
2256
|
+
chainId: this.chainId,
|
|
2257
|
+
voterDid: this.selfDid,
|
|
2258
|
+
currentView: this.currentView,
|
|
2259
|
+
lockedQC: this.lockedQC,
|
|
2260
|
+
highQC: this.highQC,
|
|
2261
|
+
lastTC: this.lastTC,
|
|
2262
|
+
pruneUncommittedUpToHeight: targetHeight
|
|
2263
|
+
});
|
|
2264
|
+
} catch {}
|
|
2265
|
+
this.scheduleViewTimeout();
|
|
2266
|
+
return {
|
|
2267
|
+
status: 200,
|
|
2268
|
+
body: {
|
|
2269
|
+
ok: true,
|
|
2270
|
+
currentView: this.currentView
|
|
2271
|
+
}
|
|
2272
|
+
};
|
|
2273
|
+
});
|
|
2274
|
+
}
|
|
2275
|
+
currentEffectiveViewTimeoutMs() {
|
|
2276
|
+
const exp = Math.min(3, Math.max(0, this.consecutiveViewTimeouts));
|
|
2277
|
+
const mult = Math.min(MAX_VIEW_TIMEOUT_BACKOFF_MULTIPLIER, 2 ** exp);
|
|
2278
|
+
return this.baseViewTimeoutMs * mult;
|
|
2279
|
+
}
|
|
2280
|
+
scheduleViewTimeout() {
|
|
2281
|
+
if (this.viewTimer) {
|
|
2282
|
+
clearTimeout(this.viewTimer);
|
|
2283
|
+
this.viewTimer = null;
|
|
2284
|
+
}
|
|
2285
|
+
if (this.stopped || this.halted) return;
|
|
2286
|
+
const scheduledView = this.currentView;
|
|
2287
|
+
const delayMs = this.currentEffectiveViewTimeoutMs();
|
|
2288
|
+
this.viewTimer = setTimeout(() => {
|
|
2289
|
+
this.onViewTimerExpired(scheduledView).catch(() => void 0);
|
|
2290
|
+
}, delayMs);
|
|
2291
|
+
if (typeof this.viewTimer.unref === "function") this.viewTimer.unref();
|
|
2292
|
+
}
|
|
2293
|
+
clearTimers() {
|
|
2294
|
+
if (this.heartbeatTimer) {
|
|
2295
|
+
clearInterval(this.heartbeatTimer);
|
|
2296
|
+
this.heartbeatTimer = null;
|
|
2297
|
+
}
|
|
2298
|
+
if (this.viewTimer) {
|
|
2299
|
+
clearTimeout(this.viewTimer);
|
|
2300
|
+
this.viewTimer = null;
|
|
2301
|
+
}
|
|
2302
|
+
}
|
|
2303
|
+
async sendHeartbeatsTick() {
|
|
2304
|
+
if (this.stopped || this.halted) return;
|
|
2305
|
+
if ((0, _arcblock_ledger_core.getLeaderForView)(this.clusterConfig, this.currentView).did !== this.selfDid) return;
|
|
2306
|
+
if (this.hasSpentTurnInView(this.currentView) || this.unleadableView === this.currentView) return;
|
|
2307
|
+
const threshold = this.clusterConfig.quorumThreshold ?? 3;
|
|
2308
|
+
let reachablePeers = 0;
|
|
2309
|
+
const viewSnapshot = this.currentView;
|
|
2310
|
+
await Promise.all(this.peerValidators.map(async (peer) => {
|
|
2311
|
+
try {
|
|
2312
|
+
const res = await this.sendConsensusJson(peer, "POST", "/consensus/heartbeat", {
|
|
2313
|
+
viewNumber: viewSnapshot,
|
|
2314
|
+
highQC: this.highQC,
|
|
2315
|
+
lastTC: this.lastTC
|
|
2316
|
+
}, Math.min(this.httpTimeoutMs, Math.max(250, this.heartbeatIntervalMs * 2)), MAX_CONTROL_ROUTE_BYTES);
|
|
2317
|
+
if (res.authenticated) reachablePeers++;
|
|
2318
|
+
const advancedQC = this.tryAbsorbRemoteQC(res.body?.highQC);
|
|
2319
|
+
const advancedTC = this.tryAbsorbRemoteTC(res.body?.lastTC);
|
|
2320
|
+
if (advancedQC || advancedTC) this.runExclusive(async () => {
|
|
2321
|
+
if (this.highQC) await this.catchUpToHeightLocked(this.highQC.height);
|
|
2322
|
+
}).catch(() => void 0);
|
|
2323
|
+
} catch {}
|
|
2324
|
+
}));
|
|
2325
|
+
if (reachablePeers + 1 >= threshold) {
|
|
2326
|
+
this.quorumContactObserved = true;
|
|
2327
|
+
this.quorumLeaseValid = true;
|
|
2328
|
+
if ((0, _arcblock_ledger_core.getLeaderForView)(this.clusterConfig, this.currentView).did === this.selfDid && !this.hasSpentTurnInView(this.currentView) && this.unleadableView !== this.currentView) {
|
|
2329
|
+
const lock = await Promise.resolve(this.stateStore.getLock(this.chainId, this.selfDid)).catch(() => null);
|
|
2330
|
+
if (!this.shouldSuppressHeartbeatKeepalive(lock, this.currentView)) this.scheduleViewTimeout();
|
|
2331
|
+
}
|
|
2332
|
+
} else this.quorumLeaseValid = false;
|
|
2333
|
+
}
|
|
2334
|
+
async onViewTimerExpired(timedOutView) {
|
|
2335
|
+
if (this.stopped || this.halted || timedOutView !== this.currentView) return;
|
|
2336
|
+
if ((0, _arcblock_ledger_core.getLeaderForView)(this.clusterConfig, timedOutView).did === this.selfDid && !this.hasSpentTurnInView(timedOutView) && this.unleadableView !== timedOutView && this.refreshQuorumLease()) {
|
|
2337
|
+
const lock = await Promise.resolve(this.stateStore.getLock(this.chainId, this.selfDid)).catch(() => null);
|
|
2338
|
+
if (!this.shouldSuppressHeartbeatKeepalive(lock, timedOutView)) {
|
|
2339
|
+
this.scheduleViewTimeout();
|
|
2340
|
+
return;
|
|
2341
|
+
}
|
|
2342
|
+
}
|
|
2343
|
+
let timeoutMsg = null;
|
|
2344
|
+
let formedTC = null;
|
|
2345
|
+
await this.runExclusive(async () => {
|
|
2346
|
+
if (this.stopped || this.halted || timedOutView !== this.currentView) return;
|
|
2347
|
+
this.consecutiveViewTimeouts++;
|
|
2348
|
+
timeoutMsg = await (0, _arcblock_ledger_core.createHotStuffTimeoutMessage)({
|
|
2349
|
+
config: this.clusterConfig,
|
|
2350
|
+
viewNumber: timedOutView,
|
|
2351
|
+
voterWallet: this.validatorWallet,
|
|
2352
|
+
voterDid: this.selfDid,
|
|
2353
|
+
highQC: this.highQC,
|
|
2354
|
+
voterStateStore: this.stateStore,
|
|
2355
|
+
usePrepareQCLock: true
|
|
2356
|
+
});
|
|
2357
|
+
formedTC = await this.recordTimeoutMessageAndMaybeAggregateLocked(timeoutMsg);
|
|
2358
|
+
this.scheduleViewTimeout();
|
|
2359
|
+
});
|
|
2360
|
+
if (formedTC) {
|
|
2361
|
+
await this.broadcastNewViewAndMaybeRecover(formedTC);
|
|
2362
|
+
return;
|
|
2363
|
+
}
|
|
2364
|
+
if (timeoutMsg) {
|
|
2365
|
+
const msgToSend = timeoutMsg;
|
|
2366
|
+
await Promise.allSettled(this.peerValidators.map(async (peer) => {
|
|
2367
|
+
const res = await this.sendConsensusJson(peer, "POST", "/consensus/timeout", { timeout: msgToSend }, this.httpTimeoutMs, MAX_CONTROL_ROUTE_BYTES);
|
|
2368
|
+
if (res.authenticated && res.body?.tc) await this.handleNewView({ tc: res.body.tc });
|
|
2369
|
+
}));
|
|
2370
|
+
}
|
|
2371
|
+
}
|
|
2372
|
+
recordTimeoutMessageAndMaybeAggregateLocked(msg) {
|
|
2373
|
+
if (!Number.isSafeInteger(msg?.viewNumber) || msg.viewNumber < 0 || !(0, _arcblock_ledger_core.verifyHotStuffTimeoutMessage)(msg, this.clusterConfig).ok) return Promise.resolve(null);
|
|
2374
|
+
if (msg.highQC) this.tryAbsorbRemoteQC(msg.highQC);
|
|
2375
|
+
for (const view of Array.from(this.collectedTimeoutsByView.keys())) if (!Number.isSafeInteger(view) || view < this.currentView - 1) this.collectedTimeoutsByView.delete(view);
|
|
2376
|
+
if (msg.viewNumber < this.currentView - 1) return Promise.resolve(null);
|
|
2377
|
+
if (msg.viewNumber === this.currentView - 1 && this.hasSpentTurnInView(this.currentView)) return Promise.resolve(this.lastTC && this.lastTC.viewNumber === msg.viewNumber ? this.lastTC : null);
|
|
2378
|
+
const maxFutureTimeoutWindow = Math.max(8, this.clusterConfig.validators.length * 2);
|
|
2379
|
+
if (msg.viewNumber > this.currentView + maxFutureTimeoutWindow) return Promise.resolve(null);
|
|
2380
|
+
let bucket = this.collectedTimeoutsByView.get(msg.viewNumber);
|
|
2381
|
+
if (!bucket && msg.viewNumber > this.currentView) {
|
|
2382
|
+
let voterFutureBuckets = 0;
|
|
2383
|
+
for (const [view, existingBucket] of this.collectedTimeoutsByView.entries()) if (view > this.currentView && existingBucket.has(msg.voterDid)) voterFutureBuckets++;
|
|
2384
|
+
if (voterFutureBuckets >= 4) return Promise.resolve(null);
|
|
2385
|
+
}
|
|
2386
|
+
if (!bucket) {
|
|
2387
|
+
bucket = /* @__PURE__ */ new Map();
|
|
2388
|
+
this.collectedTimeoutsByView.set(msg.viewNumber, bucket);
|
|
2389
|
+
if (this.collectedTimeoutsByView.size > 32) {
|
|
2390
|
+
const evictableViews = Array.from(this.collectedTimeoutsByView.keys()).filter((v) => v !== this.currentView && v !== this.currentView - 1);
|
|
2391
|
+
if (evictableViews.length > 0) {
|
|
2392
|
+
const highestFutureView = Math.max(...evictableViews);
|
|
2393
|
+
this.collectedTimeoutsByView.delete(highestFutureView);
|
|
2394
|
+
}
|
|
2395
|
+
}
|
|
2396
|
+
}
|
|
2397
|
+
const sanitizedMsg = {
|
|
2398
|
+
chainId: msg.chainId,
|
|
2399
|
+
viewNumber: msg.viewNumber,
|
|
2400
|
+
voterDid: msg.voterDid,
|
|
2401
|
+
highQC: msg.highQC ?? null,
|
|
2402
|
+
lastVotedHeight: msg.lastVotedHeight ?? null,
|
|
2403
|
+
lastVotedView: msg.lastVotedView ?? null,
|
|
2404
|
+
lastVotedCheckpointHash: msg.lastVotedCheckpointHash ?? null,
|
|
2405
|
+
...msg.lastVotedCheckpoint ? { lastVotedCheckpoint: msg.lastVotedCheckpoint } : {},
|
|
2406
|
+
...msg.lastVotedVoteSignature ? { lastVotedVoteSignature: msg.lastVotedVoteSignature } : {},
|
|
2407
|
+
...msg.lastVotedVoteTimestamp !== void 0 && msg.lastVotedVoteTimestamp !== null ? { lastVotedVoteTimestamp: msg.lastVotedVoteTimestamp } : {},
|
|
2408
|
+
...msg.priorVotesByView ? { priorVotesByView: msg.priorVotesByView } : {},
|
|
2409
|
+
...msg.highestPrepareQC !== void 0 ? { highestPrepareQC: msg.highestPrepareQC } : {},
|
|
2410
|
+
...msg.highestPrepareCheckpoint ? { highestPrepareCheckpoint: msg.highestPrepareCheckpoint } : {},
|
|
2411
|
+
...msg.observedPrepareQCs ? { observedPrepareQCs: msg.observedPrepareQCs } : {},
|
|
2412
|
+
timestamp: msg.timestamp,
|
|
2413
|
+
signature: msg.signature
|
|
2414
|
+
};
|
|
2415
|
+
bucket.set(sanitizedMsg.voterDid, sanitizedMsg);
|
|
2416
|
+
const threshold = this.clusterConfig.quorumThreshold ?? 3;
|
|
2417
|
+
if (bucket.size < threshold) return Promise.resolve(null);
|
|
2418
|
+
if (this.lastTC && (this.lastTC.viewNumber > sanitizedMsg.viewNumber || this.lastTC.viewNumber === sanitizedMsg.viewNumber && bucket.size <= this.lastTC.signatures.length)) return Promise.resolve(this.lastTC.viewNumber === sanitizedMsg.viewNumber ? this.lastTC : null);
|
|
2419
|
+
let tc;
|
|
2420
|
+
try {
|
|
2421
|
+
tc = (0, _arcblock_ledger_core.aggregateTimeoutCertificate)({
|
|
2422
|
+
config: this.clusterConfig,
|
|
2423
|
+
viewNumber: sanitizedMsg.viewNumber,
|
|
2424
|
+
messages: Array.from(bucket.values())
|
|
2425
|
+
});
|
|
2426
|
+
} catch {
|
|
2427
|
+
return Promise.resolve(null);
|
|
2428
|
+
}
|
|
2429
|
+
if (!(0, _arcblock_ledger_core.verifyTimeoutCertificate)(tc, this.clusterConfig).ok) return Promise.resolve(null);
|
|
2430
|
+
return this.applyVerifiedTimeoutCertificateLocked(tc).then(() => tc);
|
|
2431
|
+
}
|
|
2432
|
+
async applyVerifiedTimeoutCertificateLocked(tc) {
|
|
2433
|
+
if (tc.winningHighQC) {
|
|
2434
|
+
this.tryAbsorbRemoteQC(tc.winningHighQC);
|
|
2435
|
+
const persistedLock = await Promise.resolve(this.stateStore.getLock(this.chainId, this.selfDid)).catch(() => null);
|
|
2436
|
+
if (persistedLock) {
|
|
2437
|
+
this.lastSelfVotedView = Math.max(this.lastSelfVotedView, persistedLock.lastVotedView, persistedLock.lastCommitVotedView ?? -1);
|
|
2438
|
+
this.lastSelfPreparedView = Math.max(this.lastSelfPreparedView, persistedLock.lastPreparedView ?? -1);
|
|
2439
|
+
}
|
|
2440
|
+
if (((await this.resolver.ledger.getCheckpoint())?.height ?? 0) < tc.winningHighQC.height) await this.catchUpToHeightLocked(tc.winningHighQC.height);
|
|
2441
|
+
}
|
|
2442
|
+
const prevView = this.currentView;
|
|
2443
|
+
let upgradedSameViewTC = false;
|
|
2444
|
+
if (this.isBetterTimeoutCertificate(tc)) {
|
|
2445
|
+
upgradedSameViewTC = Boolean(this.lastTC && tc.viewNumber === this.lastTC.viewNumber);
|
|
2446
|
+
this.lastTC = tc;
|
|
2447
|
+
if (upgradedSameViewTC && this.unleadableView === tc.viewNumber + 1) this.unleadableView = -1;
|
|
2448
|
+
}
|
|
2449
|
+
if (tc.viewNumber + 1 > this.currentView) {
|
|
2450
|
+
this.advanceCurrentViewTo(tc.viewNumber + 1);
|
|
2451
|
+
this.consecutiveViewTimeouts = 0;
|
|
2452
|
+
if (this.unleadableView < this.currentView) this.unleadableView = -1;
|
|
2453
|
+
}
|
|
2454
|
+
for (const view of Array.from(this.collectedTimeoutsByView.keys())) if (!Number.isSafeInteger(view) || view < this.currentView - 1) this.collectedTimeoutsByView.delete(view);
|
|
2455
|
+
try {
|
|
2456
|
+
await this.stateStore.saveRuntimeState?.({
|
|
2457
|
+
chainId: this.chainId,
|
|
2458
|
+
voterDid: this.selfDid,
|
|
2459
|
+
currentView: this.currentView,
|
|
2460
|
+
lockedQC: this.lockedQC,
|
|
2461
|
+
highQC: this.highQC,
|
|
2462
|
+
lastTC: this.lastTC
|
|
2463
|
+
});
|
|
2464
|
+
} catch {}
|
|
2465
|
+
this.scheduleViewTimeout();
|
|
2466
|
+
return this.currentView > prevView || upgradedSameViewTC;
|
|
2467
|
+
}
|
|
2468
|
+
async broadcastNewViewAndMaybeRecover(tc) {
|
|
2469
|
+
await Promise.allSettled(this.peerValidators.map((peer) => this.sendConsensusJson(peer, "POST", "/consensus/new-view", { tc }, this.httpTimeoutMs, MAX_CONTROL_ROUTE_BYTES)));
|
|
2470
|
+
await this.maybeRecoverLockedCheckpointAsNewLeader(tc);
|
|
2471
|
+
}
|
|
2472
|
+
async maybeRecoverLockedCheckpointAsNewLeader(tc) {
|
|
2473
|
+
if (!tc.lockedCheckpoint || this.stopped || this.halted) return;
|
|
2474
|
+
let recovered = null;
|
|
2475
|
+
await this.runExclusive(async () => {
|
|
2476
|
+
const expectedLeader = (0, _arcblock_ledger_core.getLeaderForView)(this.clusterConfig, this.currentView);
|
|
2477
|
+
const targetHeight = (this.highQC?.height ?? 0) + 1;
|
|
2478
|
+
if (expectedLeader.did === this.selfDid && this.currentView === tc.viewNumber + 1 && tc.lockedCheckpoint && tc.lockedCheckpoint.height === targetHeight) {
|
|
2479
|
+
let recoveryFailed = false;
|
|
2480
|
+
try {
|
|
2481
|
+
recovered = await this.recoverLockedCheckpointLocked(tc);
|
|
2482
|
+
} catch {
|
|
2483
|
+
recoveryFailed = true;
|
|
2484
|
+
recovered = null;
|
|
2485
|
+
}
|
|
2486
|
+
if (recoveryFailed || !recovered) {
|
|
2487
|
+
if (((await this.resolver.ledger.getCheckpoint().catch(() => null))?.height ?? 0) < targetHeight) this.markCurrentViewUnleadableAndScheduleTimeout();
|
|
2488
|
+
}
|
|
2489
|
+
}
|
|
2490
|
+
});
|
|
2491
|
+
if (recovered) await this.broadcastCommittedSegments([recovered]).catch(() => void 0);
|
|
2492
|
+
}
|
|
2493
|
+
async handleTimeoutMessage(body) {
|
|
2494
|
+
const msg = body.timeout;
|
|
2495
|
+
if (!msg || typeof msg !== "object") return {
|
|
2496
|
+
status: 400,
|
|
2497
|
+
body: {
|
|
2498
|
+
code: "INVALID_TIMEOUT",
|
|
2499
|
+
error: "Missing timeout message"
|
|
2500
|
+
}
|
|
2501
|
+
};
|
|
2502
|
+
const check = (0, _arcblock_ledger_core.verifyHotStuffTimeoutMessage)(msg, this.clusterConfig);
|
|
2503
|
+
if (!check.ok) return {
|
|
2504
|
+
status: 400,
|
|
2505
|
+
body: {
|
|
2506
|
+
code: "INVALID_TIMEOUT_SIGNATURE",
|
|
2507
|
+
error: check.reason || "Invalid timeout"
|
|
2508
|
+
}
|
|
2509
|
+
};
|
|
2510
|
+
let formedTC = null;
|
|
2511
|
+
let outOfRangeError = null;
|
|
2512
|
+
let leaderAbdicatedView = null;
|
|
2513
|
+
await this.runExclusive(async () => {
|
|
2514
|
+
if (msg.highQC) this.tryAbsorbRemoteQC(msg.highQC);
|
|
2515
|
+
const maxFutureTimeoutWindow = Math.max(8, this.clusterConfig.validators.length * 2);
|
|
2516
|
+
if (msg.viewNumber > this.currentView + maxFutureTimeoutWindow) {
|
|
2517
|
+
outOfRangeError = `Timeout view (${msg.viewNumber}) exceeds max future window (${this.currentView + maxFutureTimeoutWindow})`;
|
|
2518
|
+
return;
|
|
2519
|
+
}
|
|
2520
|
+
const currentViewLeader = (0, _arcblock_ledger_core.getLeaderForView)(this.clusterConfig, this.currentView);
|
|
2521
|
+
const existingBucket = this.collectedTimeoutsByView.get(this.currentView);
|
|
2522
|
+
if (msg.viewNumber === this.currentView && msg.voterDid === currentViewLeader.did && msg.voterDid !== this.selfDid && !existingBucket?.has(this.selfDid)) leaderAbdicatedView = this.currentView;
|
|
2523
|
+
formedTC = await this.recordTimeoutMessageAndMaybeAggregateLocked(msg);
|
|
2524
|
+
});
|
|
2525
|
+
if (outOfRangeError) return {
|
|
2526
|
+
status: 400,
|
|
2527
|
+
body: {
|
|
2528
|
+
code: "TIMEOUT_VIEW_OUT_OF_RANGE",
|
|
2529
|
+
error: outOfRangeError,
|
|
2530
|
+
currentView: this.currentView
|
|
2531
|
+
}
|
|
2532
|
+
};
|
|
2533
|
+
if (formedTC) this.broadcastNewViewAndMaybeRecover(formedTC).catch(() => void 0);
|
|
2534
|
+
else if (leaderAbdicatedView !== null && leaderAbdicatedView === this.currentView) {
|
|
2535
|
+
const viewToExpire = leaderAbdicatedView;
|
|
2536
|
+
const t = setTimeout(() => {
|
|
2537
|
+
this.onViewTimerExpired(viewToExpire).catch(() => void 0);
|
|
2538
|
+
}, 0);
|
|
2539
|
+
if (typeof t.unref === "function") t.unref();
|
|
2540
|
+
}
|
|
2541
|
+
return {
|
|
2542
|
+
status: 200,
|
|
2543
|
+
body: {
|
|
2544
|
+
ok: true,
|
|
2545
|
+
currentView: this.currentView,
|
|
2546
|
+
tc: formedTC
|
|
2547
|
+
}
|
|
2548
|
+
};
|
|
2549
|
+
}
|
|
2550
|
+
async handleNewView(body) {
|
|
2551
|
+
const tc = body.tc;
|
|
2552
|
+
if (!tc || typeof tc !== "object") return {
|
|
2553
|
+
status: 400,
|
|
2554
|
+
body: {
|
|
2555
|
+
code: "INVALID_TC",
|
|
2556
|
+
error: "Missing TimeoutCertificate"
|
|
2557
|
+
}
|
|
2558
|
+
};
|
|
2559
|
+
const check = (0, _arcblock_ledger_core.verifyTimeoutCertificate)(tc, this.clusterConfig);
|
|
2560
|
+
if (!check.ok) return {
|
|
2561
|
+
status: 400,
|
|
2562
|
+
body: {
|
|
2563
|
+
code: "INVALID_TC",
|
|
2564
|
+
error: check.reason || "Invalid TimeoutCertificate"
|
|
2565
|
+
}
|
|
2566
|
+
};
|
|
2567
|
+
let accepted = false;
|
|
2568
|
+
await this.runExclusive(async () => {
|
|
2569
|
+
accepted = await this.applyVerifiedTimeoutCertificateLocked(tc);
|
|
2570
|
+
});
|
|
2571
|
+
if (accepted && tc.lockedCheckpoint) this.maybeRecoverLockedCheckpointAsNewLeader(tc).catch(() => void 0);
|
|
2572
|
+
return {
|
|
2573
|
+
status: 200,
|
|
2574
|
+
body: {
|
|
2575
|
+
ok: true,
|
|
2576
|
+
currentView: this.currentView,
|
|
2577
|
+
highQC: this.highQC
|
|
2578
|
+
}
|
|
2579
|
+
};
|
|
2580
|
+
}
|
|
2581
|
+
async probePeersOnStartup() {
|
|
2582
|
+
let reachablePeers = 0;
|
|
2583
|
+
await Promise.allSettled(this.peerValidators.map(async (peer) => {
|
|
2584
|
+
const res = await this.sendConsensusJson(peer, "GET", "/consensus/status", void 0, Math.min(1e3, this.httpTimeoutMs), MAX_CONTROL_ROUTE_BYTES);
|
|
2585
|
+
if (res.status === 200 && res.authenticated) {
|
|
2586
|
+
reachablePeers++;
|
|
2587
|
+
this.tryAbsorbRemoteQC(res.body.highQC);
|
|
2588
|
+
this.tryAbsorbRemoteTC(res.body.lastTC);
|
|
2589
|
+
}
|
|
2590
|
+
}));
|
|
2591
|
+
if (reachablePeers + 1 >= (this.clusterConfig.quorumThreshold ?? 3)) {
|
|
2592
|
+
this.quorumContactObserved = true;
|
|
2593
|
+
this.quorumLeaseValid = true;
|
|
2594
|
+
}
|
|
2595
|
+
if (this.highQC) await this.runExclusive(async () => {
|
|
2596
|
+
await this.catchUpToHeightLocked(this.highQC.height);
|
|
2597
|
+
});
|
|
2598
|
+
}
|
|
2599
|
+
async start() {
|
|
2600
|
+
if (this.server) return;
|
|
2601
|
+
this.stopped = false;
|
|
2602
|
+
this.startedAtMs = Date.now();
|
|
2603
|
+
await this.runExclusive(async () => {
|
|
2604
|
+
const persisted = await this.stateStore.loadRuntimeState?.(this.chainId, this.selfDid);
|
|
2605
|
+
if (persisted) {
|
|
2606
|
+
if (persisted.lockedQC && (0, _arcblock_ledger_core.verifyQuorumCertificate)(persisted.lockedQC, this.clusterConfig).ok) this.lockedQC = persisted.lockedQC;
|
|
2607
|
+
if (persisted.highQC && (0, _arcblock_ledger_core.verifyQuorumCertificate)(persisted.highQC, this.clusterConfig).ok) this.highQC = persisted.highQC;
|
|
2608
|
+
if (persisted.lastTC && (0, _arcblock_ledger_core.verifyTimeoutCertificate)(persisted.lastTC, this.clusterConfig).ok) this.lastTC = persisted.lastTC;
|
|
2609
|
+
this.currentView = Math.max(this.currentView, persisted.currentView, this.highQC ? this.highQC.viewNumber + 1 : 0, this.lastTC ? this.lastTC.viewNumber + 1 : 0);
|
|
2610
|
+
}
|
|
2611
|
+
const persistedLock = await Promise.resolve(this.stateStore.getLock(this.chainId, this.selfDid)).catch(() => null);
|
|
2612
|
+
if (persistedLock) {
|
|
2613
|
+
this.lastSelfVotedView = Math.max(this.lastSelfVotedView, persistedLock.lastVotedView, persistedLock.lastCommitVotedView ?? -1);
|
|
2614
|
+
this.lastSelfPreparedView = Math.max(this.lastSelfPreparedView, persistedLock.lastPreparedView ?? -1);
|
|
2615
|
+
}
|
|
2616
|
+
const localCp = await this.resolver.ledger.getCheckpoint();
|
|
2617
|
+
if (localCp?.qc && (0, _arcblock_ledger_core.verifyQuorumCertificate)(localCp.qc, this.clusterConfig).ok) {
|
|
2618
|
+
this.highQC = (0, _arcblock_ledger_core.selectHighestQC)([this.highQC, localCp.qc]);
|
|
2619
|
+
this.currentView = Math.max(this.currentView, localCp.qc.viewNumber + 1);
|
|
2620
|
+
}
|
|
2621
|
+
this.currentViewEnteredAtMs = Date.now();
|
|
2622
|
+
if (localCp && localCp.height > 0) this.leaderRouter.advanceMinVerifiedCheckpointHeight(localCp.height);
|
|
2623
|
+
await this.reconcileCrashWindowLocked();
|
|
2624
|
+
});
|
|
2625
|
+
const srv = node_http.default.createServer((req, res) => {
|
|
2626
|
+
this.handleConsensusHttpRequest(req, res).catch((err) => {
|
|
2627
|
+
if (!res.headersSent) {
|
|
2628
|
+
res.statusCode = 500;
|
|
2629
|
+
res.setHeader("Content-Type", "application/json; charset=utf-8");
|
|
2630
|
+
res.end(JSON.stringify({
|
|
2631
|
+
code: "INTERNAL_ERROR",
|
|
2632
|
+
error: err instanceof Error ? err.message : String(err)
|
|
2633
|
+
}));
|
|
2634
|
+
}
|
|
2635
|
+
});
|
|
2636
|
+
});
|
|
2637
|
+
await new Promise((resolve, reject) => {
|
|
2638
|
+
srv.once("error", reject);
|
|
2639
|
+
srv.listen(this.configuredConsensusPort, this.consensusBindHost, () => {
|
|
2640
|
+
srv.removeListener("error", reject);
|
|
2641
|
+
resolve();
|
|
2642
|
+
});
|
|
2643
|
+
});
|
|
2644
|
+
this.server = srv;
|
|
2645
|
+
this.boundPort = srv.address()?.port ?? this.configuredConsensusPort;
|
|
2646
|
+
await this.probePeersOnStartup().catch(() => void 0);
|
|
2647
|
+
this.heartbeatTimer = setInterval(() => {
|
|
2648
|
+
this.sendHeartbeatsTick().catch(() => void 0);
|
|
2649
|
+
}, this.heartbeatIntervalMs);
|
|
2650
|
+
if (typeof this.heartbeatTimer.unref === "function") this.heartbeatTimer.unref();
|
|
2651
|
+
this.scheduleViewTimeout();
|
|
2652
|
+
}
|
|
2653
|
+
async close() {
|
|
2654
|
+
this.stopped = true;
|
|
2655
|
+
this.clearTimers();
|
|
2656
|
+
for (const waiter of Array.from(this.viewAdvanceWaiters)) {
|
|
2657
|
+
this.viewAdvanceWaiters.delete(waiter);
|
|
2658
|
+
waiter.resolve();
|
|
2659
|
+
}
|
|
2660
|
+
if (this.server) {
|
|
2661
|
+
const srv = this.server;
|
|
2662
|
+
this.server = null;
|
|
2663
|
+
await new Promise((resolve) => srv.close(() => resolve()));
|
|
2664
|
+
}
|
|
2665
|
+
await this.mutexTail.catch(() => void 0);
|
|
2666
|
+
}
|
|
2667
|
+
};
|
|
2668
|
+
|
|
2669
|
+
//#endregion
|
|
2670
|
+
exports.ConsensusAbortError = ConsensusAbortError;
|
|
2671
|
+
exports.HotStuffNodeRuntime = HotStuffNodeRuntime;
|