@ocap/resolver 1.32.1 → 1.32.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/esm/dashboard.d.mts +4 -0
- package/esm/dashboard.mjs +5 -0
- package/esm/explorer.mjs +12 -3
- package/esm/follower.d.mts +29 -1
- package/esm/follower.mjs +98 -14
- package/esm/hooks.mjs +1 -1
- package/esm/index.mjs +1 -1
- package/esm/node-preferences.mjs +7 -6
- package/esm/replayer.d.mts +40 -3
- package/esm/replayer.mjs +96 -34
- package/lib/dashboard.cjs +5 -0
- package/lib/dashboard.d.cts +4 -0
- package/lib/explorer.cjs +13 -3
- package/lib/follower.cjs +101 -13
- package/lib/follower.d.cts +29 -1
- package/lib/hooks.cjs +2 -2
- package/lib/index.cjs +2 -2
- package/lib/node-preferences.cjs +6 -5
- package/lib/replayer.cjs +99 -33
- package/lib/replayer.d.cts +40 -3
- package/package.json +16 -15
package/esm/replayer.mjs
CHANGED
|
@@ -1,9 +1,12 @@
|
|
|
1
|
-
import { authenticateEntryDecodedFields, computeEntryHash } from "@arcblock/ledger-core";
|
|
1
|
+
import { authenticateEntryDecodedFields, computeEntryHash, isEntryChainIdAcceptable } from "@arcblock/ledger-core";
|
|
2
2
|
import { toTxHash } from "@ocap/mcrypto";
|
|
3
3
|
import { createExecutor } from "@ocap/tx-protocols";
|
|
4
4
|
import { fromBase64 } from "@ocap/util";
|
|
5
5
|
import { CustomError } from "@ocap/util/lib/error";
|
|
6
6
|
import { Blacklist } from "@ocap/state";
|
|
7
|
+
import { pipes } from "@ocap/tx-pipeline";
|
|
8
|
+
import cloneDeep from "lodash/cloneDeep.js";
|
|
9
|
+
import merge from "lodash/merge.js";
|
|
7
10
|
|
|
8
11
|
//#region src/replayer.ts
|
|
9
12
|
/**
|
|
@@ -21,6 +24,22 @@ import { Blacklist } from "@ocap/state";
|
|
|
21
24
|
* - Enforces pre-commit `expectedCommitHash` on `statedb.runAsLambda` so any state divergence rolls back
|
|
22
25
|
* SQLite atomically BEFORE `COMMIT` and before appending to `ledger.sqlite`.
|
|
23
26
|
*/
|
|
27
|
+
/**
|
|
28
|
+
* Cross-layer chainId rule for executable replay (#2029, #2039):
|
|
29
|
+
* Own-chain entries (`entry.chainId === expectedChainId`) are accepted directly; a self-declared foreign-`chainId`
|
|
30
|
+
* wire entry (`isEntryChainIdAcceptable(entry, expectedChainId)`) can ONLY execute as a force-finalized orphan
|
|
31
|
+
* (`(entry.stateCommitHash ?? '').trim() === ''`), because `VerifyTx` rejects `chainId !== config.chainId` before
|
|
32
|
+
* `PrepareState`/`CommitState` so a foreign-`chainId` entry can never commit state.
|
|
33
|
+
*/
|
|
34
|
+
function isReplayEntryChainIdAcceptable(entry, expectedChainId) {
|
|
35
|
+
if (entry.chainId === expectedChainId) return true;
|
|
36
|
+
return isEntryChainIdAcceptable(entry, expectedChainId) && (entry.stateCommitHash ?? "").trim() === "";
|
|
37
|
+
}
|
|
38
|
+
function assertReplayEntryChainIdAcceptable(entry, expectedChainId, prefix = "Ledger entry") {
|
|
39
|
+
if (isReplayEntryChainIdAcceptable(entry, expectedChainId)) return;
|
|
40
|
+
if (entry.chainId !== expectedChainId && typeof entry.txRaw === "string" && isEntryChainIdAcceptable(entry, expectedChainId) && (entry.stateCommitHash ?? "").trim() !== "") throw new CustomError("CHAIN_ID_MISMATCH", `${prefix} foreign-chainId entry at seq=${String(entry.sequence)} (chainId=${String(entry.chainId)}, local=${expectedChainId}) cannot carry a stateCommitHash`);
|
|
41
|
+
throw new CustomError("CHAIN_ID_MISMATCH", `${prefix} chainId mismatch at seq=${String(entry.sequence)}: expected ${expectedChainId}, got ${String(entry.chainId)}`);
|
|
42
|
+
}
|
|
24
43
|
/** Error code of a replayed entry refused for a timestamp ahead of the local clock (retryable, never halting). */
|
|
25
44
|
const FUTURE_LEDGER_ENTRY_TIMESTAMP = "FUTURE_LEDGER_ENTRY_TIMESTAMP";
|
|
26
45
|
function normalizeHexLower(raw) {
|
|
@@ -181,33 +200,56 @@ var LedgerEntryReplayer = class {
|
|
|
181
200
|
if (!entry || typeof entry !== "object" || !Number.isSafeInteger(entry.sequence) || entry.sequence < 1 || !Number.isSafeInteger(entry.nonce) || entry.nonce < 0 || !Number.isSafeInteger(entry.timestamp) || entry.timestamp < 0 || entry.timestamp > 864e13 || typeof entry.hash !== "string" || typeof entry.from !== "string" || typeof entry.pk !== "string" || typeof entry.signature !== "string" || typeof entry.txRaw !== "string" || typeof entry.txType !== "string" || typeof entry.chainId !== "string" || typeof entry.entryHash !== "string" || typeof entry.finalized !== "boolean" || entry.stateCommitHash !== void 0 && typeof entry.stateCommitHash !== "string" || entry.tx === null || typeof entry.tx !== "object" || Array.isArray(entry.tx)) throw new CustomError("INVALID_LEDGER_ENTRY", `Ledger entry seq=${String(entry?.sequence)} has invalid wire primitive types`);
|
|
182
201
|
this.verifyRawTxHashMatchesEntry(entry, "INVALID_LEDGER_ENTRY");
|
|
183
202
|
this.authenticateAndDecodeEntry(entry);
|
|
203
|
+
if (!(entry.stateCommitHash ?? "").trim()) this.decodeOrphanEntryContext(entry);
|
|
184
204
|
if (!options.skipFutureTimestampCheck) this.assertEntryTimestampNotInFuture(entry);
|
|
185
205
|
}
|
|
186
206
|
authenticateAndDecodeEntry(entry) {
|
|
187
207
|
const authRes = authenticateEntryDecodedFields(entry, true, { requireFullWireEntry: true });
|
|
188
|
-
if (!authRes.ok) throw new CustomError("INVALID_LEDGER_ENTRY", `Ledger entry seq=${entry.sequence} failed field authentication: ${authRes.reason}`);
|
|
208
|
+
if (!authRes.ok) throw new CustomError("INVALID_LEDGER_ENTRY", `Ledger entry seq=${entry.sequence} hash=${String(entry.hash)} failed field authentication: ${authRes.reason}`);
|
|
189
209
|
return authRes.decodedTx;
|
|
190
210
|
}
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
+
/**
|
|
212
|
+
* Synchronously run `pipes.DecodeTx` + `pipes.DecodeItx` on a force-finalized orphan entry (`stateCommitHash === ''`)
|
|
213
|
+
* so any wire byte ceiling, non-canonical `signatures[].data` JSON, undecodable nested Any, or missing `itx` is
|
|
214
|
+
* caught as a peer-disqualifying `INVALID_LEDGER_ENTRY` BEFORE `importSegment` / `appendVerifiedEntryToLedger`
|
|
215
|
+
* mutates `ledger.sqlite` (#2039).
|
|
216
|
+
*/
|
|
217
|
+
decodeOrphanEntryContext(entry) {
|
|
218
|
+
const ctx = {
|
|
219
|
+
txBase64: entry.txRaw,
|
|
220
|
+
config: this.resolver.config,
|
|
221
|
+
extra: { txTime: entry.timestamp }
|
|
222
|
+
};
|
|
223
|
+
const runPipeSync = (pipe) => {
|
|
224
|
+
let pipeErr;
|
|
225
|
+
let completed = false;
|
|
226
|
+
pipe(ctx, (err) => {
|
|
227
|
+
completed = true;
|
|
228
|
+
pipeErr = err;
|
|
229
|
+
});
|
|
230
|
+
if (!completed) throw new Error("Orphan decode pipe did not complete synchronously");
|
|
231
|
+
if (pipeErr) throw pipeErr;
|
|
232
|
+
};
|
|
233
|
+
try {
|
|
234
|
+
runPipeSync(pipes.DecodeTx);
|
|
235
|
+
runPipeSync(pipes.DecodeItx);
|
|
236
|
+
} catch (err) {
|
|
237
|
+
throw new CustomError("INVALID_LEDGER_ENTRY", `Orphan ledger entry seq=${entry.sequence} hash=${entry.hash} does not decode as the leader decodes it: ${err instanceof Error ? err.message : String(err)}`);
|
|
238
|
+
}
|
|
239
|
+
if (String(ctx.txHash ?? "").toUpperCase() !== String(entry.hash).toUpperCase()) throw new CustomError("INVALID_LEDGER_ENTRY", `Orphan ledger entry seq=${entry.sequence} hash=${entry.hash}: txRaw does not hash to its entry hash`);
|
|
240
|
+
return ctx;
|
|
241
|
+
}
|
|
242
|
+
/**
|
|
243
|
+
* Index a rejected-but-ledgered entry (finalized, no stateCommitHash) with the row the leader wrote for it: the
|
|
244
|
+
* leader's `sendTx` passes the context of its `DecodeTx` + `DecodeItx` pipes to `_indexRejectedLedgerTx`, so the
|
|
245
|
+
* follower builds that context from the entry's txRaw with the same pipes and the time pinned to the entry
|
|
246
|
+
* (#1875). Only the rejection code differs: it is not in the ledger, so the row says `ORPHAN_LEDGER_ENTRY`.
|
|
247
|
+
*/
|
|
248
|
+
async indexOrphanEntry(entry, preDecodedCtx) {
|
|
249
|
+
const ctx = preDecodedCtx ?? this.decodeOrphanEntryContext(entry);
|
|
250
|
+
const leaderCtx = merge(cloneDeep(ctx), ctx);
|
|
251
|
+
leaderCtx.ledgerSequence = entry.sequence;
|
|
252
|
+
await this.resolver._indexRejectedLedgerTx(leaderCtx, { code: "ORPHAN_LEDGER_ENTRY" });
|
|
211
253
|
await this.resolver.flushIndexWrites();
|
|
212
254
|
}
|
|
213
255
|
/**
|
|
@@ -218,8 +260,7 @@ var LedgerEntryReplayer = class {
|
|
|
218
260
|
if (!entry || typeof entry !== "object") throw new CustomError("INVALID_LEDGER_ENTRY", "Cannot replay null or non-object ledger entry");
|
|
219
261
|
if (!entry.finalized) throw new CustomError("UNFINALIZED_LEDGER_ENTRY", `Refusing to replay unfinalized entry seq=${entry.sequence}`);
|
|
220
262
|
if (!Number.isSafeInteger(entry.timestamp) || entry.timestamp < 0 || entry.timestamp > 864e13) throw new CustomError("INVALID_LEDGER_ENTRY", `Refusing to replay entry seq=${entry.sequence} with out-of-range timestamp: ${String(entry.timestamp)}`);
|
|
221
|
-
|
|
222
|
-
if (entry.chainId !== expectedChainId) throw new CustomError("CHAIN_ID_MISMATCH", `Ledger entry chainId mismatch at seq=${entry.sequence}: expected ${expectedChainId}, got ${entry.chainId}`);
|
|
263
|
+
assertReplayEntryChainIdAcceptable(entry, this.resolver.config.chainId || this.resolver.ledger.chainId, "Ledger entry");
|
|
223
264
|
const appendToLedger = options.appendToLedger !== false;
|
|
224
265
|
if (options.entryHashV2From !== void 0 && options.entryHashV2From !== "infinity" && (!Number.isSafeInteger(options.entryHashV2From) || options.entryHashV2From < 1)) throw new CustomError("INVALID_LEDGER_ENTRY", `Invalid entryHashV2From replay option: ${String(options.entryHashV2From)}`);
|
|
225
266
|
if (!Number.isSafeInteger(entry.sequence) || entry.sequence < 1 || !Number.isSafeInteger(entry.nonce) || entry.nonce < 0 || typeof entry.entryHash !== "string" || !entry.entryHash || entry.stateCommitHash !== void 0 && typeof entry.stateCommitHash !== "string") throw new CustomError("INVALID_LEDGER_ENTRY", `Ledger entry seq=${String(entry.sequence)} has invalid sequence, nonce, or entryHash types`);
|
|
@@ -240,7 +281,7 @@ var LedgerEntryReplayer = class {
|
|
|
240
281
|
}
|
|
241
282
|
if (appendToLedger) this.assertEntryTimestampNotInFuture(entry);
|
|
242
283
|
this.verifyRawTxHashMatchesEntry(entry);
|
|
243
|
-
|
|
284
|
+
this.authenticateAndDecodeEntry(entry);
|
|
244
285
|
const expectedStateCommit = (entry.stateCommitHash ?? "").trim();
|
|
245
286
|
if (appendToLedger) {
|
|
246
287
|
const v2Cutoff = options.entryHashV2From === "infinity" ? Number.POSITIVE_INFINITY : typeof options.entryHashV2From === "number" ? options.entryHashV2From : this.resolver.ledger.getEntryHashV2From?.() ?? 1;
|
|
@@ -248,8 +289,9 @@ var LedgerEntryReplayer = class {
|
|
|
248
289
|
if (entry.entryHash !== expectedEntryHash) throw new CustomError("INVALID_LEDGER_ENTRY", `Ledger entryHash mismatch before execution at seq=${entry.sequence}: expected ${expectedEntryHash}, got ${entry.entryHash}`);
|
|
249
290
|
}
|
|
250
291
|
if (!expectedStateCommit) {
|
|
292
|
+
const decodedOrphanCtx = this.decodeOrphanEntryContext(entry);
|
|
251
293
|
if (appendToLedger) await this.appendVerifiedEntryToLedger(entry, options.entryHashV2From);
|
|
252
|
-
await this.indexOrphanEntry(entry,
|
|
294
|
+
await this.indexOrphanEntry(entry, decodedOrphanCtx);
|
|
253
295
|
return {
|
|
254
296
|
status: "orphan-indexed",
|
|
255
297
|
sequence: entry.sequence,
|
|
@@ -328,7 +370,7 @@ var LedgerEntryReplayer = class {
|
|
|
328
370
|
const entry = entries[i];
|
|
329
371
|
if (!entry || typeof entry !== "object") throw new CustomError("INVALID_LEDGER_ENTRY", `Proposal entry at index ${i} is not an object`);
|
|
330
372
|
if (entry.finalized !== true) throw new CustomError("UNFINALIZED_LEDGER_ENTRY", `Proposal entry seq=${String(entry.sequence)} must be finalized`);
|
|
331
|
-
|
|
373
|
+
assertReplayEntryChainIdAcceptable(entry, expectedChainId, "Proposal entry");
|
|
332
374
|
if (entry.sequence !== baseSeq + i + 1) throw new CustomError("SEQUENCE_GAP", `Proposal entry sequence mismatch at index ${i}: expected ${baseSeq + i + 1}, got ${entry.sequence}`);
|
|
333
375
|
this.validateWireEntry(entry, { skipFutureTimestampCheck: options.skipFutureTimestampCheck });
|
|
334
376
|
const expectedStateCommit = (entry.stateCommitHash ?? "").trim();
|
|
@@ -403,9 +445,27 @@ var LedgerEntryReplayer = class {
|
|
|
403
445
|
};
|
|
404
446
|
});
|
|
405
447
|
}
|
|
448
|
+
/** Upper-cased hashes of `entries` that neither `indexdb.tx` nor `statedb.tx` holds (one `$in` count per page). */
|
|
449
|
+
async hashesMissingFromIndex(entries) {
|
|
450
|
+
const missing = /* @__PURE__ */ new Set();
|
|
451
|
+
if (entries.length === 0) return missing;
|
|
452
|
+
const hashes = [...new Set(entries.map((e) => String(e.hash).toUpperCase()))];
|
|
453
|
+
const { indexdb, statedb } = this.resolver;
|
|
454
|
+
if (await indexdb.tx.count({ hash: { $in: hashes } }) === hashes.length) return missing;
|
|
455
|
+
for (const hash of hashes) {
|
|
456
|
+
if (await indexdb.tx.count({ hash }) > 0) continue;
|
|
457
|
+
if (await Promise.resolve(statedb?.tx?.get?.(hash))) continue;
|
|
458
|
+
missing.add(hash);
|
|
459
|
+
}
|
|
460
|
+
return missing;
|
|
461
|
+
}
|
|
406
462
|
/**
|
|
407
|
-
*
|
|
408
|
-
* orphan
|
|
463
|
+
* After `rebuildIndex()`, index the force-finalized orphan entries of `[fromSeq, toSeq]` that the index does not
|
|
464
|
+
* have yet. An orphan is an entry without a `stateCommitHash` (its tx never reached the statedb, so the build
|
|
465
|
+
* from the statedb cannot see it) whose tx is in neither `indexdb.tx` nor `statedb.tx`. Entries the index already
|
|
466
|
+
* holds are skipped before any decoding: beta's ledger starts with 434k migrated entries without a
|
|
467
|
+
* `stateCommitHash` and in a legacy entry shape, all indexed by the build (#1875). Every entry that is processed
|
|
468
|
+
* is authenticated as before and fails closed.
|
|
409
469
|
*/
|
|
410
470
|
async indexExistingOrphanEntries(fromSeq = 1, toSeq) {
|
|
411
471
|
const latestSeq = toSeq ?? await this.resolver.ledger.getLatestSequence();
|
|
@@ -414,11 +474,13 @@ var LedgerEntryReplayer = class {
|
|
|
414
474
|
const pageSize = 200;
|
|
415
475
|
for (let cursor = Math.max(1, fromSeq); cursor <= latestSeq; cursor += pageSize) {
|
|
416
476
|
const end = Math.min(latestSeq, cursor + pageSize - 1);
|
|
417
|
-
const
|
|
418
|
-
|
|
477
|
+
const candidates = (await this.resolver.ledger.getRange(cursor, end)).filter((entry) => !entry.stateCommitHash || !entry.stateCommitHash.trim());
|
|
478
|
+
const missing = await this.hashesMissingFromIndex(candidates);
|
|
479
|
+
for (const entry of candidates) {
|
|
480
|
+
if (!missing.has(String(entry.hash).toUpperCase())) continue;
|
|
419
481
|
this.verifyRawTxHashMatchesEntry(entry);
|
|
420
|
-
|
|
421
|
-
await this.indexOrphanEntry(entry
|
|
482
|
+
this.authenticateAndDecodeEntry(entry);
|
|
483
|
+
await this.indexOrphanEntry(entry);
|
|
422
484
|
indexedCount++;
|
|
423
485
|
}
|
|
424
486
|
}
|
|
@@ -427,4 +489,4 @@ var LedgerEntryReplayer = class {
|
|
|
427
489
|
};
|
|
428
490
|
|
|
429
491
|
//#endregion
|
|
430
|
-
export { FUTURE_LEDGER_ENTRY_TIMESTAMP, LedgerEntryReplayer };
|
|
492
|
+
export { FUTURE_LEDGER_ENTRY_TIMESTAMP, LedgerEntryReplayer, assertReplayEntryChainIdAcceptable, isReplayEntryChainIdAcceptable };
|
package/lib/dashboard.cjs
CHANGED
|
@@ -83,6 +83,8 @@ async function collectLedgerHealthSummary(options) {
|
|
|
83
83
|
const proposerPublicKey = options.ledger.getProposerPublicKey?.() || null;
|
|
84
84
|
const checkpointSigningEnabled = typeof options.ledger.isCheckpointSigningEnabled === "function" ? options.ledger.isCheckpointSigningEnabled() : Boolean(proposerPublicKey);
|
|
85
85
|
const witnessRegistry = extLedger.getWitnessRegistry?.();
|
|
86
|
+
const daPublisher = extLedger.getDaPublisherStatus?.() ?? null;
|
|
87
|
+
const daArchive = options.runtime?.getDaArchiveStatus?.() ?? null;
|
|
86
88
|
return {
|
|
87
89
|
healthy,
|
|
88
90
|
...halted ? {
|
|
@@ -104,6 +106,8 @@ async function collectLedgerHealthSummary(options) {
|
|
|
104
106
|
witnessQuorum: witnessRegistry?.getQuorum() ?? 0,
|
|
105
107
|
witnessCircuitBroken: witnessRegistry?.isCircuitBroken() ?? false,
|
|
106
108
|
witnessFraudAlertCount: witnessRegistry?.getFraudAlerts().length ?? 0,
|
|
109
|
+
...daPublisher ? { daPublisher } : {},
|
|
110
|
+
...daArchive ? { daArchive } : {},
|
|
107
111
|
entryHashV2From,
|
|
108
112
|
latestSequence: health.latestSequence,
|
|
109
113
|
latestCheckpoint: health.latestCheckpoint ? {
|
|
@@ -658,6 +662,7 @@ function createNodeDashboardRouter(options) {
|
|
|
658
662
|
witnessQuorum: ledgerHealth.witnessQuorum,
|
|
659
663
|
witnessCircuitBroken: ledgerHealth.witnessCircuitBroken,
|
|
660
664
|
witnessFraudAlertCount: ledgerHealth.witnessFraudAlertCount,
|
|
665
|
+
...ledgerHealth.daPublisher ? { publisher: ledgerHealth.daPublisher } : {},
|
|
661
666
|
latestAnchorReceipt
|
|
662
667
|
},
|
|
663
668
|
pipeline,
|
package/lib/dashboard.d.cts
CHANGED
|
@@ -84,6 +84,10 @@ interface ILedgerHealthSummary {
|
|
|
84
84
|
witnessQuorum: number;
|
|
85
85
|
witnessCircuitBroken: boolean;
|
|
86
86
|
witnessFraudAlertCount: number;
|
|
87
|
+
/** Present only when the segment publisher runs in DA mode (#2016); never part of `healthy` */
|
|
88
|
+
daPublisher?: Record<string, unknown>;
|
|
89
|
+
/** Present only on a follower that reads the DA archive (#2017): `{ state: 'ok' }` or `{ state: 'stalled', code: 'DA_GAP', ... }`; never part of `healthy` */
|
|
90
|
+
daArchive?: Record<string, unknown>;
|
|
87
91
|
entryHashV2From: string | null;
|
|
88
92
|
latestSequence: number;
|
|
89
93
|
latestCheckpoint: (Record<string, unknown> & {
|
package/lib/explorer.cjs
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
|
+
const require_rolldown_runtime = require('./_virtual/rolldown_runtime.cjs');
|
|
1
2
|
const require_dashboard = require('./dashboard.cjs');
|
|
3
|
+
let _ocap_util_lib_client_ip = require("@ocap/util/lib/client-ip");
|
|
2
4
|
|
|
3
5
|
//#region src/explorer.ts
|
|
4
6
|
function loadNodeBuiltin(id) {
|
|
@@ -119,8 +121,15 @@ function createExplorerMiddleware(options) {
|
|
|
119
121
|
if (verifiedDistRealPath && reqPath === "/static/js/networks.js") {
|
|
120
122
|
const cfg = options.getConfig() ?? {};
|
|
121
123
|
const chainId = String(cfg.chainId || "local-chain");
|
|
122
|
-
const
|
|
123
|
-
const
|
|
124
|
+
const rawHost = String(req.get?.("host") || req.headers?.host || "").trim();
|
|
125
|
+
const safeHost = rawHost.length > 0 && rawHost.length <= 255 && (/^[A-Za-z0-9.-]+(?::\d{1,5})?$/.test(rawHost) || /^\[[0-9a-fA-F:]+\](?::\d{1,5})?$/.test(rawHost)) ? rawHost : "127.0.0.1";
|
|
126
|
+
let scheme = req.protocol === "https" ? "https" : "http";
|
|
127
|
+
if (scheme === "http" && (0, _ocap_util_lib_client_ip.isLoopbackAddress)(req.socket?.remoteAddress)) {
|
|
128
|
+
const xfProto = String(req.headers?.["x-forwarded-proto"] || "").split(",")[0].trim().toLowerCase();
|
|
129
|
+
const cfVisitor = String(req.headers?.["cf-visitor"] || "");
|
|
130
|
+
if (xfProto === "https" || /"scheme"\s*:\s*"https"/i.test(cfVisitor)) scheme = "https";
|
|
131
|
+
}
|
|
132
|
+
const endpoint = /^https?:\/\//.test(chainEndpoint) ? chainEndpoint : `${scheme}://${safeHost}${chainEndpoint.startsWith("/") ? "" : "/"}${chainEndpoint.replace(/\/+$/, "")}/`;
|
|
124
133
|
const networks = [{
|
|
125
134
|
name: chainId,
|
|
126
135
|
abbr: cfg.token?.symbol || "ARC",
|
|
@@ -130,7 +139,8 @@ function createExplorerMiddleware(options) {
|
|
|
130
139
|
}];
|
|
131
140
|
applyExplorerSecurityHeaders(res, false);
|
|
132
141
|
res.setHeader("Content-Type", "application/javascript; charset=utf-8");
|
|
133
|
-
res.setHeader("Cache-Control", "no-
|
|
142
|
+
res.setHeader("Cache-Control", "no-store");
|
|
143
|
+
res.setHeader("Vary", "Host, X-Forwarded-Proto, CF-Visitor");
|
|
134
144
|
return res.status(200).send(`window.ARCBLOCK_NETWORKS = ${safeJsonForJsLiteral(networks)};\n`);
|
|
135
145
|
}
|
|
136
146
|
if (reqPath === "/static/js/networks.js") {
|
package/lib/follower.cjs
CHANGED
|
@@ -216,11 +216,43 @@ function isLocalTrustedStorageHashError(err, depth = 0) {
|
|
|
216
216
|
function isFatalVerificationFailure(err) {
|
|
217
217
|
if (err instanceof _arcblock_ledger_core.ArchivePayloadDecodeError || err?.name === "ArchivePayloadDecodeError") return false;
|
|
218
218
|
if (isTransientSqliteLockError(err)) return false;
|
|
219
|
+
if (err?.daFatal === true) return true;
|
|
219
220
|
if (err?.authenticatedLocalFailure === true || err?.localDivergenceFailure === true) return true;
|
|
220
221
|
const code = err?.code;
|
|
221
222
|
if (isLocalTrustedStorageHashError(err)) return true;
|
|
222
223
|
return code === "TRUST_ANCHOR_REQUIRED";
|
|
223
224
|
}
|
|
225
|
+
/** The cold catch-up after a `410`: see `FollowerSyncService.coldCatchUpAfterPrune` (a pure function so it can be tested). */
|
|
226
|
+
async function catchUpAfterPrune(options) {
|
|
227
|
+
if (!options.isDaReader) return options.run();
|
|
228
|
+
try {
|
|
229
|
+
return await options.run();
|
|
230
|
+
} catch (err) {
|
|
231
|
+
if (isFatalVerificationFailure(err) || err?.mutatedLocalDbTransientLock === true) throw err;
|
|
232
|
+
options.onSwallowed(err, coldImportsBefore(err));
|
|
233
|
+
return coldImportsBefore(err);
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
/**
|
|
237
|
+
* A cold catch-up that fails after it already imported segments (a DA stall or a transient error at segment N+k) has
|
|
238
|
+
* changed the local database: the error carries how many it imported, so the callers do not treat the cycle as untouched.
|
|
239
|
+
*/
|
|
240
|
+
function noteColdImports(err, imported) {
|
|
241
|
+
if (imported > 0 && err && typeof err === "object") err.coldImportedCount = imported;
|
|
242
|
+
}
|
|
243
|
+
/** How many segments a failed cold catch-up had imported before it failed (0 when it imported none or is not annotated). */
|
|
244
|
+
function coldImportsBefore(err) {
|
|
245
|
+
const n = err?.coldImportedCount;
|
|
246
|
+
return typeof n === "number" && Number.isSafeInteger(n) && n > 0 ? n : 0;
|
|
247
|
+
}
|
|
248
|
+
/**
|
|
249
|
+
* `.catch()` handler for a best-effort cold segment lookup (to find a checkpoint, not to import): a miss or a transient
|
|
250
|
+
* error is "not found here", but a fatal DA reader error (`daFatal`, #2017) is never swallowed: it halts the follower.
|
|
251
|
+
*/
|
|
252
|
+
function coldLookupMiss(err) {
|
|
253
|
+
if (err?.daFatal === true) throw err;
|
|
254
|
+
return null;
|
|
255
|
+
}
|
|
224
256
|
function canonicalizePeerBaseUrl(rawUrl) {
|
|
225
257
|
const trimmed = rawUrl.trim();
|
|
226
258
|
if (!trimmed) return "";
|
|
@@ -429,6 +461,7 @@ var FollowerSyncService = class {
|
|
|
429
461
|
witnessEnabled: Boolean(this.witnessWallet),
|
|
430
462
|
witnessAddress: this.witnessAddress,
|
|
431
463
|
coldArchiveConfigured: Boolean(this.coldArchiveStore),
|
|
464
|
+
...this.daArchiveDiagnostics(),
|
|
432
465
|
peerCount: this.allowedBaseUrls.size,
|
|
433
466
|
...includePrivate ? {
|
|
434
467
|
upstreamUrl: this.upstreamUrl,
|
|
@@ -446,6 +479,15 @@ var FollowerSyncService = class {
|
|
|
446
479
|
} : null
|
|
447
480
|
};
|
|
448
481
|
}
|
|
482
|
+
/** The DA reader's status for `/api/health/ledger` (`daArchive`), or null for any other cold reader (#2017). */
|
|
483
|
+
getDaArchiveStatus() {
|
|
484
|
+
return this.daArchiveDiagnostics().daArchive ?? null;
|
|
485
|
+
}
|
|
486
|
+
/** DA follower only (#2017): `daArchive: { state: 'ok' | 'stalled', ... }`; nothing is added for any other cold reader. */
|
|
487
|
+
daArchiveDiagnostics() {
|
|
488
|
+
const reader = this.coldArchiveStore;
|
|
489
|
+
return reader?.isDaSegmentReader === true && typeof reader.getStatus === "function" ? { daArchive: reader.getStatus() } : {};
|
|
490
|
+
}
|
|
449
491
|
async triggerSyncOrProbe() {
|
|
450
492
|
if (this.halted) throw new _ocap_util_lib_error.CustomError("NODE_HALTED", `Follower node is halted: ${this.haltReason || "verification error"}`);
|
|
451
493
|
const promoted = this.leaderRouter?.probeAndPromoteLeader ? await this.leaderRouter.probeAndPromoteLeader() : null;
|
|
@@ -1069,8 +1111,33 @@ var FollowerSyncService = class {
|
|
|
1069
1111
|
* Stage 1B: Catch up sealed `.zst` checkpoint segments from `coldArchiveStore` when available.
|
|
1070
1112
|
*/
|
|
1071
1113
|
async catchUpFromColdSegments(maxTargetHeight, verifiedHeadCheckpoint) {
|
|
1114
|
+
const progress = { imported: 0 };
|
|
1115
|
+
try {
|
|
1116
|
+
return await this.catchUpFromColdSegmentsInner(maxTargetHeight, verifiedHeadCheckpoint, progress);
|
|
1117
|
+
} catch (err) {
|
|
1118
|
+
if (this.coldArchiveStore?.isDaSegmentReader === true) noteColdImports(err, progress.imported);
|
|
1119
|
+
throw err;
|
|
1120
|
+
}
|
|
1121
|
+
}
|
|
1122
|
+
/**
|
|
1123
|
+
* The cold catch-up of a `410` (the leader pruned what the follower needs). Only a DA reader's non-fatal failure (a stall)
|
|
1124
|
+
* is treated as "no cold source this cycle"; any other cold reader keeps the base behaviour exactly: its error propagates.
|
|
1125
|
+
*/
|
|
1126
|
+
async coldCatchUpAfterPrune(maxTargetHeight, verifiedHeadCheckpoint) {
|
|
1127
|
+
return catchUpAfterPrune({
|
|
1128
|
+
isDaReader: this.coldArchiveStore?.isDaSegmentReader === true,
|
|
1129
|
+
run: () => this.catchUpFromColdSegments(maxTargetHeight, verifiedHeadCheckpoint),
|
|
1130
|
+
onSwallowed: (err, imported) => this.resolver.logger.warn("Cold archive segment unavailable after a pruned upstream; not a cold source for this cycle", {
|
|
1131
|
+
chainId: this.chainId,
|
|
1132
|
+
imported,
|
|
1133
|
+
error: err instanceof Error ? err.message : String(err)
|
|
1134
|
+
})
|
|
1135
|
+
});
|
|
1136
|
+
}
|
|
1137
|
+
async catchUpFromColdSegmentsInner(maxTargetHeight, verifiedHeadCheckpoint, progress) {
|
|
1072
1138
|
if (!this.coldArchiveStore) return 0;
|
|
1073
1139
|
await this.hydrateDurableTrustRatchet();
|
|
1140
|
+
const isDaCold = this.coldArchiveStore.isDaSegmentReader === true;
|
|
1074
1141
|
const verifiedColdAncestors = /* @__PURE__ */ new Map();
|
|
1075
1142
|
if (verifiedHeadCheckpoint && typeof verifiedHeadCheckpoint.hash === "string" && normalizeHexDigest((0, _arcblock_ledger_core.computeCheckpointHash)(verifiedHeadCheckpoint)) === normalizeHexDigest(verifiedHeadCheckpoint.hash)) verifiedColdAncestors.set(verifiedHeadCheckpoint.height, verifiedHeadCheckpoint);
|
|
1076
1143
|
let importedCount = 0;
|
|
@@ -1112,6 +1179,7 @@ var FollowerSyncService = class {
|
|
|
1112
1179
|
checkpoint: rawSegment.checkpoint,
|
|
1113
1180
|
entries: rawSegment.entries
|
|
1114
1181
|
};
|
|
1182
|
+
if (isDaCold && verifiedHeadCheckpoint && canonicalSegment.checkpoint.height === verifiedHeadCheckpoint.height && (normalizeHexDigest(canonicalSegment.checkpoint.hash) !== normalizeHexDigest(verifiedHeadCheckpoint.hash) || normalizeHexDigest(canonicalSegment.checkpoint.stateRoot ?? "") !== normalizeHexDigest(verifiedHeadCheckpoint.stateRoot ?? ""))) throw Object.assign(new _ocap_util_lib_error.CustomError("DA_LEADER_MISMATCH", `DA segment #${nextHeight} (${canonicalSegment.checkpoint.hash}) is not the leader's signed checkpoint at that height (${verifiedHeadCheckpoint.hash})`), { daFatal: true });
|
|
1115
1183
|
if ("contentHash" in rawBundle && rawBundle.contentHash !== void 0) {
|
|
1116
1184
|
if (typeof rawBundle.contentHash !== "string" || rawBundle.contentHash.replace(/^0x/i, "").toLowerCase() !== (0, _arcblock_ledger_core.computeSegmentContentHash)(canonicalSegment).replace(/^0x/i, "").toLowerCase()) throw new _ocap_util_lib_error.CustomError("COLD_BUNDLE_WRAPPER_MISMATCH", `Cold segment #${nextHeight} unsigned contentHash wrapper does not match canonical segment digest`);
|
|
1117
1185
|
}
|
|
@@ -1133,7 +1201,7 @@ var FollowerSyncService = class {
|
|
|
1133
1201
|
}
|
|
1134
1202
|
let stepCp = h === nextHeight ? canonicalSegment.checkpoint : verifiedColdAncestors.get(h) ?? null;
|
|
1135
1203
|
if (!stepCp) {
|
|
1136
|
-
const stepSeg = await this.coldArchiveStore.getSegment(this.chainId, h).catch(
|
|
1204
|
+
const stepSeg = await this.coldArchiveStore.getSegment(this.chainId, h).catch(coldLookupMiss);
|
|
1137
1205
|
stepCp = stepSeg && stepSeg.chainId === this.chainId ? stepSeg.checkpoint : null;
|
|
1138
1206
|
}
|
|
1139
1207
|
if (!stepCp || typeof stepCp.hash !== "string" || normalizeHexDigest(stepCp.hash) !== normalizeHexDigest(expectedHash) || normalizeHexDigest((0, _arcblock_ledger_core.computeCheckpointHash)(stepCp)) !== normalizeHexDigest(expectedHash)) {
|
|
@@ -1199,7 +1267,10 @@ var FollowerSyncService = class {
|
|
|
1199
1267
|
}
|
|
1200
1268
|
throw segVerifyErr;
|
|
1201
1269
|
}
|
|
1202
|
-
for (const entry of canonicalSegment.entries)
|
|
1270
|
+
for (const entry of canonicalSegment.entries) {
|
|
1271
|
+
this.replayer.validateWireEntry(entry);
|
|
1272
|
+
require_replayer.assertReplayEntryChainIdAcceptable(entry, this.chainId, "Cold segment");
|
|
1273
|
+
}
|
|
1203
1274
|
const hadPartialLocalEntries = localSeq > canonicalSegment.checkpoint.prevSequence;
|
|
1204
1275
|
let mutatedLocalDb = false;
|
|
1205
1276
|
try {
|
|
@@ -1288,6 +1359,7 @@ var FollowerSyncService = class {
|
|
|
1288
1359
|
verifiedLocalStateRoot: verifiedColdStateRoot
|
|
1289
1360
|
});
|
|
1290
1361
|
importedCount++;
|
|
1362
|
+
progress.imported = importedCount;
|
|
1291
1363
|
if (importedCount >= MAX_PAGES_PER_SYNC_CYCLE) break;
|
|
1292
1364
|
}
|
|
1293
1365
|
if (importedCount > 0 && this.pruneController) await this.pruneController.pruneHotWindow(this.resolver.ledger);
|
|
@@ -1370,7 +1442,7 @@ var FollowerSyncService = class {
|
|
|
1370
1442
|
publicKeyOrVerifier: coldVerifyAnchor,
|
|
1371
1443
|
requireSignature: !importOpts.allowUnauthenticated,
|
|
1372
1444
|
allowUnauthenticated: importOpts.allowUnauthenticated
|
|
1373
|
-
}).catch(
|
|
1445
|
+
}).catch(coldLookupMiss);
|
|
1374
1446
|
if (seg && seg.chainId === this.chainId) {
|
|
1375
1447
|
targetCp = seg.checkpoint;
|
|
1376
1448
|
const coldAuth = await this.isColdSegmentAuthenticatedByProposer(seg);
|
|
@@ -1517,7 +1589,10 @@ var FollowerSyncService = class {
|
|
|
1517
1589
|
hotstuffClusterConfig: importOpts.hotstuffClusterConfig
|
|
1518
1590
|
});
|
|
1519
1591
|
if (importOpts.allowUnauthenticated && !trustAnchor) {
|
|
1520
|
-
for (let i = 0; i < entries.length; i++)
|
|
1592
|
+
for (let i = 0; i < entries.length; i++) {
|
|
1593
|
+
this.replayer.validateWireEntry(entries[i]);
|
|
1594
|
+
require_replayer.assertReplayEntryChainIdAcceptable(entries[i], this.chainId, "Hot-tail");
|
|
1595
|
+
}
|
|
1521
1596
|
return batchParsedV2From ?? parsedV2From;
|
|
1522
1597
|
}
|
|
1523
1598
|
if (!trustAnchor) {
|
|
@@ -1532,11 +1607,11 @@ var FollowerSyncService = class {
|
|
|
1532
1607
|
const entry = entries[i];
|
|
1533
1608
|
const expectedSeq = requestedFrom + i;
|
|
1534
1609
|
if (!entry || entry.sequence !== expectedSeq) throw new _ocap_util_lib_error.CustomError("INVALID_LEDGER_ENTRY", `Hot-tail batch sequence discontinuity at index ${i}: expected ${expectedSeq}, got ${entry?.sequence}`);
|
|
1535
|
-
if (entry
|
|
1610
|
+
if (!require_replayer.isReplayEntryChainIdAcceptable(entry, this.chainId)) require_replayer.assertReplayEntryChainIdAcceptable(entry, this.chainId, "Hot-tail");
|
|
1536
1611
|
if (entry.finalized !== true) throw new _ocap_util_lib_error.CustomError("UNFINALIZED_LEDGER_ENTRY", `Hot-tail entry seq=${entry.sequence} is not finalized`);
|
|
1537
1612
|
if (!Number.isSafeInteger(entry.timestamp) || entry.timestamp < 0 || entry.timestamp > MAX_SAFE_EPOCH_MS) throw new _ocap_util_lib_error.CustomError("INVALID_LEDGER_ENTRY", `Hot-tail entry seq=${entry.sequence} has invalid timestamp: ${String(entry.timestamp)}`);
|
|
1538
1613
|
if (typeof entry.txRaw !== "string" || !entry.txRaw || entry.txRaw !== entry.txRaw.trim()) throw new _ocap_util_lib_error.CustomError("INVALID_LEDGER_ENTRY", `Hot-tail entry seq=${entry.sequence} has missing or non-canonical txRaw`);
|
|
1539
|
-
this.replayer.validateWireEntry(entry);
|
|
1614
|
+
this.replayer.validateWireEntry(entry, { skipFutureTimestampCheck: true });
|
|
1540
1615
|
const expectedHash = (0, _arcblock_ledger_core.computeEntryHash)(entry.sequence, entry.txRaw, entry.stateCommitHash ?? "", entry.sequence >= v2Cutoff ? 2 : 1);
|
|
1541
1616
|
if (entry.entryHash !== expectedHash) throw new _ocap_util_lib_error.CustomError("LEDGER_DIVERGENCE", `Hot-tail entryHash mismatch at seq=${entry.sequence}: expected ${expectedHash}, got ${entry.entryHash}`);
|
|
1542
1617
|
entryHashes.push(entry.entryHash);
|
|
@@ -1564,6 +1639,7 @@ var FollowerSyncService = class {
|
|
|
1564
1639
|
signatureRatchetLocked: true
|
|
1565
1640
|
});
|
|
1566
1641
|
}
|
|
1642
|
+
for (const entry of entries) this.replayer.assertEntryTimestampNotInFuture(entry);
|
|
1567
1643
|
return batchParsedV2From;
|
|
1568
1644
|
}
|
|
1569
1645
|
const cp = upstreamStatus.latestCheckpoint;
|
|
@@ -1587,11 +1663,15 @@ var FollowerSyncService = class {
|
|
|
1587
1663
|
signatureRatchetLocked: true
|
|
1588
1664
|
});
|
|
1589
1665
|
}
|
|
1666
|
+
for (const entry of entries) this.replayer.assertEntryTimestampNotInFuture(entry);
|
|
1590
1667
|
return;
|
|
1591
1668
|
}
|
|
1592
1669
|
}
|
|
1593
1670
|
}
|
|
1594
|
-
if (importOpts.allowUnauthenticated && !entriesBody.batchSignature && !this.hotstuffClusterConfig)
|
|
1671
|
+
if (importOpts.allowUnauthenticated && !entriesBody.batchSignature && !this.hotstuffClusterConfig) {
|
|
1672
|
+
for (const entry of entries) this.replayer.assertEntryTimestampNotInFuture(entry);
|
|
1673
|
+
return batchParsedV2From ?? parsedV2From;
|
|
1674
|
+
}
|
|
1595
1675
|
throw new _ocap_util_lib_error.CustomError("HOT_TAIL_SIGNATURE_MISSING", "Hot-tail /api/ledger/entries response is missing a valid proposer batchSignature");
|
|
1596
1676
|
}
|
|
1597
1677
|
/**
|
|
@@ -1671,7 +1751,7 @@ var FollowerSyncService = class {
|
|
|
1671
1751
|
publicKeyOrVerifier: coldVerifyAnchor,
|
|
1672
1752
|
requireSignature: !importOpts.allowUnauthenticated,
|
|
1673
1753
|
allowUnauthenticated: importOpts.allowUnauthenticated
|
|
1674
|
-
}).catch(
|
|
1754
|
+
}).catch(coldLookupMiss);
|
|
1675
1755
|
if (seg && seg.chainId === this.chainId) targetCp = seg.checkpoint;
|
|
1676
1756
|
}
|
|
1677
1757
|
if (!targetCp) return markUnauthenticatedPeerError(new _ocap_util_lib_error.CustomError(cpRes && cpRes.status === 200 ? "UPSTREAM_BAD_RESPONSE" : "UPSTREAM_UNAVAILABLE", `Upstream checkpoint #${nextHeight} unavailable or failed hash-chain verification before buffered commit`));
|
|
@@ -1725,7 +1805,7 @@ var FollowerSyncService = class {
|
|
|
1725
1805
|
for (let i = 0; i < segEntries.length; i++) {
|
|
1726
1806
|
const entry = segEntries[i];
|
|
1727
1807
|
const expectedSeq = targetCp.prevSequence + 1 + i;
|
|
1728
|
-
if (entry.sequence !== expectedSeq || entry
|
|
1808
|
+
if (entry.sequence !== expectedSeq || !require_replayer.isReplayEntryChainIdAcceptable(entry, this.chainId) || !entry.finalized) return markUnauthenticatedPeerError(new _ocap_util_lib_error.CustomError("LEDGER_DIVERGENCE", `Buffered pre-commit checkpoint #${nextHeight} invalid entry at seq=${entry.sequence}`));
|
|
1729
1809
|
if (!hashRun.check(entry).ok) return markUnauthenticatedPeerError(new _ocap_util_lib_error.CustomError("LEDGER_DIVERGENCE", `Buffered pre-commit checkpoint #${nextHeight} entryHash mismatch at seq=${entry.sequence}`));
|
|
1730
1810
|
tree.append(entry.entryHash);
|
|
1731
1811
|
}
|
|
@@ -1783,7 +1863,7 @@ var FollowerSyncService = class {
|
|
|
1783
1863
|
const pageLimit = Math.min(MAX_HOT_TAIL_LIMIT, targetCp.sequence - cursor + 1);
|
|
1784
1864
|
const entriesRes = await this.fetchAllowlistedJson(targetBaseUrl, `/api/ledger/entries?from=${cursor}&limit=${pageLimit}`, ENTRIES_MAX_RESPONSE_BYTES);
|
|
1785
1865
|
if (entriesRes.status === 410 && this.coldArchiveStore) {
|
|
1786
|
-
if (await this.
|
|
1866
|
+
if (await this.coldCatchUpAfterPrune(nextHeight) > 0) {
|
|
1787
1867
|
localCp = await this.resolver.ledger.getCheckpoint();
|
|
1788
1868
|
localHeight = localCp?.height ?? 0;
|
|
1789
1869
|
localSeq = await this.resolver.ledger.getLatestSequence();
|
|
@@ -1796,7 +1876,8 @@ var FollowerSyncService = class {
|
|
|
1796
1876
|
for (const item of batch) {
|
|
1797
1877
|
if (item.sequence > targetCp.sequence) break;
|
|
1798
1878
|
if (item.sequence !== cursor) throw new _ocap_util_lib_error.CustomError("INVALID_LEDGER_ENTRY", `Non-contiguous entry sequence at height ${nextHeight}: expected ${cursor}, got ${item.sequence}`);
|
|
1799
|
-
this.replayer.validateWireEntry(item);
|
|
1879
|
+
this.replayer.validateWireEntry(item, { skipFutureTimestampCheck: true });
|
|
1880
|
+
require_replayer.assertReplayEntryChainIdAcceptable(item, this.chainId, "HotStuff QC segment");
|
|
1800
1881
|
segmentEntries.push(item);
|
|
1801
1882
|
cursor++;
|
|
1802
1883
|
}
|
|
@@ -1819,6 +1900,7 @@ var FollowerSyncService = class {
|
|
|
1819
1900
|
requireSignature: !importOpts.allowUnauthenticated,
|
|
1820
1901
|
allowUnauthenticated: importOpts.allowUnauthenticated
|
|
1821
1902
|
});
|
|
1903
|
+
for (const item of segmentEntries) this.replayer.assertEntryTimestampNotInFuture(item);
|
|
1822
1904
|
let mutatedLocalDb = false;
|
|
1823
1905
|
try {
|
|
1824
1906
|
if (segmentEntries.length > 0 && typeof this.resolver.statedb.runWithinDryRunSession === "function") {
|
|
@@ -2141,8 +2223,10 @@ var FollowerSyncService = class {
|
|
|
2141
2223
|
if (await this.catchUpFromColdSegments(upstreamStatus.latestCheckpoint?.height, verifiedHeadCp) > 0) mutatedLocalDbOnPeer = true;
|
|
2142
2224
|
} catch (coldErr) {
|
|
2143
2225
|
if (isFatalVerificationFailure(coldErr) || coldErr?.mutatedLocalDbTransientLock === true) throw coldErr;
|
|
2226
|
+
if (coldImportsBefore(coldErr) > 0) mutatedLocalDbOnPeer = true;
|
|
2144
2227
|
this.resolver.logger.warn("Cold archive segment unavailable or unverified; falling back to hot-tail sync", {
|
|
2145
2228
|
chainId: this.chainId,
|
|
2229
|
+
...this.coldArchiveStore?.isDaSegmentReader === true ? { imported: coldImportsBefore(coldErr) } : {},
|
|
2146
2230
|
error: coldErr instanceof Error ? coldErr.message : String(coldErr)
|
|
2147
2231
|
});
|
|
2148
2232
|
}
|
|
@@ -2215,7 +2299,7 @@ var FollowerSyncService = class {
|
|
|
2215
2299
|
const entriesRes = await this.fetchAllowlistedJson(targetBaseUrl, `/api/ledger/entries?from=${from}&limit=${MAX_HOT_TAIL_LIMIT}`, ENTRIES_MAX_RESPONSE_BYTES);
|
|
2216
2300
|
if (entriesRes.status === 410) {
|
|
2217
2301
|
if (this.coldArchiveStore && bufferedTailBatches.length === 0) {
|
|
2218
|
-
const imported = await this.
|
|
2302
|
+
const imported = await this.coldCatchUpAfterPrune(upstreamStatus.latestCheckpoint?.height, verifiedHeadCp);
|
|
2219
2303
|
localSeq = await this.resolver.ledger.getLatestSequence();
|
|
2220
2304
|
cursorSeq = localSeq;
|
|
2221
2305
|
if (imported > 0) mutatedLocalDbOnPeer = true;
|
|
@@ -2455,4 +2539,8 @@ exports.DEFAULT_FOLLOWER_MAX_SYNC_LAG = DEFAULT_FOLLOWER_MAX_SYNC_LAG;
|
|
|
2455
2539
|
exports.DEFAULT_FOLLOWER_MAX_UPSTREAM_STALE_MS = DEFAULT_FOLLOWER_MAX_UPSTREAM_STALE_MS;
|
|
2456
2540
|
exports.DEFAULT_FOLLOWER_POLL_INTERVAL_MS = DEFAULT_FOLLOWER_POLL_INTERVAL_MS;
|
|
2457
2541
|
exports.FollowerSyncService = FollowerSyncService;
|
|
2458
|
-
exports.
|
|
2542
|
+
exports.catchUpAfterPrune = catchUpAfterPrune;
|
|
2543
|
+
exports.coldImportsBefore = coldImportsBefore;
|
|
2544
|
+
exports.coldLookupMiss = coldLookupMiss;
|
|
2545
|
+
exports.isTransientSqliteLockError = isTransientSqliteLockError;
|
|
2546
|
+
exports.noteColdImports = noteColdImports;
|
package/lib/follower.d.cts
CHANGED
|
@@ -101,6 +101,24 @@ interface FollowerSyncServiceOptions {
|
|
|
101
101
|
};
|
|
102
102
|
}
|
|
103
103
|
declare function isTransientSqliteLockError(err: unknown): boolean;
|
|
104
|
+
/** The cold catch-up after a `410`: see `FollowerSyncService.coldCatchUpAfterPrune` (a pure function so it can be tested). */
|
|
105
|
+
declare function catchUpAfterPrune(options: {
|
|
106
|
+
isDaReader: boolean;
|
|
107
|
+
run: () => Promise<number>;
|
|
108
|
+
onSwallowed: (err: unknown, imported: number) => void;
|
|
109
|
+
}): Promise<number>;
|
|
110
|
+
/**
|
|
111
|
+
* A cold catch-up that fails after it already imported segments (a DA stall or a transient error at segment N+k) has
|
|
112
|
+
* changed the local database: the error carries how many it imported, so the callers do not treat the cycle as untouched.
|
|
113
|
+
*/
|
|
114
|
+
declare function noteColdImports(err: unknown, imported: number): void;
|
|
115
|
+
/** How many segments a failed cold catch-up had imported before it failed (0 when it imported none or is not annotated). */
|
|
116
|
+
declare function coldImportsBefore(err: unknown): number;
|
|
117
|
+
/**
|
|
118
|
+
* `.catch()` handler for a best-effort cold segment lookup (to find a checkpoint, not to import): a miss or a transient
|
|
119
|
+
* error is "not found here", but a fatal DA reader error (`daFatal`, #2017) is never swallowed: it halts the follower.
|
|
120
|
+
*/
|
|
121
|
+
declare function coldLookupMiss(err: unknown): null;
|
|
104
122
|
declare class FollowerSyncService implements INodeRuntimeAdapter {
|
|
105
123
|
readonly role: TNodeRole;
|
|
106
124
|
writePolicy: TNodeWritePolicy;
|
|
@@ -170,6 +188,10 @@ declare class FollowerSyncService implements INodeRuntimeAdapter {
|
|
|
170
188
|
getRuntimeDiagnostics(options?: {
|
|
171
189
|
includePrivateTopology?: boolean;
|
|
172
190
|
}): Record<string, unknown>;
|
|
191
|
+
/** The DA reader's status for `/api/health/ledger` (`daArchive`), or null for any other cold reader (#2017). */
|
|
192
|
+
getDaArchiveStatus(): Record<string, unknown> | null;
|
|
193
|
+
/** DA follower only (#2017): `daArchive: { state: 'ok' | 'stalled', ... }`; nothing is added for any other cold reader. */
|
|
194
|
+
private daArchiveDiagnostics;
|
|
173
195
|
triggerSyncOrProbe(): Promise<Record<string, unknown>>;
|
|
174
196
|
private haltFailClosed;
|
|
175
197
|
private resolveActiveUpstreamUrl;
|
|
@@ -242,6 +264,12 @@ declare class FollowerSyncService implements INodeRuntimeAdapter {
|
|
|
242
264
|
* Stage 1B: Catch up sealed `.zst` checkpoint segments from `coldArchiveStore` when available.
|
|
243
265
|
*/
|
|
244
266
|
private catchUpFromColdSegments;
|
|
267
|
+
/**
|
|
268
|
+
* The cold catch-up of a `410` (the leader pruned what the follower needs). Only a DA reader's non-fatal failure (a stall)
|
|
269
|
+
* is treated as "no cold source this cycle"; any other cold reader keeps the base behaviour exactly: its error propagates.
|
|
270
|
+
*/
|
|
271
|
+
private coldCatchUpAfterPrune;
|
|
272
|
+
private catchUpFromColdSegmentsInner;
|
|
245
273
|
private sealCaughtUpCheckpoints;
|
|
246
274
|
private tryEmitHotTailReplayFraudAlert;
|
|
247
275
|
private parseStrictEntryHashV2From;
|
|
@@ -291,4 +319,4 @@ declare class FollowerSyncService implements INodeRuntimeAdapter {
|
|
|
291
319
|
close(): Promise<void>;
|
|
292
320
|
}
|
|
293
321
|
//#endregion
|
|
294
|
-
export { DEFAULT_FOLLOWER_HTTP_TIMEOUT_MS, DEFAULT_FOLLOWER_MAX_SYNC_LAG, DEFAULT_FOLLOWER_MAX_UPSTREAM_STALE_MS, DEFAULT_FOLLOWER_POLL_INTERVAL_MS, FollowerSyncService, FollowerSyncServiceOptions, IColdSegmentReader, ILeaderRouterLike, TFollowerUpstreamFetchInterceptor, TFollowerUpstreamFetchKind, isTransientSqliteLockError };
|
|
322
|
+
export { DEFAULT_FOLLOWER_HTTP_TIMEOUT_MS, DEFAULT_FOLLOWER_MAX_SYNC_LAG, DEFAULT_FOLLOWER_MAX_UPSTREAM_STALE_MS, DEFAULT_FOLLOWER_POLL_INTERVAL_MS, FollowerSyncService, FollowerSyncServiceOptions, IColdSegmentReader, ILeaderRouterLike, TFollowerUpstreamFetchInterceptor, TFollowerUpstreamFetchKind, catchUpAfterPrune, coldImportsBefore, coldLookupMiss, isTransientSqliteLockError, noteColdImports };
|
package/lib/hooks.cjs
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
const require_rolldown_runtime = require('./_virtual/rolldown_runtime.cjs');
|
|
2
2
|
const require_package = require('./package.cjs');
|
|
3
3
|
let _ocap_util = require("@ocap/util");
|
|
4
|
+
let lodash_merge = require("lodash/merge");
|
|
5
|
+
lodash_merge = require_rolldown_runtime.__toESM(lodash_merge);
|
|
4
6
|
let _arcblock_did_util = require("@arcblock/did-util");
|
|
5
7
|
let debug = require("debug");
|
|
6
8
|
debug = require_rolldown_runtime.__toESM(debug);
|
|
7
|
-
let lodash_merge = require("lodash/merge");
|
|
8
|
-
lodash_merge = require_rolldown_runtime.__toESM(lodash_merge);
|
|
9
9
|
let lodash_pick = require("lodash/pick");
|
|
10
10
|
lodash_pick = require_rolldown_runtime.__toESM(lodash_pick);
|
|
11
11
|
|