@secondlayer/cli 22.1.0 → 22.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +278 -205
- package/dist/cli.js.map +8 -8
- package/package.json +3 -3
package/dist/cli.js
CHANGED
|
@@ -29488,7 +29488,7 @@ var init_git_status = () => {};
|
|
|
29488
29488
|
|
|
29489
29489
|
// ../../node_modules/typescript/lib/typescript.js
|
|
29490
29490
|
var require_typescript = __commonJS(function(exports, module) {
|
|
29491
|
-
var __dirname = "/Users/ryan/projects/secondlayer/.claude/worktrees/
|
|
29491
|
+
var __dirname = "/Users/ryan/projects/secondlayer/.claude/worktrees/fix312/node_modules/typescript/lib", __filename = "/Users/ryan/projects/secondlayer/.claude/worktrees/fix312/node_modules/typescript/lib/typescript.js";
|
|
29492
29492
|
/*! *****************************************************************************
|
|
29493
29493
|
Copyright (c) Microsoft Corporation. All rights reserved.
|
|
29494
29494
|
Licensed under the Apache License, Version 2.0 (the "License"); you may not use
|
|
@@ -202711,7 +202711,7 @@ var {
|
|
|
202711
202711
|
// package.json
|
|
202712
202712
|
var package_default = {
|
|
202713
202713
|
name: "@secondlayer/cli",
|
|
202714
|
-
version: "22.
|
|
202714
|
+
version: "22.2.0",
|
|
202715
202715
|
description: "Run a Stacks index on your own hardware — set it up, restore verified history, and keep it current.",
|
|
202716
202716
|
repository: {
|
|
202717
202717
|
type: "git",
|
|
@@ -202750,8 +202750,8 @@ var package_default = {
|
|
|
202750
202750
|
"@inquirer/prompts": "^8.2.0",
|
|
202751
202751
|
"@secondlayer/bundler": "^0.7.1",
|
|
202752
202752
|
"@secondlayer/scaffold": "^1.6.2",
|
|
202753
|
-
"@secondlayer/sdk": "^15.1
|
|
202754
|
-
"@secondlayer/shared": "^13.
|
|
202753
|
+
"@secondlayer/sdk": "^15.4.1",
|
|
202754
|
+
"@secondlayer/shared": "^13.14.0",
|
|
202755
202755
|
"@secondlayer/stacks": "^6.4.0",
|
|
202756
202756
|
"@secondlayer/subgraphs": "^7.2.0",
|
|
202757
202757
|
"@secondlayer/verify": "^0.6.0",
|
|
@@ -202924,6 +202924,12 @@ import {
|
|
|
202924
202924
|
copyStatement,
|
|
202925
202925
|
writeRowsToCopyStream
|
|
202926
202926
|
} from "@secondlayer/shared/archive/copy-loader";
|
|
202927
|
+
import {
|
|
202928
|
+
ARCHIVE_DATASETS as ARCHIVE_DATASETS2,
|
|
202929
|
+
archiveHeightColumn,
|
|
202930
|
+
isArchiveDataset,
|
|
202931
|
+
manifestPublishes
|
|
202932
|
+
} from "@secondlayer/shared/archive/datasets";
|
|
202927
202933
|
import {
|
|
202928
202934
|
computeRangeDigest
|
|
202929
202935
|
} from "@secondlayer/shared/archive/range-digest";
|
|
@@ -203071,7 +203077,15 @@ async function fetchVerifiedPartition(reference, partition, gate, fetchOpts) {
|
|
|
203071
203077
|
}
|
|
203072
203078
|
|
|
203073
203079
|
// src/lib/bootstrap-resume.ts
|
|
203074
|
-
|
|
203080
|
+
import {
|
|
203081
|
+
ARCHIVE_DATASETS
|
|
203082
|
+
} from "@secondlayer/shared/archive/datasets";
|
|
203083
|
+
var RESUME_DATASETS = ARCHIVE_DATASETS;
|
|
203084
|
+
var FK_PARENT = {
|
|
203085
|
+
transactions: "blocks",
|
|
203086
|
+
events: "transactions",
|
|
203087
|
+
state_writes: "blocks"
|
|
203088
|
+
};
|
|
203075
203089
|
function isResumeDataset(value) {
|
|
203076
203090
|
return RESUME_DATASETS.includes(value);
|
|
203077
203091
|
}
|
|
@@ -203085,16 +203099,8 @@ function planTornImport(input) {
|
|
|
203085
203099
|
if (RESUME_DATASETS.every((d) => input.highWater[d] === null)) {
|
|
203086
203100
|
return { action: "fresh" };
|
|
203087
203101
|
}
|
|
203088
|
-
const truncateFrom =
|
|
203089
|
-
|
|
203090
|
-
transactions: null,
|
|
203091
|
-
events: null
|
|
203092
|
-
};
|
|
203093
|
-
const skipThrough = {
|
|
203094
|
-
blocks: 0,
|
|
203095
|
-
transactions: 0,
|
|
203096
|
-
events: 0
|
|
203097
|
-
};
|
|
203102
|
+
const truncateFrom = Object.fromEntries(RESUME_DATASETS.map((d) => [d, null]));
|
|
203103
|
+
const skipThrough = Object.fromEntries(RESUME_DATASETS.map((d) => [d, 0]));
|
|
203098
203104
|
for (const dataset of RESUME_DATASETS) {
|
|
203099
203105
|
const mark = input.highWater[dataset];
|
|
203100
203106
|
if (mark === null)
|
|
@@ -203109,17 +203115,18 @@ function planTornImport(input) {
|
|
|
203109
203115
|
const defined = values.filter((v) => v !== null);
|
|
203110
203116
|
return defined.length === 0 ? null : Math.min(...defined);
|
|
203111
203117
|
};
|
|
203112
|
-
|
|
203113
|
-
|
|
203114
|
-
|
|
203115
|
-
|
|
203116
|
-
|
|
203117
|
-
|
|
203118
|
-
|
|
203119
|
-
|
|
203120
|
-
|
|
203121
|
-
|
|
203122
|
-
|
|
203118
|
+
const children = RESUME_DATASETS.filter((d) => d !== "blocks");
|
|
203119
|
+
for (const dataset of children) {
|
|
203120
|
+
truncateFrom[dataset] = minDefined(truncateFrom[FK_PARENT[dataset]], truncateFrom[dataset]);
|
|
203121
|
+
}
|
|
203122
|
+
for (const dataset of children) {
|
|
203123
|
+
const parent = FK_PARENT[dataset];
|
|
203124
|
+
if (skipThrough[dataset] > skipThrough[parent]) {
|
|
203125
|
+
skipThrough[dataset] = skipThrough[parent];
|
|
203126
|
+
truncateFrom[dataset] = minDefined(truncateFrom[dataset], skipThrough[parent] + 1);
|
|
203127
|
+
}
|
|
203128
|
+
truncateFrom[dataset] = minDefined(truncateFrom[parent], truncateFrom[dataset]);
|
|
203129
|
+
}
|
|
203123
203130
|
return { action: "resume", truncateFrom, skipThrough };
|
|
203124
203131
|
}
|
|
203125
203132
|
function partitionIsLoaded(partition, skipThrough) {
|
|
@@ -203211,41 +203218,38 @@ async function loadPartition(reference, partition, rawClient, gate) {
|
|
|
203211
203218
|
}
|
|
203212
203219
|
return written;
|
|
203213
203220
|
}
|
|
203214
|
-
|
|
203215
|
-
"
|
|
203216
|
-
|
|
203217
|
-
|
|
203218
|
-
];
|
|
203221
|
+
function restoredSummary(loaded) {
|
|
203222
|
+
return LOAD_ORDER.filter((d) => loaded[d] !== undefined).map((d) => `${(loaded[d] ?? 0).toLocaleString()} ${d}`).join(", ");
|
|
203223
|
+
}
|
|
203224
|
+
var LOAD_ORDER = ARCHIVE_DATASETS2;
|
|
203219
203225
|
function loadOrder(partitions) {
|
|
203220
203226
|
return LOAD_ORDER.flatMap((dataset) => partitions.filter((p) => p.dataset === dataset).sort((a, b) => a.from_block - b.from_block));
|
|
203221
203227
|
}
|
|
203222
203228
|
function parseVerifyDatasets(value) {
|
|
203223
203229
|
if (value === undefined || value === "all") {
|
|
203224
|
-
return [
|
|
203230
|
+
return [...ARCHIVE_DATASETS2];
|
|
203225
203231
|
}
|
|
203226
203232
|
if (value === "blocks")
|
|
203227
203233
|
return ["blocks"];
|
|
203228
203234
|
throw new Error(`--verify must be "all" or "blocks", got "${String(value)}"`);
|
|
203229
203235
|
}
|
|
203230
203236
|
async function readHighWater(db) {
|
|
203231
|
-
const
|
|
203232
|
-
const
|
|
203233
|
-
|
|
203234
|
-
|
|
203235
|
-
|
|
203236
|
-
|
|
203237
|
-
|
|
203238
|
-
|
|
203239
|
-
}
|
|
203237
|
+
const marks = {};
|
|
203238
|
+
for (const dataset of RESUME_DATASETS) {
|
|
203239
|
+
const { rows } = await sql`
|
|
203240
|
+
SELECT max(${sql.ref(archiveHeightColumn(dataset))}) AS max
|
|
203241
|
+
FROM ${sql.table(dataset)}
|
|
203242
|
+
`.execute(db);
|
|
203243
|
+
const max = rows[0]?.max;
|
|
203244
|
+
marks[dataset] = max === null || max === undefined ? null : Number(max);
|
|
203245
|
+
}
|
|
203246
|
+
return marks;
|
|
203240
203247
|
}
|
|
203241
203248
|
async function truncateDatasetFrom(db, dataset, from) {
|
|
203242
|
-
|
|
203243
|
-
|
|
203244
|
-
|
|
203245
|
-
|
|
203246
|
-
} else {
|
|
203247
|
-
await sql`DELETE FROM events WHERE block_height >= ${from}`.execute(db);
|
|
203248
|
-
}
|
|
203249
|
+
await sql`
|
|
203250
|
+
DELETE FROM ${sql.table(dataset)}
|
|
203251
|
+
WHERE ${sql.ref(archiveHeightColumn(dataset))} >= ${from}
|
|
203252
|
+
`.execute(db);
|
|
203249
203253
|
}
|
|
203250
203254
|
function attachBootstrapCommand(cmd) {
|
|
203251
203255
|
return cmd.requiredOption("--against <manifest>", "archive manifest: an https URL or a local file path").option("--to-block <n>", "stop at this height instead of the archive tip").option("--from-block <n>", "forward-only: restore from this height instead of genesis; earlier history is declared out of scope").option("--public-key <pem>", "pin a signing key; default is the archive key built into this release").option("--verify <datasets>", "digests checked after the load: all (blocks, transactions, events; adds minutes on a full chain) or blocks", "all").option("-y, --yes", "skip the confirmation prompt").option("--json", "Output as JSON").addHelpText("after", `
|
|
@@ -203259,7 +203263,7 @@ Exit codes:
|
|
|
203259
203263
|
1 restore completed but verification found divergence
|
|
203260
203264
|
2 refused (non-empty target, or an untrusted reference)`).action(async (opts) => {
|
|
203261
203265
|
try {
|
|
203262
|
-
const
|
|
203266
|
+
const requestedVerify = parseVerifyDatasets(opts.verify);
|
|
203263
203267
|
const publicKey = await resolveArchivePublicKey({
|
|
203264
203268
|
explicitPem: opts.publicKey,
|
|
203265
203269
|
envPem: process.env.ARCHIVE_SIGNING_PUBLIC_KEY ?? process.env.STREAMS_SIGNING_PUBLIC_KEY,
|
|
@@ -203269,6 +203273,7 @@ Exit codes:
|
|
|
203269
203273
|
publicKeyPem: publicKey
|
|
203270
203274
|
});
|
|
203271
203275
|
const signature = checkSignature(reference.manifest, publicKey, false);
|
|
203276
|
+
const verifyDatasets = requestedVerify.filter((d) => manifestPublishes(reference.manifest, d));
|
|
203272
203277
|
if (!signature.verified) {
|
|
203273
203278
|
printError(`Refusing to bootstrap from an untrusted archive: ${signature.reason}.`, {
|
|
203274
203279
|
hint: "Pass --public-key <pem> to pin the archive's signing key."
|
|
@@ -203280,7 +203285,14 @@ Exit codes:
|
|
|
203280
203285
|
const highWater = await readHighWater(db);
|
|
203281
203286
|
const toBlock = opts.toBlock === undefined ? undefined : Number(opts.toBlock);
|
|
203282
203287
|
const fromBlock = opts.fromBlock === undefined ? undefined : Number(opts.fromBlock);
|
|
203283
|
-
const
|
|
203288
|
+
const inScope = (reference.manifest.partitions ?? []).filter((p) => (toBlock === undefined || p.to_block <= toBlock) && (fromBlock === undefined || p.from_block >= fromBlock));
|
|
203289
|
+
const unknown = [
|
|
203290
|
+
...new Set(inScope.map((p) => p.dataset).filter((d) => !isArchiveDataset(d)))
|
|
203291
|
+
];
|
|
203292
|
+
if (unknown.length > 0) {
|
|
203293
|
+
warn(`Skipping archive datasets this release does not know: ${unknown.join(", ")}. Upgrade the CLI to load them.`);
|
|
203294
|
+
}
|
|
203295
|
+
const declared = inScope.filter((p) => isArchiveDataset(p.dataset));
|
|
203284
203296
|
if (declared.length === 0) {
|
|
203285
203297
|
printError("The archive has no partitions for that range.", {
|
|
203286
203298
|
hint: "Check --from-block and --to-block against the manifest's coverage."
|
|
@@ -203379,16 +203391,16 @@ Exit codes:
|
|
|
203379
203391
|
}
|
|
203380
203392
|
const nodeTipAtStart = await readNodeTip();
|
|
203381
203393
|
const rawClient = getRawClient("source");
|
|
203382
|
-
const loaded =
|
|
203394
|
+
const loaded = Object.fromEntries(LOAD_ORDER.filter((d) => manifestPublishes(reference.manifest, d)).map((d) => [d, 0]));
|
|
203383
203395
|
const startedAt = Date.now();
|
|
203384
203396
|
for (const dataset of LOAD_ORDER) {
|
|
203385
203397
|
const datasetPartitions = loadOrder(partitions).filter((p) => p.dataset === dataset);
|
|
203386
203398
|
for (const [index, partition] of datasetPartitions.entries()) {
|
|
203387
|
-
loaded[dataset]
|
|
203399
|
+
loaded[dataset] = (loaded[dataset] ?? 0) + await loadPartition(reference, partition, rawClient, gate);
|
|
203388
203400
|
const done = index + 1;
|
|
203389
203401
|
if (done % 10 === 0 || done === datasetPartitions.length) {
|
|
203390
203402
|
const elapsed = (Date.now() - startedAt) / 1000;
|
|
203391
|
-
note(` ${dataset}: ${done}/${datasetPartitions.length} partitions · ${loaded[dataset].toLocaleString()} rows · ${formatDuration(elapsed)} elapsed`);
|
|
203403
|
+
note(` ${dataset}: ${done}/${datasetPartitions.length} partitions · ${(loaded[dataset] ?? 0).toLocaleString()} rows · ${formatDuration(elapsed)} elapsed`);
|
|
203392
203404
|
}
|
|
203393
203405
|
}
|
|
203394
203406
|
}
|
|
@@ -203456,7 +203468,7 @@ Exit codes:
|
|
|
203456
203468
|
data: report,
|
|
203457
203469
|
human: () => {
|
|
203458
203470
|
if (divergent === 0) {
|
|
203459
|
-
success(`Restored ${
|
|
203471
|
+
success(`Restored ${restoredSummary(loaded)} in ${formatDuration(elapsed)}.`);
|
|
203460
203472
|
if (referenceDigests.length > 0) {
|
|
203461
203473
|
note(` verified ${referenceDigests.length} ${verifyDatasets.join("/")} ranges against the archive`);
|
|
203462
203474
|
} else {
|
|
@@ -203478,7 +203490,7 @@ Exit codes:
|
|
|
203478
203490
|
}
|
|
203479
203491
|
} else {
|
|
203480
203492
|
const byDataset = verifyDatasets.map((d) => `${divergentRanges.filter((r) => r.dataset === d).length} ${d}`).join(", ");
|
|
203481
|
-
warn(`Restored ${loaded.blocks.toLocaleString()} blocks but ${divergent} ranges do not match the archive (${byDataset}).`);
|
|
203493
|
+
warn(`Restored ${(loaded.blocks ?? 0).toLocaleString()} blocks but ${divergent} ranges do not match the archive (${byDataset}).`);
|
|
203482
203494
|
console.error(dim(" Investigate with: secondlayer verify --against <manifest>"));
|
|
203483
203495
|
}
|
|
203484
203496
|
}
|
|
@@ -203511,6 +203523,10 @@ import {
|
|
|
203511
203523
|
copyStatement as copyStatement2,
|
|
203512
203524
|
writeRowsToCopyStream as writeRowsToCopyStream2
|
|
203513
203525
|
} from "@secondlayer/shared/archive/copy-loader";
|
|
203526
|
+
import {
|
|
203527
|
+
ARCHIVE_DATASETS as ARCHIVE_DATASETS3,
|
|
203528
|
+
manifestPublishes as manifestPublishes2
|
|
203529
|
+
} from "@secondlayer/shared/archive/datasets";
|
|
203514
203530
|
import {
|
|
203515
203531
|
compareRangeDigests,
|
|
203516
203532
|
computeRangeDigest as computeRangeDigest2
|
|
@@ -203594,38 +203610,43 @@ async function planRange(db, reference, partition, gate) {
|
|
|
203594
203610
|
const extraHeights = [...localByHeight.keys()].sort((a, b) => a - b);
|
|
203595
203611
|
return { fixes, extraHeights };
|
|
203596
203612
|
}
|
|
203597
|
-
var CHILD_DATASETS =
|
|
203598
|
-
var
|
|
203599
|
-
"
|
|
203600
|
-
"transactions",
|
|
203601
|
-
"
|
|
203602
|
-
|
|
203613
|
+
var CHILD_DATASETS = ARCHIVE_DATASETS3.filter((d) => d !== "blocks");
|
|
203614
|
+
var REWRITE_TOGETHER = {
|
|
203615
|
+
transactions: ["transactions", "events"],
|
|
203616
|
+
events: ["transactions", "events"],
|
|
203617
|
+
state_writes: ["state_writes"]
|
|
203618
|
+
};
|
|
203619
|
+
function publishedChildren(manifest) {
|
|
203620
|
+
return CHILD_DATASETS.filter((d) => manifestPublishes2(manifest, d));
|
|
203621
|
+
}
|
|
203622
|
+
function reportedDatasets(children) {
|
|
203623
|
+
return ARCHIVE_DATASETS3.filter((d) => d === "blocks" || children.includes(d));
|
|
203624
|
+
}
|
|
203625
|
+
function zeroCounts(datasets) {
|
|
203626
|
+
return Object.fromEntries(datasets.map((d) => [d, 0]));
|
|
203627
|
+
}
|
|
203603
203628
|
function childPartitionFor(partitions, dataset, height) {
|
|
203604
203629
|
return partitions.find((p) => p.dataset === dataset && p.from_block <= height && p.to_block >= height);
|
|
203605
203630
|
}
|
|
203606
|
-
function planChildRewrite(partitions, heights) {
|
|
203607
|
-
const missing = {
|
|
203608
|
-
transactions: [],
|
|
203609
|
-
events: []
|
|
203610
|
-
};
|
|
203631
|
+
function planChildRewrite(partitions, heights, children = ["transactions", "events"]) {
|
|
203632
|
+
const missing = {};
|
|
203611
203633
|
const found = new Map;
|
|
203612
|
-
for (const dataset of
|
|
203634
|
+
for (const dataset of children) {
|
|
203635
|
+
missing[dataset] = [];
|
|
203613
203636
|
for (const height of heights) {
|
|
203614
203637
|
const partition = childPartitionFor(partitions, dataset, height);
|
|
203615
203638
|
if (!partition) {
|
|
203616
|
-
missing[dataset]
|
|
203639
|
+
missing[dataset]?.push(height);
|
|
203617
203640
|
continue;
|
|
203618
203641
|
}
|
|
203619
203642
|
found.set(height, { ...found.get(height), [dataset]: partition });
|
|
203620
203643
|
}
|
|
203621
203644
|
}
|
|
203622
|
-
const rewritable =
|
|
203623
|
-
const entry = found.get(h);
|
|
203624
|
-
return !!entry?.transactions && !!entry?.events;
|
|
203625
|
-
});
|
|
203645
|
+
const rewritable = {};
|
|
203626
203646
|
const byPartition = new Map;
|
|
203627
|
-
for (const
|
|
203628
|
-
|
|
203647
|
+
for (const dataset of children) {
|
|
203648
|
+
rewritable[dataset] = heights.filter((h) => REWRITE_TOGETHER[dataset].every((d) => !!found.get(h)?.[d]));
|
|
203649
|
+
for (const height of rewritable[dataset] ?? []) {
|
|
203629
203650
|
const partition = found.get(height)?.[dataset];
|
|
203630
203651
|
if (!partition)
|
|
203631
203652
|
continue;
|
|
@@ -203647,11 +203668,9 @@ async function* onlyHeights(rows, heights) {
|
|
|
203647
203668
|
}
|
|
203648
203669
|
async function applyFixes(rawClient, reference, partition, fixes, gate) {
|
|
203649
203670
|
const applied = {
|
|
203650
|
-
|
|
203651
|
-
|
|
203652
|
-
|
|
203653
|
-
missing: { transactions: [], events: [] },
|
|
203654
|
-
rewritten: []
|
|
203671
|
+
rows: Object.fromEntries(ARCHIVE_DATASETS3.map((d) => [d, 0])),
|
|
203672
|
+
missing: {},
|
|
203673
|
+
rewritten: {}
|
|
203655
203674
|
};
|
|
203656
203675
|
if (fixes.length === 0)
|
|
203657
203676
|
return applied;
|
|
@@ -203660,20 +203679,20 @@ async function applyFixes(rawClient, reference, partition, fixes, gate) {
|
|
|
203660
203679
|
const byHeight = new Map(archiveBlocks.map((b) => [b.height, b]));
|
|
203661
203680
|
const targets = fixes.map((fix) => byHeight.get(fix.height)).filter((b) => b !== undefined);
|
|
203662
203681
|
const heights = targets.map((b) => b.height);
|
|
203663
|
-
const children = planChildRewrite(reference.manifest.partitions ?? [], heights);
|
|
203682
|
+
const children = planChildRewrite(reference.manifest.partitions ?? [], heights, publishedChildren(reference.manifest));
|
|
203664
203683
|
applied.missing = children.missing;
|
|
203665
203684
|
applied.rewritten = children.rewritable;
|
|
203666
|
-
const rewritable = new Set(children.rewritable);
|
|
203667
203685
|
const childBytes = new Map;
|
|
203668
203686
|
for (const [path, { partition: child }] of children.byPartition) {
|
|
203669
203687
|
childBytes.set(path, await fetchVerifiedPartition(reference, child, gate));
|
|
203670
203688
|
}
|
|
203671
203689
|
await rawClient.begin(async (tx) => {
|
|
203672
|
-
for (const
|
|
203673
|
-
|
|
203674
|
-
|
|
203675
|
-
|
|
203676
|
-
|
|
203690
|
+
for (const dataset of [...CHILD_DATASETS].reverse()) {
|
|
203691
|
+
for (const height of children.rewritable[dataset] ?? []) {
|
|
203692
|
+
await tx.unsafe(`DELETE FROM ${dataset} WHERE block_height = $1`, [
|
|
203693
|
+
height
|
|
203694
|
+
]);
|
|
203695
|
+
}
|
|
203677
203696
|
}
|
|
203678
203697
|
for (const block of targets) {
|
|
203679
203698
|
await tx.unsafe(`INSERT INTO blocks (
|
|
@@ -203697,8 +203716,9 @@ async function applyFixes(rawClient, reference, partition, fixes, gate) {
|
|
|
203697
203716
|
block.timestamp
|
|
203698
203717
|
]);
|
|
203699
203718
|
}
|
|
203700
|
-
applied.blocks = targets.length;
|
|
203719
|
+
applied.rows.blocks = targets.length;
|
|
203701
203720
|
for (const dataset of CHILD_DATASETS) {
|
|
203721
|
+
const keep = new Set(children.rewritable[dataset] ?? []);
|
|
203702
203722
|
for (const [path, { partition: child }] of children.byPartition) {
|
|
203703
203723
|
if (child.dataset !== dataset)
|
|
203704
203724
|
continue;
|
|
@@ -203706,24 +203726,20 @@ async function applyFixes(rawClient, reference, partition, fixes, gate) {
|
|
|
203706
203726
|
if (!bytes)
|
|
203707
203727
|
continue;
|
|
203708
203728
|
const writable = await tx.unsafe(copyStatement2(dataset)).writable();
|
|
203709
|
-
applied[dataset] += await writeRowsToCopyStream2({
|
|
203729
|
+
applied.rows[dataset] += await writeRowsToCopyStream2({
|
|
203710
203730
|
writable,
|
|
203711
203731
|
dataset,
|
|
203712
|
-
rows: onlyHeights(readPartitionRows(bytes, `repair-${dataset}-${child.from_block}`),
|
|
203732
|
+
rows: onlyHeights(readPartitionRows(bytes, `repair-${dataset}-${child.from_block}`), keep)
|
|
203713
203733
|
});
|
|
203714
203734
|
}
|
|
203715
203735
|
}
|
|
203716
203736
|
});
|
|
203717
203737
|
return applied;
|
|
203718
203738
|
}
|
|
203719
|
-
async function reverify(db, reference, ranges) {
|
|
203720
|
-
const remaining = {
|
|
203721
|
-
blocks: 0,
|
|
203722
|
-
transactions: 0,
|
|
203723
|
-
events: 0
|
|
203724
|
-
};
|
|
203739
|
+
async function reverify(db, reference, ranges, datasets) {
|
|
203740
|
+
const remaining = {};
|
|
203725
203741
|
const published = reference.manifest.range_digests ?? [];
|
|
203726
|
-
for (const dataset of
|
|
203742
|
+
for (const dataset of datasets) {
|
|
203727
203743
|
const expected = published.filter((d) => d.dataset === dataset && ranges.some((r) => r.from_block === d.from_block && r.to_block === d.to_block));
|
|
203728
203744
|
const actual = [];
|
|
203729
203745
|
for (const range of expected) {
|
|
@@ -203739,10 +203755,16 @@ function findChildPartition(partitions, dataset, fromBlock, toBlock) {
|
|
|
203739
203755
|
function expectedZeroRows(zeroRecordRanges, dataset, fromBlock, toBlock) {
|
|
203740
203756
|
return zeroRecordRanges.some((r) => r.dataset === dataset && r.from_block === fromBlock && r.to_block === toBlock);
|
|
203741
203757
|
}
|
|
203742
|
-
|
|
203758
|
+
var CHILD_LABEL = {
|
|
203759
|
+
transactions: "TX",
|
|
203760
|
+
events: "EVENTS",
|
|
203761
|
+
state_writes: "WRITES"
|
|
203762
|
+
};
|
|
203763
|
+
async function planChildPartitionDivergence(db, partitions, zeroRecordRanges, inRange, children = ["transactions", "events"]) {
|
|
203764
|
+
const isChild = (dataset) => children.includes(dataset);
|
|
203743
203765
|
const ranges = new Map;
|
|
203744
203766
|
for (const p of partitions) {
|
|
203745
|
-
if (p.dataset
|
|
203767
|
+
if (!isChild(p.dataset))
|
|
203746
203768
|
continue;
|
|
203747
203769
|
if (!inRange(p.from_block, p.to_block))
|
|
203748
203770
|
continue;
|
|
@@ -203753,59 +203775,74 @@ async function planChildPartitionDivergence(db, partitions, zeroRecordRanges, in
|
|
|
203753
203775
|
}
|
|
203754
203776
|
const mismatches = [];
|
|
203755
203777
|
for (const { from, to } of ranges.values()) {
|
|
203756
|
-
const
|
|
203757
|
-
|
|
203758
|
-
|
|
203759
|
-
const expectedEv = evPartition?.row_count ?? (expectedZeroRows(zeroRecordRanges, "events", from, to) ? 0 : null);
|
|
203760
|
-
if (expectedTx === null && expectedEv === null)
|
|
203761
|
-
continue;
|
|
203762
|
-
const actualTx = (await computeRangeDigest2(db, "transactions", from, to)).row_count;
|
|
203763
|
-
const actualEv = (await computeRangeDigest2(db, "events", from, to)).row_count;
|
|
203764
|
-
if (expectedTx !== null && actualTx !== expectedTx || expectedEv !== null && actualEv !== expectedEv) {
|
|
203765
|
-
mismatches.push({
|
|
203766
|
-
from_block: from,
|
|
203767
|
-
to_block: to,
|
|
203768
|
-
transactions: { expected: expectedTx ?? actualTx, actual: actualTx },
|
|
203769
|
-
events: { expected: expectedEv ?? actualEv, actual: actualEv }
|
|
203770
|
-
});
|
|
203778
|
+
const expected = new Map;
|
|
203779
|
+
for (const dataset of children) {
|
|
203780
|
+
expected.set(dataset, findChildPartition(partitions, dataset, from, to)?.row_count ?? (expectedZeroRows(zeroRecordRanges, dataset, from, to) ? 0 : null));
|
|
203771
203781
|
}
|
|
203782
|
+
if ([...expected.values()].every((count) => count === null))
|
|
203783
|
+
continue;
|
|
203784
|
+
const mismatch = { from_block: from, to_block: to };
|
|
203785
|
+
let diverged = false;
|
|
203786
|
+
for (const dataset of children) {
|
|
203787
|
+
const actual = (await computeRangeDigest2(db, dataset, from, to)).row_count;
|
|
203788
|
+
const want = expected.get(dataset) ?? actual;
|
|
203789
|
+
mismatch[dataset] = { expected: want, actual };
|
|
203790
|
+
if (want !== actual)
|
|
203791
|
+
diverged = true;
|
|
203792
|
+
}
|
|
203793
|
+
if (diverged)
|
|
203794
|
+
mismatches.push(mismatch);
|
|
203772
203795
|
}
|
|
203773
203796
|
return mismatches;
|
|
203774
203797
|
}
|
|
203798
|
+
function childRewritesFor(mismatch) {
|
|
203799
|
+
const rewrite = new Set;
|
|
203800
|
+
for (const dataset of CHILD_DATASETS) {
|
|
203801
|
+
const count = mismatch[dataset];
|
|
203802
|
+
if (!count || count.expected === count.actual)
|
|
203803
|
+
continue;
|
|
203804
|
+
for (const partner of REWRITE_TOGETHER[dataset]) {
|
|
203805
|
+
if (mismatch[partner])
|
|
203806
|
+
rewrite.add(partner);
|
|
203807
|
+
}
|
|
203808
|
+
}
|
|
203809
|
+
return CHILD_DATASETS.filter((d) => rewrite.has(d));
|
|
203810
|
+
}
|
|
203775
203811
|
async function applyChildRangeFix(rawClient, reference, mismatch, gate) {
|
|
203776
203812
|
const partitions = reference.manifest.partitions ?? [];
|
|
203777
|
-
const
|
|
203778
|
-
const
|
|
203779
|
-
const
|
|
203780
|
-
|
|
203813
|
+
const rewrite = childRewritesFor(mismatch);
|
|
203814
|
+
const bytes = new Map;
|
|
203815
|
+
for (const dataset of rewrite) {
|
|
203816
|
+
const partition = findChildPartition(partitions, dataset, mismatch.from_block, mismatch.to_block);
|
|
203817
|
+
if (partition) {
|
|
203818
|
+
bytes.set(dataset, await fetchVerifiedPartition(reference, partition, gate));
|
|
203819
|
+
}
|
|
203820
|
+
}
|
|
203781
203821
|
const written = {
|
|
203782
|
-
|
|
203783
|
-
events: 0,
|
|
203822
|
+
rows: Object.fromEntries(rewrite.map((d) => [d, 0])),
|
|
203784
203823
|
vm_events: { restored: 0, dropped: 0 }
|
|
203785
203824
|
};
|
|
203825
|
+
const parksVmEvents = rewrite.includes("transactions");
|
|
203786
203826
|
await rawClient.begin(async (tx) => {
|
|
203787
|
-
|
|
203788
|
-
|
|
203789
|
-
mismatch.from_block,
|
|
203790
|
-
mismatch.to_block
|
|
203791
|
-
]);
|
|
203792
|
-
await tx.unsafe("DELETE FROM transactions WHERE block_height BETWEEN $1 AND $2", [mismatch.from_block, mismatch.to_block]);
|
|
203793
|
-
if (txBytes) {
|
|
203794
|
-
const writable = await tx.unsafe(copyStatement2("transactions")).writable();
|
|
203795
|
-
written.transactions = await writeRowsToCopyStream2({
|
|
203796
|
-
writable,
|
|
203797
|
-
dataset: "transactions",
|
|
203798
|
-
rows: readPartitionRows(txBytes, `repair-transactions-${mismatch.from_block}`)
|
|
203799
|
-
});
|
|
203827
|
+
if (parksVmEvents) {
|
|
203828
|
+
await tx.unsafe("CREATE TEMP TABLE repair_vm_events_keep ON COMMIT DROP AS SELECT * FROM vm_events WHERE block_height BETWEEN $1 AND $2", [mismatch.from_block, mismatch.to_block]);
|
|
203800
203829
|
}
|
|
203801
|
-
|
|
203802
|
-
|
|
203803
|
-
|
|
203830
|
+
for (const dataset of [...rewrite].reverse()) {
|
|
203831
|
+
await tx.unsafe(`DELETE FROM ${dataset} WHERE block_height BETWEEN $1 AND $2`, [mismatch.from_block, mismatch.to_block]);
|
|
203832
|
+
}
|
|
203833
|
+
for (const dataset of rewrite) {
|
|
203834
|
+
const partitionBytes = bytes.get(dataset);
|
|
203835
|
+
if (!partitionBytes)
|
|
203836
|
+
continue;
|
|
203837
|
+
const writable = await tx.unsafe(copyStatement2(dataset)).writable();
|
|
203838
|
+
written.rows[dataset] = await writeRowsToCopyStream2({
|
|
203804
203839
|
writable,
|
|
203805
|
-
dataset
|
|
203806
|
-
rows: readPartitionRows(
|
|
203840
|
+
dataset,
|
|
203841
|
+
rows: readPartitionRows(partitionBytes, `repair-${dataset}-${mismatch.from_block}`)
|
|
203807
203842
|
});
|
|
203808
203843
|
}
|
|
203844
|
+
if (!parksVmEvents)
|
|
203845
|
+
return;
|
|
203809
203846
|
const kept = await tx.unsafe("SELECT count(*)::int AS n FROM repair_vm_events_keep");
|
|
203810
203847
|
const keptCount = Number(kept[0]?.n ?? 0);
|
|
203811
203848
|
if (keptCount > 0) {
|
|
@@ -203817,18 +203854,14 @@ async function applyChildRangeFix(rawClient, reference, mismatch, gate) {
|
|
|
203817
203854
|
return written;
|
|
203818
203855
|
}
|
|
203819
203856
|
async function runChildOnlyRepair(params) {
|
|
203820
|
-
const { reference, childMismatches, apply, json, yes } = params;
|
|
203857
|
+
const { reference, childMismatches, children, apply, json, yes } = params;
|
|
203821
203858
|
if (childMismatches.length === 0) {
|
|
203822
203859
|
success("Nothing to repair — local data matches the archive.");
|
|
203823
203860
|
output({ json, data: { status: "clean", fixes: [] }, human: () => {} });
|
|
203824
203861
|
process.exit(REPAIR_EXIT.OK);
|
|
203825
203862
|
}
|
|
203826
203863
|
const partitions = reference.manifest.partitions ?? [];
|
|
203827
|
-
const fetchPaths = childMismatches.flatMap((m) =>
|
|
203828
|
-
const tx = findChildPartition(partitions, "transactions", m.from_block, m.to_block);
|
|
203829
|
-
const ev = findChildPartition(partitions, "events", m.from_block, m.to_block);
|
|
203830
|
-
return [tx?.path, ev?.path].filter((p) => p !== undefined);
|
|
203831
|
-
});
|
|
203864
|
+
const fetchPaths = childMismatches.flatMap((m) => childRewritesFor(m).map((dataset) => findChildPartition(partitions, dataset, m.from_block, m.to_block)?.path).filter((p) => p !== undefined));
|
|
203832
203865
|
let gate;
|
|
203833
203866
|
let quoteLine;
|
|
203834
203867
|
if (apply && isOfficialArchive(reference)) {
|
|
@@ -203859,14 +203892,20 @@ async function runChildOnlyRepair(params) {
|
|
|
203859
203892
|
}
|
|
203860
203893
|
gate = createGatedFetcher(fetchPaths, "repair");
|
|
203861
203894
|
}
|
|
203862
|
-
const applied =
|
|
203895
|
+
const applied = zeroCounts(children);
|
|
203896
|
+
const rewritten = new Set;
|
|
203863
203897
|
const vmEvents = { restored: 0, dropped: 0 };
|
|
203864
203898
|
if (apply) {
|
|
203865
203899
|
const rawClient = getRawClient2("source");
|
|
203866
203900
|
for (const mismatch of childMismatches) {
|
|
203867
203901
|
const result = await applyChildRangeFix(rawClient, reference, mismatch, gate);
|
|
203868
|
-
|
|
203869
|
-
|
|
203902
|
+
for (const dataset of CHILD_DATASETS) {
|
|
203903
|
+
const rows = result.rows[dataset];
|
|
203904
|
+
if (rows === undefined)
|
|
203905
|
+
continue;
|
|
203906
|
+
applied[dataset] = (applied[dataset] ?? 0) + rows;
|
|
203907
|
+
rewritten.add(dataset);
|
|
203908
|
+
}
|
|
203870
203909
|
vmEvents.restored += result.vm_events.restored;
|
|
203871
203910
|
vmEvents.dropped += result.vm_events.dropped;
|
|
203872
203911
|
}
|
|
@@ -203876,13 +203915,17 @@ async function runChildOnlyRepair(params) {
|
|
|
203876
203915
|
if (apply) {
|
|
203877
203916
|
remaining = 0;
|
|
203878
203917
|
for (const mismatch of childMismatches) {
|
|
203879
|
-
|
|
203880
|
-
|
|
203881
|
-
|
|
203882
|
-
|
|
203883
|
-
|
|
203884
|
-
|
|
203918
|
+
let stillDiverges = false;
|
|
203919
|
+
for (const dataset of children) {
|
|
203920
|
+
const count = mismatch[dataset];
|
|
203921
|
+
if (!count)
|
|
203922
|
+
continue;
|
|
203923
|
+
const actual = await computeRangeDigest2(db, dataset, mismatch.from_block, mismatch.to_block);
|
|
203924
|
+
if (actual.row_count !== count.expected)
|
|
203925
|
+
stillDiverges = true;
|
|
203885
203926
|
}
|
|
203927
|
+
if (stillDiverges)
|
|
203928
|
+
remaining++;
|
|
203886
203929
|
}
|
|
203887
203930
|
}
|
|
203888
203931
|
const complete = apply && remaining === 0;
|
|
@@ -203895,7 +203938,7 @@ async function runChildOnlyRepair(params) {
|
|
|
203895
203938
|
local_only_heights: [],
|
|
203896
203939
|
applied: 0,
|
|
203897
203940
|
rows_written: { blocks: 0, ...applied },
|
|
203898
|
-
datasets_rewritten:
|
|
203941
|
+
datasets_rewritten: CHILD_DATASETS.filter((d) => rewritten.has(d)),
|
|
203899
203942
|
heights_missing_child_partitions: [],
|
|
203900
203943
|
child_partition_mismatches: childMismatches,
|
|
203901
203944
|
fixes: [],
|
|
@@ -203910,27 +203953,27 @@ async function runChildOnlyRepair(params) {
|
|
|
203910
203953
|
if (vmEvents.dropped > 0) {
|
|
203911
203954
|
warn(`Dropped ${vmEvents.dropped} vm_events row(s) whose transaction is not in the archive; vm_events can't be refetched from the archive.`);
|
|
203912
203955
|
}
|
|
203913
|
-
const written = `${applied
|
|
203956
|
+
const written = children.map((d) => `${applied[d] ?? 0} ${d}`).join(", ");
|
|
203914
203957
|
if (complete) {
|
|
203915
203958
|
success(`Repaired ${childMismatches.length} range(s) (rewrote ${written}); blocks were already correct.`);
|
|
203916
203959
|
} else {
|
|
203917
203960
|
warn(`Rewrote ${written} but ${remaining} range(s) still diverge from the archive.`);
|
|
203918
203961
|
}
|
|
203919
203962
|
} else {
|
|
203920
|
-
warn(`${childMismatches.length} range(s) have
|
|
203963
|
+
warn(`${childMismatches.length} range(s) have ${children.join("/")} that diverge from the archive, while their blocks match.`);
|
|
203921
203964
|
console.error("");
|
|
203922
203965
|
console.error(formatTable([
|
|
203923
203966
|
"RANGE",
|
|
203924
|
-
|
|
203925
|
-
|
|
203926
|
-
|
|
203927
|
-
|
|
203967
|
+
...children.flatMap((d) => [
|
|
203968
|
+
`${CHILD_LABEL[d]} EXPECTED`,
|
|
203969
|
+
`${CHILD_LABEL[d]} LOCAL`
|
|
203970
|
+
])
|
|
203928
203971
|
], childMismatches.slice(0, 20).map((m) => [
|
|
203929
203972
|
`${m.from_block}-${m.to_block}`,
|
|
203930
|
-
|
|
203931
|
-
|
|
203932
|
-
|
|
203933
|
-
|
|
203973
|
+
...children.flatMap((d) => [
|
|
203974
|
+
String(m[d]?.expected ?? "—"),
|
|
203975
|
+
String(m[d]?.actual ?? "—")
|
|
203976
|
+
])
|
|
203934
203977
|
])));
|
|
203935
203978
|
console.error("");
|
|
203936
203979
|
console.error(dim("(dry-run — pass --apply to write the repair)"));
|
|
@@ -203946,15 +203989,16 @@ Examples:
|
|
|
203946
203989
|
$ secondlayer repair --against ./snapshot.json --apply # write the fix
|
|
203947
203990
|
$ secondlayer repair --against ./snapshot.json --from-block 8500000 --to-block 8549999
|
|
203948
203991
|
|
|
203949
|
-
A fixed block is rewritten together with its transactions and events
|
|
203950
|
-
|
|
203951
|
-
|
|
203952
|
-
the run names the height, and it
|
|
203992
|
+
A fixed block is rewritten together with its transactions and events (and
|
|
203993
|
+
state_writes, when the archive publishes them) from the archive's partitions
|
|
203994
|
+
for that height. When the reference carries no partition of one of those for
|
|
203995
|
+
a height, the block is rewritten alone, the run names the height, and it
|
|
203996
|
+
exits 1.
|
|
203953
203997
|
|
|
203954
|
-
When blocks already match the archive but a
|
|
203955
|
-
|
|
203956
|
-
is no block or fork boundary to preserve, only stale rows
|
|
203957
|
-
was never wrong.
|
|
203998
|
+
When blocks already match the archive but a child partition does not, that
|
|
203999
|
+
range's diverging rows are replaced outright (transactions and events always
|
|
204000
|
+
as a pair) — there is no block or fork boundary to preserve, only stale rows
|
|
204001
|
+
underneath one that was never wrong.
|
|
203958
204002
|
|
|
203959
204003
|
Exit codes:
|
|
203960
204004
|
0 nothing to repair, or the repair completed and re-verified clean
|
|
@@ -203992,11 +204036,14 @@ Exit codes:
|
|
|
203992
204036
|
local.push(await computeRangeDigest2(db, "blocks", range.from_block, range.to_block));
|
|
203993
204037
|
}
|
|
203994
204038
|
const divergent = compareRangeDigests(local, referenceDigests).filter((c) => c.status !== "match");
|
|
204039
|
+
const children = publishedChildren(reference.manifest);
|
|
204040
|
+
const reported = reportedDatasets(children);
|
|
203995
204041
|
if (divergent.length === 0) {
|
|
203996
|
-
const childMismatches = await planChildPartitionDivergence(db, reference.manifest.partitions ?? [], reference.manifest.zero_record_ranges ?? [], inRange);
|
|
204042
|
+
const childMismatches = await planChildPartitionDivergence(db, reference.manifest.partitions ?? [], reference.manifest.zero_record_ranges ?? [], inRange, children);
|
|
203997
204043
|
await runChildOnlyRepair({
|
|
203998
204044
|
reference,
|
|
203999
204045
|
childMismatches,
|
|
204046
|
+
children,
|
|
204000
204047
|
apply: !!opts.apply,
|
|
204001
204048
|
json: !!opts.json,
|
|
204002
204049
|
yes: !!opts.yes
|
|
@@ -204004,7 +204051,7 @@ Exit codes:
|
|
|
204004
204051
|
return;
|
|
204005
204052
|
}
|
|
204006
204053
|
const partitions = (reference.manifest.partitions ?? []).filter((p) => p.dataset === "blocks" && divergent.some((d) => d.from_block === p.from_block && d.to_block === p.to_block));
|
|
204007
|
-
const childPartitions = opts.apply ? (reference.manifest.partitions ?? []).filter((p) => (p.dataset
|
|
204054
|
+
const childPartitions = opts.apply ? (reference.manifest.partitions ?? []).filter((p) => children.includes(p.dataset) && divergent.some((d) => p.from_block <= d.to_block && p.to_block >= d.from_block)) : [];
|
|
204008
204055
|
const fetchPaths = [...partitions, ...childPartitions].map((p) => p.path);
|
|
204009
204056
|
let gate;
|
|
204010
204057
|
let quoteLine;
|
|
@@ -204047,37 +204094,34 @@ Exit codes:
|
|
|
204047
204094
|
allFixes.push(...fixes);
|
|
204048
204095
|
allExtra.push(...extraHeights);
|
|
204049
204096
|
}
|
|
204050
|
-
const applied =
|
|
204051
|
-
const missingChild =
|
|
204052
|
-
|
|
204053
|
-
events: []
|
|
204054
|
-
};
|
|
204055
|
-
let childrenRewritten = 0;
|
|
204097
|
+
const applied = zeroCounts(reported);
|
|
204098
|
+
const missingChild = new Map(children.map((d) => [d, []]));
|
|
204099
|
+
const rewrittenHeights = new Map;
|
|
204056
204100
|
if (opts.apply) {
|
|
204057
204101
|
const rawClient = getRawClient2("source");
|
|
204058
204102
|
for (const { partition, fixes } of planned) {
|
|
204059
204103
|
const result = await applyFixes(rawClient, reference, partition, fixes, gate);
|
|
204060
|
-
|
|
204061
|
-
|
|
204062
|
-
|
|
204063
|
-
|
|
204064
|
-
|
|
204065
|
-
|
|
204104
|
+
for (const dataset of reported) {
|
|
204105
|
+
applied[dataset] = (applied[dataset] ?? 0) + result.rows[dataset];
|
|
204106
|
+
}
|
|
204107
|
+
for (const dataset of children) {
|
|
204108
|
+
missingChild.get(dataset)?.push(...result.missing[dataset] ?? []);
|
|
204109
|
+
rewrittenHeights.set(dataset, (rewrittenHeights.get(dataset) ?? 0) + (result.rewritten[dataset]?.length ?? 0));
|
|
204110
|
+
}
|
|
204066
204111
|
}
|
|
204067
204112
|
}
|
|
204068
204113
|
const missingHeights = [
|
|
204069
|
-
...new Set([...missingChild.
|
|
204114
|
+
...new Set([...missingChild.values()].flat())
|
|
204070
204115
|
].sort((a, b) => a - b);
|
|
204071
|
-
const rewritten = opts.apply ?
|
|
204116
|
+
const rewritten = opts.apply ? reported.filter((d) => d === "blocks" || (rewrittenHeights.get(d) ?? 0) > 0) : [];
|
|
204072
204117
|
let remainingByDataset = {
|
|
204073
|
-
|
|
204074
|
-
|
|
204075
|
-
events: 0
|
|
204118
|
+
...zeroCounts(reported),
|
|
204119
|
+
blocks: divergent.length
|
|
204076
204120
|
};
|
|
204077
204121
|
if (opts.apply) {
|
|
204078
|
-
remainingByDataset = await reverify(db, reference, referenceDigests);
|
|
204122
|
+
remainingByDataset = await reverify(db, reference, referenceDigests, reported);
|
|
204079
204123
|
}
|
|
204080
|
-
const remaining =
|
|
204124
|
+
const remaining = reported.reduce((sum, d) => sum + (remainingByDataset[d] ?? 0), 0);
|
|
204081
204125
|
const complete = !!opts.apply && remaining === 0 && missingHeights.length === 0;
|
|
204082
204126
|
const report = {
|
|
204083
204127
|
status: opts.apply ? complete ? "repaired" : "incomplete" : "plan",
|
|
@@ -204086,7 +204130,7 @@ Exit codes:
|
|
|
204086
204130
|
blocks_to_replace: allFixes.filter((f) => f.kind === "replace").length,
|
|
204087
204131
|
blocks_to_insert: allFixes.filter((f) => f.kind === "insert").length,
|
|
204088
204132
|
local_only_heights: allExtra,
|
|
204089
|
-
applied: applied.blocks,
|
|
204133
|
+
applied: applied.blocks ?? 0,
|
|
204090
204134
|
rows_written: applied,
|
|
204091
204135
|
datasets_rewritten: rewritten,
|
|
204092
204136
|
heights_missing_child_partitions: missingHeights,
|
|
@@ -204095,14 +204139,15 @@ Exit codes:
|
|
|
204095
204139
|
metered: quoteLine ?? null
|
|
204096
204140
|
};
|
|
204097
204141
|
if (missingHeights.length > 0) {
|
|
204098
|
-
for (const dataset of
|
|
204099
|
-
const heights = missingChild[
|
|
204142
|
+
for (const dataset of children) {
|
|
204143
|
+
const heights = missingChild.get(dataset) ?? [];
|
|
204100
204144
|
if (heights.length === 0)
|
|
204101
204145
|
continue;
|
|
204102
204146
|
warn(` the reference has no ${dataset} partition for ${heights.length} height(s): ${heights.slice(0, 5).join(", ")}${heights.length > 5 ? ", …" : ""}`);
|
|
204103
204147
|
}
|
|
204104
204148
|
for (const height of missingHeights.slice(0, 5)) {
|
|
204105
|
-
|
|
204149
|
+
const lacking = children.filter((d) => missingChild.get(d)?.includes(height)).join("/");
|
|
204150
|
+
console.error(dim(` the archive has no ${lacking} at ${height} either: it cannot be repaired in place; re-ingest ${height} from your own node, or bootstrap a fresh database from the archive`));
|
|
204106
204151
|
}
|
|
204107
204152
|
}
|
|
204108
204153
|
output({
|
|
@@ -204112,14 +204157,14 @@ Exit codes:
|
|
|
204112
204157
|
const replace = report.blocks_to_replace;
|
|
204113
204158
|
const insert = report.blocks_to_insert;
|
|
204114
204159
|
if (opts.apply) {
|
|
204115
|
-
const written = `${applied
|
|
204160
|
+
const written = reported.map((d) => `${applied[d] ?? 0} ${d}`).join(", ");
|
|
204116
204161
|
if (complete) {
|
|
204117
204162
|
success(`Repaired ${applied.blocks} heights across ${divergent.length} ranges (rewrote ${written}); re-verified clean on ${rewritten.join(", ")}.`);
|
|
204118
204163
|
} else if (remaining > 0) {
|
|
204119
|
-
const byDataset =
|
|
204164
|
+
const byDataset = reported.map((d) => `${remainingByDataset[d] ?? 0} ${d}`).join(", ");
|
|
204120
204165
|
warn(`Rewrote ${written} but ${remaining} ranges still diverge (${byDataset}).`);
|
|
204121
204166
|
} else {
|
|
204122
|
-
warn(`Rewrote ${written}; blocks re-verified clean, but
|
|
204167
|
+
warn(`Rewrote ${written}; blocks re-verified clean, but child rows at ${missingHeights.length} height(s) were not rewritten.`);
|
|
204123
204168
|
}
|
|
204124
204169
|
} else {
|
|
204125
204170
|
warn(`${divergent.length} ranges diverge — ${replace} blocks to replace, ${insert} to insert.`);
|
|
@@ -204170,6 +204215,7 @@ function registerRepairCommand(program) {
|
|
|
204170
204215
|
}
|
|
204171
204216
|
|
|
204172
204217
|
// src/commands/verify.ts
|
|
204218
|
+
import { isArchiveDataset as isArchiveDataset2 } from "@secondlayer/shared/archive/datasets";
|
|
204173
204219
|
import {
|
|
204174
204220
|
compareRangeDigests as compareRangeDigests2,
|
|
204175
204221
|
computeRangeDigest as computeRangeDigest3
|
|
@@ -204186,6 +204232,7 @@ import {
|
|
|
204186
204232
|
import { getDb as getDb3 } from "@secondlayer/shared/db";
|
|
204187
204233
|
|
|
204188
204234
|
// src/lib/archive-heights.ts
|
|
204235
|
+
import { assertNeverDataset } from "@secondlayer/shared/archive/datasets";
|
|
204189
204236
|
import {
|
|
204190
204237
|
PerHeightDigestAccumulator
|
|
204191
204238
|
} from "@secondlayer/shared/archive/per-height-digest";
|
|
@@ -204197,6 +204244,18 @@ var PART_FOR_DATASET = {
|
|
|
204197
204244
|
transactions: "transactions",
|
|
204198
204245
|
events: "events"
|
|
204199
204246
|
};
|
|
204247
|
+
function sidecarNarrows(dataset) {
|
|
204248
|
+
switch (dataset) {
|
|
204249
|
+
case "blocks":
|
|
204250
|
+
case "transactions":
|
|
204251
|
+
case "events":
|
|
204252
|
+
return true;
|
|
204253
|
+
case "state_writes":
|
|
204254
|
+
return false;
|
|
204255
|
+
default:
|
|
204256
|
+
return assertNeverDataset(dataset);
|
|
204257
|
+
}
|
|
204258
|
+
}
|
|
204200
204259
|
async function localPerHeightDigests(db, fromBlock, toBlock, datasets = ["blocks", "transactions", "events"]) {
|
|
204201
204260
|
const accumulator = new PerHeightDigestAccumulator;
|
|
204202
204261
|
for await (const { height, digest } of streamHeightDigests(db, "blocks", fromBlock, toBlock)) {
|
|
@@ -204261,15 +204320,21 @@ function compareHeights(params) {
|
|
|
204261
204320
|
return out;
|
|
204262
204321
|
}
|
|
204263
204322
|
function partsMatch(dataset, a, b) {
|
|
204264
|
-
|
|
204265
|
-
|
|
204266
|
-
|
|
204267
|
-
|
|
204323
|
+
switch (dataset) {
|
|
204324
|
+
case "blocks":
|
|
204325
|
+
return a.block_digest === b.block_digest;
|
|
204326
|
+
case "transactions":
|
|
204327
|
+
return a.transactions_rollup === b.transactions_rollup;
|
|
204328
|
+
case "events":
|
|
204329
|
+
return a.events_rollup === b.events_rollup;
|
|
204330
|
+
default:
|
|
204331
|
+
return assertNeverDataset(dataset);
|
|
204268
204332
|
}
|
|
204269
|
-
return a.events_rollup === b.events_rollup;
|
|
204270
204333
|
}
|
|
204271
204334
|
async function divergentHeightsForRange(params) {
|
|
204272
204335
|
const { db, reference, dataset, fromBlock, toBlock } = params;
|
|
204336
|
+
if (!sidecarNarrows(dataset))
|
|
204337
|
+
return null;
|
|
204273
204338
|
const sidecars = sidecarsCovering(params.digestIndex, fromBlock, toBlock);
|
|
204274
204339
|
if (sidecars.length === 0)
|
|
204275
204340
|
return null;
|
|
@@ -204812,6 +204877,8 @@ Exit codes:
|
|
|
204812
204877
|
const countChecks = [];
|
|
204813
204878
|
if (opts.counts && manifest.partitions) {
|
|
204814
204879
|
for (const partition of manifest.partitions) {
|
|
204880
|
+
if (!isArchiveDataset2(partition.dataset))
|
|
204881
|
+
continue;
|
|
204815
204882
|
if (partition.dataset === "blocks")
|
|
204816
204883
|
continue;
|
|
204817
204884
|
if (fromBlock !== undefined && partition.to_block < fromBlock)
|
|
@@ -204833,6 +204900,8 @@ Exit codes:
|
|
|
204833
204900
|
}
|
|
204834
204901
|
const semanticChecks = [];
|
|
204835
204902
|
const referenceSemantic = (manifest.partition_semantic_digests ?? []).filter((d) => {
|
|
204903
|
+
if (!isArchiveDataset2(d.dataset))
|
|
204904
|
+
return false;
|
|
204836
204905
|
if (!datasetMatchesTarget(d.dataset, target))
|
|
204837
204906
|
return false;
|
|
204838
204907
|
if (fromBlock !== undefined && d.to_block < fromBlock)
|
|
@@ -204879,6 +204948,10 @@ Exit codes:
|
|
|
204879
204948
|
heightNotes.push("This snapshot has no per-height digests; exact diverging heights need a newer snapshot.");
|
|
204880
204949
|
} else {
|
|
204881
204950
|
for (const range of diverged) {
|
|
204951
|
+
if (!sidecarNarrows(range.dataset)) {
|
|
204952
|
+
heightNotes.push(`${range.dataset} ${range.from_block}-${range.to_block} is checked as a whole range: per-height digests carry no ${range.dataset} part.`);
|
|
204953
|
+
continue;
|
|
204954
|
+
}
|
|
204882
204955
|
try {
|
|
204883
204956
|
const heights = await divergentHeightsForRange({
|
|
204884
204957
|
db,
|
|
@@ -208854,5 +208927,5 @@ registerCreditsCommand(program);
|
|
|
208854
208927
|
registerArchiveCommand(program);
|
|
208855
208928
|
program.parse();
|
|
208856
208929
|
|
|
208857
|
-
//# debugId=
|
|
208930
|
+
//# debugId=68E0C291ECE1B27C64756E2164756E21
|
|
208858
208931
|
//# sourceMappingURL=cli.js.map
|