@cogcoin/client 0.5.5 → 0.5.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -2
- package/dist/bitcoind/bootstrap/chainstate.d.ts +2 -1
- package/dist/bitcoind/bootstrap/chainstate.js +4 -1
- package/dist/bitcoind/bootstrap/controller.d.ts +4 -1
- package/dist/bitcoind/bootstrap/controller.js +42 -5
- package/dist/bitcoind/bootstrap/getblock-archive.d.ts +39 -0
- package/dist/bitcoind/bootstrap/getblock-archive.js +548 -0
- package/dist/bitcoind/bootstrap/headers.d.ts +12 -0
- package/dist/bitcoind/bootstrap/headers.js +95 -10
- package/dist/bitcoind/bootstrap.d.ts +1 -0
- package/dist/bitcoind/bootstrap.js +1 -0
- package/dist/bitcoind/client/factory.js +91 -28
- package/dist/bitcoind/client/managed-client.d.ts +1 -1
- package/dist/bitcoind/client/managed-client.js +4 -3
- package/dist/bitcoind/client/sync-engine.js +55 -13
- package/dist/bitcoind/errors.js +18 -0
- package/dist/bitcoind/indexer-daemon-main.js +78 -0
- package/dist/bitcoind/indexer-daemon.d.ts +10 -1
- package/dist/bitcoind/indexer-daemon.js +44 -28
- package/dist/bitcoind/node.js +2 -0
- package/dist/bitcoind/processing-start-height.d.ts +7 -0
- package/dist/bitcoind/processing-start-height.js +9 -0
- package/dist/bitcoind/progress/constants.d.ts +1 -0
- package/dist/bitcoind/progress/constants.js +1 -0
- package/dist/bitcoind/progress/controller.d.ts +22 -0
- package/dist/bitcoind/progress/controller.js +49 -23
- package/dist/bitcoind/progress/formatting.js +29 -1
- package/dist/bitcoind/progress/render-policy.d.ts +35 -0
- package/dist/bitcoind/progress/render-policy.js +81 -0
- package/dist/bitcoind/retryable-rpc.d.ts +11 -0
- package/dist/bitcoind/retryable-rpc.js +30 -0
- package/dist/bitcoind/service-paths.js +2 -6
- package/dist/bitcoind/service.d.ts +21 -2
- package/dist/bitcoind/service.js +274 -122
- package/dist/bitcoind/testing.d.ts +2 -2
- package/dist/bitcoind/testing.js +2 -2
- package/dist/bitcoind/types.d.ts +36 -1
- package/dist/cli/commands/follow.js +11 -0
- package/dist/cli/commands/getblock-archive-restart.d.ts +5 -0
- package/dist/cli/commands/getblock-archive-restart.js +15 -0
- package/dist/cli/commands/mining-admin.js +4 -0
- package/dist/cli/commands/mining-read.js +8 -5
- package/dist/cli/commands/mining-runtime.js +4 -0
- package/dist/cli/commands/service-runtime.js +150 -134
- package/dist/cli/commands/status.js +2 -0
- package/dist/cli/commands/sync.js +11 -0
- package/dist/cli/commands/wallet-admin.js +106 -24
- package/dist/cli/commands/wallet-mutation.js +57 -4
- package/dist/cli/commands/wallet-read.js +2 -0
- package/dist/cli/context.js +8 -4
- package/dist/cli/mutation-command-groups.d.ts +2 -1
- package/dist/cli/mutation-command-groups.js +5 -0
- package/dist/cli/mutation-json.d.ts +18 -2
- package/dist/cli/mutation-json.js +49 -0
- package/dist/cli/mutation-success.d.ts +1 -0
- package/dist/cli/mutation-success.js +2 -2
- package/dist/cli/output.js +86 -1
- package/dist/cli/parse.d.ts +1 -1
- package/dist/cli/parse.js +133 -3
- package/dist/cli/preview-json.d.ts +10 -1
- package/dist/cli/preview-json.js +32 -0
- package/dist/cli/prompt.js +1 -1
- package/dist/cli/runner.js +4 -0
- package/dist/cli/types.d.ts +15 -5
- package/dist/cli/types.js +1 -1
- package/dist/cli/wallet-format.js +140 -14
- package/dist/wallet/lifecycle.d.ts +21 -1
- package/dist/wallet/lifecycle.js +252 -116
- package/dist/wallet/mining/visualizer.d.ts +11 -6
- package/dist/wallet/mining/visualizer.js +32 -15
- package/dist/wallet/read/context.js +10 -4
- package/dist/wallet/reset.d.ts +61 -2
- package/dist/wallet/reset.js +246 -89
- package/dist/wallet/root-resolution.d.ts +20 -0
- package/dist/wallet/root-resolution.js +37 -0
- package/dist/wallet/runtime.d.ts +13 -1
- package/dist/wallet/runtime.js +54 -11
- package/dist/wallet/state/crypto.d.ts +3 -0
- package/dist/wallet/state/crypto.js +3 -0
- package/dist/wallet/state/provider.d.ts +1 -0
- package/dist/wallet/state/provider.js +119 -3
- package/dist/wallet/state/seed-index.d.ts +43 -0
- package/dist/wallet/state/seed-index.js +151 -0
- package/dist/wallet/state/storage.d.ts +7 -1
- package/dist/wallet/state/storage.js +39 -0
- package/dist/wallet/tx/anchor.d.ts +22 -0
- package/dist/wallet/tx/anchor.js +215 -8
- package/dist/wallet/tx/index.d.ts +1 -1
- package/dist/wallet/tx/index.js +1 -1
- package/dist/wallet/types.d.ts +1 -0
- package/package.json +1 -1
|
@@ -0,0 +1,548 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { mkdir, open, readFile, rename, rm, stat, writeFile } from "node:fs/promises";
|
|
3
|
+
import { dirname, join } from "node:path";
|
|
4
|
+
import { DOWNLOAD_RETRY_BASE_MS, DOWNLOAD_RETRY_MAX_MS } from "./constants.js";
|
|
5
|
+
const GETBLOCK_ARCHIVE_STATE_VERSION = 1;
|
|
6
|
+
const GETBLOCK_ARCHIVE_BASE_HEIGHT = 910_000;
|
|
7
|
+
const GETBLOCK_ARCHIVE_FILENAME = "getblock-910000-latest.dat";
|
|
8
|
+
const GETBLOCK_ARCHIVE_MANIFEST_FILENAME = "getblock-910000-latest.json";
|
|
9
|
+
const GETBLOCK_ARCHIVE_REMOTE_DATA_URL = "https://snapshots.cogcoin.org/getblock-910000-latest.dat";
|
|
10
|
+
const GETBLOCK_ARCHIVE_REMOTE_MANIFEST_URL = "https://snapshots.cogcoin.org/getblock-910000-latest.json";
|
|
11
|
+
const TRUSTED_FRONTIER_REVERIFY_CHUNKS = 2;
|
|
12
|
+
const HASH_READ_BUFFER_BYTES = 1024 * 1024;
|
|
13
|
+
const IMPORT_POLL_MS = 2_000;
|
|
14
|
+
function createInitialState() {
|
|
15
|
+
return {
|
|
16
|
+
metadataVersion: GETBLOCK_ARCHIVE_STATE_VERSION,
|
|
17
|
+
formatVersion: 0,
|
|
18
|
+
endHeight: null,
|
|
19
|
+
artifactSizeBytes: 0,
|
|
20
|
+
artifactSha256: null,
|
|
21
|
+
chunkSizeBytes: 0,
|
|
22
|
+
verifiedChunkCount: 0,
|
|
23
|
+
downloadedBytes: 0,
|
|
24
|
+
validated: false,
|
|
25
|
+
lastError: null,
|
|
26
|
+
updatedAt: Date.now(),
|
|
27
|
+
};
|
|
28
|
+
}
|
|
29
|
+
function assertManifestShape(parsed) {
|
|
30
|
+
if (typeof parsed !== "object" || parsed === null) {
|
|
31
|
+
throw new Error("managed_getblock_archive_manifest_invalid");
|
|
32
|
+
}
|
|
33
|
+
const manifest = parsed;
|
|
34
|
+
if (manifest.chain !== "main"
|
|
35
|
+
|| manifest.baseSnapshotHeight !== GETBLOCK_ARCHIVE_BASE_HEIGHT
|
|
36
|
+
|| manifest.artifactFilename !== GETBLOCK_ARCHIVE_FILENAME
|
|
37
|
+
|| typeof manifest.firstBlockHeight !== "number"
|
|
38
|
+
|| typeof manifest.endHeight !== "number"
|
|
39
|
+
|| typeof manifest.blockCount !== "number"
|
|
40
|
+
|| typeof manifest.artifactSizeBytes !== "number"
|
|
41
|
+
|| typeof manifest.artifactSha256 !== "string"
|
|
42
|
+
|| typeof manifest.chunkSizeBytes !== "number"
|
|
43
|
+
|| !Array.isArray(manifest.chunkSha256s)
|
|
44
|
+
|| !Array.isArray(manifest.blocks)) {
|
|
45
|
+
throw new Error("managed_getblock_archive_manifest_invalid");
|
|
46
|
+
}
|
|
47
|
+
return manifest;
|
|
48
|
+
}
|
|
49
|
+
function resolvePaths(dataDir) {
|
|
50
|
+
const directory = join(dataDir, "bootstrap", "getblock");
|
|
51
|
+
return {
|
|
52
|
+
directory,
|
|
53
|
+
artifactPath: join(directory, GETBLOCK_ARCHIVE_FILENAME),
|
|
54
|
+
partialArtifactPath: join(directory, `${GETBLOCK_ARCHIVE_FILENAME}.part`),
|
|
55
|
+
manifestPath: join(directory, GETBLOCK_ARCHIVE_MANIFEST_FILENAME),
|
|
56
|
+
partialManifestPath: join(directory, `${GETBLOCK_ARCHIVE_MANIFEST_FILENAME}.part`),
|
|
57
|
+
statePath: join(directory, "state.json"),
|
|
58
|
+
};
|
|
59
|
+
}
|
|
60
|
+
async function statOrNull(path) {
|
|
61
|
+
try {
|
|
62
|
+
return await stat(path);
|
|
63
|
+
}
|
|
64
|
+
catch (error) {
|
|
65
|
+
if (error instanceof Error && "code" in error && error.code === "ENOENT") {
|
|
66
|
+
return null;
|
|
67
|
+
}
|
|
68
|
+
throw error;
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
async function writeJsonAtomic(path, payload) {
|
|
72
|
+
await mkdir(dirname(path), { recursive: true });
|
|
73
|
+
const tempPath = `${path}.tmp`;
|
|
74
|
+
await writeFile(tempPath, JSON.stringify(payload, null, 2));
|
|
75
|
+
await rename(tempPath, path);
|
|
76
|
+
}
|
|
77
|
+
async function readManifest(path) {
|
|
78
|
+
try {
|
|
79
|
+
return assertManifestShape(JSON.parse(await readFile(path, "utf8")));
|
|
80
|
+
}
|
|
81
|
+
catch (error) {
|
|
82
|
+
if (error instanceof Error && "code" in error && error.code === "ENOENT") {
|
|
83
|
+
return null;
|
|
84
|
+
}
|
|
85
|
+
throw error;
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
async function loadState(paths) {
|
|
89
|
+
try {
|
|
90
|
+
const parsed = JSON.parse(await readFile(paths.statePath, "utf8"));
|
|
91
|
+
if (typeof parsed.downloadedBytes !== "number" || typeof parsed.verifiedChunkCount !== "number") {
|
|
92
|
+
throw new Error("managed_getblock_archive_state_invalid");
|
|
93
|
+
}
|
|
94
|
+
return {
|
|
95
|
+
metadataVersion: GETBLOCK_ARCHIVE_STATE_VERSION,
|
|
96
|
+
formatVersion: typeof parsed.formatVersion === "number" ? parsed.formatVersion : 0,
|
|
97
|
+
endHeight: typeof parsed.endHeight === "number" ? parsed.endHeight : null,
|
|
98
|
+
artifactSizeBytes: typeof parsed.artifactSizeBytes === "number" ? parsed.artifactSizeBytes : 0,
|
|
99
|
+
artifactSha256: typeof parsed.artifactSha256 === "string" ? parsed.artifactSha256 : null,
|
|
100
|
+
chunkSizeBytes: typeof parsed.chunkSizeBytes === "number" ? parsed.chunkSizeBytes : 0,
|
|
101
|
+
verifiedChunkCount: parsed.verifiedChunkCount,
|
|
102
|
+
downloadedBytes: parsed.downloadedBytes,
|
|
103
|
+
validated: parsed.validated === true,
|
|
104
|
+
lastError: typeof parsed.lastError === "string" ? parsed.lastError : null,
|
|
105
|
+
updatedAt: typeof parsed.updatedAt === "number" ? parsed.updatedAt : Date.now(),
|
|
106
|
+
};
|
|
107
|
+
}
|
|
108
|
+
catch (error) {
|
|
109
|
+
if (error instanceof Error && "code" in error && error.code !== "ENOENT") {
|
|
110
|
+
// Ignore corrupt state and start cleanly.
|
|
111
|
+
}
|
|
112
|
+
const state = createInitialState();
|
|
113
|
+
await saveState(paths, state);
|
|
114
|
+
return state;
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
async function saveState(paths, state) {
|
|
118
|
+
state.updatedAt = Date.now();
|
|
119
|
+
await mkdir(paths.directory, { recursive: true });
|
|
120
|
+
await writeJsonAtomic(paths.statePath, state);
|
|
121
|
+
}
|
|
122
|
+
function resolveChunkSize(manifest, chunkIndex) {
|
|
123
|
+
const lastChunkIndex = manifest.chunkSha256s.length - 1;
|
|
124
|
+
if (chunkIndex < lastChunkIndex) {
|
|
125
|
+
return manifest.chunkSizeBytes;
|
|
126
|
+
}
|
|
127
|
+
const trailingBytes = manifest.artifactSizeBytes % manifest.chunkSizeBytes;
|
|
128
|
+
return trailingBytes === 0 ? manifest.chunkSizeBytes : trailingBytes;
|
|
129
|
+
}
|
|
130
|
+
function resolveVerifiedBytes(manifest, verifiedChunkCount) {
|
|
131
|
+
if (verifiedChunkCount <= 0) {
|
|
132
|
+
return 0;
|
|
133
|
+
}
|
|
134
|
+
if (verifiedChunkCount >= manifest.chunkSha256s.length) {
|
|
135
|
+
return manifest.artifactSizeBytes;
|
|
136
|
+
}
|
|
137
|
+
return verifiedChunkCount * manifest.chunkSizeBytes;
|
|
138
|
+
}
|
|
139
|
+
function resolveVerifiedChunkCountFromBytes(manifest, bytes) {
|
|
140
|
+
let verifiedChunkCount = 0;
|
|
141
|
+
while (verifiedChunkCount < manifest.chunkSha256s.length
|
|
142
|
+
&& resolveVerifiedBytes(manifest, verifiedChunkCount + 1) <= bytes) {
|
|
143
|
+
verifiedChunkCount += 1;
|
|
144
|
+
}
|
|
145
|
+
return verifiedChunkCount;
|
|
146
|
+
}
|
|
147
|
+
function stateMatchesManifest(state, manifest) {
|
|
148
|
+
return state.endHeight === manifest.endHeight
|
|
149
|
+
&& state.artifactSha256 === manifest.artifactSha256
|
|
150
|
+
&& state.artifactSizeBytes === manifest.artifactSizeBytes
|
|
151
|
+
&& state.chunkSizeBytes === manifest.chunkSizeBytes
|
|
152
|
+
&& state.formatVersion === manifest.formatVersion;
|
|
153
|
+
}
|
|
154
|
+
async function hashChunkRange(path, manifest, chunkIndex) {
|
|
155
|
+
const file = await open(path, "r");
|
|
156
|
+
const chunkSizeBytes = resolveChunkSize(manifest, chunkIndex);
|
|
157
|
+
const buffer = Buffer.allocUnsafe(Math.min(HASH_READ_BUFFER_BYTES, chunkSizeBytes));
|
|
158
|
+
const hash = createHash("sha256");
|
|
159
|
+
let remainingBytes = chunkSizeBytes;
|
|
160
|
+
let position = resolveVerifiedBytes(manifest, chunkIndex);
|
|
161
|
+
try {
|
|
162
|
+
while (remainingBytes > 0) {
|
|
163
|
+
const readLength = Math.min(buffer.length, remainingBytes);
|
|
164
|
+
const { bytesRead } = await file.read(buffer, 0, readLength, position);
|
|
165
|
+
if (bytesRead === 0) {
|
|
166
|
+
return null;
|
|
167
|
+
}
|
|
168
|
+
hash.update(buffer.subarray(0, bytesRead));
|
|
169
|
+
remainingBytes -= bytesRead;
|
|
170
|
+
position += bytesRead;
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
finally {
|
|
174
|
+
await file.close();
|
|
175
|
+
}
|
|
176
|
+
return hash.digest("hex");
|
|
177
|
+
}
|
|
178
|
+
async function scanVerifiedPrefix(path, manifest, fileSize) {
|
|
179
|
+
for (let chunkIndex = 0; chunkIndex < manifest.chunkSha256s.length; chunkIndex += 1) {
|
|
180
|
+
const chunkEnd = resolveVerifiedBytes(manifest, chunkIndex + 1);
|
|
181
|
+
if (fileSize < chunkEnd) {
|
|
182
|
+
return chunkIndex;
|
|
183
|
+
}
|
|
184
|
+
const actualSha256 = await hashChunkRange(path, manifest, chunkIndex);
|
|
185
|
+
if (actualSha256 !== manifest.chunkSha256s[chunkIndex]) {
|
|
186
|
+
return chunkIndex;
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
return manifest.chunkSha256s.length;
|
|
190
|
+
}
|
|
191
|
+
async function reverifyTrustedFrontier(path, manifest, verifiedChunkCount, fileSize) {
|
|
192
|
+
const maxCompleteChunkCount = resolveVerifiedChunkCountFromBytes(manifest, Math.min(fileSize, manifest.artifactSizeBytes));
|
|
193
|
+
const tentative = Math.min(Math.max(0, verifiedChunkCount), maxCompleteChunkCount);
|
|
194
|
+
const startChunk = Math.max(0, tentative - TRUSTED_FRONTIER_REVERIFY_CHUNKS);
|
|
195
|
+
for (let chunkIndex = startChunk; chunkIndex < tentative; chunkIndex += 1) {
|
|
196
|
+
const actualSha256 = await hashChunkRange(path, manifest, chunkIndex);
|
|
197
|
+
if (actualSha256 !== manifest.chunkSha256s[chunkIndex]) {
|
|
198
|
+
return chunkIndex;
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
return tentative;
|
|
202
|
+
}
|
|
203
|
+
async function truncateFile(path, size) {
|
|
204
|
+
const file = await open(path, "a+");
|
|
205
|
+
try {
|
|
206
|
+
await file.truncate(size);
|
|
207
|
+
await file.sync();
|
|
208
|
+
}
|
|
209
|
+
finally {
|
|
210
|
+
await file.close();
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
async function reconcilePartialDownloadArtifacts(paths, manifest, state) {
|
|
214
|
+
const partialManifest = await readManifest(paths.partialManifestPath).catch(() => null);
|
|
215
|
+
if (partialManifest === null || partialManifest.endHeight !== manifest.endHeight || partialManifest.artifactSha256 !== manifest.artifactSha256) {
|
|
216
|
+
await rm(paths.partialArtifactPath, { force: true }).catch(() => undefined);
|
|
217
|
+
await rm(paths.partialManifestPath, { force: true }).catch(() => undefined);
|
|
218
|
+
state.formatVersion = manifest.formatVersion;
|
|
219
|
+
state.endHeight = manifest.endHeight;
|
|
220
|
+
state.artifactSizeBytes = manifest.artifactSizeBytes;
|
|
221
|
+
state.artifactSha256 = manifest.artifactSha256;
|
|
222
|
+
state.chunkSizeBytes = manifest.chunkSizeBytes;
|
|
223
|
+
state.verifiedChunkCount = 0;
|
|
224
|
+
state.downloadedBytes = 0;
|
|
225
|
+
state.validated = false;
|
|
226
|
+
state.lastError = null;
|
|
227
|
+
await writeJsonAtomic(paths.partialManifestPath, manifest);
|
|
228
|
+
await saveState(paths, state);
|
|
229
|
+
return;
|
|
230
|
+
}
|
|
231
|
+
const partialInfo = await statOrNull(paths.partialArtifactPath);
|
|
232
|
+
if (partialInfo === null) {
|
|
233
|
+
state.verifiedChunkCount = 0;
|
|
234
|
+
state.downloadedBytes = 0;
|
|
235
|
+
state.validated = false;
|
|
236
|
+
state.lastError = null;
|
|
237
|
+
await saveState(paths, state);
|
|
238
|
+
return;
|
|
239
|
+
}
|
|
240
|
+
const partialSize = Number(partialInfo.size);
|
|
241
|
+
const verifiedChunkCount = stateMatchesManifest(state, manifest)
|
|
242
|
+
? await reverifyTrustedFrontier(paths.partialArtifactPath, manifest, state.verifiedChunkCount, partialSize)
|
|
243
|
+
: await scanVerifiedPrefix(paths.partialArtifactPath, manifest, partialSize);
|
|
244
|
+
const verifiedBytes = resolveVerifiedBytes(manifest, verifiedChunkCount);
|
|
245
|
+
if (partialSize !== verifiedBytes) {
|
|
246
|
+
await truncateFile(paths.partialArtifactPath, verifiedBytes);
|
|
247
|
+
}
|
|
248
|
+
state.formatVersion = manifest.formatVersion;
|
|
249
|
+
state.endHeight = manifest.endHeight;
|
|
250
|
+
state.artifactSizeBytes = manifest.artifactSizeBytes;
|
|
251
|
+
state.artifactSha256 = manifest.artifactSha256;
|
|
252
|
+
state.chunkSizeBytes = manifest.chunkSizeBytes;
|
|
253
|
+
state.verifiedChunkCount = verifiedChunkCount;
|
|
254
|
+
state.downloadedBytes = verifiedBytes;
|
|
255
|
+
state.validated = false;
|
|
256
|
+
state.lastError = null;
|
|
257
|
+
await saveState(paths, state);
|
|
258
|
+
}
|
|
259
|
+
async function validateWholeFile(path, manifest) {
|
|
260
|
+
const file = await open(path, "r");
|
|
261
|
+
const hash = createHash("sha256");
|
|
262
|
+
const buffer = Buffer.allocUnsafe(HASH_READ_BUFFER_BYTES);
|
|
263
|
+
let position = 0;
|
|
264
|
+
try {
|
|
265
|
+
while (position < manifest.artifactSizeBytes) {
|
|
266
|
+
const length = Math.min(buffer.length, manifest.artifactSizeBytes - position);
|
|
267
|
+
const { bytesRead } = await file.read(buffer, 0, length, position);
|
|
268
|
+
if (bytesRead === 0) {
|
|
269
|
+
throw new Error("managed_getblock_archive_truncated");
|
|
270
|
+
}
|
|
271
|
+
hash.update(buffer.subarray(0, bytesRead));
|
|
272
|
+
position += bytesRead;
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
finally {
|
|
276
|
+
await file.close();
|
|
277
|
+
}
|
|
278
|
+
if (hash.digest("hex") !== manifest.artifactSha256) {
|
|
279
|
+
throw new Error("managed_getblock_archive_sha256_mismatch");
|
|
280
|
+
}
|
|
281
|
+
}
|
|
282
|
+
async function sleep(ms, signal) {
|
|
283
|
+
if (ms <= 0) {
|
|
284
|
+
return;
|
|
285
|
+
}
|
|
286
|
+
if (signal?.aborted) {
|
|
287
|
+
throw new Error("managed_getblock_archive_aborted");
|
|
288
|
+
}
|
|
289
|
+
await new Promise((resolve, reject) => {
|
|
290
|
+
const onAbort = () => {
|
|
291
|
+
clearTimeout(timer);
|
|
292
|
+
signal?.removeEventListener("abort", onAbort);
|
|
293
|
+
reject(new Error("managed_getblock_archive_aborted"));
|
|
294
|
+
};
|
|
295
|
+
const timer = setTimeout(() => {
|
|
296
|
+
signal?.removeEventListener("abort", onAbort);
|
|
297
|
+
resolve();
|
|
298
|
+
}, ms);
|
|
299
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
300
|
+
});
|
|
301
|
+
}
|
|
302
|
+
async function updateDownloadProgress(progress, manifest, state, startedAtUnixMs, attemptStartBytes, downloadedBytes) {
|
|
303
|
+
const elapsedSeconds = Math.max(0.001, (Date.now() - startedAtUnixMs) / 1000);
|
|
304
|
+
const bytesPerSecond = Math.max(0, (downloadedBytes - attemptStartBytes) / elapsedSeconds);
|
|
305
|
+
const remaining = Math.max(0, manifest.artifactSizeBytes - downloadedBytes);
|
|
306
|
+
await progress.setPhase("getblock_archive_download", {
|
|
307
|
+
message: "Downloading getblock archive.",
|
|
308
|
+
resumed: downloadedBytes > 0,
|
|
309
|
+
downloadedBytes,
|
|
310
|
+
totalBytes: manifest.artifactSizeBytes,
|
|
311
|
+
percent: manifest.artifactSizeBytes > 0 ? (downloadedBytes / manifest.artifactSizeBytes) * 100 : 0,
|
|
312
|
+
bytesPerSecond,
|
|
313
|
+
etaSeconds: bytesPerSecond > 0 ? remaining / bytesPerSecond : null,
|
|
314
|
+
targetHeight: manifest.endHeight,
|
|
315
|
+
lastError: state.lastError,
|
|
316
|
+
});
|
|
317
|
+
}
|
|
318
|
+
async function downloadRemoteArchive(paths, manifest, state, progress, fetchImpl, signal) {
|
|
319
|
+
await reconcilePartialDownloadArtifacts(paths, manifest, state);
|
|
320
|
+
if (state.downloadedBytes >= manifest.artifactSizeBytes) {
|
|
321
|
+
await validateWholeFile(paths.partialArtifactPath, manifest);
|
|
322
|
+
await rename(paths.partialArtifactPath, paths.artifactPath);
|
|
323
|
+
await rename(paths.partialManifestPath, paths.manifestPath);
|
|
324
|
+
state.validated = true;
|
|
325
|
+
state.verifiedChunkCount = manifest.chunkSha256s.length;
|
|
326
|
+
state.downloadedBytes = manifest.artifactSizeBytes;
|
|
327
|
+
state.lastError = null;
|
|
328
|
+
await saveState(paths, state);
|
|
329
|
+
return;
|
|
330
|
+
}
|
|
331
|
+
let retryDelayMs = DOWNLOAD_RETRY_BASE_MS;
|
|
332
|
+
while (true) {
|
|
333
|
+
const startOffset = state.downloadedBytes;
|
|
334
|
+
try {
|
|
335
|
+
const response = await fetchImpl(`${GETBLOCK_ARCHIVE_REMOTE_DATA_URL}?end=${manifest.endHeight}`, {
|
|
336
|
+
headers: startOffset > 0 ? { Range: `bytes=${startOffset}-` } : undefined,
|
|
337
|
+
signal,
|
|
338
|
+
});
|
|
339
|
+
if (startOffset > 0 && response.status !== 206) {
|
|
340
|
+
throw new Error(response.status === 200
|
|
341
|
+
? "managed_getblock_archive_resume_requires_partial_content"
|
|
342
|
+
: `managed_getblock_archive_http_${response.status}`);
|
|
343
|
+
}
|
|
344
|
+
if (startOffset === 0 && response.status !== 200) {
|
|
345
|
+
throw new Error(`managed_getblock_archive_http_${response.status}`);
|
|
346
|
+
}
|
|
347
|
+
if (response.body === null) {
|
|
348
|
+
throw new Error("managed_getblock_archive_response_body_missing");
|
|
349
|
+
}
|
|
350
|
+
const file = await open(paths.partialArtifactPath, startOffset > 0 ? "a" : "w");
|
|
351
|
+
const reader = response.body.getReader();
|
|
352
|
+
const startedAtUnixMs = Date.now();
|
|
353
|
+
let downloadedBytes = startOffset;
|
|
354
|
+
let currentChunkIndex = state.verifiedChunkCount;
|
|
355
|
+
let currentChunkBytes = 0;
|
|
356
|
+
let currentChunkHash = createHash("sha256");
|
|
357
|
+
try {
|
|
358
|
+
await updateDownloadProgress(progress, manifest, state, startedAtUnixMs, startOffset, downloadedBytes);
|
|
359
|
+
while (true) {
|
|
360
|
+
const { done, value } = await reader.read();
|
|
361
|
+
if (done) {
|
|
362
|
+
break;
|
|
363
|
+
}
|
|
364
|
+
if (value === undefined || value.byteLength === 0) {
|
|
365
|
+
continue;
|
|
366
|
+
}
|
|
367
|
+
let offset = 0;
|
|
368
|
+
while (offset < value.byteLength) {
|
|
369
|
+
const chunkSizeBytes = resolveChunkSize(manifest, currentChunkIndex);
|
|
370
|
+
const remainingChunkBytes = chunkSizeBytes - currentChunkBytes;
|
|
371
|
+
const writeLength = Math.min(remainingChunkBytes, value.byteLength - offset);
|
|
372
|
+
const segment = value.subarray(offset, offset + writeLength);
|
|
373
|
+
await file.write(segment);
|
|
374
|
+
currentChunkHash.update(segment);
|
|
375
|
+
currentChunkBytes += segment.byteLength;
|
|
376
|
+
downloadedBytes += segment.byteLength;
|
|
377
|
+
offset += writeLength;
|
|
378
|
+
if (currentChunkBytes === chunkSizeBytes) {
|
|
379
|
+
const actualSha256 = currentChunkHash.digest("hex");
|
|
380
|
+
if (actualSha256 !== manifest.chunkSha256s[currentChunkIndex]) {
|
|
381
|
+
const verifiedBytes = resolveVerifiedBytes(manifest, currentChunkIndex);
|
|
382
|
+
await file.truncate(verifiedBytes);
|
|
383
|
+
await file.sync();
|
|
384
|
+
throw new Error(`managed_getblock_archive_chunk_sha256_mismatch_${currentChunkIndex}`);
|
|
385
|
+
}
|
|
386
|
+
currentChunkIndex += 1;
|
|
387
|
+
currentChunkBytes = 0;
|
|
388
|
+
currentChunkHash = createHash("sha256");
|
|
389
|
+
state.verifiedChunkCount = currentChunkIndex;
|
|
390
|
+
state.downloadedBytes = resolveVerifiedBytes(manifest, currentChunkIndex);
|
|
391
|
+
state.lastError = null;
|
|
392
|
+
await file.sync();
|
|
393
|
+
await saveState(paths, state);
|
|
394
|
+
}
|
|
395
|
+
}
|
|
396
|
+
await updateDownloadProgress(progress, manifest, state, startedAtUnixMs, startOffset, downloadedBytes);
|
|
397
|
+
}
|
|
398
|
+
}
|
|
399
|
+
finally {
|
|
400
|
+
await file.close();
|
|
401
|
+
}
|
|
402
|
+
await validateWholeFile(paths.partialArtifactPath, manifest);
|
|
403
|
+
await rename(paths.partialArtifactPath, paths.artifactPath);
|
|
404
|
+
await rename(paths.partialManifestPath, paths.manifestPath);
|
|
405
|
+
state.validated = true;
|
|
406
|
+
state.verifiedChunkCount = manifest.chunkSha256s.length;
|
|
407
|
+
state.downloadedBytes = manifest.artifactSizeBytes;
|
|
408
|
+
state.lastError = null;
|
|
409
|
+
await saveState(paths, state);
|
|
410
|
+
await progress.setPhase("getblock_archive_download", {
|
|
411
|
+
message: "Downloading getblock archive.",
|
|
412
|
+
resumed: startOffset > 0,
|
|
413
|
+
downloadedBytes: manifest.artifactSizeBytes,
|
|
414
|
+
totalBytes: manifest.artifactSizeBytes,
|
|
415
|
+
percent: 100,
|
|
416
|
+
bytesPerSecond: null,
|
|
417
|
+
etaSeconds: 0,
|
|
418
|
+
targetHeight: manifest.endHeight,
|
|
419
|
+
lastError: null,
|
|
420
|
+
});
|
|
421
|
+
return;
|
|
422
|
+
}
|
|
423
|
+
catch (error) {
|
|
424
|
+
state.lastError = error instanceof Error ? error.message : String(error);
|
|
425
|
+
await saveState(paths, state);
|
|
426
|
+
if (signal?.aborted) {
|
|
427
|
+
throw error;
|
|
428
|
+
}
|
|
429
|
+
await progress.setPhase("getblock_archive_download", {
|
|
430
|
+
message: "Downloading getblock archive.",
|
|
431
|
+
resumed: startOffset > 0,
|
|
432
|
+
downloadedBytes: state.downloadedBytes,
|
|
433
|
+
totalBytes: manifest.artifactSizeBytes,
|
|
434
|
+
percent: manifest.artifactSizeBytes > 0 ? (state.downloadedBytes / manifest.artifactSizeBytes) * 100 : 0,
|
|
435
|
+
bytesPerSecond: null,
|
|
436
|
+
etaSeconds: null,
|
|
437
|
+
targetHeight: manifest.endHeight,
|
|
438
|
+
lastError: state.lastError,
|
|
439
|
+
});
|
|
440
|
+
await sleep(retryDelayMs, signal);
|
|
441
|
+
retryDelayMs = Math.min(retryDelayMs * 2, DOWNLOAD_RETRY_MAX_MS);
|
|
442
|
+
}
|
|
443
|
+
}
|
|
444
|
+
}
|
|
445
|
+
async function fetchLatestManifest(fetchImpl, cacheBustEnd, signal) {
|
|
446
|
+
const url = cacheBustEnd === null
|
|
447
|
+
? GETBLOCK_ARCHIVE_REMOTE_MANIFEST_URL
|
|
448
|
+
: `${GETBLOCK_ARCHIVE_REMOTE_MANIFEST_URL}?end=${cacheBustEnd}`;
|
|
449
|
+
const response = await fetchImpl(url, { signal });
|
|
450
|
+
if (!response.ok) {
|
|
451
|
+
throw new Error(`managed_getblock_archive_manifest_http_${response.status}`);
|
|
452
|
+
}
|
|
453
|
+
return assertManifestShape(await response.json());
|
|
454
|
+
}
|
|
455
|
+
export function resolveGetblockArchivePathsForTesting(dataDir) {
|
|
456
|
+
return resolvePaths(dataDir);
|
|
457
|
+
}
|
|
458
|
+
export async function resolveReadyGetblockArchiveForTesting(dataDir) {
|
|
459
|
+
return resolveReadyLocalGetblockArchive(resolvePaths(dataDir));
|
|
460
|
+
}
|
|
461
|
+
async function resolveReadyLocalGetblockArchive(paths, state = null) {
|
|
462
|
+
const manifest = await readManifest(paths.manifestPath).catch(() => null);
|
|
463
|
+
if (manifest === null) {
|
|
464
|
+
return null;
|
|
465
|
+
}
|
|
466
|
+
const info = await statOrNull(paths.artifactPath);
|
|
467
|
+
if (info === null || info.size !== manifest.artifactSizeBytes) {
|
|
468
|
+
return null;
|
|
469
|
+
}
|
|
470
|
+
const loadedState = state ?? await loadState(paths);
|
|
471
|
+
if (!(loadedState.validated && stateMatchesManifest(loadedState, manifest))) {
|
|
472
|
+
try {
|
|
473
|
+
await validateWholeFile(paths.artifactPath, manifest);
|
|
474
|
+
loadedState.formatVersion = manifest.formatVersion;
|
|
475
|
+
loadedState.endHeight = manifest.endHeight;
|
|
476
|
+
loadedState.artifactSizeBytes = manifest.artifactSizeBytes;
|
|
477
|
+
loadedState.artifactSha256 = manifest.artifactSha256;
|
|
478
|
+
loadedState.chunkSizeBytes = manifest.chunkSizeBytes;
|
|
479
|
+
loadedState.verifiedChunkCount = manifest.chunkSha256s.length;
|
|
480
|
+
loadedState.downloadedBytes = manifest.artifactSizeBytes;
|
|
481
|
+
loadedState.validated = true;
|
|
482
|
+
loadedState.lastError = null;
|
|
483
|
+
await saveState(paths, loadedState);
|
|
484
|
+
}
|
|
485
|
+
catch {
|
|
486
|
+
return null;
|
|
487
|
+
}
|
|
488
|
+
}
|
|
489
|
+
return {
|
|
490
|
+
manifest,
|
|
491
|
+
artifactPath: paths.artifactPath,
|
|
492
|
+
manifestPath: paths.manifestPath,
|
|
493
|
+
};
|
|
494
|
+
}
|
|
495
|
+
export async function prepareLatestGetblockArchive(options) {
|
|
496
|
+
const paths = resolvePaths(options.dataDir);
|
|
497
|
+
await mkdir(paths.directory, { recursive: true });
|
|
498
|
+
const state = await loadState(paths);
|
|
499
|
+
const readyLocal = await resolveReadyLocalGetblockArchive(paths, state);
|
|
500
|
+
const fetchImpl = options.fetchImpl ?? fetch;
|
|
501
|
+
let remoteManifest;
|
|
502
|
+
try {
|
|
503
|
+
remoteManifest = await fetchLatestManifest(fetchImpl, readyLocal?.manifest.endHeight ?? null, options.signal);
|
|
504
|
+
}
|
|
505
|
+
catch {
|
|
506
|
+
return readyLocal;
|
|
507
|
+
}
|
|
508
|
+
if (readyLocal !== null
|
|
509
|
+
&& readyLocal.manifest.endHeight === remoteManifest.endHeight
|
|
510
|
+
&& readyLocal.manifest.artifactSha256 === remoteManifest.artifactSha256) {
|
|
511
|
+
return readyLocal;
|
|
512
|
+
}
|
|
513
|
+
if (readyLocal !== null && readyLocal.manifest.endHeight > remoteManifest.endHeight) {
|
|
514
|
+
return readyLocal;
|
|
515
|
+
}
|
|
516
|
+
await writeJsonAtomic(paths.partialManifestPath, remoteManifest);
|
|
517
|
+
try {
|
|
518
|
+
await downloadRemoteArchive(paths, remoteManifest, state, options.progress, fetchImpl, options.signal);
|
|
519
|
+
}
|
|
520
|
+
catch {
|
|
521
|
+
return readyLocal;
|
|
522
|
+
}
|
|
523
|
+
return resolveReadyLocalGetblockArchive(paths, state);
|
|
524
|
+
}
|
|
525
|
+
export const prepareLatestGetblockArchiveForTesting = prepareLatestGetblockArchive;
|
|
526
|
+
export async function waitForGetblockArchiveImportForTesting(rpc, progress, targetEndHeight, signal) {
|
|
527
|
+
await waitForGetblockArchiveImport(rpc, progress, targetEndHeight, signal);
|
|
528
|
+
}
|
|
529
|
+
export async function waitForGetblockArchiveImport(rpc, progress, targetEndHeight, signal) {
|
|
530
|
+
while (true) {
|
|
531
|
+
if (signal?.aborted) {
|
|
532
|
+
throw new Error("managed_getblock_archive_aborted");
|
|
533
|
+
}
|
|
534
|
+
const info = await rpc.getBlockchainInfo();
|
|
535
|
+
await progress.setPhase("getblock_archive_import", {
|
|
536
|
+
message: "Bitcoin Core is importing getblock archive blocks.",
|
|
537
|
+
blocks: info.blocks,
|
|
538
|
+
headers: info.headers,
|
|
539
|
+
targetHeight: targetEndHeight,
|
|
540
|
+
etaSeconds: null,
|
|
541
|
+
lastError: null,
|
|
542
|
+
});
|
|
543
|
+
if (info.blocks >= targetEndHeight) {
|
|
544
|
+
return;
|
|
545
|
+
}
|
|
546
|
+
await sleep(IMPORT_POLL_MS, signal);
|
|
547
|
+
}
|
|
548
|
+
}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { type ManagedRpcRetryState } from "../retryable-rpc.js";
|
|
1
2
|
import type { BitcoinRpcClient } from "../rpc.js";
|
|
2
3
|
import type { ManagedProgressController } from "../progress.js";
|
|
3
4
|
import type { SnapshotMetadata } from "../types.js";
|
|
@@ -6,10 +7,21 @@ export declare function waitForHeaders(rpc: Pick<BitcoinRpcClient, "getBlockchai
|
|
|
6
7
|
sleep?: (ms: number, signal?: AbortSignal) => Promise<void>;
|
|
7
8
|
noPeerTimeoutMs?: number;
|
|
8
9
|
signal?: AbortSignal;
|
|
10
|
+
retryState?: ManagedRpcRetryState;
|
|
11
|
+
debugLogPath?: string;
|
|
12
|
+
readDebugLogProgress?: (debugLogPath: string) => Promise<{
|
|
13
|
+
height: number;
|
|
14
|
+
message: string;
|
|
15
|
+
} | null>;
|
|
9
16
|
}): Promise<void>;
|
|
10
17
|
export declare function waitForHeadersForTesting(rpc: Pick<BitcoinRpcClient, "getBlockchainInfo" | "getNetworkInfo">, snapshot: SnapshotMetadata | undefined, progress: Pick<ManagedProgressController, "setPhase">, options?: {
|
|
11
18
|
now?: () => number;
|
|
12
19
|
sleep?: (ms: number, signal?: AbortSignal) => Promise<void>;
|
|
13
20
|
noPeerTimeoutMs?: number;
|
|
14
21
|
signal?: AbortSignal;
|
|
22
|
+
debugLogPath?: string;
|
|
23
|
+
readDebugLogProgress?: (debugLogPath: string) => Promise<{
|
|
24
|
+
height: number;
|
|
25
|
+
message: string;
|
|
26
|
+
} | null>;
|
|
15
27
|
}): Promise<void>;
|