@aztec/blob-client 0.0.1-commit.03f7ef2 → 0.0.1-commit.04d373f
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -1
- package/dest/archive/blobscan_archive_client.d.ts +8 -92
- package/dest/archive/blobscan_archive_client.d.ts.map +1 -1
- package/dest/archive/config.js +1 -1
- package/dest/archive/instrumentation.d.ts +1 -1
- package/dest/archive/instrumentation.d.ts.map +1 -1
- package/dest/archive/instrumentation.js +13 -13
- package/dest/blobstore/blob_store_test_suite.js +9 -9
- package/dest/client/config.d.ts +11 -1
- package/dest/client/config.d.ts.map +1 -1
- package/dest/client/config.js +22 -2
- package/dest/client/factory.d.ts +5 -6
- package/dest/client/factory.d.ts.map +1 -1
- package/dest/client/factory.js +13 -4
- package/dest/client/http.d.ts +34 -10
- package/dest/client/http.d.ts.map +1 -1
- package/dest/client/http.js +304 -139
- package/dest/client/interface.d.ts +19 -4
- package/dest/client/interface.d.ts.map +1 -1
- package/dest/client/local.d.ts +3 -1
- package/dest/client/local.d.ts.map +1 -1
- package/dest/client/local.js +3 -0
- package/dest/client/tests.js +3 -3
- package/dest/filestore/factory.d.ts +5 -4
- package/dest/filestore/factory.d.ts.map +1 -1
- package/dest/filestore/factory.js +4 -4
- package/dest/filestore/filestore_blob_client.d.ts +14 -2
- package/dest/filestore/filestore_blob_client.d.ts.map +1 -1
- package/dest/filestore/filestore_blob_client.js +29 -5
- package/dest/filestore/healthcheck.d.ts +5 -0
- package/dest/filestore/healthcheck.d.ts.map +1 -0
- package/dest/filestore/healthcheck.js +3 -0
- package/package.json +9 -9
- package/src/archive/config.ts +1 -1
- package/src/archive/instrumentation.ts +22 -13
- package/src/blobstore/blob_store_test_suite.ts +9 -9
- package/src/client/config.ts +36 -1
- package/src/client/factory.ts +17 -5
- package/src/client/http.ts +357 -134
- package/src/client/interface.ts +18 -3
- package/src/client/local.ts +5 -0
- package/src/client/tests.ts +2 -2
- package/src/filestore/factory.ts +7 -2
- package/src/filestore/filestore_blob_client.ts +33 -5
- package/src/filestore/healthcheck.ts +5 -0
package/dest/client/http.js
CHANGED
|
@@ -1,10 +1,12 @@
|
|
|
1
1
|
import { Blob, computeEthVersionedBlobHash } from '@aztec/blob-lib';
|
|
2
|
+
import { makeL1HttpTransport } from '@aztec/ethereum/client';
|
|
2
3
|
import { shuffle } from '@aztec/foundation/array';
|
|
3
4
|
import { createLogger } from '@aztec/foundation/log';
|
|
4
5
|
import { makeBackoff, retry } from '@aztec/foundation/retry';
|
|
5
6
|
import { bufferToHex, hexToBuffer } from '@aztec/foundation/string';
|
|
6
|
-
import { createPublicClient
|
|
7
|
+
import { createPublicClient } from 'viem';
|
|
7
8
|
import { createBlobArchiveClient } from '../archive/factory.js';
|
|
9
|
+
import { DEFAULT_HEALTHCHECK_UPLOAD_INTERVAL_MINUTES } from '../filestore/healthcheck.js';
|
|
8
10
|
import { getBlobClientConfigFromEnv } from './config.js';
|
|
9
11
|
export class HttpBlobClient {
|
|
10
12
|
opts;
|
|
@@ -15,6 +17,10 @@ export class HttpBlobClient {
|
|
|
15
17
|
fileStoreClients;
|
|
16
18
|
fileStoreUploadClient;
|
|
17
19
|
disabled;
|
|
20
|
+
healthcheckUploadIntervalId;
|
|
21
|
+
/** Cached beacon genesis time (seconds since Unix epoch). Fetched once at startup. */ beaconGenesisTime;
|
|
22
|
+
/** Cached beacon slot duration in seconds. Fetched once at startup. */ beaconSecondsPerSlot;
|
|
23
|
+
/** Indexes of consensus hosts that serve blob sidecars (supernodes). Populated by testSources(). */ superNodeHostIndexes;
|
|
18
24
|
constructor(config, opts = {}){
|
|
19
25
|
this.opts = opts;
|
|
20
26
|
this.disabled = false;
|
|
@@ -64,22 +70,52 @@ export class HttpBlobClient {
|
|
|
64
70
|
l1ConsensusHostUrls,
|
|
65
71
|
archiveUrl
|
|
66
72
|
});
|
|
67
|
-
let
|
|
73
|
+
let consensusSuperNodes = 0;
|
|
74
|
+
let consensusNonSuperNodes = 0;
|
|
75
|
+
let archiveSources = 0;
|
|
76
|
+
let blobSinks = 0;
|
|
77
|
+
const detectedSuperNodes = new Set();
|
|
68
78
|
if (l1ConsensusHostUrls && l1ConsensusHostUrls.length > 0) {
|
|
69
79
|
for(let l1ConsensusHostIndex = 0; l1ConsensusHostIndex < l1ConsensusHostUrls.length; l1ConsensusHostIndex++){
|
|
70
80
|
const l1ConsensusHostUrl = l1ConsensusHostUrls[l1ConsensusHostIndex];
|
|
71
81
|
try {
|
|
72
|
-
const { url, ...options } = getBeaconNodeFetchOptions(`${l1ConsensusHostUrl}/eth/v1/beacon/headers`, this.config, l1ConsensusHostIndex);
|
|
82
|
+
const { url, ...options } = getBeaconNodeFetchOptions(`${l1ConsensusHostUrl}/eth/v1/beacon/headers/head`, this.config, l1ConsensusHostIndex);
|
|
73
83
|
const res = await this.fetch(url, options);
|
|
74
|
-
if (res.ok) {
|
|
75
|
-
this.log.
|
|
84
|
+
if (!res.ok) {
|
|
85
|
+
this.log.error(`Failure reaching L1 consensus host: ${res.statusText} (${res.status})`, {
|
|
76
86
|
l1ConsensusHostUrl
|
|
77
87
|
});
|
|
78
|
-
|
|
88
|
+
continue;
|
|
89
|
+
}
|
|
90
|
+
this.log.info(`L1 consensus host is reachable`, {
|
|
91
|
+
l1ConsensusHostUrl
|
|
92
|
+
});
|
|
93
|
+
// Check if the host serves blob sidecars (supernode/semi-supernode).
|
|
94
|
+
// Post-Fusaka (PeerDAS), non-supernode beacon nodes no longer serve the
|
|
95
|
+
// blob sidecar endpoint. A 200 response (even with an empty data array
|
|
96
|
+
// for a slot with no blobs) means the node supports serving blob sidecars.
|
|
97
|
+
const body = await res.json();
|
|
98
|
+
const headSlot = body?.data?.header?.message?.slot;
|
|
99
|
+
if (headSlot) {
|
|
100
|
+
const { url: blobUrl, ...blobOptions } = getBeaconNodeFetchOptions(`${l1ConsensusHostUrl}/eth/v1/beacon/blobs/${headSlot}`, this.config, l1ConsensusHostIndex);
|
|
101
|
+
const blobRes = await this.fetch(blobUrl, blobOptions);
|
|
102
|
+
if (blobRes.ok) {
|
|
103
|
+
this.log.info(`L1 consensus host serves blob sidecars (supernode)`, {
|
|
104
|
+
l1ConsensusHostUrl
|
|
105
|
+
});
|
|
106
|
+
detectedSuperNodes.add(l1ConsensusHostIndex);
|
|
107
|
+
consensusSuperNodes++;
|
|
108
|
+
} else {
|
|
109
|
+
this.log.info(`L1 consensus host does not serve blob sidecars, skipping for blob fetching`, {
|
|
110
|
+
l1ConsensusHostUrl
|
|
111
|
+
});
|
|
112
|
+
consensusNonSuperNodes++;
|
|
113
|
+
}
|
|
79
114
|
} else {
|
|
80
|
-
this.log.
|
|
115
|
+
this.log.info(`L1 consensus host is reachable but could not determine head slot`, {
|
|
81
116
|
l1ConsensusHostUrl
|
|
82
117
|
});
|
|
118
|
+
consensusNonSuperNodes++;
|
|
83
119
|
}
|
|
84
120
|
} catch (err) {
|
|
85
121
|
this.log.error(`Error reaching L1 consensus host`, err, {
|
|
@@ -87,9 +123,8 @@ export class HttpBlobClient {
|
|
|
87
123
|
});
|
|
88
124
|
}
|
|
89
125
|
}
|
|
90
|
-
} else {
|
|
91
|
-
this.log.warn('No L1 consensus host urls configured');
|
|
92
126
|
}
|
|
127
|
+
this.superNodeHostIndexes = detectedSuperNodes;
|
|
93
128
|
if (this.archiveClient) {
|
|
94
129
|
try {
|
|
95
130
|
const latest = await this.archiveClient.getLatestBlock();
|
|
@@ -97,14 +132,12 @@ export class HttpBlobClient {
|
|
|
97
132
|
latest,
|
|
98
133
|
archiveUrl
|
|
99
134
|
});
|
|
100
|
-
|
|
135
|
+
archiveSources++;
|
|
101
136
|
} catch (err) {
|
|
102
137
|
this.log.error(`Error reaching archive client`, err, {
|
|
103
138
|
archiveUrl
|
|
104
139
|
});
|
|
105
140
|
}
|
|
106
|
-
} else {
|
|
107
|
-
this.log.warn('No archive client configured');
|
|
108
141
|
}
|
|
109
142
|
if (this.fileStoreClients.length > 0) {
|
|
110
143
|
for (const fileStoreClient of this.fileStoreClients){
|
|
@@ -114,7 +147,7 @@ export class HttpBlobClient {
|
|
|
114
147
|
this.log.info(`FileStore is reachable`, {
|
|
115
148
|
url: fileStoreClient.getBaseUrl()
|
|
116
149
|
});
|
|
117
|
-
|
|
150
|
+
blobSinks++;
|
|
118
151
|
} else {
|
|
119
152
|
this.log.warn(`FileStore is not accessible`, {
|
|
120
153
|
url: fileStoreClient.getBaseUrl()
|
|
@@ -127,12 +160,22 @@ export class HttpBlobClient {
|
|
|
127
160
|
}
|
|
128
161
|
}
|
|
129
162
|
}
|
|
163
|
+
// Emit a single summary after validating all sources
|
|
164
|
+
const successfulSourceCount = consensusSuperNodes + archiveSources + blobSinks;
|
|
165
|
+
let summary = `Blob client running with consensusSuperNodes=${consensusSuperNodes} archiveSources=${archiveSources} blobSinks=${blobSinks}`;
|
|
166
|
+
if (consensusNonSuperNodes > 0) {
|
|
167
|
+
summary += `. ${consensusNonSuperNodes} consensus client(s) ignored because they are not running in supernode or semi-supernode mode`;
|
|
168
|
+
}
|
|
130
169
|
if (successfulSourceCount === 0) {
|
|
131
170
|
if (this.config.blobAllowEmptySources) {
|
|
132
|
-
this.log.warn(
|
|
171
|
+
this.log.warn(summary);
|
|
133
172
|
} else {
|
|
134
|
-
throw new Error(
|
|
173
|
+
throw new Error(summary);
|
|
135
174
|
}
|
|
175
|
+
} else if (consensusSuperNodes === 0) {
|
|
176
|
+
this.log.warn(summary);
|
|
177
|
+
} else {
|
|
178
|
+
this.log.info(summary);
|
|
136
179
|
}
|
|
137
180
|
}
|
|
138
181
|
async sendBlobsToFilestore(blobs) {
|
|
@@ -154,35 +197,31 @@ export class HttpBlobClient {
|
|
|
154
197
|
}
|
|
155
198
|
}
|
|
156
199
|
/**
|
|
157
|
-
* Get the blob sidecar
|
|
158
|
-
*
|
|
159
|
-
* If requesting from the blob client, we send the blobkHash
|
|
160
|
-
* If requesting from the beacon node, we send the slot number
|
|
200
|
+
* Get the blob sidecar.
|
|
161
201
|
*
|
|
162
|
-
*
|
|
163
|
-
*
|
|
164
|
-
*
|
|
202
|
+
* Alternates between two primary sources (consensus and filestore) in a retry loop,
|
|
203
|
+
* then falls back to archive if blobs are still missing. The order of the primary
|
|
204
|
+
* sources is configurable via `blobPreferFilestores`.
|
|
165
205
|
*
|
|
166
206
|
* @param blockHash - The block hash
|
|
167
207
|
* @param blobHashes - The blob hashes to fetch
|
|
168
|
-
* @param opts - Options
|
|
208
|
+
* @param opts - Options for slot resolution
|
|
169
209
|
* @returns The blobs
|
|
170
210
|
*/ async getBlobSidecar(blockHash, blobHashes, opts) {
|
|
171
211
|
if (this.disabled) {
|
|
172
212
|
this.log.warn('Blob storage is disabled, returning empty blob sidecar');
|
|
173
213
|
return [];
|
|
174
214
|
}
|
|
175
|
-
const isHistoricalSync = opts?.isHistoricalSync ?? false;
|
|
176
215
|
// Accumulate blobs across sources, preserving order and handling duplicates
|
|
177
216
|
// resultBlobs[i] will contain the blob for blobHashes[i], or undefined if not yet found
|
|
178
217
|
const resultBlobs = new Array(blobHashes.length).fill(undefined);
|
|
179
|
-
// Helper to get
|
|
218
|
+
// Helper to get missing blob hashes that we still need to fetch
|
|
180
219
|
const getMissingBlobHashes = ()=>blobHashes.map((bh, i)=>resultBlobs[i] === undefined ? bh : undefined).filter((bh)=>bh !== undefined);
|
|
181
220
|
// Return the result, ignoring any undefined ones
|
|
182
221
|
const getFilledBlobs = ()=>resultBlobs.filter((b)=>b !== undefined);
|
|
183
222
|
// Helper to fill in results from fetched blobs
|
|
184
|
-
const fillResults = (fetchedBlobs)=>{
|
|
185
|
-
const blobs = processFetchedBlobs(fetchedBlobs, blobHashes, this.log);
|
|
223
|
+
const fillResults = async (fetchedBlobs)=>{
|
|
224
|
+
const blobs = await processFetchedBlobs(fetchedBlobs, blobHashes, this.log);
|
|
186
225
|
// Fill in any missing positions with matching blobs
|
|
187
226
|
for(let i = 0; i < blobHashes.length; i++){
|
|
188
227
|
if (resultBlobs[i] === undefined) {
|
|
@@ -198,86 +237,75 @@ export class HttpBlobClient {
|
|
|
198
237
|
}
|
|
199
238
|
return blobs;
|
|
200
239
|
};
|
|
201
|
-
const { l1ConsensusHostUrls } = this.config;
|
|
202
240
|
const ctx = {
|
|
203
241
|
blockHash,
|
|
204
242
|
blobHashes: blobHashes.map(bufferToHex)
|
|
205
243
|
};
|
|
206
|
-
//
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
244
|
+
// Lazily resolve the slot number — only resolved when consensus hosts are actually tried.
|
|
245
|
+
let slotNumber;
|
|
246
|
+
let slotResolved = false;
|
|
247
|
+
const getSlotNumber = async ()=>{
|
|
248
|
+
if (!slotResolved) {
|
|
249
|
+
slotNumber = await this.resolveSlotNumber(blockHash, opts);
|
|
250
|
+
slotResolved = true;
|
|
211
251
|
}
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
if (result.length === blobHashes.length) {
|
|
244
|
-
return returnWithCallback(result);
|
|
245
|
-
}
|
|
252
|
+
return slotNumber;
|
|
253
|
+
};
|
|
254
|
+
// Build the two source-try functions. The order depends on the config.
|
|
255
|
+
const tryConsensus = ()=>this.tryConsensusHosts(getSlotNumber, getMissingBlobHashes, fillResults, ctx);
|
|
256
|
+
const tryFilestores = ()=>this.tryFileStores(getMissingBlobHashes, fillResults, ctx);
|
|
257
|
+
const preferFilestores = this.config.blobPreferFilestores ?? false;
|
|
258
|
+
const [trySourceA, trySourceB] = preferFilestores ? [
|
|
259
|
+
tryFilestores,
|
|
260
|
+
tryConsensus
|
|
261
|
+
] : [
|
|
262
|
+
tryConsensus,
|
|
263
|
+
tryFilestores
|
|
264
|
+
];
|
|
265
|
+
// Historical sync: blobs should already exist, use shorter backoff for transient errors.
|
|
266
|
+
// Near-tip sync: blobs may still be uploading, use longer backoff for eventual consistency.
|
|
267
|
+
const isHistoricalSync = opts?.isHistoricalSync ?? false;
|
|
268
|
+
const backoff = isHistoricalSync ? [
|
|
269
|
+
1,
|
|
270
|
+
1
|
|
271
|
+
] : [
|
|
272
|
+
1,
|
|
273
|
+
1,
|
|
274
|
+
1,
|
|
275
|
+
2,
|
|
276
|
+
2
|
|
277
|
+
];
|
|
278
|
+
// Retry loop: alternate between the two primary sources with backoff.
|
|
279
|
+
try {
|
|
280
|
+
await retry(async ()=>{
|
|
281
|
+
if (getMissingBlobHashes().length > 0) {
|
|
282
|
+
await trySourceA();
|
|
246
283
|
}
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
}
|
|
258
|
-
}, 'filestore blob retrieval', makeBackoff([
|
|
259
|
-
1,
|
|
260
|
-
1,
|
|
261
|
-
2
|
|
262
|
-
]), this.log, true);
|
|
263
|
-
return returnWithCallback(getFilledBlobs());
|
|
264
|
-
} catch {
|
|
265
|
-
// Exhausted retries, continue to archive fallback
|
|
266
|
-
}
|
|
284
|
+
if (getMissingBlobHashes().length > 0) {
|
|
285
|
+
await trySourceB();
|
|
286
|
+
}
|
|
287
|
+
if (getMissingBlobHashes().length > 0) {
|
|
288
|
+
throw new Error('Still missing blobs after trying all primary sources');
|
|
289
|
+
}
|
|
290
|
+
}, 'blob retrieval', makeBackoff(backoff), this.log, true);
|
|
291
|
+
return returnWithCallback(getFilledBlobs());
|
|
292
|
+
} catch {
|
|
293
|
+
// Exhausted retries, continue to archive fallback
|
|
267
294
|
}
|
|
268
|
-
|
|
269
|
-
|
|
295
|
+
// Archive fallback
|
|
296
|
+
const missingAfterPrimary = getMissingBlobHashes();
|
|
297
|
+
if (missingAfterPrimary.length > 0 && this.archiveClient) {
|
|
270
298
|
const archiveCtx = {
|
|
271
299
|
archiveUrl: this.archiveClient.getBaseUrl(),
|
|
272
300
|
...ctx
|
|
273
301
|
};
|
|
274
|
-
this.log.trace(`Attempting to get ${
|
|
302
|
+
this.log.trace(`Attempting to get ${missingAfterPrimary.length} blobs from archive`, archiveCtx);
|
|
275
303
|
const allBlobs = await this.archiveClient.getBlobsFromBlock(blockHash);
|
|
276
304
|
if (!allBlobs) {
|
|
277
305
|
this.log.debug('No blobs found from archive client', archiveCtx);
|
|
278
306
|
} else {
|
|
279
307
|
this.log.trace(`Got ${allBlobs.length} blobs from archive client before filtering`, archiveCtx);
|
|
280
|
-
const result = fillResults(allBlobs);
|
|
308
|
+
const result = await fillResults(allBlobs);
|
|
281
309
|
this.log.debug(`Got ${allBlobs.length} blobs from archive client (total: ${result.length}/${blobHashes.length})`, archiveCtx);
|
|
282
310
|
if (result.length === blobHashes.length) {
|
|
283
311
|
return returnWithCallback(result);
|
|
@@ -287,13 +315,63 @@ export class HttpBlobClient {
|
|
|
287
315
|
const result = getFilledBlobs();
|
|
288
316
|
if (result.length < blobHashes.length) {
|
|
289
317
|
this.log.warn(`Failed to fetch all blobs for ${blockHash} from all blob sources (got ${result.length}/${blobHashes.length})`, {
|
|
290
|
-
l1ConsensusHostUrls,
|
|
318
|
+
l1ConsensusHostUrls: this.config.l1ConsensusHostUrls,
|
|
291
319
|
archiveUrl: this.archiveClient?.getBaseUrl(),
|
|
292
320
|
fileStoreUrls: this.fileStoreClients.map((c)=>c.getBaseUrl())
|
|
293
321
|
});
|
|
294
322
|
}
|
|
295
323
|
return returnWithCallback(result);
|
|
296
324
|
}
|
|
325
|
+
/** Resolves the beacon slot number for the given block hash. Returns undefined if no consensus hosts. */ resolveSlotNumber(blockHash, opts) {
|
|
326
|
+
const { l1ConsensusHostUrls } = this.config;
|
|
327
|
+
if (!l1ConsensusHostUrls || l1ConsensusHostUrls.length === 0) {
|
|
328
|
+
return undefined;
|
|
329
|
+
}
|
|
330
|
+
// If no supernodes, no point resolving the slot
|
|
331
|
+
if (this.superNodeHostIndexes && this.superNodeHostIndexes.size === 0) {
|
|
332
|
+
return undefined;
|
|
333
|
+
}
|
|
334
|
+
return this.getSlotNumber(blockHash, opts?.parentBeaconBlockRoot, opts?.l1BlockTimestamp);
|
|
335
|
+
}
|
|
336
|
+
/**
|
|
337
|
+
* Try all supernode consensus hosts for blob sidecars.
|
|
338
|
+
* Skips hosts that were detected as non-supernodes during testSources().
|
|
339
|
+
*/ async tryConsensusHosts(getSlotNumber, getMissingBlobHashes, fillResults, ctx) {
|
|
340
|
+
const { l1ConsensusHostUrls } = this.config;
|
|
341
|
+
if (!l1ConsensusHostUrls || l1ConsensusHostUrls.length === 0) {
|
|
342
|
+
return;
|
|
343
|
+
}
|
|
344
|
+
const slotNumber = await getSlotNumber();
|
|
345
|
+
if (!slotNumber) {
|
|
346
|
+
return;
|
|
347
|
+
}
|
|
348
|
+
for(let l1ConsensusHostIndex = 0; l1ConsensusHostIndex < l1ConsensusHostUrls.length; l1ConsensusHostIndex++){
|
|
349
|
+
const missingHashes = getMissingBlobHashes();
|
|
350
|
+
if (missingHashes.length === 0) {
|
|
351
|
+
break;
|
|
352
|
+
}
|
|
353
|
+
// Skip non-supernode hosts if we've already detected supernodes
|
|
354
|
+
if (this.superNodeHostIndexes && !this.superNodeHostIndexes.has(l1ConsensusHostIndex)) {
|
|
355
|
+
this.log.trace(`Skipping non-supernode consensus host`, {
|
|
356
|
+
l1ConsensusHostUrl: l1ConsensusHostUrls[l1ConsensusHostIndex]
|
|
357
|
+
});
|
|
358
|
+
continue;
|
|
359
|
+
}
|
|
360
|
+
const l1ConsensusHostUrl = l1ConsensusHostUrls[l1ConsensusHostIndex];
|
|
361
|
+
this.log.trace(`Attempting to get ${missingHashes.length} blobs from consensus host`, {
|
|
362
|
+
slotNumber,
|
|
363
|
+
l1ConsensusHostUrl,
|
|
364
|
+
...ctx
|
|
365
|
+
});
|
|
366
|
+
const blobs = await this.getBlobsFromHost(l1ConsensusHostUrl, slotNumber, l1ConsensusHostIndex, missingHashes);
|
|
367
|
+
const result = await fillResults(blobs);
|
|
368
|
+
this.log.debug(`Got ${blobs.length} blobs from consensus host (total: ${result.length}/${ctx.blobHashes.length})`, {
|
|
369
|
+
slotNumber,
|
|
370
|
+
l1ConsensusHostUrl,
|
|
371
|
+
...ctx
|
|
372
|
+
});
|
|
373
|
+
}
|
|
374
|
+
}
|
|
297
375
|
/**
|
|
298
376
|
* Try all filestores once (shuffled for load distribution).
|
|
299
377
|
* @param getMissingBlobHashes - Function to get remaining blob hashes to fetch
|
|
@@ -318,7 +396,7 @@ export class HttpBlobClient {
|
|
|
318
396
|
});
|
|
319
397
|
const blobs = await client.getBlobsByHashes(blobHashStrings);
|
|
320
398
|
if (blobs.length > 0) {
|
|
321
|
-
const result = fillResults(blobs);
|
|
399
|
+
const result = await fillResults(blobs);
|
|
322
400
|
this.log.debug(`Got ${blobs.length} blobs from filestore (total: ${result.length}/${ctx.blobHashes.length})`, {
|
|
323
401
|
url: client.getBaseUrl(),
|
|
324
402
|
...ctx
|
|
@@ -332,14 +410,14 @@ export class HttpBlobClient {
|
|
|
332
410
|
}
|
|
333
411
|
}
|
|
334
412
|
async getBlobSidecarFrom(hostUrl, blockHashOrSlot, blobHashes = [], l1ConsensusHostIndex) {
|
|
335
|
-
const blobs = await this.getBlobsFromHost(hostUrl, blockHashOrSlot, l1ConsensusHostIndex);
|
|
336
|
-
return processFetchedBlobs(blobs, blobHashes, this.log).filter((b)=>b !== undefined);
|
|
413
|
+
const blobs = await this.getBlobsFromHost(hostUrl, blockHashOrSlot, l1ConsensusHostIndex, blobHashes);
|
|
414
|
+
return (await processFetchedBlobs(blobs, blobHashes, this.log)).filter((b)=>b !== undefined);
|
|
337
415
|
}
|
|
338
|
-
async getBlobsFromHost(hostUrl, blockHashOrSlot, l1ConsensusHostIndex) {
|
|
416
|
+
async getBlobsFromHost(hostUrl, blockHashOrSlot, l1ConsensusHostIndex, blobHashes) {
|
|
339
417
|
try {
|
|
340
|
-
let res = await this.fetchBlobSidecars(hostUrl, blockHashOrSlot, l1ConsensusHostIndex);
|
|
418
|
+
let res = await this.fetchBlobSidecars(hostUrl, blockHashOrSlot, l1ConsensusHostIndex, blobHashes);
|
|
341
419
|
if (res.ok) {
|
|
342
|
-
return parseBlobJsonsFromResponse(await res.json(), this.log);
|
|
420
|
+
return await parseBlobJsonsFromResponse(await res.json(), this.log);
|
|
343
421
|
}
|
|
344
422
|
if (res.status === 404 && typeof blockHashOrSlot === 'number') {
|
|
345
423
|
const latestSlot = await this.getLatestSlotNumber(hostUrl, l1ConsensusHostIndex);
|
|
@@ -352,9 +430,9 @@ export class HttpBlobClient {
|
|
|
352
430
|
let currentSlot = blockHashOrSlot + 1;
|
|
353
431
|
while(res.status === 404 && maxRetries > 0 && latestSlot !== undefined && currentSlot <= latestSlot){
|
|
354
432
|
this.log.debug(`Trying slot ${currentSlot}`);
|
|
355
|
-
res = await this.fetchBlobSidecars(hostUrl, currentSlot, l1ConsensusHostIndex);
|
|
433
|
+
res = await this.fetchBlobSidecars(hostUrl, currentSlot, l1ConsensusHostIndex, blobHashes);
|
|
356
434
|
if (res.ok) {
|
|
357
|
-
return parseBlobJsonsFromResponse(await res.json(), this.log);
|
|
435
|
+
return await parseBlobJsonsFromResponse(await res.json(), this.log);
|
|
358
436
|
}
|
|
359
437
|
currentSlot++;
|
|
360
438
|
maxRetries--;
|
|
@@ -371,21 +449,29 @@ export class HttpBlobClient {
|
|
|
371
449
|
return [];
|
|
372
450
|
}
|
|
373
451
|
}
|
|
374
|
-
fetchBlobSidecars(hostUrl, blockHashOrSlot, l1ConsensusHostIndex) {
|
|
375
|
-
|
|
376
|
-
|
|
452
|
+
fetchBlobSidecars(hostUrl, blockHashOrSlot, l1ConsensusHostIndex, blobHashes) {
|
|
453
|
+
let baseUrl = `${hostUrl}/eth/v1/beacon/blobs/${blockHashOrSlot}`;
|
|
454
|
+
if (blobHashes && blobHashes.length > 0) {
|
|
455
|
+
const params = new URLSearchParams();
|
|
456
|
+
for (const hash of blobHashes){
|
|
457
|
+
params.append('versioned_hashes', `0x${hash.toString('hex')}`);
|
|
458
|
+
}
|
|
459
|
+
baseUrl += `?${params.toString()}`;
|
|
460
|
+
}
|
|
461
|
+
const { url, logSafeUrl, ...options } = getBeaconNodeFetchOptions(baseUrl, this.config, l1ConsensusHostIndex);
|
|
377
462
|
this.log.debug(`Fetching blob sidecar for ${blockHashOrSlot}`, {
|
|
378
|
-
url,
|
|
463
|
+
url: logSafeUrl,
|
|
379
464
|
...options
|
|
380
465
|
});
|
|
381
|
-
|
|
466
|
+
// No retry here — this is called inside the main retry loop in getBlobSidecar
|
|
467
|
+
return fetch(url, options);
|
|
382
468
|
}
|
|
383
469
|
async getLatestSlotNumber(hostUrl, l1ConsensusHostIndex) {
|
|
384
470
|
try {
|
|
385
471
|
const baseUrl = `${hostUrl}/eth/v1/beacon/headers/head`;
|
|
386
|
-
const { url, ...options } = getBeaconNodeFetchOptions(baseUrl, this.config, l1ConsensusHostIndex);
|
|
472
|
+
const { url, logSafeUrl, ...options } = getBeaconNodeFetchOptions(baseUrl, this.config, l1ConsensusHostIndex);
|
|
387
473
|
this.log.debug(`Fetching latest slot number`, {
|
|
388
|
-
url,
|
|
474
|
+
url: logSafeUrl,
|
|
389
475
|
...options
|
|
390
476
|
});
|
|
391
477
|
const res = await this.fetch(url, options);
|
|
@@ -418,36 +504,45 @@ export class HttpBlobClient {
|
|
|
418
504
|
*
|
|
419
505
|
* @param blockHash - The block hash
|
|
420
506
|
* @returns The slot number
|
|
421
|
-
*/ async getSlotNumber(blockHash) {
|
|
507
|
+
*/ async getSlotNumber(blockHash, parentBeaconBlockRoot, l1BlockTimestamp) {
|
|
422
508
|
const { l1ConsensusHostUrls, l1RpcUrls } = this.config;
|
|
423
509
|
if (!l1ConsensusHostUrls || l1ConsensusHostUrls.length === 0) {
|
|
424
510
|
this.log.debug('No consensus host url configured');
|
|
425
511
|
return undefined;
|
|
426
512
|
}
|
|
427
|
-
if (
|
|
428
|
-
|
|
429
|
-
|
|
513
|
+
// Primary path: compute slot from timestamp if genesis config is cached (no network call needed)
|
|
514
|
+
if (l1BlockTimestamp !== undefined && this.beaconGenesisTime !== undefined && this.beaconSecondsPerSlot !== undefined) {
|
|
515
|
+
const slot = Number((l1BlockTimestamp - this.beaconGenesisTime) / BigInt(this.beaconSecondsPerSlot));
|
|
516
|
+
this.log.debug(`Computed slot ${slot} from L1 block timestamp`, {
|
|
517
|
+
l1BlockTimestamp
|
|
518
|
+
});
|
|
519
|
+
return slot;
|
|
430
520
|
}
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
params: [
|
|
442
|
-
blockHash,
|
|
443
|
-
/*tx flag*/ false
|
|
444
|
-
]
|
|
521
|
+
if (!parentBeaconBlockRoot) {
|
|
522
|
+
// parentBeaconBlockRoot not provided by caller — fetch it from the execution RPC
|
|
523
|
+
if (!l1RpcUrls || l1RpcUrls.length === 0) {
|
|
524
|
+
this.log.debug('No execution host url configured');
|
|
525
|
+
return undefined;
|
|
526
|
+
}
|
|
527
|
+
const client = createPublicClient({
|
|
528
|
+
transport: makeL1HttpTransport(l1RpcUrls, {
|
|
529
|
+
timeout: this.config.l1HttpTimeoutMS
|
|
530
|
+
})
|
|
445
531
|
});
|
|
446
|
-
|
|
447
|
-
|
|
532
|
+
try {
|
|
533
|
+
const res = await client.request({
|
|
534
|
+
method: 'eth_getBlockByHash',
|
|
535
|
+
params: [
|
|
536
|
+
blockHash,
|
|
537
|
+
/*tx flag*/ false
|
|
538
|
+
]
|
|
539
|
+
});
|
|
540
|
+
if (res.parentBeaconBlockRoot) {
|
|
541
|
+
parentBeaconBlockRoot = res.parentBeaconBlockRoot;
|
|
542
|
+
}
|
|
543
|
+
} catch (err) {
|
|
544
|
+
this.log.error(`Error getting parent beacon block root`, err);
|
|
448
545
|
}
|
|
449
|
-
} catch (err) {
|
|
450
|
-
this.log.error(`Error getting parent beacon block root`, err);
|
|
451
546
|
}
|
|
452
547
|
if (!parentBeaconBlockRoot) {
|
|
453
548
|
this.log.error(`No parent beacon block root found for block ${blockHash}`);
|
|
@@ -474,11 +569,78 @@ export class HttpBlobClient {
|
|
|
474
569
|
/** @internal - exposed for testing */ getArchiveClient() {
|
|
475
570
|
return this.archiveClient;
|
|
476
571
|
}
|
|
572
|
+
/** Returns true if this client can upload blobs to filestore. */ canUpload() {
|
|
573
|
+
return this.fileStoreUploadClient !== undefined;
|
|
574
|
+
}
|
|
575
|
+
/**
|
|
576
|
+
* Start the blob client.
|
|
577
|
+
* Fetches and caches beacon genesis config for timestamp-based slot resolution,
|
|
578
|
+
* then uploads the initial healthcheck file (awaited) and starts periodic uploads.
|
|
579
|
+
*/ async start() {
|
|
580
|
+
await this.fetchBeaconConfig();
|
|
581
|
+
if (!this.fileStoreUploadClient) {
|
|
582
|
+
return;
|
|
583
|
+
}
|
|
584
|
+
await this.fileStoreUploadClient.uploadHealthcheck();
|
|
585
|
+
this.log.debug('Initial healthcheck file uploaded');
|
|
586
|
+
this.startPeriodicHealthcheckUpload();
|
|
587
|
+
}
|
|
588
|
+
/**
|
|
589
|
+
* Start periodic healthcheck upload to the file store to ensure it remains available even if accidentally deleted.
|
|
590
|
+
*/ startPeriodicHealthcheckUpload() {
|
|
591
|
+
const intervalMs = (this.config.blobHealthcheckUploadIntervalMinutes ?? DEFAULT_HEALTHCHECK_UPLOAD_INTERVAL_MINUTES) * 60 * 1000;
|
|
592
|
+
this.healthcheckUploadIntervalId = setInterval(()=>{
|
|
593
|
+
void this.fileStoreUploadClient.uploadHealthcheck().catch((err)=>{
|
|
594
|
+
this.log.warn('Failed to upload periodic healthcheck file', err);
|
|
595
|
+
});
|
|
596
|
+
}, intervalMs);
|
|
597
|
+
}
|
|
598
|
+
/**
|
|
599
|
+
* Fetches and caches beacon genesis time and slot duration from the first available consensus host.
|
|
600
|
+
* These static values enable timestamp-based slot resolution, eliminating the per-fetch headers call.
|
|
601
|
+
* Logs a warning and leaves fields undefined if all hosts fail, callers fall back gracefully.
|
|
602
|
+
*/ async fetchBeaconConfig() {
|
|
603
|
+
const { l1ConsensusHostUrls } = this.config;
|
|
604
|
+
if (!l1ConsensusHostUrls || l1ConsensusHostUrls.length === 0) {
|
|
605
|
+
return;
|
|
606
|
+
}
|
|
607
|
+
for(let i = 0; i < l1ConsensusHostUrls.length; i++){
|
|
608
|
+
try {
|
|
609
|
+
const { url: genesisUrl, ...genesisOptions } = getBeaconNodeFetchOptions(`${l1ConsensusHostUrls[i]}/eth/v1/config/genesis`, this.config, i);
|
|
610
|
+
const { url: specUrl, ...specOptions } = getBeaconNodeFetchOptions(`${l1ConsensusHostUrls[i]}/eth/v1/config/spec`, this.config, i);
|
|
611
|
+
const [genesisRes, specRes] = await Promise.all([
|
|
612
|
+
this.fetch(genesisUrl, genesisOptions),
|
|
613
|
+
this.fetch(specUrl, specOptions)
|
|
614
|
+
]);
|
|
615
|
+
if (genesisRes.ok && specRes.ok) {
|
|
616
|
+
const genesis = await genesisRes.json();
|
|
617
|
+
const spec = await specRes.json();
|
|
618
|
+
this.beaconGenesisTime = BigInt(genesis.data.genesisTime);
|
|
619
|
+
this.beaconSecondsPerSlot = parseInt(spec.data.secondsPerSlot);
|
|
620
|
+
this.log.debug(`Fetched beacon genesis config`, {
|
|
621
|
+
genesisTime: this.beaconGenesisTime,
|
|
622
|
+
secondsPerSlot: this.beaconSecondsPerSlot
|
|
623
|
+
});
|
|
624
|
+
return;
|
|
625
|
+
}
|
|
626
|
+
} catch (err) {
|
|
627
|
+
this.log.warn(`Failed to fetch beacon config from host ${l1ConsensusHostUrls[i]}`, err);
|
|
628
|
+
}
|
|
629
|
+
}
|
|
630
|
+
this.log.warn('Could not fetch beacon genesis config from any consensus host — will use headers call fallback');
|
|
631
|
+
}
|
|
632
|
+
/**
|
|
633
|
+
* Stop the blob client, clearing any periodic tasks.
|
|
634
|
+
*/ stop() {
|
|
635
|
+
if (this.healthcheckUploadIntervalId) {
|
|
636
|
+
clearInterval(this.healthcheckUploadIntervalId);
|
|
637
|
+
this.healthcheckUploadIntervalId = undefined;
|
|
638
|
+
}
|
|
639
|
+
}
|
|
477
640
|
}
|
|
478
|
-
function parseBlobJsonsFromResponse(response, logger) {
|
|
641
|
+
async function parseBlobJsonsFromResponse(response, logger) {
|
|
479
642
|
try {
|
|
480
|
-
|
|
481
|
-
return blobs;
|
|
643
|
+
return await Promise.all(response.data.map(parseBlobJson));
|
|
482
644
|
} catch (err) {
|
|
483
645
|
logger.error(`Error parsing blob json from response`, err);
|
|
484
646
|
return [];
|
|
@@ -488,15 +650,14 @@ function parseBlobJsonsFromResponse(response, logger) {
|
|
|
488
650
|
// https://ethereum.github.io/beacon-APIs/?urls.primaryName=dev#/Beacon/getBlobSidecars
|
|
489
651
|
// Here we attempt to parse the response data to Buffer, and check the lengths (via Blob's constructor), to avoid
|
|
490
652
|
// throwing an error down the line when calling Blob.fromJson().
|
|
491
|
-
function parseBlobJson(
|
|
492
|
-
const blobBuffer = Buffer.from(
|
|
493
|
-
const
|
|
494
|
-
const blob = new Blob(blobBuffer, commitmentBuffer);
|
|
653
|
+
async function parseBlobJson(rawHex) {
|
|
654
|
+
const blobBuffer = Buffer.from(rawHex.slice(2), 'hex');
|
|
655
|
+
const blob = await Blob.fromBlobBuffer(blobBuffer);
|
|
495
656
|
return blob.toJSON();
|
|
496
657
|
}
|
|
497
658
|
// Returns an array that maps each blob hash to the corresponding blob, or undefined if the blob is not found
|
|
498
659
|
// or the data does not match the commitment.
|
|
499
|
-
function processFetchedBlobs(blobs, blobHashes, logger) {
|
|
660
|
+
async function processFetchedBlobs(blobs, blobHashes, logger) {
|
|
500
661
|
const requestedBlobHashes = new Set(blobHashes.map(bufferToHex));
|
|
501
662
|
const hashToBlob = new Map();
|
|
502
663
|
for (const blobJson of blobs){
|
|
@@ -505,7 +666,7 @@ function processFetchedBlobs(blobs, blobHashes, logger) {
|
|
|
505
666
|
continue;
|
|
506
667
|
}
|
|
507
668
|
try {
|
|
508
|
-
const blob = Blob.fromJson(blobJson);
|
|
669
|
+
const blob = await Blob.fromJson(blobJson);
|
|
509
670
|
hashToBlob.set(hashHex, blob);
|
|
510
671
|
} catch (err) {
|
|
511
672
|
// If the above throws, it's likely that the blob commitment does not match the hash of the blob data.
|
|
@@ -519,11 +680,15 @@ function getBeaconNodeFetchOptions(url, config, l1ConsensusHostIndex) {
|
|
|
519
680
|
const l1ConsensusHostApiKey = l1ConsensusHostIndex !== undefined && l1ConsensusHostApiKeys && l1ConsensusHostApiKeys[l1ConsensusHostIndex];
|
|
520
681
|
const l1ConsensusHostApiKeyHeader = l1ConsensusHostIndex !== undefined && l1ConsensusHostApiKeyHeaders && l1ConsensusHostApiKeyHeaders[l1ConsensusHostIndex];
|
|
521
682
|
let formattedUrl = url;
|
|
683
|
+
let logSafeUrl = url;
|
|
522
684
|
if (l1ConsensusHostApiKey && l1ConsensusHostApiKey.getValue() !== '' && !l1ConsensusHostApiKeyHeader) {
|
|
523
|
-
|
|
685
|
+
const separator = formattedUrl.includes('?') ? '&' : '?';
|
|
686
|
+
formattedUrl += `${separator}key=${l1ConsensusHostApiKey.getValue()}`;
|
|
687
|
+
logSafeUrl += `${separator}key=[REDACTED]`;
|
|
524
688
|
}
|
|
525
689
|
return {
|
|
526
690
|
url: formattedUrl,
|
|
691
|
+
logSafeUrl,
|
|
527
692
|
...l1ConsensusHostApiKey && l1ConsensusHostApiKeyHeader && {
|
|
528
693
|
headers: {
|
|
529
694
|
[l1ConsensusHostApiKeyHeader]: l1ConsensusHostApiKey.getValue()
|