@aztec/blob-client 0.0.1-commit.03f7ef2 → 0.0.1-commit.04d373f

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. package/README.md +10 -1
  2. package/dest/archive/blobscan_archive_client.d.ts +8 -92
  3. package/dest/archive/blobscan_archive_client.d.ts.map +1 -1
  4. package/dest/archive/config.js +1 -1
  5. package/dest/archive/instrumentation.d.ts +1 -1
  6. package/dest/archive/instrumentation.d.ts.map +1 -1
  7. package/dest/archive/instrumentation.js +13 -13
  8. package/dest/blobstore/blob_store_test_suite.js +9 -9
  9. package/dest/client/config.d.ts +11 -1
  10. package/dest/client/config.d.ts.map +1 -1
  11. package/dest/client/config.js +22 -2
  12. package/dest/client/factory.d.ts +5 -6
  13. package/dest/client/factory.d.ts.map +1 -1
  14. package/dest/client/factory.js +13 -4
  15. package/dest/client/http.d.ts +34 -10
  16. package/dest/client/http.d.ts.map +1 -1
  17. package/dest/client/http.js +304 -139
  18. package/dest/client/interface.d.ts +19 -4
  19. package/dest/client/interface.d.ts.map +1 -1
  20. package/dest/client/local.d.ts +3 -1
  21. package/dest/client/local.d.ts.map +1 -1
  22. package/dest/client/local.js +3 -0
  23. package/dest/client/tests.js +3 -3
  24. package/dest/filestore/factory.d.ts +5 -4
  25. package/dest/filestore/factory.d.ts.map +1 -1
  26. package/dest/filestore/factory.js +4 -4
  27. package/dest/filestore/filestore_blob_client.d.ts +14 -2
  28. package/dest/filestore/filestore_blob_client.d.ts.map +1 -1
  29. package/dest/filestore/filestore_blob_client.js +29 -5
  30. package/dest/filestore/healthcheck.d.ts +5 -0
  31. package/dest/filestore/healthcheck.d.ts.map +1 -0
  32. package/dest/filestore/healthcheck.js +3 -0
  33. package/package.json +9 -9
  34. package/src/archive/config.ts +1 -1
  35. package/src/archive/instrumentation.ts +22 -13
  36. package/src/blobstore/blob_store_test_suite.ts +9 -9
  37. package/src/client/config.ts +36 -1
  38. package/src/client/factory.ts +17 -5
  39. package/src/client/http.ts +357 -134
  40. package/src/client/interface.ts +18 -3
  41. package/src/client/local.ts +5 -0
  42. package/src/client/tests.ts +2 -2
  43. package/src/filestore/factory.ts +7 -2
  44. package/src/filestore/filestore_blob_client.ts +33 -5
  45. package/src/filestore/healthcheck.ts +5 -0
@@ -1,10 +1,12 @@
1
1
  import { Blob, computeEthVersionedBlobHash } from '@aztec/blob-lib';
2
+ import { makeL1HttpTransport } from '@aztec/ethereum/client';
2
3
  import { shuffle } from '@aztec/foundation/array';
3
4
  import { createLogger } from '@aztec/foundation/log';
4
5
  import { makeBackoff, retry } from '@aztec/foundation/retry';
5
6
  import { bufferToHex, hexToBuffer } from '@aztec/foundation/string';
6
- import { createPublicClient, fallback, http } from 'viem';
7
+ import { createPublicClient } from 'viem';
7
8
  import { createBlobArchiveClient } from '../archive/factory.js';
9
+ import { DEFAULT_HEALTHCHECK_UPLOAD_INTERVAL_MINUTES } from '../filestore/healthcheck.js';
8
10
  import { getBlobClientConfigFromEnv } from './config.js';
9
11
  export class HttpBlobClient {
10
12
  opts;
@@ -15,6 +17,10 @@ export class HttpBlobClient {
15
17
  fileStoreClients;
16
18
  fileStoreUploadClient;
17
19
  disabled;
20
+ healthcheckUploadIntervalId;
21
+ /** Cached beacon genesis time (seconds since Unix epoch). Fetched once at startup. */ beaconGenesisTime;
22
+ /** Cached beacon slot duration in seconds. Fetched once at startup. */ beaconSecondsPerSlot;
23
+ /** Indexes of consensus hosts that serve blob sidecars (supernodes). Populated by testSources(). */ superNodeHostIndexes;
18
24
  constructor(config, opts = {}){
19
25
  this.opts = opts;
20
26
  this.disabled = false;
@@ -64,22 +70,52 @@ export class HttpBlobClient {
64
70
  l1ConsensusHostUrls,
65
71
  archiveUrl
66
72
  });
67
- let successfulSourceCount = 0;
73
+ let consensusSuperNodes = 0;
74
+ let consensusNonSuperNodes = 0;
75
+ let archiveSources = 0;
76
+ let blobSinks = 0;
77
+ const detectedSuperNodes = new Set();
68
78
  if (l1ConsensusHostUrls && l1ConsensusHostUrls.length > 0) {
69
79
  for(let l1ConsensusHostIndex = 0; l1ConsensusHostIndex < l1ConsensusHostUrls.length; l1ConsensusHostIndex++){
70
80
  const l1ConsensusHostUrl = l1ConsensusHostUrls[l1ConsensusHostIndex];
71
81
  try {
72
- const { url, ...options } = getBeaconNodeFetchOptions(`${l1ConsensusHostUrl}/eth/v1/beacon/headers`, this.config, l1ConsensusHostIndex);
82
+ const { url, ...options } = getBeaconNodeFetchOptions(`${l1ConsensusHostUrl}/eth/v1/beacon/headers/head`, this.config, l1ConsensusHostIndex);
73
83
  const res = await this.fetch(url, options);
74
- if (res.ok) {
75
- this.log.info(`L1 consensus host is reachable`, {
84
+ if (!res.ok) {
85
+ this.log.error(`Failure reaching L1 consensus host: ${res.statusText} (${res.status})`, {
76
86
  l1ConsensusHostUrl
77
87
  });
78
- successfulSourceCount++;
88
+ continue;
89
+ }
90
+ this.log.info(`L1 consensus host is reachable`, {
91
+ l1ConsensusHostUrl
92
+ });
93
+ // Check if the host serves blob sidecars (supernode/semi-supernode).
94
+ // Post-Fusaka (PeerDAS), non-supernode beacon nodes no longer serve the
95
+ // blob sidecar endpoint. A 200 response (even with an empty data array
96
+ // for a slot with no blobs) means the node supports serving blob sidecars.
97
+ const body = await res.json();
98
+ const headSlot = body?.data?.header?.message?.slot;
99
+ if (headSlot) {
100
+ const { url: blobUrl, ...blobOptions } = getBeaconNodeFetchOptions(`${l1ConsensusHostUrl}/eth/v1/beacon/blobs/${headSlot}`, this.config, l1ConsensusHostIndex);
101
+ const blobRes = await this.fetch(blobUrl, blobOptions);
102
+ if (blobRes.ok) {
103
+ this.log.info(`L1 consensus host serves blob sidecars (supernode)`, {
104
+ l1ConsensusHostUrl
105
+ });
106
+ detectedSuperNodes.add(l1ConsensusHostIndex);
107
+ consensusSuperNodes++;
108
+ } else {
109
+ this.log.info(`L1 consensus host does not serve blob sidecars, skipping for blob fetching`, {
110
+ l1ConsensusHostUrl
111
+ });
112
+ consensusNonSuperNodes++;
113
+ }
79
114
  } else {
80
- this.log.error(`Failure reaching L1 consensus host: ${res.statusText} (${res.status})`, {
115
+ this.log.info(`L1 consensus host is reachable but could not determine head slot`, {
81
116
  l1ConsensusHostUrl
82
117
  });
118
+ consensusNonSuperNodes++;
83
119
  }
84
120
  } catch (err) {
85
121
  this.log.error(`Error reaching L1 consensus host`, err, {
@@ -87,9 +123,8 @@ export class HttpBlobClient {
87
123
  });
88
124
  }
89
125
  }
90
- } else {
91
- this.log.warn('No L1 consensus host urls configured');
92
126
  }
127
+ this.superNodeHostIndexes = detectedSuperNodes;
93
128
  if (this.archiveClient) {
94
129
  try {
95
130
  const latest = await this.archiveClient.getLatestBlock();
@@ -97,14 +132,12 @@ export class HttpBlobClient {
97
132
  latest,
98
133
  archiveUrl
99
134
  });
100
- successfulSourceCount++;
135
+ archiveSources++;
101
136
  } catch (err) {
102
137
  this.log.error(`Error reaching archive client`, err, {
103
138
  archiveUrl
104
139
  });
105
140
  }
106
- } else {
107
- this.log.warn('No archive client configured');
108
141
  }
109
142
  if (this.fileStoreClients.length > 0) {
110
143
  for (const fileStoreClient of this.fileStoreClients){
@@ -114,7 +147,7 @@ export class HttpBlobClient {
114
147
  this.log.info(`FileStore is reachable`, {
115
148
  url: fileStoreClient.getBaseUrl()
116
149
  });
117
- successfulSourceCount++;
150
+ blobSinks++;
118
151
  } else {
119
152
  this.log.warn(`FileStore is not accessible`, {
120
153
  url: fileStoreClient.getBaseUrl()
@@ -127,12 +160,22 @@ export class HttpBlobClient {
127
160
  }
128
161
  }
129
162
  }
163
+ // Emit a single summary after validating all sources
164
+ const successfulSourceCount = consensusSuperNodes + archiveSources + blobSinks;
165
+ let summary = `Blob client running with consensusSuperNodes=${consensusSuperNodes} archiveSources=${archiveSources} blobSinks=${blobSinks}`;
166
+ if (consensusNonSuperNodes > 0) {
167
+ summary += `. ${consensusNonSuperNodes} consensus client(s) ignored because they are not running in supernode or semi-supernode mode`;
168
+ }
130
169
  if (successfulSourceCount === 0) {
131
170
  if (this.config.blobAllowEmptySources) {
132
- this.log.warn('No blob sources are reachable');
171
+ this.log.warn(summary);
133
172
  } else {
134
- throw new Error('No blob sources are reachable');
173
+ throw new Error(summary);
135
174
  }
175
+ } else if (consensusSuperNodes === 0) {
176
+ this.log.warn(summary);
177
+ } else {
178
+ this.log.info(summary);
136
179
  }
137
180
  }
138
181
  async sendBlobsToFilestore(blobs) {
@@ -154,35 +197,31 @@ export class HttpBlobClient {
154
197
  }
155
198
  }
156
199
  /**
157
- * Get the blob sidecar
158
- *
159
- * If requesting from the blob client, we send the blobkHash
160
- * If requesting from the beacon node, we send the slot number
200
+ * Get the blob sidecar.
161
201
  *
162
- * Source ordering depends on sync state:
163
- * - Historical sync: blob client FileStore L1 consensus Archive
164
- * - Near tip sync: blob client → FileStore → L1 consensus → FileStore (with retries) → Archive (eg blobscan)
202
+ * Alternates between two primary sources (consensus and filestore) in a retry loop,
203
+ * then falls back to archive if blobs are still missing. The order of the primary
204
+ * sources is configurable via `blobPreferFilestores`.
165
205
  *
166
206
  * @param blockHash - The block hash
167
207
  * @param blobHashes - The blob hashes to fetch
168
- * @param opts - Options including isHistoricalSync flag
208
+ * @param opts - Options for slot resolution
169
209
  * @returns The blobs
170
210
  */ async getBlobSidecar(blockHash, blobHashes, opts) {
171
211
  if (this.disabled) {
172
212
  this.log.warn('Blob storage is disabled, returning empty blob sidecar');
173
213
  return [];
174
214
  }
175
- const isHistoricalSync = opts?.isHistoricalSync ?? false;
176
215
  // Accumulate blobs across sources, preserving order and handling duplicates
177
216
  // resultBlobs[i] will contain the blob for blobHashes[i], or undefined if not yet found
178
217
  const resultBlobs = new Array(blobHashes.length).fill(undefined);
179
- // Helper to get missing blob hashes that we still need to fetch
218
+ // Helper to get missing blob hashes that we still need to fetch
180
219
  const getMissingBlobHashes = ()=>blobHashes.map((bh, i)=>resultBlobs[i] === undefined ? bh : undefined).filter((bh)=>bh !== undefined);
181
220
  // Return the result, ignoring any undefined ones
182
221
  const getFilledBlobs = ()=>resultBlobs.filter((b)=>b !== undefined);
183
222
  // Helper to fill in results from fetched blobs
184
- const fillResults = (fetchedBlobs)=>{
185
- const blobs = processFetchedBlobs(fetchedBlobs, blobHashes, this.log);
223
+ const fillResults = async (fetchedBlobs)=>{
224
+ const blobs = await processFetchedBlobs(fetchedBlobs, blobHashes, this.log);
186
225
  // Fill in any missing positions with matching blobs
187
226
  for(let i = 0; i < blobHashes.length; i++){
188
227
  if (resultBlobs[i] === undefined) {
@@ -198,86 +237,75 @@ export class HttpBlobClient {
198
237
  }
199
238
  return blobs;
200
239
  };
201
- const { l1ConsensusHostUrls } = this.config;
202
240
  const ctx = {
203
241
  blockHash,
204
242
  blobHashes: blobHashes.map(bufferToHex)
205
243
  };
206
- // Try filestore (quick, no retries) - useful for both historical and near-tip sync
207
- if (this.fileStoreClients.length > 0 && getMissingBlobHashes().length > 0) {
208
- await this.tryFileStores(getMissingBlobHashes, fillResults, ctx);
209
- if (getMissingBlobHashes().length === 0) {
210
- return returnWithCallback(getFilledBlobs());
244
+ // Lazily resolve the slot number only resolved when consensus hosts are actually tried.
245
+ let slotNumber;
246
+ let slotResolved = false;
247
+ const getSlotNumber = async ()=>{
248
+ if (!slotResolved) {
249
+ slotNumber = await this.resolveSlotNumber(blockHash, opts);
250
+ slotResolved = true;
211
251
  }
212
- }
213
- const missingAfterSink = getMissingBlobHashes();
214
- if (missingAfterSink.length > 0 && l1ConsensusHostUrls && l1ConsensusHostUrls.length > 0) {
215
- // The beacon api can query by slot number, so we get that first
216
- const consensusCtx = {
217
- l1ConsensusHostUrls,
218
- ...ctx
219
- };
220
- this.log.trace(`Attempting to get slot number for block hash`, consensusCtx);
221
- const slotNumber = await this.getSlotNumber(blockHash);
222
- this.log.debug(`Got slot number ${slotNumber} from consensus host for querying blobs`, consensusCtx);
223
- if (slotNumber) {
224
- let l1ConsensusHostUrl;
225
- for(let l1ConsensusHostIndex = 0; l1ConsensusHostIndex < l1ConsensusHostUrls.length; l1ConsensusHostIndex++){
226
- const missingHashes = getMissingBlobHashes();
227
- if (missingHashes.length === 0) {
228
- break;
229
- }
230
- l1ConsensusHostUrl = l1ConsensusHostUrls[l1ConsensusHostIndex];
231
- this.log.trace(`Attempting to get ${missingHashes.length} blobs from consensus host`, {
232
- slotNumber,
233
- l1ConsensusHostUrl,
234
- ...ctx
235
- });
236
- const blobs = await this.getBlobsFromHost(l1ConsensusHostUrl, slotNumber, l1ConsensusHostIndex);
237
- const result = fillResults(blobs);
238
- this.log.debug(`Got ${blobs.length} blobs from consensus host (total: ${result.length}/${blobHashes.length})`, {
239
- slotNumber,
240
- l1ConsensusHostUrl,
241
- ...ctx
242
- });
243
- if (result.length === blobHashes.length) {
244
- return returnWithCallback(result);
245
- }
252
+ return slotNumber;
253
+ };
254
+ // Build the two source-try functions. The order depends on the config.
255
+ const tryConsensus = ()=>this.tryConsensusHosts(getSlotNumber, getMissingBlobHashes, fillResults, ctx);
256
+ const tryFilestores = ()=>this.tryFileStores(getMissingBlobHashes, fillResults, ctx);
257
+ const preferFilestores = this.config.blobPreferFilestores ?? false;
258
+ const [trySourceA, trySourceB] = preferFilestores ? [
259
+ tryFilestores,
260
+ tryConsensus
261
+ ] : [
262
+ tryConsensus,
263
+ tryFilestores
264
+ ];
265
+ // Historical sync: blobs should already exist, use shorter backoff for transient errors.
266
+ // Near-tip sync: blobs may still be uploading, use longer backoff for eventual consistency.
267
+ const isHistoricalSync = opts?.isHistoricalSync ?? false;
268
+ const backoff = isHistoricalSync ? [
269
+ 1,
270
+ 1
271
+ ] : [
272
+ 1,
273
+ 1,
274
+ 1,
275
+ 2,
276
+ 2
277
+ ];
278
+ // Retry loop: alternate between the two primary sources with backoff.
279
+ try {
280
+ await retry(async ()=>{
281
+ if (getMissingBlobHashes().length > 0) {
282
+ await trySourceA();
246
283
  }
247
- }
248
- }
249
- // For near-tip sync, retry filestores with backoff (eventual consistency)
250
- // This handles the case where blobs are still being uploaded by other validators
251
- if (!isHistoricalSync && this.fileStoreClients.length > 0 && getMissingBlobHashes().length > 0) {
252
- try {
253
- await retry(async ()=>{
254
- await this.tryFileStores(getMissingBlobHashes, fillResults, ctx);
255
- if (getMissingBlobHashes().length > 0) {
256
- throw new Error('Still missing blobs from filestores');
257
- }
258
- }, 'filestore blob retrieval', makeBackoff([
259
- 1,
260
- 1,
261
- 2
262
- ]), this.log, true);
263
- return returnWithCallback(getFilledBlobs());
264
- } catch {
265
- // Exhausted retries, continue to archive fallback
266
- }
284
+ if (getMissingBlobHashes().length > 0) {
285
+ await trySourceB();
286
+ }
287
+ if (getMissingBlobHashes().length > 0) {
288
+ throw new Error('Still missing blobs after trying all primary sources');
289
+ }
290
+ }, 'blob retrieval', makeBackoff(backoff), this.log, true);
291
+ return returnWithCallback(getFilledBlobs());
292
+ } catch {
293
+ // Exhausted retries, continue to archive fallback
267
294
  }
268
- const missingAfterConsensus = getMissingBlobHashes();
269
- if (missingAfterConsensus.length > 0 && this.archiveClient) {
295
+ // Archive fallback
296
+ const missingAfterPrimary = getMissingBlobHashes();
297
+ if (missingAfterPrimary.length > 0 && this.archiveClient) {
270
298
  const archiveCtx = {
271
299
  archiveUrl: this.archiveClient.getBaseUrl(),
272
300
  ...ctx
273
301
  };
274
- this.log.trace(`Attempting to get ${missingAfterConsensus.length} blobs from archive`, archiveCtx);
302
+ this.log.trace(`Attempting to get ${missingAfterPrimary.length} blobs from archive`, archiveCtx);
275
303
  const allBlobs = await this.archiveClient.getBlobsFromBlock(blockHash);
276
304
  if (!allBlobs) {
277
305
  this.log.debug('No blobs found from archive client', archiveCtx);
278
306
  } else {
279
307
  this.log.trace(`Got ${allBlobs.length} blobs from archive client before filtering`, archiveCtx);
280
- const result = fillResults(allBlobs);
308
+ const result = await fillResults(allBlobs);
281
309
  this.log.debug(`Got ${allBlobs.length} blobs from archive client (total: ${result.length}/${blobHashes.length})`, archiveCtx);
282
310
  if (result.length === blobHashes.length) {
283
311
  return returnWithCallback(result);
@@ -287,13 +315,63 @@ export class HttpBlobClient {
287
315
  const result = getFilledBlobs();
288
316
  if (result.length < blobHashes.length) {
289
317
  this.log.warn(`Failed to fetch all blobs for ${blockHash} from all blob sources (got ${result.length}/${blobHashes.length})`, {
290
- l1ConsensusHostUrls,
318
+ l1ConsensusHostUrls: this.config.l1ConsensusHostUrls,
291
319
  archiveUrl: this.archiveClient?.getBaseUrl(),
292
320
  fileStoreUrls: this.fileStoreClients.map((c)=>c.getBaseUrl())
293
321
  });
294
322
  }
295
323
  return returnWithCallback(result);
296
324
  }
325
+ /** Resolves the beacon slot number for the given block hash. Returns undefined if no consensus hosts. */ resolveSlotNumber(blockHash, opts) {
326
+ const { l1ConsensusHostUrls } = this.config;
327
+ if (!l1ConsensusHostUrls || l1ConsensusHostUrls.length === 0) {
328
+ return undefined;
329
+ }
330
+ // If no supernodes, no point resolving the slot
331
+ if (this.superNodeHostIndexes && this.superNodeHostIndexes.size === 0) {
332
+ return undefined;
333
+ }
334
+ return this.getSlotNumber(blockHash, opts?.parentBeaconBlockRoot, opts?.l1BlockTimestamp);
335
+ }
336
+ /**
337
+ * Try all supernode consensus hosts for blob sidecars.
338
+ * Skips hosts that were detected as non-supernodes during testSources().
339
+ */ async tryConsensusHosts(getSlotNumber, getMissingBlobHashes, fillResults, ctx) {
340
+ const { l1ConsensusHostUrls } = this.config;
341
+ if (!l1ConsensusHostUrls || l1ConsensusHostUrls.length === 0) {
342
+ return;
343
+ }
344
+ const slotNumber = await getSlotNumber();
345
+ if (!slotNumber) {
346
+ return;
347
+ }
348
+ for(let l1ConsensusHostIndex = 0; l1ConsensusHostIndex < l1ConsensusHostUrls.length; l1ConsensusHostIndex++){
349
+ const missingHashes = getMissingBlobHashes();
350
+ if (missingHashes.length === 0) {
351
+ break;
352
+ }
353
+ // Skip non-supernode hosts if we've already detected supernodes
354
+ if (this.superNodeHostIndexes && !this.superNodeHostIndexes.has(l1ConsensusHostIndex)) {
355
+ this.log.trace(`Skipping non-supernode consensus host`, {
356
+ l1ConsensusHostUrl: l1ConsensusHostUrls[l1ConsensusHostIndex]
357
+ });
358
+ continue;
359
+ }
360
+ const l1ConsensusHostUrl = l1ConsensusHostUrls[l1ConsensusHostIndex];
361
+ this.log.trace(`Attempting to get ${missingHashes.length} blobs from consensus host`, {
362
+ slotNumber,
363
+ l1ConsensusHostUrl,
364
+ ...ctx
365
+ });
366
+ const blobs = await this.getBlobsFromHost(l1ConsensusHostUrl, slotNumber, l1ConsensusHostIndex, missingHashes);
367
+ const result = await fillResults(blobs);
368
+ this.log.debug(`Got ${blobs.length} blobs from consensus host (total: ${result.length}/${ctx.blobHashes.length})`, {
369
+ slotNumber,
370
+ l1ConsensusHostUrl,
371
+ ...ctx
372
+ });
373
+ }
374
+ }
297
375
  /**
298
376
  * Try all filestores once (shuffled for load distribution).
299
377
  * @param getMissingBlobHashes - Function to get remaining blob hashes to fetch
@@ -318,7 +396,7 @@ export class HttpBlobClient {
318
396
  });
319
397
  const blobs = await client.getBlobsByHashes(blobHashStrings);
320
398
  if (blobs.length > 0) {
321
- const result = fillResults(blobs);
399
+ const result = await fillResults(blobs);
322
400
  this.log.debug(`Got ${blobs.length} blobs from filestore (total: ${result.length}/${ctx.blobHashes.length})`, {
323
401
  url: client.getBaseUrl(),
324
402
  ...ctx
@@ -332,14 +410,14 @@ export class HttpBlobClient {
332
410
  }
333
411
  }
334
412
  async getBlobSidecarFrom(hostUrl, blockHashOrSlot, blobHashes = [], l1ConsensusHostIndex) {
335
- const blobs = await this.getBlobsFromHost(hostUrl, blockHashOrSlot, l1ConsensusHostIndex);
336
- return processFetchedBlobs(blobs, blobHashes, this.log).filter((b)=>b !== undefined);
413
+ const blobs = await this.getBlobsFromHost(hostUrl, blockHashOrSlot, l1ConsensusHostIndex, blobHashes);
414
+ return (await processFetchedBlobs(blobs, blobHashes, this.log)).filter((b)=>b !== undefined);
337
415
  }
338
- async getBlobsFromHost(hostUrl, blockHashOrSlot, l1ConsensusHostIndex) {
416
+ async getBlobsFromHost(hostUrl, blockHashOrSlot, l1ConsensusHostIndex, blobHashes) {
339
417
  try {
340
- let res = await this.fetchBlobSidecars(hostUrl, blockHashOrSlot, l1ConsensusHostIndex);
418
+ let res = await this.fetchBlobSidecars(hostUrl, blockHashOrSlot, l1ConsensusHostIndex, blobHashes);
341
419
  if (res.ok) {
342
- return parseBlobJsonsFromResponse(await res.json(), this.log);
420
+ return await parseBlobJsonsFromResponse(await res.json(), this.log);
343
421
  }
344
422
  if (res.status === 404 && typeof blockHashOrSlot === 'number') {
345
423
  const latestSlot = await this.getLatestSlotNumber(hostUrl, l1ConsensusHostIndex);
@@ -352,9 +430,9 @@ export class HttpBlobClient {
352
430
  let currentSlot = blockHashOrSlot + 1;
353
431
  while(res.status === 404 && maxRetries > 0 && latestSlot !== undefined && currentSlot <= latestSlot){
354
432
  this.log.debug(`Trying slot ${currentSlot}`);
355
- res = await this.fetchBlobSidecars(hostUrl, currentSlot, l1ConsensusHostIndex);
433
+ res = await this.fetchBlobSidecars(hostUrl, currentSlot, l1ConsensusHostIndex, blobHashes);
356
434
  if (res.ok) {
357
- return parseBlobJsonsFromResponse(await res.json(), this.log);
435
+ return await parseBlobJsonsFromResponse(await res.json(), this.log);
358
436
  }
359
437
  currentSlot++;
360
438
  maxRetries--;
@@ -371,21 +449,29 @@ export class HttpBlobClient {
371
449
  return [];
372
450
  }
373
451
  }
374
- fetchBlobSidecars(hostUrl, blockHashOrSlot, l1ConsensusHostIndex) {
375
- const baseUrl = `${hostUrl}/eth/v1/beacon/blob_sidecars/${blockHashOrSlot}`;
376
- const { url, ...options } = getBeaconNodeFetchOptions(baseUrl, this.config, l1ConsensusHostIndex);
452
+ fetchBlobSidecars(hostUrl, blockHashOrSlot, l1ConsensusHostIndex, blobHashes) {
453
+ let baseUrl = `${hostUrl}/eth/v1/beacon/blobs/${blockHashOrSlot}`;
454
+ if (blobHashes && blobHashes.length > 0) {
455
+ const params = new URLSearchParams();
456
+ for (const hash of blobHashes){
457
+ params.append('versioned_hashes', `0x${hash.toString('hex')}`);
458
+ }
459
+ baseUrl += `?${params.toString()}`;
460
+ }
461
+ const { url, logSafeUrl, ...options } = getBeaconNodeFetchOptions(baseUrl, this.config, l1ConsensusHostIndex);
377
462
  this.log.debug(`Fetching blob sidecar for ${blockHashOrSlot}`, {
378
- url,
463
+ url: logSafeUrl,
379
464
  ...options
380
465
  });
381
- return this.fetch(url, options);
466
+ // No retry here — this is called inside the main retry loop in getBlobSidecar
467
+ return fetch(url, options);
382
468
  }
383
469
  async getLatestSlotNumber(hostUrl, l1ConsensusHostIndex) {
384
470
  try {
385
471
  const baseUrl = `${hostUrl}/eth/v1/beacon/headers/head`;
386
- const { url, ...options } = getBeaconNodeFetchOptions(baseUrl, this.config, l1ConsensusHostIndex);
472
+ const { url, logSafeUrl, ...options } = getBeaconNodeFetchOptions(baseUrl, this.config, l1ConsensusHostIndex);
387
473
  this.log.debug(`Fetching latest slot number`, {
388
- url,
474
+ url: logSafeUrl,
389
475
  ...options
390
476
  });
391
477
  const res = await this.fetch(url, options);
@@ -418,36 +504,45 @@ export class HttpBlobClient {
418
504
  *
419
505
  * @param blockHash - The block hash
420
506
  * @returns The slot number
421
- */ async getSlotNumber(blockHash) {
507
+ */ async getSlotNumber(blockHash, parentBeaconBlockRoot, l1BlockTimestamp) {
422
508
  const { l1ConsensusHostUrls, l1RpcUrls } = this.config;
423
509
  if (!l1ConsensusHostUrls || l1ConsensusHostUrls.length === 0) {
424
510
  this.log.debug('No consensus host url configured');
425
511
  return undefined;
426
512
  }
427
- if (!l1RpcUrls || l1RpcUrls.length === 0) {
428
- this.log.debug('No execution host url configured');
429
- return undefined;
513
+ // Primary path: compute slot from timestamp if genesis config is cached (no network call needed)
514
+ if (l1BlockTimestamp !== undefined && this.beaconGenesisTime !== undefined && this.beaconSecondsPerSlot !== undefined) {
515
+ const slot = Number((l1BlockTimestamp - this.beaconGenesisTime) / BigInt(this.beaconSecondsPerSlot));
516
+ this.log.debug(`Computed slot ${slot} from L1 block timestamp`, {
517
+ l1BlockTimestamp
518
+ });
519
+ return slot;
430
520
  }
431
- // Ping execution node to get the parentBeaconBlockRoot for this block
432
- let parentBeaconBlockRoot;
433
- const client = createPublicClient({
434
- transport: fallback(l1RpcUrls.map((url)=>http(url, {
435
- batch: false
436
- })))
437
- });
438
- try {
439
- const res = await client.request({
440
- method: 'eth_getBlockByHash',
441
- params: [
442
- blockHash,
443
- /*tx flag*/ false
444
- ]
521
+ if (!parentBeaconBlockRoot) {
522
+ // parentBeaconBlockRoot not provided by caller — fetch it from the execution RPC
523
+ if (!l1RpcUrls || l1RpcUrls.length === 0) {
524
+ this.log.debug('No execution host url configured');
525
+ return undefined;
526
+ }
527
+ const client = createPublicClient({
528
+ transport: makeL1HttpTransport(l1RpcUrls, {
529
+ timeout: this.config.l1HttpTimeoutMS
530
+ })
445
531
  });
446
- if (res.parentBeaconBlockRoot) {
447
- parentBeaconBlockRoot = res.parentBeaconBlockRoot;
532
+ try {
533
+ const res = await client.request({
534
+ method: 'eth_getBlockByHash',
535
+ params: [
536
+ blockHash,
537
+ /*tx flag*/ false
538
+ ]
539
+ });
540
+ if (res.parentBeaconBlockRoot) {
541
+ parentBeaconBlockRoot = res.parentBeaconBlockRoot;
542
+ }
543
+ } catch (err) {
544
+ this.log.error(`Error getting parent beacon block root`, err);
448
545
  }
449
- } catch (err) {
450
- this.log.error(`Error getting parent beacon block root`, err);
451
546
  }
452
547
  if (!parentBeaconBlockRoot) {
453
548
  this.log.error(`No parent beacon block root found for block ${blockHash}`);
@@ -474,11 +569,78 @@ export class HttpBlobClient {
474
569
  /** @internal - exposed for testing */ getArchiveClient() {
475
570
  return this.archiveClient;
476
571
  }
572
+ /** Returns true if this client can upload blobs to filestore. */ canUpload() {
573
+ return this.fileStoreUploadClient !== undefined;
574
+ }
575
+ /**
576
+ * Start the blob client.
577
+ * Fetches and caches beacon genesis config for timestamp-based slot resolution,
578
+ * then uploads the initial healthcheck file (awaited) and starts periodic uploads.
579
+ */ async start() {
580
+ await this.fetchBeaconConfig();
581
+ if (!this.fileStoreUploadClient) {
582
+ return;
583
+ }
584
+ await this.fileStoreUploadClient.uploadHealthcheck();
585
+ this.log.debug('Initial healthcheck file uploaded');
586
+ this.startPeriodicHealthcheckUpload();
587
+ }
588
+ /**
589
+ * Start periodic healthcheck upload to the file store to ensure it remains available even if accidentally deleted.
590
+ */ startPeriodicHealthcheckUpload() {
591
+ const intervalMs = (this.config.blobHealthcheckUploadIntervalMinutes ?? DEFAULT_HEALTHCHECK_UPLOAD_INTERVAL_MINUTES) * 60 * 1000;
592
+ this.healthcheckUploadIntervalId = setInterval(()=>{
593
+ void this.fileStoreUploadClient.uploadHealthcheck().catch((err)=>{
594
+ this.log.warn('Failed to upload periodic healthcheck file', err);
595
+ });
596
+ }, intervalMs);
597
+ }
598
+ /**
599
+ * Fetches and caches beacon genesis time and slot duration from the first available consensus host.
600
+ * These static values enable timestamp-based slot resolution, eliminating the per-fetch headers call.
601
+ * Logs a warning and leaves fields undefined if all hosts fail, callers fall back gracefully.
602
+ */ async fetchBeaconConfig() {
603
+ const { l1ConsensusHostUrls } = this.config;
604
+ if (!l1ConsensusHostUrls || l1ConsensusHostUrls.length === 0) {
605
+ return;
606
+ }
607
+ for(let i = 0; i < l1ConsensusHostUrls.length; i++){
608
+ try {
609
+ const { url: genesisUrl, ...genesisOptions } = getBeaconNodeFetchOptions(`${l1ConsensusHostUrls[i]}/eth/v1/config/genesis`, this.config, i);
610
+ const { url: specUrl, ...specOptions } = getBeaconNodeFetchOptions(`${l1ConsensusHostUrls[i]}/eth/v1/config/spec`, this.config, i);
611
+ const [genesisRes, specRes] = await Promise.all([
612
+ this.fetch(genesisUrl, genesisOptions),
613
+ this.fetch(specUrl, specOptions)
614
+ ]);
615
+ if (genesisRes.ok && specRes.ok) {
616
+ const genesis = await genesisRes.json();
617
+ const spec = await specRes.json();
618
+ this.beaconGenesisTime = BigInt(genesis.data.genesisTime);
619
+ this.beaconSecondsPerSlot = parseInt(spec.data.secondsPerSlot);
620
+ this.log.debug(`Fetched beacon genesis config`, {
621
+ genesisTime: this.beaconGenesisTime,
622
+ secondsPerSlot: this.beaconSecondsPerSlot
623
+ });
624
+ return;
625
+ }
626
+ } catch (err) {
627
+ this.log.warn(`Failed to fetch beacon config from host ${l1ConsensusHostUrls[i]}`, err);
628
+ }
629
+ }
630
+ this.log.warn('Could not fetch beacon genesis config from any consensus host — will use headers call fallback');
631
+ }
632
+ /**
633
+ * Stop the blob client, clearing any periodic tasks.
634
+ */ stop() {
635
+ if (this.healthcheckUploadIntervalId) {
636
+ clearInterval(this.healthcheckUploadIntervalId);
637
+ this.healthcheckUploadIntervalId = undefined;
638
+ }
639
+ }
477
640
  }
478
- function parseBlobJsonsFromResponse(response, logger) {
641
+ async function parseBlobJsonsFromResponse(response, logger) {
479
642
  try {
480
- const blobs = response.data.map(parseBlobJson);
481
- return blobs;
643
+ return await Promise.all(response.data.map(parseBlobJson));
482
644
  } catch (err) {
483
645
  logger.error(`Error parsing blob json from response`, err);
484
646
  return [];
@@ -488,15 +650,14 @@ function parseBlobJsonsFromResponse(response, logger) {
488
650
  // https://ethereum.github.io/beacon-APIs/?urls.primaryName=dev#/Beacon/getBlobSidecars
489
651
  // Here we attempt to parse the response data to Buffer, and check the lengths (via Blob's constructor), to avoid
490
652
  // throwing an error down the line when calling Blob.fromJson().
491
- function parseBlobJson(data) {
492
- const blobBuffer = Buffer.from(data.blob.slice(2), 'hex');
493
- const commitmentBuffer = Buffer.from(data.kzg_commitment.slice(2), 'hex');
494
- const blob = new Blob(blobBuffer, commitmentBuffer);
653
+ async function parseBlobJson(rawHex) {
654
+ const blobBuffer = Buffer.from(rawHex.slice(2), 'hex');
655
+ const blob = await Blob.fromBlobBuffer(blobBuffer);
495
656
  return blob.toJSON();
496
657
  }
497
658
  // Returns an array that maps each blob hash to the corresponding blob, or undefined if the blob is not found
498
659
  // or the data does not match the commitment.
499
- function processFetchedBlobs(blobs, blobHashes, logger) {
660
+ async function processFetchedBlobs(blobs, blobHashes, logger) {
500
661
  const requestedBlobHashes = new Set(blobHashes.map(bufferToHex));
501
662
  const hashToBlob = new Map();
502
663
  for (const blobJson of blobs){
@@ -505,7 +666,7 @@ function processFetchedBlobs(blobs, blobHashes, logger) {
505
666
  continue;
506
667
  }
507
668
  try {
508
- const blob = Blob.fromJson(blobJson);
669
+ const blob = await Blob.fromJson(blobJson);
509
670
  hashToBlob.set(hashHex, blob);
510
671
  } catch (err) {
511
672
  // If the above throws, it's likely that the blob commitment does not match the hash of the blob data.
@@ -519,11 +680,15 @@ function getBeaconNodeFetchOptions(url, config, l1ConsensusHostIndex) {
519
680
  const l1ConsensusHostApiKey = l1ConsensusHostIndex !== undefined && l1ConsensusHostApiKeys && l1ConsensusHostApiKeys[l1ConsensusHostIndex];
520
681
  const l1ConsensusHostApiKeyHeader = l1ConsensusHostIndex !== undefined && l1ConsensusHostApiKeyHeaders && l1ConsensusHostApiKeyHeaders[l1ConsensusHostIndex];
521
682
  let formattedUrl = url;
683
+ let logSafeUrl = url;
522
684
  if (l1ConsensusHostApiKey && l1ConsensusHostApiKey.getValue() !== '' && !l1ConsensusHostApiKeyHeader) {
523
- formattedUrl += `${formattedUrl.includes('?') ? '&' : '?'}key=${l1ConsensusHostApiKey.getValue()}`;
685
+ const separator = formattedUrl.includes('?') ? '&' : '?';
686
+ formattedUrl += `${separator}key=${l1ConsensusHostApiKey.getValue()}`;
687
+ logSafeUrl += `${separator}key=[REDACTED]`;
524
688
  }
525
689
  return {
526
690
  url: formattedUrl,
691
+ logSafeUrl,
527
692
  ...l1ConsensusHostApiKey && l1ConsensusHostApiKeyHeader && {
528
693
  headers: {
529
694
  [l1ConsensusHostApiKeyHeader]: l1ConsensusHostApiKey.getValue()