@aztec/blob-client 0.0.1-commit.9ef841308 → 0.0.1-commit.a89ec08
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dest/client/config.d.ts +1 -5
- package/dest/client/config.d.ts.map +1 -1
- package/dest/client/config.js +0 -10
- package/dest/client/factory.d.ts +1 -1
- package/dest/client/factory.d.ts.map +1 -1
- package/dest/client/factory.js +1 -7
- package/dest/client/http.d.ts +11 -19
- package/dest/client/http.d.ts.map +1 -1
- package/dest/client/http.js +114 -206
- package/dest/client/interface.d.ts +4 -13
- package/dest/client/interface.d.ts.map +1 -1
- package/dest/filestore/factory.d.ts +4 -5
- package/dest/filestore/factory.d.ts.map +1 -1
- package/dest/filestore/factory.js +4 -4
- package/package.json +7 -7
- package/src/client/config.ts +0 -16
- package/src/client/factory.ts +1 -8
- package/src/client/http.ts +108 -240
- package/src/client/interface.ts +3 -12
- package/src/filestore/factory.ts +2 -7
package/src/client/config.ts
CHANGED
|
@@ -59,12 +59,6 @@ export interface BlobClientConfig extends BlobArchiveApiConfig {
|
|
|
59
59
|
|
|
60
60
|
/** Timeout for HTTP requests to the L1 RPC node in ms. */
|
|
61
61
|
l1HttpTimeoutMS?: number;
|
|
62
|
-
|
|
63
|
-
/** Whether to prefer filestores over consensus clients when fetching blobs. Default: false (consensus first). */
|
|
64
|
-
blobPreferFilestores?: boolean;
|
|
65
|
-
|
|
66
|
-
/** Timeout in ms for HTTP requests to the blob file store. Default: 10000 (10s). */
|
|
67
|
-
blobFileStoreTimeoutMs?: number;
|
|
68
62
|
}
|
|
69
63
|
|
|
70
64
|
export const blobClientConfigMapping: ConfigMappingsType<BlobClientConfig> = {
|
|
@@ -123,16 +117,6 @@ export const blobClientConfigMapping: ConfigMappingsType<BlobClientConfig> = {
|
|
|
123
117
|
description: 'Timeout for HTTP requests to the L1 RPC node in ms.',
|
|
124
118
|
...optionalNumberConfigHelper(),
|
|
125
119
|
},
|
|
126
|
-
blobPreferFilestores: {
|
|
127
|
-
env: 'BLOB_PREFER_FILESTORES',
|
|
128
|
-
description: 'Whether to prefer filestores over consensus clients when fetching blobs. Default: false.',
|
|
129
|
-
...booleanConfigHelper(false),
|
|
130
|
-
},
|
|
131
|
-
blobFileStoreTimeoutMs: {
|
|
132
|
-
env: 'BLOB_FILE_STORE_TIMEOUT_MS',
|
|
133
|
-
description: 'Timeout in ms for HTTP requests to the blob file store. Default: 10000 (10s).',
|
|
134
|
-
...optionalNumberConfigHelper(),
|
|
135
|
-
},
|
|
136
120
|
...blobArchiveApiConfigMappings,
|
|
137
121
|
};
|
|
138
122
|
|
package/src/client/factory.ts
CHANGED
|
@@ -76,15 +76,8 @@ export async function createBlobClientWithFileStores(
|
|
|
76
76
|
rollupAddress: config.l1Contracts.rollupAddress.toString(),
|
|
77
77
|
};
|
|
78
78
|
|
|
79
|
-
// Disable internal retries for blob file stores — retry logic is handled by HttpBlobClient.
|
|
80
|
-
// Set a configurable timeout (default 10s) to avoid hanging on slow stores.
|
|
81
|
-
const httpOptions = {
|
|
82
|
-
retryBackoff: [] as number[],
|
|
83
|
-
timeoutMs: config.blobFileStoreTimeoutMs ?? 10_000,
|
|
84
|
-
};
|
|
85
|
-
|
|
86
79
|
const [fileStoreClients, fileStoreUploadClient] = await Promise.all([
|
|
87
|
-
createReadOnlyFileStoreBlobClients(config.blobFileStoreUrls, fileStoreMetadata, log
|
|
80
|
+
createReadOnlyFileStoreBlobClients(config.blobFileStoreUrls, fileStoreMetadata, log),
|
|
88
81
|
createWritableFileStoreBlobClient(config.blobFileStoreUploadUrl, fileStoreMetadata, log),
|
|
89
82
|
]);
|
|
90
83
|
|
package/src/client/http.ts
CHANGED
|
@@ -25,14 +25,6 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
25
25
|
private disabled = false;
|
|
26
26
|
private healthcheckUploadIntervalId?: NodeJS.Timeout;
|
|
27
27
|
|
|
28
|
-
/** Cached beacon genesis time (seconds since Unix epoch). Fetched once at startup. */
|
|
29
|
-
private beaconGenesisTime?: bigint;
|
|
30
|
-
/** Cached beacon slot duration in seconds. Fetched once at startup. */
|
|
31
|
-
private beaconSecondsPerSlot?: number;
|
|
32
|
-
|
|
33
|
-
/** Indexes of consensus hosts that serve blob sidecars (supernodes). Populated by testSources(). */
|
|
34
|
-
private superNodeHostIndexes?: Set<number>;
|
|
35
|
-
|
|
36
28
|
constructor(
|
|
37
29
|
config?: BlobClientConfig,
|
|
38
30
|
private readonly opts: {
|
|
@@ -103,8 +95,6 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
103
95
|
let archiveSources = 0;
|
|
104
96
|
let blobSinks = 0;
|
|
105
97
|
|
|
106
|
-
const detectedSuperNodes = new Set<number>();
|
|
107
|
-
|
|
108
98
|
if (l1ConsensusHostUrls && l1ConsensusHostUrls.length > 0) {
|
|
109
99
|
for (let l1ConsensusHostIndex = 0; l1ConsensusHostIndex < l1ConsensusHostUrls.length; l1ConsensusHostIndex++) {
|
|
110
100
|
const l1ConsensusHostUrl = l1ConsensusHostUrls[l1ConsensusHostIndex];
|
|
@@ -139,12 +129,9 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
139
129
|
const blobRes = await this.fetch(blobUrl, blobOptions);
|
|
140
130
|
if (blobRes.ok) {
|
|
141
131
|
this.log.info(`L1 consensus host serves blob sidecars (supernode)`, { l1ConsensusHostUrl });
|
|
142
|
-
detectedSuperNodes.add(l1ConsensusHostIndex);
|
|
143
132
|
consensusSuperNodes++;
|
|
144
133
|
} else {
|
|
145
|
-
this.log.info(`L1 consensus host does not serve blob sidecars
|
|
146
|
-
l1ConsensusHostUrl,
|
|
147
|
-
});
|
|
134
|
+
this.log.info(`L1 consensus host does not serve blob sidecars`, { l1ConsensusHostUrl });
|
|
148
135
|
consensusNonSuperNodes++;
|
|
149
136
|
}
|
|
150
137
|
} else {
|
|
@@ -157,8 +144,6 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
157
144
|
}
|
|
158
145
|
}
|
|
159
146
|
|
|
160
|
-
this.superNodeHostIndexes = detectedSuperNodes;
|
|
161
|
-
|
|
162
147
|
if (this.archiveClient) {
|
|
163
148
|
try {
|
|
164
149
|
const latest = await this.archiveClient.getLatestBlock();
|
|
@@ -228,15 +213,18 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
228
213
|
}
|
|
229
214
|
|
|
230
215
|
/**
|
|
231
|
-
* Get the blob sidecar
|
|
216
|
+
* Get the blob sidecar
|
|
232
217
|
*
|
|
233
|
-
*
|
|
234
|
-
*
|
|
235
|
-
*
|
|
218
|
+
* If requesting from the blob client, we send the blobkHash
|
|
219
|
+
* If requesting from the beacon node, we send the slot number
|
|
220
|
+
*
|
|
221
|
+
* Source ordering depends on sync state:
|
|
222
|
+
* - Historical sync: blob client → FileStore → L1 consensus → Archive
|
|
223
|
+
* - Near tip sync: blob client → FileStore → L1 consensus → FileStore (with retries) → Archive (eg blobscan)
|
|
236
224
|
*
|
|
237
225
|
* @param blockHash - The block hash
|
|
238
226
|
* @param blobHashes - The blob hashes to fetch
|
|
239
|
-
* @param opts - Options
|
|
227
|
+
* @param opts - Options including isHistoricalSync flag
|
|
240
228
|
* @returns The blobs
|
|
241
229
|
*/
|
|
242
230
|
public async getBlobSidecar(
|
|
@@ -249,11 +237,12 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
249
237
|
return [];
|
|
250
238
|
}
|
|
251
239
|
|
|
240
|
+
const isHistoricalSync = opts?.isHistoricalSync ?? false;
|
|
252
241
|
// Accumulate blobs across sources, preserving order and handling duplicates
|
|
253
242
|
// resultBlobs[i] will contain the blob for blobHashes[i], or undefined if not yet found
|
|
254
243
|
const resultBlobs: (Blob | undefined)[] = new Array(blobHashes.length).fill(undefined);
|
|
255
244
|
|
|
256
|
-
// Helper to get
|
|
245
|
+
// Helper to get missing blob hashes that we still need to fetch
|
|
257
246
|
const getMissingBlobHashes = (): Buffer[] =>
|
|
258
247
|
blobHashes
|
|
259
248
|
.map((bh, i) => (resultBlobs[i] === undefined ? bh : undefined))
|
|
@@ -282,60 +271,79 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
282
271
|
return blobs;
|
|
283
272
|
};
|
|
284
273
|
|
|
274
|
+
const { l1ConsensusHostUrls } = this.config;
|
|
275
|
+
|
|
285
276
|
const ctx = { blockHash, blobHashes: blobHashes.map(bufferToHex) };
|
|
286
277
|
|
|
287
|
-
//
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
slotNumber = await this.resolveSlotNumber(blockHash, opts);
|
|
293
|
-
slotResolved = true;
|
|
278
|
+
// Try filestore (quick, no retries) - useful for both historical and near-tip sync
|
|
279
|
+
if (this.fileStoreClients.length > 0 && getMissingBlobHashes().length > 0) {
|
|
280
|
+
await this.tryFileStores(getMissingBlobHashes, fillResults, ctx);
|
|
281
|
+
if (getMissingBlobHashes().length === 0) {
|
|
282
|
+
return returnWithCallback(getFilledBlobs());
|
|
294
283
|
}
|
|
295
|
-
|
|
296
|
-
};
|
|
297
|
-
|
|
298
|
-
// Build the two source-try functions. The order depends on the config.
|
|
299
|
-
const tryConsensus = () => this.tryConsensusHosts(getSlotNumber, getMissingBlobHashes, fillResults, ctx);
|
|
300
|
-
const tryFilestores = () => this.tryFileStores(getMissingBlobHashes, fillResults, ctx);
|
|
301
|
-
|
|
302
|
-
const preferFilestores = this.config.blobPreferFilestores ?? false;
|
|
303
|
-
const [trySourceA, trySourceB] = preferFilestores ? [tryFilestores, tryConsensus] : [tryConsensus, tryFilestores];
|
|
304
|
-
|
|
305
|
-
// Historical sync: blobs should already exist, use shorter backoff for transient errors.
|
|
306
|
-
// Near-tip sync: blobs may still be uploading, use longer backoff for eventual consistency.
|
|
307
|
-
const isHistoricalSync = opts?.isHistoricalSync ?? false;
|
|
308
|
-
const backoff = isHistoricalSync ? [1, 1] : [1, 1, 1, 2, 2];
|
|
284
|
+
}
|
|
309
285
|
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
286
|
+
const missingAfterSink = getMissingBlobHashes();
|
|
287
|
+
if (missingAfterSink.length > 0 && l1ConsensusHostUrls && l1ConsensusHostUrls.length > 0) {
|
|
288
|
+
// The beacon api can query by slot number, so we get that first
|
|
289
|
+
const consensusCtx = { l1ConsensusHostUrls, ...ctx };
|
|
290
|
+
this.log.trace(`Attempting to get slot number for block hash`, consensusCtx);
|
|
291
|
+
const slotNumber = await this.getSlotNumber(blockHash);
|
|
292
|
+
this.log.debug(`Got slot number ${slotNumber} from consensus host for querying blobs`, consensusCtx);
|
|
293
|
+
|
|
294
|
+
if (slotNumber) {
|
|
295
|
+
let l1ConsensusHostUrl: string;
|
|
296
|
+
for (let l1ConsensusHostIndex = 0; l1ConsensusHostIndex < l1ConsensusHostUrls.length; l1ConsensusHostIndex++) {
|
|
297
|
+
const missingHashes = getMissingBlobHashes();
|
|
298
|
+
if (missingHashes.length === 0) {
|
|
299
|
+
break;
|
|
316
300
|
}
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
}
|
|
320
|
-
|
|
321
|
-
|
|
301
|
+
|
|
302
|
+
l1ConsensusHostUrl = l1ConsensusHostUrls[l1ConsensusHostIndex];
|
|
303
|
+
this.log.trace(`Attempting to get ${missingHashes.length} blobs from consensus host`, {
|
|
304
|
+
slotNumber,
|
|
305
|
+
l1ConsensusHostUrl,
|
|
306
|
+
...ctx,
|
|
307
|
+
});
|
|
308
|
+
const blobs = await this.getBlobsFromHost(l1ConsensusHostUrl, slotNumber, l1ConsensusHostIndex);
|
|
309
|
+
const result = await fillResults(blobs);
|
|
310
|
+
this.log.debug(
|
|
311
|
+
`Got ${blobs.length} blobs from consensus host (total: ${result.length}/${blobHashes.length})`,
|
|
312
|
+
{ slotNumber, l1ConsensusHostUrl, ...ctx },
|
|
313
|
+
);
|
|
314
|
+
if (result.length === blobHashes.length) {
|
|
315
|
+
return returnWithCallback(result);
|
|
322
316
|
}
|
|
323
|
-
}
|
|
324
|
-
|
|
325
|
-
makeBackoff(backoff),
|
|
326
|
-
this.log,
|
|
327
|
-
true, // failSilently — expected during eventual consistency
|
|
328
|
-
);
|
|
329
|
-
return returnWithCallback(getFilledBlobs());
|
|
330
|
-
} catch {
|
|
331
|
-
// Exhausted retries, continue to archive fallback
|
|
317
|
+
}
|
|
318
|
+
}
|
|
332
319
|
}
|
|
333
320
|
|
|
334
|
-
//
|
|
335
|
-
|
|
336
|
-
if (
|
|
321
|
+
// For near-tip sync, retry filestores with backoff (eventual consistency)
|
|
322
|
+
// This handles the case where blobs are still being uploaded by other validators
|
|
323
|
+
if (!isHistoricalSync && this.fileStoreClients.length > 0 && getMissingBlobHashes().length > 0) {
|
|
324
|
+
try {
|
|
325
|
+
await retry(
|
|
326
|
+
async () => {
|
|
327
|
+
await this.tryFileStores(getMissingBlobHashes, fillResults, ctx);
|
|
328
|
+
if (getMissingBlobHashes().length > 0) {
|
|
329
|
+
throw new Error('Still missing blobs from filestores');
|
|
330
|
+
}
|
|
331
|
+
},
|
|
332
|
+
'filestore blob retrieval',
|
|
333
|
+
makeBackoff([1, 1, 2]),
|
|
334
|
+
this.log,
|
|
335
|
+
true, // failSilently - expected to fail during eventual consistency
|
|
336
|
+
);
|
|
337
|
+
return returnWithCallback(getFilledBlobs());
|
|
338
|
+
} catch {
|
|
339
|
+
// Exhausted retries, continue to archive fallback
|
|
340
|
+
}
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
const missingAfterConsensus = getMissingBlobHashes();
|
|
344
|
+
if (missingAfterConsensus.length > 0 && this.archiveClient) {
|
|
337
345
|
const archiveCtx = { archiveUrl: this.archiveClient.getBaseUrl(), ...ctx };
|
|
338
|
-
this.log.trace(`Attempting to get ${
|
|
346
|
+
this.log.trace(`Attempting to get ${missingAfterConsensus.length} blobs from archive`, archiveCtx);
|
|
339
347
|
const allBlobs = await this.archiveClient.getBlobsFromBlock(blockHash);
|
|
340
348
|
if (!allBlobs) {
|
|
341
349
|
this.log.debug('No blobs found from archive client', archiveCtx);
|
|
@@ -357,7 +365,7 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
357
365
|
this.log.warn(
|
|
358
366
|
`Failed to fetch all blobs for ${blockHash} from all blob sources (got ${result.length}/${blobHashes.length})`,
|
|
359
367
|
{
|
|
360
|
-
l1ConsensusHostUrls
|
|
368
|
+
l1ConsensusHostUrls,
|
|
361
369
|
archiveUrl: this.archiveClient?.getBaseUrl(),
|
|
362
370
|
fileStoreUrls: this.fileStoreClients.map(c => c.getBaseUrl()),
|
|
363
371
|
},
|
|
@@ -366,71 +374,6 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
366
374
|
return returnWithCallback(result);
|
|
367
375
|
}
|
|
368
376
|
|
|
369
|
-
/** Resolves the beacon slot number for the given block hash. Returns undefined if no consensus hosts. */
|
|
370
|
-
private resolveSlotNumber(
|
|
371
|
-
blockHash: `0x${string}`,
|
|
372
|
-
opts?: GetBlobSidecarOptions,
|
|
373
|
-
): Promise<number | undefined> | undefined {
|
|
374
|
-
const { l1ConsensusHostUrls } = this.config;
|
|
375
|
-
if (!l1ConsensusHostUrls || l1ConsensusHostUrls.length === 0) {
|
|
376
|
-
return undefined;
|
|
377
|
-
}
|
|
378
|
-
// If no supernodes, no point resolving the slot
|
|
379
|
-
if (this.superNodeHostIndexes && this.superNodeHostIndexes.size === 0) {
|
|
380
|
-
return undefined;
|
|
381
|
-
}
|
|
382
|
-
return this.getSlotNumber(blockHash, opts?.parentBeaconBlockRoot, opts?.l1BlockTimestamp);
|
|
383
|
-
}
|
|
384
|
-
|
|
385
|
-
/**
|
|
386
|
-
* Try all supernode consensus hosts for blob sidecars.
|
|
387
|
-
* Skips hosts that were detected as non-supernodes during testSources().
|
|
388
|
-
*/
|
|
389
|
-
private async tryConsensusHosts(
|
|
390
|
-
getSlotNumber: () => Promise<number | undefined>,
|
|
391
|
-
getMissingBlobHashes: () => Buffer[],
|
|
392
|
-
fillResults: (blobs: BlobJson[]) => Promise<Blob[]>,
|
|
393
|
-
ctx: { blockHash: string; blobHashes: string[] },
|
|
394
|
-
): Promise<void> {
|
|
395
|
-
const { l1ConsensusHostUrls } = this.config;
|
|
396
|
-
if (!l1ConsensusHostUrls || l1ConsensusHostUrls.length === 0) {
|
|
397
|
-
return;
|
|
398
|
-
}
|
|
399
|
-
|
|
400
|
-
const slotNumber = await getSlotNumber();
|
|
401
|
-
if (!slotNumber) {
|
|
402
|
-
return;
|
|
403
|
-
}
|
|
404
|
-
|
|
405
|
-
for (let l1ConsensusHostIndex = 0; l1ConsensusHostIndex < l1ConsensusHostUrls.length; l1ConsensusHostIndex++) {
|
|
406
|
-
const missingHashes = getMissingBlobHashes();
|
|
407
|
-
if (missingHashes.length === 0) {
|
|
408
|
-
break;
|
|
409
|
-
}
|
|
410
|
-
|
|
411
|
-
// Skip non-supernode hosts if we've already detected supernodes
|
|
412
|
-
if (this.superNodeHostIndexes && !this.superNodeHostIndexes.has(l1ConsensusHostIndex)) {
|
|
413
|
-
this.log.trace(`Skipping non-supernode consensus host`, {
|
|
414
|
-
l1ConsensusHostUrl: l1ConsensusHostUrls[l1ConsensusHostIndex],
|
|
415
|
-
});
|
|
416
|
-
continue;
|
|
417
|
-
}
|
|
418
|
-
|
|
419
|
-
const l1ConsensusHostUrl = l1ConsensusHostUrls[l1ConsensusHostIndex];
|
|
420
|
-
this.log.trace(`Attempting to get ${missingHashes.length} blobs from consensus host`, {
|
|
421
|
-
slotNumber,
|
|
422
|
-
l1ConsensusHostUrl,
|
|
423
|
-
...ctx,
|
|
424
|
-
});
|
|
425
|
-
const blobs = await this.getBlobsFromHost(l1ConsensusHostUrl, slotNumber, l1ConsensusHostIndex, missingHashes);
|
|
426
|
-
const result = await fillResults(blobs);
|
|
427
|
-
this.log.debug(
|
|
428
|
-
`Got ${blobs.length} blobs from consensus host (total: ${result.length}/${ctx.blobHashes.length})`,
|
|
429
|
-
{ slotNumber, l1ConsensusHostUrl, ...ctx },
|
|
430
|
-
);
|
|
431
|
-
}
|
|
432
|
-
}
|
|
433
|
-
|
|
434
377
|
/**
|
|
435
378
|
* Try all filestores once (shuffled for load distribution).
|
|
436
379
|
* @param getMissingBlobHashes - Function to get remaining blob hashes to fetch
|
|
@@ -481,7 +424,7 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
481
424
|
blobHashes: Buffer[] = [],
|
|
482
425
|
l1ConsensusHostIndex?: number,
|
|
483
426
|
): Promise<Blob[]> {
|
|
484
|
-
const blobs = await this.getBlobsFromHost(hostUrl, blockHashOrSlot, l1ConsensusHostIndex
|
|
427
|
+
const blobs = await this.getBlobsFromHost(hostUrl, blockHashOrSlot, l1ConsensusHostIndex);
|
|
485
428
|
return (await processFetchedBlobs(blobs, blobHashes, this.log)).filter((b): b is Blob => b !== undefined);
|
|
486
429
|
}
|
|
487
430
|
|
|
@@ -489,12 +432,11 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
489
432
|
hostUrl: string,
|
|
490
433
|
blockHashOrSlot: string | number,
|
|
491
434
|
l1ConsensusHostIndex?: number,
|
|
492
|
-
blobHashes?: Buffer[],
|
|
493
435
|
): Promise<BlobJson[]> {
|
|
494
436
|
try {
|
|
495
|
-
let res = await this.fetchBlobSidecars(hostUrl, blockHashOrSlot, l1ConsensusHostIndex
|
|
437
|
+
let res = await this.fetchBlobSidecars(hostUrl, blockHashOrSlot, l1ConsensusHostIndex);
|
|
496
438
|
if (res.ok) {
|
|
497
|
-
return
|
|
439
|
+
return parseBlobJsonsFromResponse(await res.json(), this.log);
|
|
498
440
|
}
|
|
499
441
|
|
|
500
442
|
if (res.status === 404 && typeof blockHashOrSlot === 'number') {
|
|
@@ -509,9 +451,9 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
509
451
|
let currentSlot = blockHashOrSlot + 1;
|
|
510
452
|
while (res.status === 404 && maxRetries > 0 && latestSlot !== undefined && currentSlot <= latestSlot) {
|
|
511
453
|
this.log.debug(`Trying slot ${currentSlot}`);
|
|
512
|
-
res = await this.fetchBlobSidecars(hostUrl, currentSlot, l1ConsensusHostIndex
|
|
454
|
+
res = await this.fetchBlobSidecars(hostUrl, currentSlot, l1ConsensusHostIndex);
|
|
513
455
|
if (res.ok) {
|
|
514
|
-
return
|
|
456
|
+
return parseBlobJsonsFromResponse(await res.json(), this.log);
|
|
515
457
|
}
|
|
516
458
|
currentSlot++;
|
|
517
459
|
maxRetries--;
|
|
@@ -534,22 +476,12 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
534
476
|
hostUrl: string,
|
|
535
477
|
blockHashOrSlot: string | number,
|
|
536
478
|
l1ConsensusHostIndex?: number,
|
|
537
|
-
blobHashes?: Buffer[],
|
|
538
479
|
): Promise<Response> {
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
if (blobHashes && blobHashes.length > 0) {
|
|
542
|
-
const params = new URLSearchParams();
|
|
543
|
-
for (const hash of blobHashes) {
|
|
544
|
-
params.append('versioned_hashes', `0x${hash.toString('hex')}`);
|
|
545
|
-
}
|
|
546
|
-
baseUrl += `?${params.toString()}`;
|
|
547
|
-
}
|
|
480
|
+
const baseUrl = `${hostUrl}/eth/v1/beacon/blob_sidecars/${blockHashOrSlot}`;
|
|
548
481
|
|
|
549
482
|
const { url, ...options } = getBeaconNodeFetchOptions(baseUrl, this.config, l1ConsensusHostIndex);
|
|
550
483
|
this.log.debug(`Fetching blob sidecar for ${blockHashOrSlot}`, { url, ...options });
|
|
551
|
-
|
|
552
|
-
return fetch(url, options);
|
|
484
|
+
return this.fetch(url, options);
|
|
553
485
|
}
|
|
554
486
|
|
|
555
487
|
private async getLatestSlotNumber(hostUrl: string, l1ConsensusHostIndex?: number): Promise<number | undefined> {
|
|
@@ -587,50 +519,34 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
587
519
|
* @param blockHash - The block hash
|
|
588
520
|
* @returns The slot number
|
|
589
521
|
*/
|
|
590
|
-
private async getSlotNumber(
|
|
591
|
-
blockHash: `0x${string}`,
|
|
592
|
-
parentBeaconBlockRoot?: string,
|
|
593
|
-
l1BlockTimestamp?: bigint,
|
|
594
|
-
): Promise<number | undefined> {
|
|
522
|
+
private async getSlotNumber(blockHash: `0x${string}`): Promise<number | undefined> {
|
|
595
523
|
const { l1ConsensusHostUrls, l1RpcUrls } = this.config;
|
|
596
524
|
if (!l1ConsensusHostUrls || l1ConsensusHostUrls.length === 0) {
|
|
597
525
|
this.log.debug('No consensus host url configured');
|
|
598
526
|
return undefined;
|
|
599
527
|
}
|
|
600
528
|
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
this.beaconGenesisTime !== undefined &&
|
|
605
|
-
this.beaconSecondsPerSlot !== undefined
|
|
606
|
-
) {
|
|
607
|
-
const slot = Number((l1BlockTimestamp - this.beaconGenesisTime) / BigInt(this.beaconSecondsPerSlot));
|
|
608
|
-
this.log.debug(`Computed slot ${slot} from L1 block timestamp`, { l1BlockTimestamp });
|
|
609
|
-
return slot;
|
|
529
|
+
if (!l1RpcUrls || l1RpcUrls.length === 0) {
|
|
530
|
+
this.log.debug('No execution host url configured');
|
|
531
|
+
return undefined;
|
|
610
532
|
}
|
|
611
533
|
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
534
|
+
// Ping execution node to get the parentBeaconBlockRoot for this block
|
|
535
|
+
let parentBeaconBlockRoot: string | undefined;
|
|
536
|
+
const client = createPublicClient({
|
|
537
|
+
transport: makeL1HttpTransport(l1RpcUrls, { timeout: this.config.l1HttpTimeoutMS }),
|
|
538
|
+
});
|
|
539
|
+
try {
|
|
540
|
+
const res: RpcBlock = await client.request({
|
|
541
|
+
method: 'eth_getBlockByHash',
|
|
542
|
+
params: [blockHash, /*tx flag*/ false],
|
|
621
543
|
});
|
|
622
|
-
try {
|
|
623
|
-
const res: RpcBlock = await client.request({
|
|
624
|
-
method: 'eth_getBlockByHash',
|
|
625
|
-
params: [blockHash, /*tx flag*/ false],
|
|
626
|
-
});
|
|
627
544
|
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
}
|
|
631
|
-
} catch (err) {
|
|
632
|
-
this.log.error(`Error getting parent beacon block root`, err);
|
|
545
|
+
if (res.parentBeaconBlockRoot) {
|
|
546
|
+
parentBeaconBlockRoot = res.parentBeaconBlockRoot;
|
|
633
547
|
}
|
|
548
|
+
} catch (err) {
|
|
549
|
+
this.log.error(`Error getting parent beacon block root`, err);
|
|
634
550
|
}
|
|
635
551
|
|
|
636
552
|
if (!parentBeaconBlockRoot) {
|
|
@@ -676,12 +592,9 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
676
592
|
|
|
677
593
|
/**
|
|
678
594
|
* Start the blob client.
|
|
679
|
-
*
|
|
680
|
-
* then uploads the initial healthcheck file (awaited) and starts periodic uploads.
|
|
595
|
+
* Uploads the initial healthcheck file (awaited) and starts periodic uploads.
|
|
681
596
|
*/
|
|
682
597
|
public async start(): Promise<void> {
|
|
683
|
-
await this.fetchBeaconConfig();
|
|
684
|
-
|
|
685
598
|
if (!this.fileStoreUploadClient) {
|
|
686
599
|
return;
|
|
687
600
|
}
|
|
@@ -706,53 +619,6 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
706
619
|
}, intervalMs);
|
|
707
620
|
}
|
|
708
621
|
|
|
709
|
-
/**
|
|
710
|
-
* Fetches and caches beacon genesis time and slot duration from the first available consensus host.
|
|
711
|
-
* These static values enable timestamp-based slot resolution, eliminating the per-fetch headers call.
|
|
712
|
-
* Logs a warning and leaves fields undefined if all hosts fail, callers fall back gracefully.
|
|
713
|
-
*/
|
|
714
|
-
private async fetchBeaconConfig(): Promise<void> {
|
|
715
|
-
const { l1ConsensusHostUrls } = this.config;
|
|
716
|
-
if (!l1ConsensusHostUrls || l1ConsensusHostUrls.length === 0) {
|
|
717
|
-
return;
|
|
718
|
-
}
|
|
719
|
-
|
|
720
|
-
for (let i = 0; i < l1ConsensusHostUrls.length; i++) {
|
|
721
|
-
try {
|
|
722
|
-
const { url: genesisUrl, ...genesisOptions } = getBeaconNodeFetchOptions(
|
|
723
|
-
`${l1ConsensusHostUrls[i]}/eth/v1/config/genesis`,
|
|
724
|
-
this.config,
|
|
725
|
-
i,
|
|
726
|
-
);
|
|
727
|
-
const { url: specUrl, ...specOptions } = getBeaconNodeFetchOptions(
|
|
728
|
-
`${l1ConsensusHostUrls[i]}/eth/v1/config/spec`,
|
|
729
|
-
this.config,
|
|
730
|
-
i,
|
|
731
|
-
);
|
|
732
|
-
|
|
733
|
-
const [genesisRes, specRes] = await Promise.all([
|
|
734
|
-
this.fetch(genesisUrl, genesisOptions),
|
|
735
|
-
this.fetch(specUrl, specOptions),
|
|
736
|
-
]);
|
|
737
|
-
|
|
738
|
-
if (genesisRes.ok && specRes.ok) {
|
|
739
|
-
const genesis = await genesisRes.json();
|
|
740
|
-
const spec = await specRes.json();
|
|
741
|
-
this.beaconGenesisTime = BigInt(genesis.data.genesisTime);
|
|
742
|
-
this.beaconSecondsPerSlot = parseInt(spec.data.secondsPerSlot);
|
|
743
|
-
this.log.debug(`Fetched beacon genesis config`, {
|
|
744
|
-
genesisTime: this.beaconGenesisTime,
|
|
745
|
-
secondsPerSlot: this.beaconSecondsPerSlot,
|
|
746
|
-
});
|
|
747
|
-
return;
|
|
748
|
-
}
|
|
749
|
-
} catch (err) {
|
|
750
|
-
this.log.warn(`Failed to fetch beacon config from host ${l1ConsensusHostUrls[i]}`, err);
|
|
751
|
-
}
|
|
752
|
-
}
|
|
753
|
-
this.log.warn('Could not fetch beacon genesis config from any consensus host — will use headers call fallback');
|
|
754
|
-
}
|
|
755
|
-
|
|
756
622
|
/**
|
|
757
623
|
* Stop the blob client, clearing any periodic tasks.
|
|
758
624
|
*/
|
|
@@ -764,9 +630,10 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
764
630
|
}
|
|
765
631
|
}
|
|
766
632
|
|
|
767
|
-
|
|
633
|
+
function parseBlobJsonsFromResponse(response: any, logger: Logger): BlobJson[] {
|
|
768
634
|
try {
|
|
769
|
-
|
|
635
|
+
const blobs = response.data.map(parseBlobJson);
|
|
636
|
+
return blobs;
|
|
770
637
|
} catch (err) {
|
|
771
638
|
logger.error(`Error parsing blob json from response`, err);
|
|
772
639
|
return [];
|
|
@@ -777,9 +644,10 @@ async function parseBlobJsonsFromResponse(response: any, logger: Logger): Promis
|
|
|
777
644
|
// https://ethereum.github.io/beacon-APIs/?urls.primaryName=dev#/Beacon/getBlobSidecars
|
|
778
645
|
// Here we attempt to parse the response data to Buffer, and check the lengths (via Blob's constructor), to avoid
|
|
779
646
|
// throwing an error down the line when calling Blob.fromJson().
|
|
780
|
-
|
|
781
|
-
const blobBuffer = Buffer.from(
|
|
782
|
-
const
|
|
647
|
+
function parseBlobJson(data: any): BlobJson {
|
|
648
|
+
const blobBuffer = Buffer.from(data.blob.slice(2), 'hex');
|
|
649
|
+
const commitmentBuffer = Buffer.from(data.kzg_commitment.slice(2), 'hex');
|
|
650
|
+
const blob = new Blob(blobBuffer, commitmentBuffer);
|
|
783
651
|
return blob.toJSON();
|
|
784
652
|
}
|
|
785
653
|
|
package/src/client/interface.ts
CHANGED
|
@@ -6,20 +6,11 @@ import type { Blob } from '@aztec/blob-lib';
|
|
|
6
6
|
export interface GetBlobSidecarOptions {
|
|
7
7
|
/**
|
|
8
8
|
* True if the archiver is catching up (historical sync), false if near tip.
|
|
9
|
-
*
|
|
9
|
+
* This affects source ordering:
|
|
10
|
+
* - Historical: FileStore first (data should exist), then L1 consensus, then archive (eg. blobscan)
|
|
11
|
+
* - Near tip: FileStore first with no retries (data should exist), L1 consensus second (freshest data), then FileStore with retries, then archive (eg. blobscan)
|
|
10
12
|
*/
|
|
11
13
|
isHistoricalSync?: boolean;
|
|
12
|
-
/**
|
|
13
|
-
* The parent beacon block root for the L1 block containing the blobs.
|
|
14
|
-
* If provided, skips the eth_getBlockByHash execution RPC call inside getSlotNumber.
|
|
15
|
-
*/
|
|
16
|
-
parentBeaconBlockRoot?: string;
|
|
17
|
-
/**
|
|
18
|
-
* The timestamp of the L1 execution block containing the blobs.
|
|
19
|
-
* When provided alongside a cached beacon genesis config (fetched at startup), allows computing
|
|
20
|
-
* the beacon slot directly via timestamp math, skipping the beacon headers network call entirely.
|
|
21
|
-
*/
|
|
22
|
-
l1BlockTimestamp?: bigint;
|
|
23
14
|
}
|
|
24
15
|
|
|
25
16
|
export interface BlobClientInterface {
|
package/src/filestore/factory.ts
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import { type Logger, createLogger } from '@aztec/foundation/log';
|
|
2
2
|
import {
|
|
3
3
|
type FileStore,
|
|
4
|
-
type HttpFileStoreOptions,
|
|
5
4
|
type ReadOnlyFileStore,
|
|
6
5
|
createFileStore,
|
|
7
6
|
createReadOnlyFileStore,
|
|
@@ -45,19 +44,16 @@ export async function createReadOnlyFileStoreBlobClient(
|
|
|
45
44
|
storeUrl: string,
|
|
46
45
|
metadata: BlobFileStoreMetadata,
|
|
47
46
|
logger?: Logger,
|
|
48
|
-
httpOptions?: HttpFileStoreOptions,
|
|
49
47
|
): Promise<FileStoreBlobClient>;
|
|
50
48
|
export async function createReadOnlyFileStoreBlobClient(
|
|
51
49
|
storeUrl: string | undefined,
|
|
52
50
|
metadata: BlobFileStoreMetadata,
|
|
53
51
|
logger?: Logger,
|
|
54
|
-
httpOptions?: HttpFileStoreOptions,
|
|
55
52
|
): Promise<FileStoreBlobClient | undefined>;
|
|
56
53
|
export async function createReadOnlyFileStoreBlobClient(
|
|
57
54
|
storeUrl: string | undefined,
|
|
58
55
|
metadata: BlobFileStoreMetadata,
|
|
59
56
|
logger?: Logger,
|
|
60
|
-
httpOptions?: HttpFileStoreOptions,
|
|
61
57
|
): Promise<FileStoreBlobClient | undefined> {
|
|
62
58
|
if (!storeUrl) {
|
|
63
59
|
return undefined;
|
|
@@ -68,7 +64,7 @@ export async function createReadOnlyFileStoreBlobClient(
|
|
|
68
64
|
|
|
69
65
|
log.debug(`Creating read-only filestore blob client`, { storeUrl, basePath });
|
|
70
66
|
|
|
71
|
-
const store: ReadOnlyFileStore = await createReadOnlyFileStore(storeUrl, log
|
|
67
|
+
const store: ReadOnlyFileStore = await createReadOnlyFileStore(storeUrl, log);
|
|
72
68
|
return new FileStoreBlobClient(store, basePath, log);
|
|
73
69
|
}
|
|
74
70
|
|
|
@@ -84,7 +80,6 @@ export async function createReadOnlyFileStoreBlobClients(
|
|
|
84
80
|
storeUrls: string[] | undefined,
|
|
85
81
|
metadata: BlobFileStoreMetadata,
|
|
86
82
|
logger?: Logger,
|
|
87
|
-
httpOptions?: HttpFileStoreOptions,
|
|
88
83
|
): Promise<FileStoreBlobClient[]> {
|
|
89
84
|
if (!storeUrls || storeUrls.length === 0) {
|
|
90
85
|
return [];
|
|
@@ -95,7 +90,7 @@ export async function createReadOnlyFileStoreBlobClients(
|
|
|
95
90
|
|
|
96
91
|
for (const storeUrl of storeUrls) {
|
|
97
92
|
try {
|
|
98
|
-
const client = await createReadOnlyFileStoreBlobClient(storeUrl, metadata, log
|
|
93
|
+
const client = await createReadOnlyFileStoreBlobClient(storeUrl, metadata, log);
|
|
99
94
|
if (client) {
|
|
100
95
|
clients.push(client);
|
|
101
96
|
}
|