@aztec/blob-client 0.0.1-commit.8c0b8ff → 0.0.1-commit.8cb2d04d8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dest/client/config.d.ts +5 -1
- package/dest/client/config.d.ts.map +1 -1
- package/dest/client/config.js +12 -2
- package/dest/client/factory.d.ts +1 -1
- package/dest/client/factory.d.ts.map +1 -1
- package/dest/client/factory.js +7 -1
- package/dest/client/http.d.ts +19 -11
- package/dest/client/http.d.ts.map +1 -1
- package/dest/client/http.js +215 -119
- package/dest/client/interface.d.ts +13 -4
- package/dest/client/interface.d.ts.map +1 -1
- package/dest/filestore/factory.d.ts +5 -4
- package/dest/filestore/factory.d.ts.map +1 -1
- package/dest/filestore/factory.js +4 -4
- package/package.json +7 -7
- package/src/client/config.ts +18 -2
- package/src/client/factory.ts +8 -1
- package/src/client/http.ts +249 -113
- package/src/client/interface.ts +12 -3
- package/src/filestore/factory.ts +7 -2
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@aztec/blob-client",
|
|
3
|
-
"version": "0.0.1-commit.
|
|
3
|
+
"version": "0.0.1-commit.8cb2d04d8",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"bin": "./dest/client/bin/index.js",
|
|
6
6
|
"exports": {
|
|
@@ -56,12 +56,12 @@
|
|
|
56
56
|
]
|
|
57
57
|
},
|
|
58
58
|
"dependencies": {
|
|
59
|
-
"@aztec/blob-lib": "0.0.1-commit.
|
|
60
|
-
"@aztec/ethereum": "0.0.1-commit.
|
|
61
|
-
"@aztec/foundation": "0.0.1-commit.
|
|
62
|
-
"@aztec/kv-store": "0.0.1-commit.
|
|
63
|
-
"@aztec/stdlib": "0.0.1-commit.
|
|
64
|
-
"@aztec/telemetry-client": "0.0.1-commit.
|
|
59
|
+
"@aztec/blob-lib": "0.0.1-commit.8cb2d04d8",
|
|
60
|
+
"@aztec/ethereum": "0.0.1-commit.8cb2d04d8",
|
|
61
|
+
"@aztec/foundation": "0.0.1-commit.8cb2d04d8",
|
|
62
|
+
"@aztec/kv-store": "0.0.1-commit.8cb2d04d8",
|
|
63
|
+
"@aztec/stdlib": "0.0.1-commit.8cb2d04d8",
|
|
64
|
+
"@aztec/telemetry-client": "0.0.1-commit.8cb2d04d8",
|
|
65
65
|
"express": "^4.21.2",
|
|
66
66
|
"snappy": "^7.2.2",
|
|
67
67
|
"source-map-support": "^0.5.21",
|
package/src/client/config.ts
CHANGED
|
@@ -59,6 +59,12 @@ export interface BlobClientConfig extends BlobArchiveApiConfig {
|
|
|
59
59
|
|
|
60
60
|
/** Timeout for HTTP requests to the L1 RPC node in ms. */
|
|
61
61
|
l1HttpTimeoutMS?: number;
|
|
62
|
+
|
|
63
|
+
/** Whether to prefer filestores over consensus clients when fetching blobs. Default: false (consensus first). */
|
|
64
|
+
blobPreferFilestores?: boolean;
|
|
65
|
+
|
|
66
|
+
/** Timeout in ms for HTTP requests to the blob file store. Default: 10000 (10s). */
|
|
67
|
+
blobFileStoreTimeoutMs?: number;
|
|
62
68
|
}
|
|
63
69
|
|
|
64
70
|
export const blobClientConfigMapping: ConfigMappingsType<BlobClientConfig> = {
|
|
@@ -87,7 +93,7 @@ export const blobClientConfigMapping: ConfigMappingsType<BlobClientConfig> = {
|
|
|
87
93
|
blobSinkMapSizeKb: {
|
|
88
94
|
env: 'BLOB_SINK_MAP_SIZE_KB',
|
|
89
95
|
description: 'The maximum possible size of the blob sink DB in KB. Overwrites the general dataStoreMapSizeKb.',
|
|
90
|
-
parseEnv: (val: string
|
|
96
|
+
parseEnv: (val: string) => +val,
|
|
91
97
|
},
|
|
92
98
|
blobAllowEmptySources: {
|
|
93
99
|
env: 'BLOB_ALLOW_EMPTY_SOURCES',
|
|
@@ -110,13 +116,23 @@ export const blobClientConfigMapping: ConfigMappingsType<BlobClientConfig> = {
|
|
|
110
116
|
blobHealthcheckUploadIntervalMinutes: {
|
|
111
117
|
env: 'BLOB_HEALTHCHECK_UPLOAD_INTERVAL_MINUTES',
|
|
112
118
|
description: 'Interval in minutes for uploading healthcheck file to file store (default: 60 = 1 hour)',
|
|
113
|
-
parseEnv: (val: string
|
|
119
|
+
parseEnv: (val: string) => +val,
|
|
114
120
|
},
|
|
115
121
|
l1HttpTimeoutMS: {
|
|
116
122
|
env: 'ETHEREUM_HTTP_TIMEOUT_MS',
|
|
117
123
|
description: 'Timeout for HTTP requests to the L1 RPC node in ms.',
|
|
118
124
|
...optionalNumberConfigHelper(),
|
|
119
125
|
},
|
|
126
|
+
blobPreferFilestores: {
|
|
127
|
+
env: 'BLOB_PREFER_FILESTORES',
|
|
128
|
+
description: 'Whether to prefer filestores over consensus clients when fetching blobs. Default: false.',
|
|
129
|
+
...booleanConfigHelper(false),
|
|
130
|
+
},
|
|
131
|
+
blobFileStoreTimeoutMs: {
|
|
132
|
+
env: 'BLOB_FILE_STORE_TIMEOUT_MS',
|
|
133
|
+
description: 'Timeout in ms for HTTP requests to the blob file store. Default: 10000 (10s).',
|
|
134
|
+
...optionalNumberConfigHelper(),
|
|
135
|
+
},
|
|
120
136
|
...blobArchiveApiConfigMappings,
|
|
121
137
|
};
|
|
122
138
|
|
package/src/client/factory.ts
CHANGED
|
@@ -76,8 +76,15 @@ export async function createBlobClientWithFileStores(
|
|
|
76
76
|
rollupAddress: config.l1Contracts.rollupAddress.toString(),
|
|
77
77
|
};
|
|
78
78
|
|
|
79
|
+
// Disable internal retries for blob file stores — retry logic is handled by HttpBlobClient.
|
|
80
|
+
// Set a configurable timeout (default 10s) to avoid hanging on slow stores.
|
|
81
|
+
const httpOptions = {
|
|
82
|
+
retryBackoff: [] as number[],
|
|
83
|
+
timeoutMs: config.blobFileStoreTimeoutMs ?? 10_000,
|
|
84
|
+
};
|
|
85
|
+
|
|
79
86
|
const [fileStoreClients, fileStoreUploadClient] = await Promise.all([
|
|
80
|
-
createReadOnlyFileStoreBlobClients(config.blobFileStoreUrls, fileStoreMetadata, log),
|
|
87
|
+
createReadOnlyFileStoreBlobClients(config.blobFileStoreUrls, fileStoreMetadata, log, httpOptions),
|
|
81
88
|
createWritableFileStoreBlobClient(config.blobFileStoreUploadUrl, fileStoreMetadata, log),
|
|
82
89
|
]);
|
|
83
90
|
|
package/src/client/http.ts
CHANGED
|
@@ -25,6 +25,14 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
25
25
|
private disabled = false;
|
|
26
26
|
private healthcheckUploadIntervalId?: NodeJS.Timeout;
|
|
27
27
|
|
|
28
|
+
/** Cached beacon genesis time (seconds since Unix epoch). Fetched once at startup. */
|
|
29
|
+
private beaconGenesisTime?: bigint;
|
|
30
|
+
/** Cached beacon slot duration in seconds. Fetched once at startup. */
|
|
31
|
+
private beaconSecondsPerSlot?: number;
|
|
32
|
+
|
|
33
|
+
/** Indexes of consensus hosts that serve blob sidecars (supernodes). Populated by testSources(). */
|
|
34
|
+
private superNodeHostIndexes?: Set<number>;
|
|
35
|
+
|
|
28
36
|
constructor(
|
|
29
37
|
config?: BlobClientConfig,
|
|
30
38
|
private readonly opts: {
|
|
@@ -95,6 +103,8 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
95
103
|
let archiveSources = 0;
|
|
96
104
|
let blobSinks = 0;
|
|
97
105
|
|
|
106
|
+
const detectedSuperNodes = new Set<number>();
|
|
107
|
+
|
|
98
108
|
if (l1ConsensusHostUrls && l1ConsensusHostUrls.length > 0) {
|
|
99
109
|
for (let l1ConsensusHostIndex = 0; l1ConsensusHostIndex < l1ConsensusHostUrls.length; l1ConsensusHostIndex++) {
|
|
100
110
|
const l1ConsensusHostUrl = l1ConsensusHostUrls[l1ConsensusHostIndex];
|
|
@@ -129,9 +139,12 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
129
139
|
const blobRes = await this.fetch(blobUrl, blobOptions);
|
|
130
140
|
if (blobRes.ok) {
|
|
131
141
|
this.log.info(`L1 consensus host serves blob sidecars (supernode)`, { l1ConsensusHostUrl });
|
|
142
|
+
detectedSuperNodes.add(l1ConsensusHostIndex);
|
|
132
143
|
consensusSuperNodes++;
|
|
133
144
|
} else {
|
|
134
|
-
this.log.info(`L1 consensus host does not serve blob sidecars`, {
|
|
145
|
+
this.log.info(`L1 consensus host does not serve blob sidecars, skipping for blob fetching`, {
|
|
146
|
+
l1ConsensusHostUrl,
|
|
147
|
+
});
|
|
135
148
|
consensusNonSuperNodes++;
|
|
136
149
|
}
|
|
137
150
|
} else {
|
|
@@ -144,6 +157,8 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
144
157
|
}
|
|
145
158
|
}
|
|
146
159
|
|
|
160
|
+
this.superNodeHostIndexes = detectedSuperNodes;
|
|
161
|
+
|
|
147
162
|
if (this.archiveClient) {
|
|
148
163
|
try {
|
|
149
164
|
const latest = await this.archiveClient.getLatestBlock();
|
|
@@ -213,18 +228,15 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
213
228
|
}
|
|
214
229
|
|
|
215
230
|
/**
|
|
216
|
-
* Get the blob sidecar
|
|
231
|
+
* Get the blob sidecar.
|
|
217
232
|
*
|
|
218
|
-
*
|
|
219
|
-
*
|
|
220
|
-
*
|
|
221
|
-
* Source ordering depends on sync state:
|
|
222
|
-
* - Historical sync: blob client → FileStore → L1 consensus → Archive
|
|
223
|
-
* - Near tip sync: blob client → FileStore → L1 consensus → FileStore (with retries) → Archive (eg blobscan)
|
|
233
|
+
* Alternates between two primary sources (consensus and filestore) in a retry loop,
|
|
234
|
+
* then falls back to archive if blobs are still missing. The order of the primary
|
|
235
|
+
* sources is configurable via `blobPreferFilestores`.
|
|
224
236
|
*
|
|
225
237
|
* @param blockHash - The block hash
|
|
226
238
|
* @param blobHashes - The blob hashes to fetch
|
|
227
|
-
* @param opts - Options
|
|
239
|
+
* @param opts - Options for slot resolution
|
|
228
240
|
* @returns The blobs
|
|
229
241
|
*/
|
|
230
242
|
public async getBlobSidecar(
|
|
@@ -237,12 +249,11 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
237
249
|
return [];
|
|
238
250
|
}
|
|
239
251
|
|
|
240
|
-
const isHistoricalSync = opts?.isHistoricalSync ?? false;
|
|
241
252
|
// Accumulate blobs across sources, preserving order and handling duplicates
|
|
242
253
|
// resultBlobs[i] will contain the blob for blobHashes[i], or undefined if not yet found
|
|
243
254
|
const resultBlobs: (Blob | undefined)[] = new Array(blobHashes.length).fill(undefined);
|
|
244
255
|
|
|
245
|
-
// Helper to get
|
|
256
|
+
// Helper to get missing blob hashes that we still need to fetch
|
|
246
257
|
const getMissingBlobHashes = (): Buffer[] =>
|
|
247
258
|
blobHashes
|
|
248
259
|
.map((bh, i) => (resultBlobs[i] === undefined ? bh : undefined))
|
|
@@ -271,79 +282,60 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
271
282
|
return blobs;
|
|
272
283
|
};
|
|
273
284
|
|
|
274
|
-
const { l1ConsensusHostUrls } = this.config;
|
|
275
|
-
|
|
276
285
|
const ctx = { blockHash, blobHashes: blobHashes.map(bufferToHex) };
|
|
277
286
|
|
|
278
|
-
//
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
287
|
+
// Lazily resolve the slot number — only resolved when consensus hosts are actually tried.
|
|
288
|
+
let slotNumber: number | undefined;
|
|
289
|
+
let slotResolved = false;
|
|
290
|
+
const getSlotNumber = async (): Promise<number | undefined> => {
|
|
291
|
+
if (!slotResolved) {
|
|
292
|
+
slotNumber = await this.resolveSlotNumber(blockHash, opts);
|
|
293
|
+
slotResolved = true;
|
|
283
294
|
}
|
|
284
|
-
|
|
295
|
+
return slotNumber;
|
|
296
|
+
};
|
|
285
297
|
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
const consensusCtx = { l1ConsensusHostUrls, ...ctx };
|
|
290
|
-
this.log.trace(`Attempting to get slot number for block hash`, consensusCtx);
|
|
291
|
-
const slotNumber = await this.getSlotNumber(blockHash);
|
|
292
|
-
this.log.debug(`Got slot number ${slotNumber} from consensus host for querying blobs`, consensusCtx);
|
|
293
|
-
|
|
294
|
-
if (slotNumber) {
|
|
295
|
-
let l1ConsensusHostUrl: string;
|
|
296
|
-
for (let l1ConsensusHostIndex = 0; l1ConsensusHostIndex < l1ConsensusHostUrls.length; l1ConsensusHostIndex++) {
|
|
297
|
-
const missingHashes = getMissingBlobHashes();
|
|
298
|
-
if (missingHashes.length === 0) {
|
|
299
|
-
break;
|
|
300
|
-
}
|
|
298
|
+
// Build the two source-try functions. The order depends on the config.
|
|
299
|
+
const tryConsensus = () => this.tryConsensusHosts(getSlotNumber, getMissingBlobHashes, fillResults, ctx);
|
|
300
|
+
const tryFilestores = () => this.tryFileStores(getMissingBlobHashes, fillResults, ctx);
|
|
301
301
|
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
slotNumber,
|
|
305
|
-
l1ConsensusHostUrl,
|
|
306
|
-
...ctx,
|
|
307
|
-
});
|
|
308
|
-
const blobs = await this.getBlobsFromHost(l1ConsensusHostUrl, slotNumber, l1ConsensusHostIndex);
|
|
309
|
-
const result = await fillResults(blobs);
|
|
310
|
-
this.log.debug(
|
|
311
|
-
`Got ${blobs.length} blobs from consensus host (total: ${result.length}/${blobHashes.length})`,
|
|
312
|
-
{ slotNumber, l1ConsensusHostUrl, ...ctx },
|
|
313
|
-
);
|
|
314
|
-
if (result.length === blobHashes.length) {
|
|
315
|
-
return returnWithCallback(result);
|
|
316
|
-
}
|
|
317
|
-
}
|
|
318
|
-
}
|
|
319
|
-
}
|
|
302
|
+
const preferFilestores = this.config.blobPreferFilestores ?? false;
|
|
303
|
+
const [trySourceA, trySourceB] = preferFilestores ? [tryFilestores, tryConsensus] : [tryConsensus, tryFilestores];
|
|
320
304
|
|
|
321
|
-
//
|
|
322
|
-
//
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
305
|
+
// Historical sync: blobs should already exist, use shorter backoff for transient errors.
|
|
306
|
+
// Near-tip sync: blobs may still be uploading, use longer backoff for eventual consistency.
|
|
307
|
+
const isHistoricalSync = opts?.isHistoricalSync ?? false;
|
|
308
|
+
const backoff = isHistoricalSync ? [1, 1] : [1, 1, 1, 2, 2];
|
|
309
|
+
|
|
310
|
+
// Retry loop: alternate between the two primary sources with backoff.
|
|
311
|
+
try {
|
|
312
|
+
await retry(
|
|
313
|
+
async () => {
|
|
314
|
+
if (getMissingBlobHashes().length > 0) {
|
|
315
|
+
await trySourceA();
|
|
316
|
+
}
|
|
317
|
+
if (getMissingBlobHashes().length > 0) {
|
|
318
|
+
await trySourceB();
|
|
319
|
+
}
|
|
320
|
+
if (getMissingBlobHashes().length > 0) {
|
|
321
|
+
throw new Error('Still missing blobs after trying all primary sources');
|
|
322
|
+
}
|
|
323
|
+
},
|
|
324
|
+
'blob retrieval',
|
|
325
|
+
makeBackoff(backoff),
|
|
326
|
+
this.log,
|
|
327
|
+
true, // failSilently — expected during eventual consistency
|
|
328
|
+
);
|
|
329
|
+
return returnWithCallback(getFilledBlobs());
|
|
330
|
+
} catch {
|
|
331
|
+
// Exhausted retries, continue to archive fallback
|
|
341
332
|
}
|
|
342
333
|
|
|
343
|
-
|
|
344
|
-
|
|
334
|
+
// Archive fallback
|
|
335
|
+
const missingAfterPrimary = getMissingBlobHashes();
|
|
336
|
+
if (missingAfterPrimary.length > 0 && this.archiveClient) {
|
|
345
337
|
const archiveCtx = { archiveUrl: this.archiveClient.getBaseUrl(), ...ctx };
|
|
346
|
-
this.log.trace(`Attempting to get ${
|
|
338
|
+
this.log.trace(`Attempting to get ${missingAfterPrimary.length} blobs from archive`, archiveCtx);
|
|
347
339
|
const allBlobs = await this.archiveClient.getBlobsFromBlock(blockHash);
|
|
348
340
|
if (!allBlobs) {
|
|
349
341
|
this.log.debug('No blobs found from archive client', archiveCtx);
|
|
@@ -365,7 +357,7 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
365
357
|
this.log.warn(
|
|
366
358
|
`Failed to fetch all blobs for ${blockHash} from all blob sources (got ${result.length}/${blobHashes.length})`,
|
|
367
359
|
{
|
|
368
|
-
l1ConsensusHostUrls,
|
|
360
|
+
l1ConsensusHostUrls: this.config.l1ConsensusHostUrls,
|
|
369
361
|
archiveUrl: this.archiveClient?.getBaseUrl(),
|
|
370
362
|
fileStoreUrls: this.fileStoreClients.map(c => c.getBaseUrl()),
|
|
371
363
|
},
|
|
@@ -374,6 +366,71 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
374
366
|
return returnWithCallback(result);
|
|
375
367
|
}
|
|
376
368
|
|
|
369
|
+
/** Resolves the beacon slot number for the given block hash. Returns undefined if no consensus hosts. */
|
|
370
|
+
private resolveSlotNumber(
|
|
371
|
+
blockHash: `0x${string}`,
|
|
372
|
+
opts?: GetBlobSidecarOptions,
|
|
373
|
+
): Promise<number | undefined> | undefined {
|
|
374
|
+
const { l1ConsensusHostUrls } = this.config;
|
|
375
|
+
if (!l1ConsensusHostUrls || l1ConsensusHostUrls.length === 0) {
|
|
376
|
+
return undefined;
|
|
377
|
+
}
|
|
378
|
+
// If no supernodes, no point resolving the slot
|
|
379
|
+
if (this.superNodeHostIndexes && this.superNodeHostIndexes.size === 0) {
|
|
380
|
+
return undefined;
|
|
381
|
+
}
|
|
382
|
+
return this.getSlotNumber(blockHash, opts?.parentBeaconBlockRoot, opts?.l1BlockTimestamp);
|
|
383
|
+
}
|
|
384
|
+
|
|
385
|
+
/**
|
|
386
|
+
* Try all supernode consensus hosts for blob sidecars.
|
|
387
|
+
* Skips hosts that were detected as non-supernodes during testSources().
|
|
388
|
+
*/
|
|
389
|
+
private async tryConsensusHosts(
|
|
390
|
+
getSlotNumber: () => Promise<number | undefined>,
|
|
391
|
+
getMissingBlobHashes: () => Buffer[],
|
|
392
|
+
fillResults: (blobs: BlobJson[]) => Promise<Blob[]>,
|
|
393
|
+
ctx: { blockHash: string; blobHashes: string[] },
|
|
394
|
+
): Promise<void> {
|
|
395
|
+
const { l1ConsensusHostUrls } = this.config;
|
|
396
|
+
if (!l1ConsensusHostUrls || l1ConsensusHostUrls.length === 0) {
|
|
397
|
+
return;
|
|
398
|
+
}
|
|
399
|
+
|
|
400
|
+
const slotNumber = await getSlotNumber();
|
|
401
|
+
if (!slotNumber) {
|
|
402
|
+
return;
|
|
403
|
+
}
|
|
404
|
+
|
|
405
|
+
for (let l1ConsensusHostIndex = 0; l1ConsensusHostIndex < l1ConsensusHostUrls.length; l1ConsensusHostIndex++) {
|
|
406
|
+
const missingHashes = getMissingBlobHashes();
|
|
407
|
+
if (missingHashes.length === 0) {
|
|
408
|
+
break;
|
|
409
|
+
}
|
|
410
|
+
|
|
411
|
+
// Skip non-supernode hosts if we've already detected supernodes
|
|
412
|
+
if (this.superNodeHostIndexes && !this.superNodeHostIndexes.has(l1ConsensusHostIndex)) {
|
|
413
|
+
this.log.trace(`Skipping non-supernode consensus host`, {
|
|
414
|
+
l1ConsensusHostUrl: l1ConsensusHostUrls[l1ConsensusHostIndex],
|
|
415
|
+
});
|
|
416
|
+
continue;
|
|
417
|
+
}
|
|
418
|
+
|
|
419
|
+
const l1ConsensusHostUrl = l1ConsensusHostUrls[l1ConsensusHostIndex];
|
|
420
|
+
this.log.trace(`Attempting to get ${missingHashes.length} blobs from consensus host`, {
|
|
421
|
+
slotNumber,
|
|
422
|
+
l1ConsensusHostUrl,
|
|
423
|
+
...ctx,
|
|
424
|
+
});
|
|
425
|
+
const blobs = await this.getBlobsFromHost(l1ConsensusHostUrl, slotNumber, l1ConsensusHostIndex, missingHashes);
|
|
426
|
+
const result = await fillResults(blobs);
|
|
427
|
+
this.log.debug(
|
|
428
|
+
`Got ${blobs.length} blobs from consensus host (total: ${result.length}/${ctx.blobHashes.length})`,
|
|
429
|
+
{ slotNumber, l1ConsensusHostUrl, ...ctx },
|
|
430
|
+
);
|
|
431
|
+
}
|
|
432
|
+
}
|
|
433
|
+
|
|
377
434
|
/**
|
|
378
435
|
* Try all filestores once (shuffled for load distribution).
|
|
379
436
|
* @param getMissingBlobHashes - Function to get remaining blob hashes to fetch
|
|
@@ -424,7 +481,7 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
424
481
|
blobHashes: Buffer[] = [],
|
|
425
482
|
l1ConsensusHostIndex?: number,
|
|
426
483
|
): Promise<Blob[]> {
|
|
427
|
-
const blobs = await this.getBlobsFromHost(hostUrl, blockHashOrSlot, l1ConsensusHostIndex);
|
|
484
|
+
const blobs = await this.getBlobsFromHost(hostUrl, blockHashOrSlot, l1ConsensusHostIndex, blobHashes);
|
|
428
485
|
return (await processFetchedBlobs(blobs, blobHashes, this.log)).filter((b): b is Blob => b !== undefined);
|
|
429
486
|
}
|
|
430
487
|
|
|
@@ -432,11 +489,12 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
432
489
|
hostUrl: string,
|
|
433
490
|
blockHashOrSlot: string | number,
|
|
434
491
|
l1ConsensusHostIndex?: number,
|
|
492
|
+
blobHashes?: Buffer[],
|
|
435
493
|
): Promise<BlobJson[]> {
|
|
436
494
|
try {
|
|
437
|
-
let res = await this.fetchBlobSidecars(hostUrl, blockHashOrSlot, l1ConsensusHostIndex);
|
|
495
|
+
let res = await this.fetchBlobSidecars(hostUrl, blockHashOrSlot, l1ConsensusHostIndex, blobHashes);
|
|
438
496
|
if (res.ok) {
|
|
439
|
-
return parseBlobJsonsFromResponse(await res.json(), this.log);
|
|
497
|
+
return await parseBlobJsonsFromResponse(await res.json(), this.log);
|
|
440
498
|
}
|
|
441
499
|
|
|
442
500
|
if (res.status === 404 && typeof blockHashOrSlot === 'number') {
|
|
@@ -451,9 +509,9 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
451
509
|
let currentSlot = blockHashOrSlot + 1;
|
|
452
510
|
while (res.status === 404 && maxRetries > 0 && latestSlot !== undefined && currentSlot <= latestSlot) {
|
|
453
511
|
this.log.debug(`Trying slot ${currentSlot}`);
|
|
454
|
-
res = await this.fetchBlobSidecars(hostUrl, currentSlot, l1ConsensusHostIndex);
|
|
512
|
+
res = await this.fetchBlobSidecars(hostUrl, currentSlot, l1ConsensusHostIndex, blobHashes);
|
|
455
513
|
if (res.ok) {
|
|
456
|
-
return parseBlobJsonsFromResponse(await res.json(), this.log);
|
|
514
|
+
return await parseBlobJsonsFromResponse(await res.json(), this.log);
|
|
457
515
|
}
|
|
458
516
|
currentSlot++;
|
|
459
517
|
maxRetries--;
|
|
@@ -476,19 +534,29 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
476
534
|
hostUrl: string,
|
|
477
535
|
blockHashOrSlot: string | number,
|
|
478
536
|
l1ConsensusHostIndex?: number,
|
|
537
|
+
blobHashes?: Buffer[],
|
|
479
538
|
): Promise<Response> {
|
|
480
|
-
|
|
539
|
+
let baseUrl = `${hostUrl}/eth/v1/beacon/blobs/${blockHashOrSlot}`;
|
|
481
540
|
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
541
|
+
if (blobHashes && blobHashes.length > 0) {
|
|
542
|
+
const params = new URLSearchParams();
|
|
543
|
+
for (const hash of blobHashes) {
|
|
544
|
+
params.append('versioned_hashes', `0x${hash.toString('hex')}`);
|
|
545
|
+
}
|
|
546
|
+
baseUrl += `?${params.toString()}`;
|
|
547
|
+
}
|
|
548
|
+
|
|
549
|
+
const { url, logSafeUrl, ...options } = getBeaconNodeFetchOptions(baseUrl, this.config, l1ConsensusHostIndex);
|
|
550
|
+
this.log.debug(`Fetching blob sidecar for ${blockHashOrSlot}`, { url: logSafeUrl, ...options });
|
|
551
|
+
// No retry here — this is called inside the main retry loop in getBlobSidecar
|
|
552
|
+
return fetch(url, options);
|
|
485
553
|
}
|
|
486
554
|
|
|
487
555
|
private async getLatestSlotNumber(hostUrl: string, l1ConsensusHostIndex?: number): Promise<number | undefined> {
|
|
488
556
|
try {
|
|
489
557
|
const baseUrl = `${hostUrl}/eth/v1/beacon/headers/head`;
|
|
490
|
-
const { url, ...options } = getBeaconNodeFetchOptions(baseUrl, this.config, l1ConsensusHostIndex);
|
|
491
|
-
this.log.debug(`Fetching latest slot number`, { url, ...options });
|
|
558
|
+
const { url, logSafeUrl, ...options } = getBeaconNodeFetchOptions(baseUrl, this.config, l1ConsensusHostIndex);
|
|
559
|
+
this.log.debug(`Fetching latest slot number`, { url: logSafeUrl, ...options });
|
|
492
560
|
const res = await this.fetch(url, options);
|
|
493
561
|
if (res.ok) {
|
|
494
562
|
const body = await res.json();
|
|
@@ -519,34 +587,50 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
519
587
|
* @param blockHash - The block hash
|
|
520
588
|
* @returns The slot number
|
|
521
589
|
*/
|
|
522
|
-
private async getSlotNumber(
|
|
590
|
+
private async getSlotNumber(
|
|
591
|
+
blockHash: `0x${string}`,
|
|
592
|
+
parentBeaconBlockRoot?: string,
|
|
593
|
+
l1BlockTimestamp?: bigint,
|
|
594
|
+
): Promise<number | undefined> {
|
|
523
595
|
const { l1ConsensusHostUrls, l1RpcUrls } = this.config;
|
|
524
596
|
if (!l1ConsensusHostUrls || l1ConsensusHostUrls.length === 0) {
|
|
525
597
|
this.log.debug('No consensus host url configured');
|
|
526
598
|
return undefined;
|
|
527
599
|
}
|
|
528
600
|
|
|
529
|
-
if (
|
|
530
|
-
|
|
531
|
-
|
|
601
|
+
// Primary path: compute slot from timestamp if genesis config is cached (no network call needed)
|
|
602
|
+
if (
|
|
603
|
+
l1BlockTimestamp !== undefined &&
|
|
604
|
+
this.beaconGenesisTime !== undefined &&
|
|
605
|
+
this.beaconSecondsPerSlot !== undefined
|
|
606
|
+
) {
|
|
607
|
+
const slot = Number((l1BlockTimestamp - this.beaconGenesisTime) / BigInt(this.beaconSecondsPerSlot));
|
|
608
|
+
this.log.debug(`Computed slot ${slot} from L1 block timestamp`, { l1BlockTimestamp });
|
|
609
|
+
return slot;
|
|
532
610
|
}
|
|
533
611
|
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
612
|
+
if (!parentBeaconBlockRoot) {
|
|
613
|
+
// parentBeaconBlockRoot not provided by caller — fetch it from the execution RPC
|
|
614
|
+
if (!l1RpcUrls || l1RpcUrls.length === 0) {
|
|
615
|
+
this.log.debug('No execution host url configured');
|
|
616
|
+
return undefined;
|
|
617
|
+
}
|
|
618
|
+
|
|
619
|
+
const client = createPublicClient({
|
|
620
|
+
transport: makeL1HttpTransport(l1RpcUrls, { timeout: this.config.l1HttpTimeoutMS }),
|
|
543
621
|
});
|
|
622
|
+
try {
|
|
623
|
+
const res: RpcBlock = await client.request({
|
|
624
|
+
method: 'eth_getBlockByHash',
|
|
625
|
+
params: [blockHash, /*tx flag*/ false],
|
|
626
|
+
});
|
|
544
627
|
|
|
545
|
-
|
|
546
|
-
|
|
628
|
+
if (res.parentBeaconBlockRoot) {
|
|
629
|
+
parentBeaconBlockRoot = res.parentBeaconBlockRoot;
|
|
630
|
+
}
|
|
631
|
+
} catch (err) {
|
|
632
|
+
this.log.error(`Error getting parent beacon block root`, err);
|
|
547
633
|
}
|
|
548
|
-
} catch (err) {
|
|
549
|
-
this.log.error(`Error getting parent beacon block root`, err);
|
|
550
634
|
}
|
|
551
635
|
|
|
552
636
|
if (!parentBeaconBlockRoot) {
|
|
@@ -592,9 +676,12 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
592
676
|
|
|
593
677
|
/**
|
|
594
678
|
* Start the blob client.
|
|
595
|
-
*
|
|
679
|
+
* Fetches and caches beacon genesis config for timestamp-based slot resolution,
|
|
680
|
+
* then uploads the initial healthcheck file (awaited) and starts periodic uploads.
|
|
596
681
|
*/
|
|
597
682
|
public async start(): Promise<void> {
|
|
683
|
+
await this.fetchBeaconConfig();
|
|
684
|
+
|
|
598
685
|
if (!this.fileStoreUploadClient) {
|
|
599
686
|
return;
|
|
600
687
|
}
|
|
@@ -619,6 +706,53 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
619
706
|
}, intervalMs);
|
|
620
707
|
}
|
|
621
708
|
|
|
709
|
+
/**
|
|
710
|
+
* Fetches and caches beacon genesis time and slot duration from the first available consensus host.
|
|
711
|
+
* These static values enable timestamp-based slot resolution, eliminating the per-fetch headers call.
|
|
712
|
+
* Logs a warning and leaves fields undefined if all hosts fail, callers fall back gracefully.
|
|
713
|
+
*/
|
|
714
|
+
private async fetchBeaconConfig(): Promise<void> {
|
|
715
|
+
const { l1ConsensusHostUrls } = this.config;
|
|
716
|
+
if (!l1ConsensusHostUrls || l1ConsensusHostUrls.length === 0) {
|
|
717
|
+
return;
|
|
718
|
+
}
|
|
719
|
+
|
|
720
|
+
for (let i = 0; i < l1ConsensusHostUrls.length; i++) {
|
|
721
|
+
try {
|
|
722
|
+
const { url: genesisUrl, ...genesisOptions } = getBeaconNodeFetchOptions(
|
|
723
|
+
`${l1ConsensusHostUrls[i]}/eth/v1/config/genesis`,
|
|
724
|
+
this.config,
|
|
725
|
+
i,
|
|
726
|
+
);
|
|
727
|
+
const { url: specUrl, ...specOptions } = getBeaconNodeFetchOptions(
|
|
728
|
+
`${l1ConsensusHostUrls[i]}/eth/v1/config/spec`,
|
|
729
|
+
this.config,
|
|
730
|
+
i,
|
|
731
|
+
);
|
|
732
|
+
|
|
733
|
+
const [genesisRes, specRes] = await Promise.all([
|
|
734
|
+
this.fetch(genesisUrl, genesisOptions),
|
|
735
|
+
this.fetch(specUrl, specOptions),
|
|
736
|
+
]);
|
|
737
|
+
|
|
738
|
+
if (genesisRes.ok && specRes.ok) {
|
|
739
|
+
const genesis = await genesisRes.json();
|
|
740
|
+
const spec = await specRes.json();
|
|
741
|
+
this.beaconGenesisTime = BigInt(genesis.data.genesisTime);
|
|
742
|
+
this.beaconSecondsPerSlot = parseInt(spec.data.secondsPerSlot);
|
|
743
|
+
this.log.debug(`Fetched beacon genesis config`, {
|
|
744
|
+
genesisTime: this.beaconGenesisTime,
|
|
745
|
+
secondsPerSlot: this.beaconSecondsPerSlot,
|
|
746
|
+
});
|
|
747
|
+
return;
|
|
748
|
+
}
|
|
749
|
+
} catch (err) {
|
|
750
|
+
this.log.warn(`Failed to fetch beacon config from host ${l1ConsensusHostUrls[i]}`, err);
|
|
751
|
+
}
|
|
752
|
+
}
|
|
753
|
+
this.log.warn('Could not fetch beacon genesis config from any consensus host — will use headers call fallback');
|
|
754
|
+
}
|
|
755
|
+
|
|
622
756
|
/**
|
|
623
757
|
* Stop the blob client, clearing any periodic tasks.
|
|
624
758
|
*/
|
|
@@ -630,10 +764,9 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
630
764
|
}
|
|
631
765
|
}
|
|
632
766
|
|
|
633
|
-
function parseBlobJsonsFromResponse(response: any, logger: Logger): BlobJson[] {
|
|
767
|
+
async function parseBlobJsonsFromResponse(response: any, logger: Logger): Promise<BlobJson[]> {
|
|
634
768
|
try {
|
|
635
|
-
|
|
636
|
-
return blobs;
|
|
769
|
+
return await Promise.all((response.data as string[]).map(parseBlobJson));
|
|
637
770
|
} catch (err) {
|
|
638
771
|
logger.error(`Error parsing blob json from response`, err);
|
|
639
772
|
return [];
|
|
@@ -644,10 +777,9 @@ function parseBlobJsonsFromResponse(response: any, logger: Logger): BlobJson[] {
|
|
|
644
777
|
// https://ethereum.github.io/beacon-APIs/?urls.primaryName=dev#/Beacon/getBlobSidecars
|
|
645
778
|
// Here we attempt to parse the response data to Buffer, and check the lengths (via Blob's constructor), to avoid
|
|
646
779
|
// throwing an error down the line when calling Blob.fromJson().
|
|
647
|
-
function parseBlobJson(
|
|
648
|
-
const blobBuffer = Buffer.from(
|
|
649
|
-
const
|
|
650
|
-
const blob = new Blob(blobBuffer, commitmentBuffer);
|
|
780
|
+
async function parseBlobJson(rawHex: string): Promise<BlobJson> {
|
|
781
|
+
const blobBuffer = Buffer.from(rawHex.slice(2), 'hex');
|
|
782
|
+
const blob = await Blob.fromBlobBuffer(blobBuffer);
|
|
651
783
|
return blob.toJSON();
|
|
652
784
|
}
|
|
653
785
|
|
|
@@ -687,12 +819,16 @@ function getBeaconNodeFetchOptions(url: string, config: BlobClientConfig, l1Cons
|
|
|
687
819
|
l1ConsensusHostApiKeyHeaders[l1ConsensusHostIndex];
|
|
688
820
|
|
|
689
821
|
let formattedUrl = url;
|
|
822
|
+
let logSafeUrl = url;
|
|
690
823
|
if (l1ConsensusHostApiKey && l1ConsensusHostApiKey.getValue() !== '' && !l1ConsensusHostApiKeyHeader) {
|
|
691
|
-
|
|
824
|
+
const separator = formattedUrl.includes('?') ? '&' : '?';
|
|
825
|
+
formattedUrl += `${separator}key=${l1ConsensusHostApiKey.getValue()}`;
|
|
826
|
+
logSafeUrl += `${separator}key=[REDACTED]`;
|
|
692
827
|
}
|
|
693
828
|
|
|
694
829
|
return {
|
|
695
830
|
url: formattedUrl,
|
|
831
|
+
logSafeUrl,
|
|
696
832
|
...(l1ConsensusHostApiKey &&
|
|
697
833
|
l1ConsensusHostApiKeyHeader && {
|
|
698
834
|
headers: {
|
package/src/client/interface.ts
CHANGED
|
@@ -6,11 +6,20 @@ import type { Blob } from '@aztec/blob-lib';
|
|
|
6
6
|
export interface GetBlobSidecarOptions {
|
|
7
7
|
/**
|
|
8
8
|
* True if the archiver is catching up (historical sync), false if near tip.
|
|
9
|
-
*
|
|
10
|
-
* - Historical: FileStore first (data should exist), then L1 consensus, then archive (eg. blobscan)
|
|
11
|
-
* - Near tip: FileStore first with no retries (data should exist), L1 consensus second (freshest data), then FileStore with retries, then archive (eg. blobscan)
|
|
9
|
+
* Historical sync uses a shorter retry backoff since blobs should already exist.
|
|
12
10
|
*/
|
|
13
11
|
isHistoricalSync?: boolean;
|
|
12
|
+
/**
|
|
13
|
+
* The parent beacon block root for the L1 block containing the blobs.
|
|
14
|
+
* If provided, skips the eth_getBlockByHash execution RPC call inside getSlotNumber.
|
|
15
|
+
*/
|
|
16
|
+
parentBeaconBlockRoot?: string;
|
|
17
|
+
/**
|
|
18
|
+
* The timestamp of the L1 execution block containing the blobs.
|
|
19
|
+
* When provided alongside a cached beacon genesis config (fetched at startup), allows computing
|
|
20
|
+
* the beacon slot directly via timestamp math, skipping the beacon headers network call entirely.
|
|
21
|
+
*/
|
|
22
|
+
l1BlockTimestamp?: bigint;
|
|
14
23
|
}
|
|
15
24
|
|
|
16
25
|
export interface BlobClientInterface {
|