@aztec/blob-client 0.0.1-commit.e2b2873ed → 0.0.1-commit.e304674f1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dest/blobstore/blob_store_test_suite.js +9 -9
- package/dest/client/config.d.ts +7 -1
- package/dest/client/config.d.ts.map +1 -1
- package/dest/client/config.js +16 -1
- package/dest/client/factory.d.ts +1 -1
- package/dest/client/factory.d.ts.map +1 -1
- package/dest/client/factory.js +7 -1
- package/dest/client/http.d.ts +19 -11
- package/dest/client/http.d.ts.map +1 -1
- package/dest/client/http.js +271 -140
- package/dest/client/interface.d.ts +13 -4
- package/dest/client/interface.d.ts.map +1 -1
- package/dest/client/tests.js +3 -3
- package/dest/filestore/factory.d.ts +5 -4
- package/dest/filestore/factory.d.ts.map +1 -1
- package/dest/filestore/factory.js +4 -4
- package/package.json +7 -7
- package/src/blobstore/blob_store_test_suite.ts +9 -9
- package/src/client/config.ts +25 -0
- package/src/client/factory.ts +8 -1
- package/src/client/http.ts +312 -135
- package/src/client/interface.ts +12 -3
- package/src/client/tests.ts +2 -2
- package/src/filestore/factory.ts +7 -2
package/src/client/http.ts
CHANGED
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
import { Blob, type BlobJson, computeEthVersionedBlobHash } from '@aztec/blob-lib';
|
|
2
|
+
import { makeL1HttpTransport } from '@aztec/ethereum/client';
|
|
2
3
|
import { shuffle } from '@aztec/foundation/array';
|
|
3
4
|
import { type Logger, createLogger } from '@aztec/foundation/log';
|
|
4
5
|
import { makeBackoff, retry } from '@aztec/foundation/retry';
|
|
5
6
|
import { bufferToHex, hexToBuffer } from '@aztec/foundation/string';
|
|
6
7
|
|
|
7
|
-
import { type RpcBlock, createPublicClient
|
|
8
|
+
import { type RpcBlock, createPublicClient } from 'viem';
|
|
8
9
|
|
|
9
10
|
import { createBlobArchiveClient } from '../archive/factory.js';
|
|
10
11
|
import type { BlobArchiveClient } from '../archive/interface.js';
|
|
@@ -24,6 +25,14 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
24
25
|
private disabled = false;
|
|
25
26
|
private healthcheckUploadIntervalId?: NodeJS.Timeout;
|
|
26
27
|
|
|
28
|
+
/** Cached beacon genesis time (seconds since Unix epoch). Fetched once at startup. */
|
|
29
|
+
private beaconGenesisTime?: bigint;
|
|
30
|
+
/** Cached beacon slot duration in seconds. Fetched once at startup. */
|
|
31
|
+
private beaconSecondsPerSlot?: number;
|
|
32
|
+
|
|
33
|
+
/** Indexes of consensus hosts that serve blob sidecars (supernodes). Populated by testSources(). */
|
|
34
|
+
private superNodeHostIndexes?: Set<number>;
|
|
35
|
+
|
|
27
36
|
constructor(
|
|
28
37
|
config?: BlobClientConfig,
|
|
29
38
|
private readonly opts: {
|
|
@@ -89,44 +98,75 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
89
98
|
const archiveUrl = this.archiveClient?.getBaseUrl();
|
|
90
99
|
this.log.info(`Testing configured blob sources`, { l1ConsensusHostUrls, archiveUrl });
|
|
91
100
|
|
|
92
|
-
let
|
|
101
|
+
let consensusSuperNodes = 0;
|
|
102
|
+
let consensusNonSuperNodes = 0;
|
|
103
|
+
let archiveSources = 0;
|
|
104
|
+
let blobSinks = 0;
|
|
105
|
+
|
|
106
|
+
const detectedSuperNodes = new Set<number>();
|
|
93
107
|
|
|
94
108
|
if (l1ConsensusHostUrls && l1ConsensusHostUrls.length > 0) {
|
|
95
109
|
for (let l1ConsensusHostIndex = 0; l1ConsensusHostIndex < l1ConsensusHostUrls.length; l1ConsensusHostIndex++) {
|
|
96
110
|
const l1ConsensusHostUrl = l1ConsensusHostUrls[l1ConsensusHostIndex];
|
|
97
111
|
try {
|
|
98
112
|
const { url, ...options } = getBeaconNodeFetchOptions(
|
|
99
|
-
`${l1ConsensusHostUrl}/eth/v1/beacon/headers`,
|
|
113
|
+
`${l1ConsensusHostUrl}/eth/v1/beacon/headers/head`,
|
|
100
114
|
this.config,
|
|
101
115
|
l1ConsensusHostIndex,
|
|
102
116
|
);
|
|
103
117
|
const res = await this.fetch(url, options);
|
|
104
|
-
if (res.ok) {
|
|
105
|
-
this.log.info(`L1 consensus host is reachable`, { l1ConsensusHostUrl });
|
|
106
|
-
successfulSourceCount++;
|
|
107
|
-
} else {
|
|
118
|
+
if (!res.ok) {
|
|
108
119
|
this.log.error(`Failure reaching L1 consensus host: ${res.statusText} (${res.status})`, {
|
|
109
120
|
l1ConsensusHostUrl,
|
|
110
121
|
});
|
|
122
|
+
continue;
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
this.log.info(`L1 consensus host is reachable`, { l1ConsensusHostUrl });
|
|
126
|
+
|
|
127
|
+
// Check if the host serves blob sidecars (supernode/semi-supernode).
|
|
128
|
+
// Post-Fusaka (PeerDAS), non-supernode beacon nodes no longer serve the
|
|
129
|
+
// blob sidecar endpoint. A 200 response (even with an empty data array
|
|
130
|
+
// for a slot with no blobs) means the node supports serving blob sidecars.
|
|
131
|
+
const body = await res.json();
|
|
132
|
+
const headSlot = body?.data?.header?.message?.slot;
|
|
133
|
+
if (headSlot) {
|
|
134
|
+
const { url: blobUrl, ...blobOptions } = getBeaconNodeFetchOptions(
|
|
135
|
+
`${l1ConsensusHostUrl}/eth/v1/beacon/blobs/${headSlot}`,
|
|
136
|
+
this.config,
|
|
137
|
+
l1ConsensusHostIndex,
|
|
138
|
+
);
|
|
139
|
+
const blobRes = await this.fetch(blobUrl, blobOptions);
|
|
140
|
+
if (blobRes.ok) {
|
|
141
|
+
this.log.info(`L1 consensus host serves blob sidecars (supernode)`, { l1ConsensusHostUrl });
|
|
142
|
+
detectedSuperNodes.add(l1ConsensusHostIndex);
|
|
143
|
+
consensusSuperNodes++;
|
|
144
|
+
} else {
|
|
145
|
+
this.log.info(`L1 consensus host does not serve blob sidecars, skipping for blob fetching`, {
|
|
146
|
+
l1ConsensusHostUrl,
|
|
147
|
+
});
|
|
148
|
+
consensusNonSuperNodes++;
|
|
149
|
+
}
|
|
150
|
+
} else {
|
|
151
|
+
this.log.info(`L1 consensus host is reachable but could not determine head slot`, { l1ConsensusHostUrl });
|
|
152
|
+
consensusNonSuperNodes++;
|
|
111
153
|
}
|
|
112
154
|
} catch (err) {
|
|
113
155
|
this.log.error(`Error reaching L1 consensus host`, err, { l1ConsensusHostUrl });
|
|
114
156
|
}
|
|
115
157
|
}
|
|
116
|
-
} else {
|
|
117
|
-
this.log.warn('No L1 consensus host urls configured');
|
|
118
158
|
}
|
|
119
159
|
|
|
160
|
+
this.superNodeHostIndexes = detectedSuperNodes;
|
|
161
|
+
|
|
120
162
|
if (this.archiveClient) {
|
|
121
163
|
try {
|
|
122
164
|
const latest = await this.archiveClient.getLatestBlock();
|
|
123
165
|
this.log.info(`Archive client is reachable and synced to L1 block ${latest.number}`, { latest, archiveUrl });
|
|
124
|
-
|
|
166
|
+
archiveSources++;
|
|
125
167
|
} catch (err) {
|
|
126
168
|
this.log.error(`Error reaching archive client`, err, { archiveUrl });
|
|
127
169
|
}
|
|
128
|
-
} else {
|
|
129
|
-
this.log.warn('No archive client configured');
|
|
130
170
|
}
|
|
131
171
|
|
|
132
172
|
if (this.fileStoreClients.length > 0) {
|
|
@@ -135,7 +175,7 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
135
175
|
const accessible = await fileStoreClient.testConnection();
|
|
136
176
|
if (accessible) {
|
|
137
177
|
this.log.info(`FileStore is reachable`, { url: fileStoreClient.getBaseUrl() });
|
|
138
|
-
|
|
178
|
+
blobSinks++;
|
|
139
179
|
} else {
|
|
140
180
|
this.log.warn(`FileStore is not accessible`, { url: fileStoreClient.getBaseUrl() });
|
|
141
181
|
}
|
|
@@ -145,12 +185,24 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
145
185
|
}
|
|
146
186
|
}
|
|
147
187
|
|
|
188
|
+
// Emit a single summary after validating all sources
|
|
189
|
+
const successfulSourceCount = consensusSuperNodes + archiveSources + blobSinks;
|
|
190
|
+
|
|
191
|
+
let summary = `Blob client running with consensusSuperNodes=${consensusSuperNodes} archiveSources=${archiveSources} blobSinks=${blobSinks}`;
|
|
192
|
+
if (consensusNonSuperNodes > 0) {
|
|
193
|
+
summary += `. ${consensusNonSuperNodes} consensus client(s) ignored because they are not running in supernode or semi-supernode mode`;
|
|
194
|
+
}
|
|
195
|
+
|
|
148
196
|
if (successfulSourceCount === 0) {
|
|
149
197
|
if (this.config.blobAllowEmptySources) {
|
|
150
|
-
this.log.warn(
|
|
198
|
+
this.log.warn(summary);
|
|
151
199
|
} else {
|
|
152
|
-
throw new Error(
|
|
200
|
+
throw new Error(summary);
|
|
153
201
|
}
|
|
202
|
+
} else if (consensusSuperNodes === 0) {
|
|
203
|
+
this.log.warn(summary);
|
|
204
|
+
} else {
|
|
205
|
+
this.log.info(summary);
|
|
154
206
|
}
|
|
155
207
|
}
|
|
156
208
|
|
|
@@ -176,18 +228,15 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
176
228
|
}
|
|
177
229
|
|
|
178
230
|
/**
|
|
179
|
-
* Get the blob sidecar
|
|
180
|
-
*
|
|
181
|
-
* If requesting from the blob client, we send the blobkHash
|
|
182
|
-
* If requesting from the beacon node, we send the slot number
|
|
231
|
+
* Get the blob sidecar.
|
|
183
232
|
*
|
|
184
|
-
*
|
|
185
|
-
*
|
|
186
|
-
*
|
|
233
|
+
* Alternates between two primary sources (consensus and filestore) in a retry loop,
|
|
234
|
+
* then falls back to archive if blobs are still missing. The order of the primary
|
|
235
|
+
* sources is configurable via `blobPreferFilestores`.
|
|
187
236
|
*
|
|
188
237
|
* @param blockHash - The block hash
|
|
189
238
|
* @param blobHashes - The blob hashes to fetch
|
|
190
|
-
* @param opts - Options
|
|
239
|
+
* @param opts - Options for slot resolution
|
|
191
240
|
* @returns The blobs
|
|
192
241
|
*/
|
|
193
242
|
public async getBlobSidecar(
|
|
@@ -200,12 +249,11 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
200
249
|
return [];
|
|
201
250
|
}
|
|
202
251
|
|
|
203
|
-
const isHistoricalSync = opts?.isHistoricalSync ?? false;
|
|
204
252
|
// Accumulate blobs across sources, preserving order and handling duplicates
|
|
205
253
|
// resultBlobs[i] will contain the blob for blobHashes[i], or undefined if not yet found
|
|
206
254
|
const resultBlobs: (Blob | undefined)[] = new Array(blobHashes.length).fill(undefined);
|
|
207
255
|
|
|
208
|
-
// Helper to get
|
|
256
|
+
// Helper to get missing blob hashes that we still need to fetch
|
|
209
257
|
const getMissingBlobHashes = (): Buffer[] =>
|
|
210
258
|
blobHashes
|
|
211
259
|
.map((bh, i) => (resultBlobs[i] === undefined ? bh : undefined))
|
|
@@ -215,8 +263,8 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
215
263
|
const getFilledBlobs = (): Blob[] => resultBlobs.filter((b): b is Blob => b !== undefined);
|
|
216
264
|
|
|
217
265
|
// Helper to fill in results from fetched blobs
|
|
218
|
-
const fillResults = (fetchedBlobs: BlobJson[]): Blob[] => {
|
|
219
|
-
const blobs = processFetchedBlobs(fetchedBlobs, blobHashes, this.log);
|
|
266
|
+
const fillResults = async (fetchedBlobs: BlobJson[]): Promise<Blob[]> => {
|
|
267
|
+
const blobs = await processFetchedBlobs(fetchedBlobs, blobHashes, this.log);
|
|
220
268
|
// Fill in any missing positions with matching blobs
|
|
221
269
|
for (let i = 0; i < blobHashes.length; i++) {
|
|
222
270
|
if (resultBlobs[i] === undefined) {
|
|
@@ -234,85 +282,66 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
234
282
|
return blobs;
|
|
235
283
|
};
|
|
236
284
|
|
|
237
|
-
const { l1ConsensusHostUrls } = this.config;
|
|
238
|
-
|
|
239
285
|
const ctx = { blockHash, blobHashes: blobHashes.map(bufferToHex) };
|
|
240
286
|
|
|
241
|
-
//
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
287
|
+
// Lazily resolve the slot number — only resolved when consensus hosts are actually tried.
|
|
288
|
+
let slotNumber: number | undefined;
|
|
289
|
+
let slotResolved = false;
|
|
290
|
+
const getSlotNumber = async (): Promise<number | undefined> => {
|
|
291
|
+
if (!slotResolved) {
|
|
292
|
+
slotNumber = await this.resolveSlotNumber(blockHash, opts);
|
|
293
|
+
slotResolved = true;
|
|
246
294
|
}
|
|
247
|
-
|
|
295
|
+
return slotNumber;
|
|
296
|
+
};
|
|
248
297
|
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
const consensusCtx = { l1ConsensusHostUrls, ...ctx };
|
|
253
|
-
this.log.trace(`Attempting to get slot number for block hash`, consensusCtx);
|
|
254
|
-
const slotNumber = await this.getSlotNumber(blockHash);
|
|
255
|
-
this.log.debug(`Got slot number ${slotNumber} from consensus host for querying blobs`, consensusCtx);
|
|
256
|
-
|
|
257
|
-
if (slotNumber) {
|
|
258
|
-
let l1ConsensusHostUrl: string;
|
|
259
|
-
for (let l1ConsensusHostIndex = 0; l1ConsensusHostIndex < l1ConsensusHostUrls.length; l1ConsensusHostIndex++) {
|
|
260
|
-
const missingHashes = getMissingBlobHashes();
|
|
261
|
-
if (missingHashes.length === 0) {
|
|
262
|
-
break;
|
|
263
|
-
}
|
|
298
|
+
// Build the two source-try functions. The order depends on the config.
|
|
299
|
+
const tryConsensus = () => this.tryConsensusHosts(getSlotNumber, getMissingBlobHashes, fillResults, ctx);
|
|
300
|
+
const tryFilestores = () => this.tryFileStores(getMissingBlobHashes, fillResults, ctx);
|
|
264
301
|
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
slotNumber,
|
|
268
|
-
l1ConsensusHostUrl,
|
|
269
|
-
...ctx,
|
|
270
|
-
});
|
|
271
|
-
const blobs = await this.getBlobsFromHost(l1ConsensusHostUrl, slotNumber, l1ConsensusHostIndex);
|
|
272
|
-
const result = fillResults(blobs);
|
|
273
|
-
this.log.debug(
|
|
274
|
-
`Got ${blobs.length} blobs from consensus host (total: ${result.length}/${blobHashes.length})`,
|
|
275
|
-
{ slotNumber, l1ConsensusHostUrl, ...ctx },
|
|
276
|
-
);
|
|
277
|
-
if (result.length === blobHashes.length) {
|
|
278
|
-
return returnWithCallback(result);
|
|
279
|
-
}
|
|
280
|
-
}
|
|
281
|
-
}
|
|
282
|
-
}
|
|
302
|
+
const preferFilestores = this.config.blobPreferFilestores ?? false;
|
|
303
|
+
const [trySourceA, trySourceB] = preferFilestores ? [tryFilestores, tryConsensus] : [tryConsensus, tryFilestores];
|
|
283
304
|
|
|
284
|
-
//
|
|
285
|
-
//
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
305
|
+
// Historical sync: blobs should already exist, use shorter backoff for transient errors.
|
|
306
|
+
// Near-tip sync: blobs may still be uploading, use longer backoff for eventual consistency.
|
|
307
|
+
const isHistoricalSync = opts?.isHistoricalSync ?? false;
|
|
308
|
+
const backoff = isHistoricalSync ? [1, 1] : [1, 1, 1, 2, 2];
|
|
309
|
+
|
|
310
|
+
// Retry loop: alternate between the two primary sources with backoff.
|
|
311
|
+
try {
|
|
312
|
+
await retry(
|
|
313
|
+
async () => {
|
|
314
|
+
if (getMissingBlobHashes().length > 0) {
|
|
315
|
+
await trySourceA();
|
|
316
|
+
}
|
|
317
|
+
if (getMissingBlobHashes().length > 0) {
|
|
318
|
+
await trySourceB();
|
|
319
|
+
}
|
|
320
|
+
if (getMissingBlobHashes().length > 0) {
|
|
321
|
+
throw new Error('Still missing blobs after trying all primary sources');
|
|
322
|
+
}
|
|
323
|
+
},
|
|
324
|
+
'blob retrieval',
|
|
325
|
+
makeBackoff(backoff),
|
|
326
|
+
this.log,
|
|
327
|
+
true, // failSilently — expected during eventual consistency
|
|
328
|
+
);
|
|
329
|
+
return returnWithCallback(getFilledBlobs());
|
|
330
|
+
} catch {
|
|
331
|
+
// Exhausted retries, continue to archive fallback
|
|
304
332
|
}
|
|
305
333
|
|
|
306
|
-
|
|
307
|
-
|
|
334
|
+
// Archive fallback
|
|
335
|
+
const missingAfterPrimary = getMissingBlobHashes();
|
|
336
|
+
if (missingAfterPrimary.length > 0 && this.archiveClient) {
|
|
308
337
|
const archiveCtx = { archiveUrl: this.archiveClient.getBaseUrl(), ...ctx };
|
|
309
|
-
this.log.trace(`Attempting to get ${
|
|
338
|
+
this.log.trace(`Attempting to get ${missingAfterPrimary.length} blobs from archive`, archiveCtx);
|
|
310
339
|
const allBlobs = await this.archiveClient.getBlobsFromBlock(blockHash);
|
|
311
340
|
if (!allBlobs) {
|
|
312
341
|
this.log.debug('No blobs found from archive client', archiveCtx);
|
|
313
342
|
} else {
|
|
314
343
|
this.log.trace(`Got ${allBlobs.length} blobs from archive client before filtering`, archiveCtx);
|
|
315
|
-
const result = fillResults(allBlobs);
|
|
344
|
+
const result = await fillResults(allBlobs);
|
|
316
345
|
this.log.debug(
|
|
317
346
|
`Got ${allBlobs.length} blobs from archive client (total: ${result.length}/${blobHashes.length})`,
|
|
318
347
|
archiveCtx,
|
|
@@ -328,7 +357,7 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
328
357
|
this.log.warn(
|
|
329
358
|
`Failed to fetch all blobs for ${blockHash} from all blob sources (got ${result.length}/${blobHashes.length})`,
|
|
330
359
|
{
|
|
331
|
-
l1ConsensusHostUrls,
|
|
360
|
+
l1ConsensusHostUrls: this.config.l1ConsensusHostUrls,
|
|
332
361
|
archiveUrl: this.archiveClient?.getBaseUrl(),
|
|
333
362
|
fileStoreUrls: this.fileStoreClients.map(c => c.getBaseUrl()),
|
|
334
363
|
},
|
|
@@ -337,6 +366,71 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
337
366
|
return returnWithCallback(result);
|
|
338
367
|
}
|
|
339
368
|
|
|
369
|
+
/** Resolves the beacon slot number for the given block hash. Returns undefined if no consensus hosts. */
|
|
370
|
+
private resolveSlotNumber(
|
|
371
|
+
blockHash: `0x${string}`,
|
|
372
|
+
opts?: GetBlobSidecarOptions,
|
|
373
|
+
): Promise<number | undefined> | undefined {
|
|
374
|
+
const { l1ConsensusHostUrls } = this.config;
|
|
375
|
+
if (!l1ConsensusHostUrls || l1ConsensusHostUrls.length === 0) {
|
|
376
|
+
return undefined;
|
|
377
|
+
}
|
|
378
|
+
// If no supernodes, no point resolving the slot
|
|
379
|
+
if (this.superNodeHostIndexes && this.superNodeHostIndexes.size === 0) {
|
|
380
|
+
return undefined;
|
|
381
|
+
}
|
|
382
|
+
return this.getSlotNumber(blockHash, opts?.parentBeaconBlockRoot, opts?.l1BlockTimestamp);
|
|
383
|
+
}
|
|
384
|
+
|
|
385
|
+
/**
|
|
386
|
+
* Try all supernode consensus hosts for blob sidecars.
|
|
387
|
+
* Skips hosts that were detected as non-supernodes during testSources().
|
|
388
|
+
*/
|
|
389
|
+
private async tryConsensusHosts(
|
|
390
|
+
getSlotNumber: () => Promise<number | undefined>,
|
|
391
|
+
getMissingBlobHashes: () => Buffer[],
|
|
392
|
+
fillResults: (blobs: BlobJson[]) => Promise<Blob[]>,
|
|
393
|
+
ctx: { blockHash: string; blobHashes: string[] },
|
|
394
|
+
): Promise<void> {
|
|
395
|
+
const { l1ConsensusHostUrls } = this.config;
|
|
396
|
+
if (!l1ConsensusHostUrls || l1ConsensusHostUrls.length === 0) {
|
|
397
|
+
return;
|
|
398
|
+
}
|
|
399
|
+
|
|
400
|
+
const slotNumber = await getSlotNumber();
|
|
401
|
+
if (!slotNumber) {
|
|
402
|
+
return;
|
|
403
|
+
}
|
|
404
|
+
|
|
405
|
+
for (let l1ConsensusHostIndex = 0; l1ConsensusHostIndex < l1ConsensusHostUrls.length; l1ConsensusHostIndex++) {
|
|
406
|
+
const missingHashes = getMissingBlobHashes();
|
|
407
|
+
if (missingHashes.length === 0) {
|
|
408
|
+
break;
|
|
409
|
+
}
|
|
410
|
+
|
|
411
|
+
// Skip non-supernode hosts if we've already detected supernodes
|
|
412
|
+
if (this.superNodeHostIndexes && !this.superNodeHostIndexes.has(l1ConsensusHostIndex)) {
|
|
413
|
+
this.log.trace(`Skipping non-supernode consensus host`, {
|
|
414
|
+
l1ConsensusHostUrl: l1ConsensusHostUrls[l1ConsensusHostIndex],
|
|
415
|
+
});
|
|
416
|
+
continue;
|
|
417
|
+
}
|
|
418
|
+
|
|
419
|
+
const l1ConsensusHostUrl = l1ConsensusHostUrls[l1ConsensusHostIndex];
|
|
420
|
+
this.log.trace(`Attempting to get ${missingHashes.length} blobs from consensus host`, {
|
|
421
|
+
slotNumber,
|
|
422
|
+
l1ConsensusHostUrl,
|
|
423
|
+
...ctx,
|
|
424
|
+
});
|
|
425
|
+
const blobs = await this.getBlobsFromHost(l1ConsensusHostUrl, slotNumber, l1ConsensusHostIndex, missingHashes);
|
|
426
|
+
const result = await fillResults(blobs);
|
|
427
|
+
this.log.debug(
|
|
428
|
+
`Got ${blobs.length} blobs from consensus host (total: ${result.length}/${ctx.blobHashes.length})`,
|
|
429
|
+
{ slotNumber, l1ConsensusHostUrl, ...ctx },
|
|
430
|
+
);
|
|
431
|
+
}
|
|
432
|
+
}
|
|
433
|
+
|
|
340
434
|
/**
|
|
341
435
|
* Try all filestores once (shuffled for load distribution).
|
|
342
436
|
* @param getMissingBlobHashes - Function to get remaining blob hashes to fetch
|
|
@@ -345,7 +439,7 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
345
439
|
*/
|
|
346
440
|
private async tryFileStores(
|
|
347
441
|
getMissingBlobHashes: () => Buffer[],
|
|
348
|
-
fillResults: (blobs: BlobJson[]) => Blob[]
|
|
442
|
+
fillResults: (blobs: BlobJson[]) => Promise<Blob[]>,
|
|
349
443
|
ctx: { blockHash: string; blobHashes: string[] },
|
|
350
444
|
): Promise<void> {
|
|
351
445
|
// Shuffle clients for load distribution
|
|
@@ -366,7 +460,7 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
366
460
|
});
|
|
367
461
|
const blobs = await client.getBlobsByHashes(blobHashStrings);
|
|
368
462
|
if (blobs.length > 0) {
|
|
369
|
-
const result = fillResults(blobs);
|
|
463
|
+
const result = await fillResults(blobs);
|
|
370
464
|
this.log.debug(
|
|
371
465
|
`Got ${blobs.length} blobs from filestore (total: ${result.length}/${ctx.blobHashes.length})`,
|
|
372
466
|
{
|
|
@@ -387,19 +481,20 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
387
481
|
blobHashes: Buffer[] = [],
|
|
388
482
|
l1ConsensusHostIndex?: number,
|
|
389
483
|
): Promise<Blob[]> {
|
|
390
|
-
const blobs = await this.getBlobsFromHost(hostUrl, blockHashOrSlot, l1ConsensusHostIndex);
|
|
391
|
-
return processFetchedBlobs(blobs, blobHashes, this.log).filter((b): b is Blob => b !== undefined);
|
|
484
|
+
const blobs = await this.getBlobsFromHost(hostUrl, blockHashOrSlot, l1ConsensusHostIndex, blobHashes);
|
|
485
|
+
return (await processFetchedBlobs(blobs, blobHashes, this.log)).filter((b): b is Blob => b !== undefined);
|
|
392
486
|
}
|
|
393
487
|
|
|
394
488
|
public async getBlobsFromHost(
|
|
395
489
|
hostUrl: string,
|
|
396
490
|
blockHashOrSlot: string | number,
|
|
397
491
|
l1ConsensusHostIndex?: number,
|
|
492
|
+
blobHashes?: Buffer[],
|
|
398
493
|
): Promise<BlobJson[]> {
|
|
399
494
|
try {
|
|
400
|
-
let res = await this.fetchBlobSidecars(hostUrl, blockHashOrSlot, l1ConsensusHostIndex);
|
|
495
|
+
let res = await this.fetchBlobSidecars(hostUrl, blockHashOrSlot, l1ConsensusHostIndex, blobHashes);
|
|
401
496
|
if (res.ok) {
|
|
402
|
-
return parseBlobJsonsFromResponse(await res.json(), this.log);
|
|
497
|
+
return await parseBlobJsonsFromResponse(await res.json(), this.log);
|
|
403
498
|
}
|
|
404
499
|
|
|
405
500
|
if (res.status === 404 && typeof blockHashOrSlot === 'number') {
|
|
@@ -414,9 +509,9 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
414
509
|
let currentSlot = blockHashOrSlot + 1;
|
|
415
510
|
while (res.status === 404 && maxRetries > 0 && latestSlot !== undefined && currentSlot <= latestSlot) {
|
|
416
511
|
this.log.debug(`Trying slot ${currentSlot}`);
|
|
417
|
-
res = await this.fetchBlobSidecars(hostUrl, currentSlot, l1ConsensusHostIndex);
|
|
512
|
+
res = await this.fetchBlobSidecars(hostUrl, currentSlot, l1ConsensusHostIndex, blobHashes);
|
|
418
513
|
if (res.ok) {
|
|
419
|
-
return parseBlobJsonsFromResponse(await res.json(), this.log);
|
|
514
|
+
return await parseBlobJsonsFromResponse(await res.json(), this.log);
|
|
420
515
|
}
|
|
421
516
|
currentSlot++;
|
|
422
517
|
maxRetries--;
|
|
@@ -439,19 +534,29 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
439
534
|
hostUrl: string,
|
|
440
535
|
blockHashOrSlot: string | number,
|
|
441
536
|
l1ConsensusHostIndex?: number,
|
|
537
|
+
blobHashes?: Buffer[],
|
|
442
538
|
): Promise<Response> {
|
|
443
|
-
|
|
539
|
+
let baseUrl = `${hostUrl}/eth/v1/beacon/blobs/${blockHashOrSlot}`;
|
|
540
|
+
|
|
541
|
+
if (blobHashes && blobHashes.length > 0) {
|
|
542
|
+
const params = new URLSearchParams();
|
|
543
|
+
for (const hash of blobHashes) {
|
|
544
|
+
params.append('versioned_hashes', `0x${hash.toString('hex')}`);
|
|
545
|
+
}
|
|
546
|
+
baseUrl += `?${params.toString()}`;
|
|
547
|
+
}
|
|
444
548
|
|
|
445
|
-
const { url, ...options } = getBeaconNodeFetchOptions(baseUrl, this.config, l1ConsensusHostIndex);
|
|
446
|
-
this.log.debug(`Fetching blob sidecar for ${blockHashOrSlot}`, { url, ...options });
|
|
447
|
-
|
|
549
|
+
const { url, logSafeUrl, ...options } = getBeaconNodeFetchOptions(baseUrl, this.config, l1ConsensusHostIndex);
|
|
550
|
+
this.log.debug(`Fetching blob sidecar for ${blockHashOrSlot}`, { url: logSafeUrl, ...options });
|
|
551
|
+
// No retry here — this is called inside the main retry loop in getBlobSidecar
|
|
552
|
+
return fetch(url, options);
|
|
448
553
|
}
|
|
449
554
|
|
|
450
555
|
private async getLatestSlotNumber(hostUrl: string, l1ConsensusHostIndex?: number): Promise<number | undefined> {
|
|
451
556
|
try {
|
|
452
557
|
const baseUrl = `${hostUrl}/eth/v1/beacon/headers/head`;
|
|
453
|
-
const { url, ...options } = getBeaconNodeFetchOptions(baseUrl, this.config, l1ConsensusHostIndex);
|
|
454
|
-
this.log.debug(`Fetching latest slot number`, { url, ...options });
|
|
558
|
+
const { url, logSafeUrl, ...options } = getBeaconNodeFetchOptions(baseUrl, this.config, l1ConsensusHostIndex);
|
|
559
|
+
this.log.debug(`Fetching latest slot number`, { url: logSafeUrl, ...options });
|
|
455
560
|
const res = await this.fetch(url, options);
|
|
456
561
|
if (res.ok) {
|
|
457
562
|
const body = await res.json();
|
|
@@ -482,34 +587,50 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
482
587
|
* @param blockHash - The block hash
|
|
483
588
|
* @returns The slot number
|
|
484
589
|
*/
|
|
485
|
-
private async getSlotNumber(
|
|
590
|
+
private async getSlotNumber(
|
|
591
|
+
blockHash: `0x${string}`,
|
|
592
|
+
parentBeaconBlockRoot?: string,
|
|
593
|
+
l1BlockTimestamp?: bigint,
|
|
594
|
+
): Promise<number | undefined> {
|
|
486
595
|
const { l1ConsensusHostUrls, l1RpcUrls } = this.config;
|
|
487
596
|
if (!l1ConsensusHostUrls || l1ConsensusHostUrls.length === 0) {
|
|
488
597
|
this.log.debug('No consensus host url configured');
|
|
489
598
|
return undefined;
|
|
490
599
|
}
|
|
491
600
|
|
|
492
|
-
if (
|
|
493
|
-
|
|
494
|
-
|
|
601
|
+
// Primary path: compute slot from timestamp if genesis config is cached (no network call needed)
|
|
602
|
+
if (
|
|
603
|
+
l1BlockTimestamp !== undefined &&
|
|
604
|
+
this.beaconGenesisTime !== undefined &&
|
|
605
|
+
this.beaconSecondsPerSlot !== undefined
|
|
606
|
+
) {
|
|
607
|
+
const slot = Number((l1BlockTimestamp - this.beaconGenesisTime) / BigInt(this.beaconSecondsPerSlot));
|
|
608
|
+
this.log.debug(`Computed slot ${slot} from L1 block timestamp`, { l1BlockTimestamp });
|
|
609
|
+
return slot;
|
|
495
610
|
}
|
|
496
611
|
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
612
|
+
if (!parentBeaconBlockRoot) {
|
|
613
|
+
// parentBeaconBlockRoot not provided by caller — fetch it from the execution RPC
|
|
614
|
+
if (!l1RpcUrls || l1RpcUrls.length === 0) {
|
|
615
|
+
this.log.debug('No execution host url configured');
|
|
616
|
+
return undefined;
|
|
617
|
+
}
|
|
618
|
+
|
|
619
|
+
const client = createPublicClient({
|
|
620
|
+
transport: makeL1HttpTransport(l1RpcUrls, { timeout: this.config.l1HttpTimeoutMS }),
|
|
506
621
|
});
|
|
622
|
+
try {
|
|
623
|
+
const res: RpcBlock = await client.request({
|
|
624
|
+
method: 'eth_getBlockByHash',
|
|
625
|
+
params: [blockHash, /*tx flag*/ false],
|
|
626
|
+
});
|
|
507
627
|
|
|
508
|
-
|
|
509
|
-
|
|
628
|
+
if (res.parentBeaconBlockRoot) {
|
|
629
|
+
parentBeaconBlockRoot = res.parentBeaconBlockRoot;
|
|
630
|
+
}
|
|
631
|
+
} catch (err) {
|
|
632
|
+
this.log.error(`Error getting parent beacon block root`, err);
|
|
510
633
|
}
|
|
511
|
-
} catch (err) {
|
|
512
|
-
this.log.error(`Error getting parent beacon block root`, err);
|
|
513
634
|
}
|
|
514
635
|
|
|
515
636
|
if (!parentBeaconBlockRoot) {
|
|
@@ -555,9 +676,12 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
555
676
|
|
|
556
677
|
/**
|
|
557
678
|
* Start the blob client.
|
|
558
|
-
*
|
|
679
|
+
* Fetches and caches beacon genesis config for timestamp-based slot resolution,
|
|
680
|
+
* then uploads the initial healthcheck file (awaited) and starts periodic uploads.
|
|
559
681
|
*/
|
|
560
682
|
public async start(): Promise<void> {
|
|
683
|
+
await this.fetchBeaconConfig();
|
|
684
|
+
|
|
561
685
|
if (!this.fileStoreUploadClient) {
|
|
562
686
|
return;
|
|
563
687
|
}
|
|
@@ -582,6 +706,53 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
582
706
|
}, intervalMs);
|
|
583
707
|
}
|
|
584
708
|
|
|
709
|
+
/**
|
|
710
|
+
* Fetches and caches beacon genesis time and slot duration from the first available consensus host.
|
|
711
|
+
* These static values enable timestamp-based slot resolution, eliminating the per-fetch headers call.
|
|
712
|
+
* Logs a warning and leaves fields undefined if all hosts fail, callers fall back gracefully.
|
|
713
|
+
*/
|
|
714
|
+
private async fetchBeaconConfig(): Promise<void> {
|
|
715
|
+
const { l1ConsensusHostUrls } = this.config;
|
|
716
|
+
if (!l1ConsensusHostUrls || l1ConsensusHostUrls.length === 0) {
|
|
717
|
+
return;
|
|
718
|
+
}
|
|
719
|
+
|
|
720
|
+
for (let i = 0; i < l1ConsensusHostUrls.length; i++) {
|
|
721
|
+
try {
|
|
722
|
+
const { url: genesisUrl, ...genesisOptions } = getBeaconNodeFetchOptions(
|
|
723
|
+
`${l1ConsensusHostUrls[i]}/eth/v1/config/genesis`,
|
|
724
|
+
this.config,
|
|
725
|
+
i,
|
|
726
|
+
);
|
|
727
|
+
const { url: specUrl, ...specOptions } = getBeaconNodeFetchOptions(
|
|
728
|
+
`${l1ConsensusHostUrls[i]}/eth/v1/config/spec`,
|
|
729
|
+
this.config,
|
|
730
|
+
i,
|
|
731
|
+
);
|
|
732
|
+
|
|
733
|
+
const [genesisRes, specRes] = await Promise.all([
|
|
734
|
+
this.fetch(genesisUrl, genesisOptions),
|
|
735
|
+
this.fetch(specUrl, specOptions),
|
|
736
|
+
]);
|
|
737
|
+
|
|
738
|
+
if (genesisRes.ok && specRes.ok) {
|
|
739
|
+
const genesis = await genesisRes.json();
|
|
740
|
+
const spec = await specRes.json();
|
|
741
|
+
this.beaconGenesisTime = BigInt(genesis.data.genesisTime);
|
|
742
|
+
this.beaconSecondsPerSlot = parseInt(spec.data.secondsPerSlot);
|
|
743
|
+
this.log.debug(`Fetched beacon genesis config`, {
|
|
744
|
+
genesisTime: this.beaconGenesisTime,
|
|
745
|
+
secondsPerSlot: this.beaconSecondsPerSlot,
|
|
746
|
+
});
|
|
747
|
+
return;
|
|
748
|
+
}
|
|
749
|
+
} catch (err) {
|
|
750
|
+
this.log.warn(`Failed to fetch beacon config from host ${l1ConsensusHostUrls[i]}`, err);
|
|
751
|
+
}
|
|
752
|
+
}
|
|
753
|
+
this.log.warn('Could not fetch beacon genesis config from any consensus host — will use headers call fallback');
|
|
754
|
+
}
|
|
755
|
+
|
|
585
756
|
/**
|
|
586
757
|
* Stop the blob client, clearing any periodic tasks.
|
|
587
758
|
*/
|
|
@@ -593,10 +764,9 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
593
764
|
}
|
|
594
765
|
}
|
|
595
766
|
|
|
596
|
-
function parseBlobJsonsFromResponse(response: any, logger: Logger): BlobJson[] {
|
|
767
|
+
async function parseBlobJsonsFromResponse(response: any, logger: Logger): Promise<BlobJson[]> {
|
|
597
768
|
try {
|
|
598
|
-
|
|
599
|
-
return blobs;
|
|
769
|
+
return await Promise.all((response.data as string[]).map(parseBlobJson));
|
|
600
770
|
} catch (err) {
|
|
601
771
|
logger.error(`Error parsing blob json from response`, err);
|
|
602
772
|
return [];
|
|
@@ -607,16 +777,19 @@ function parseBlobJsonsFromResponse(response: any, logger: Logger): BlobJson[] {
|
|
|
607
777
|
// https://ethereum.github.io/beacon-APIs/?urls.primaryName=dev#/Beacon/getBlobSidecars
|
|
608
778
|
// Here we attempt to parse the response data to Buffer, and check the lengths (via Blob's constructor), to avoid
|
|
609
779
|
// throwing an error down the line when calling Blob.fromJson().
|
|
610
|
-
function parseBlobJson(
|
|
611
|
-
const blobBuffer = Buffer.from(
|
|
612
|
-
const
|
|
613
|
-
const blob = new Blob(blobBuffer, commitmentBuffer);
|
|
780
|
+
async function parseBlobJson(rawHex: string): Promise<BlobJson> {
|
|
781
|
+
const blobBuffer = Buffer.from(rawHex.slice(2), 'hex');
|
|
782
|
+
const blob = await Blob.fromBlobBuffer(blobBuffer);
|
|
614
783
|
return blob.toJSON();
|
|
615
784
|
}
|
|
616
785
|
|
|
617
786
|
// Returns an array that maps each blob hash to the corresponding blob, or undefined if the blob is not found
|
|
618
787
|
// or the data does not match the commitment.
|
|
619
|
-
function processFetchedBlobs(
|
|
788
|
+
async function processFetchedBlobs(
|
|
789
|
+
blobs: BlobJson[],
|
|
790
|
+
blobHashes: Buffer[],
|
|
791
|
+
logger: Logger,
|
|
792
|
+
): Promise<(Blob | undefined)[]> {
|
|
620
793
|
const requestedBlobHashes = new Set<string>(blobHashes.map(bufferToHex));
|
|
621
794
|
const hashToBlob = new Map<string, Blob>();
|
|
622
795
|
for (const blobJson of blobs) {
|
|
@@ -626,7 +799,7 @@ function processFetchedBlobs(blobs: BlobJson[], blobHashes: Buffer[], logger: Lo
|
|
|
626
799
|
}
|
|
627
800
|
|
|
628
801
|
try {
|
|
629
|
-
const blob = Blob.fromJson(blobJson);
|
|
802
|
+
const blob = await Blob.fromJson(blobJson);
|
|
630
803
|
hashToBlob.set(hashHex, blob);
|
|
631
804
|
} catch (err) {
|
|
632
805
|
// If the above throws, it's likely that the blob commitment does not match the hash of the blob data.
|
|
@@ -646,12 +819,16 @@ function getBeaconNodeFetchOptions(url: string, config: BlobClientConfig, l1Cons
|
|
|
646
819
|
l1ConsensusHostApiKeyHeaders[l1ConsensusHostIndex];
|
|
647
820
|
|
|
648
821
|
let formattedUrl = url;
|
|
822
|
+
let logSafeUrl = url;
|
|
649
823
|
if (l1ConsensusHostApiKey && l1ConsensusHostApiKey.getValue() !== '' && !l1ConsensusHostApiKeyHeader) {
|
|
650
|
-
|
|
824
|
+
const separator = formattedUrl.includes('?') ? '&' : '?';
|
|
825
|
+
formattedUrl += `${separator}key=${l1ConsensusHostApiKey.getValue()}`;
|
|
826
|
+
logSafeUrl += `${separator}key=[REDACTED]`;
|
|
651
827
|
}
|
|
652
828
|
|
|
653
829
|
return {
|
|
654
830
|
url: formattedUrl,
|
|
831
|
+
logSafeUrl,
|
|
655
832
|
...(l1ConsensusHostApiKey &&
|
|
656
833
|
l1ConsensusHostApiKeyHeader && {
|
|
657
834
|
headers: {
|