@aztec/blob-client 0.0.1-commit.03f7ef2 → 0.0.1-commit.04d373f
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -1
- package/dest/archive/blobscan_archive_client.d.ts +8 -92
- package/dest/archive/blobscan_archive_client.d.ts.map +1 -1
- package/dest/archive/config.js +1 -1
- package/dest/archive/instrumentation.d.ts +1 -1
- package/dest/archive/instrumentation.d.ts.map +1 -1
- package/dest/archive/instrumentation.js +13 -13
- package/dest/blobstore/blob_store_test_suite.js +9 -9
- package/dest/client/config.d.ts +11 -1
- package/dest/client/config.d.ts.map +1 -1
- package/dest/client/config.js +22 -2
- package/dest/client/factory.d.ts +5 -6
- package/dest/client/factory.d.ts.map +1 -1
- package/dest/client/factory.js +13 -4
- package/dest/client/http.d.ts +34 -10
- package/dest/client/http.d.ts.map +1 -1
- package/dest/client/http.js +304 -139
- package/dest/client/interface.d.ts +19 -4
- package/dest/client/interface.d.ts.map +1 -1
- package/dest/client/local.d.ts +3 -1
- package/dest/client/local.d.ts.map +1 -1
- package/dest/client/local.js +3 -0
- package/dest/client/tests.js +3 -3
- package/dest/filestore/factory.d.ts +5 -4
- package/dest/filestore/factory.d.ts.map +1 -1
- package/dest/filestore/factory.js +4 -4
- package/dest/filestore/filestore_blob_client.d.ts +14 -2
- package/dest/filestore/filestore_blob_client.d.ts.map +1 -1
- package/dest/filestore/filestore_blob_client.js +29 -5
- package/dest/filestore/healthcheck.d.ts +5 -0
- package/dest/filestore/healthcheck.d.ts.map +1 -0
- package/dest/filestore/healthcheck.js +3 -0
- package/package.json +9 -9
- package/src/archive/config.ts +1 -1
- package/src/archive/instrumentation.ts +22 -13
- package/src/blobstore/blob_store_test_suite.ts +9 -9
- package/src/client/config.ts +36 -1
- package/src/client/factory.ts +17 -5
- package/src/client/http.ts +357 -134
- package/src/client/interface.ts +18 -3
- package/src/client/local.ts +5 -0
- package/src/client/tests.ts +2 -2
- package/src/filestore/factory.ts +7 -2
- package/src/filestore/filestore_blob_client.ts +33 -5
- package/src/filestore/healthcheck.ts +5 -0
package/src/client/http.ts
CHANGED
|
@@ -1,14 +1,16 @@
|
|
|
1
1
|
import { Blob, type BlobJson, computeEthVersionedBlobHash } from '@aztec/blob-lib';
|
|
2
|
+
import { makeL1HttpTransport } from '@aztec/ethereum/client';
|
|
2
3
|
import { shuffle } from '@aztec/foundation/array';
|
|
3
4
|
import { type Logger, createLogger } from '@aztec/foundation/log';
|
|
4
5
|
import { makeBackoff, retry } from '@aztec/foundation/retry';
|
|
5
6
|
import { bufferToHex, hexToBuffer } from '@aztec/foundation/string';
|
|
6
7
|
|
|
7
|
-
import { type RpcBlock, createPublicClient
|
|
8
|
+
import { type RpcBlock, createPublicClient } from 'viem';
|
|
8
9
|
|
|
9
10
|
import { createBlobArchiveClient } from '../archive/factory.js';
|
|
10
11
|
import type { BlobArchiveClient } from '../archive/interface.js';
|
|
11
12
|
import type { FileStoreBlobClient } from '../filestore/filestore_blob_client.js';
|
|
13
|
+
import { DEFAULT_HEALTHCHECK_UPLOAD_INTERVAL_MINUTES } from '../filestore/healthcheck.js';
|
|
12
14
|
import { type BlobClientConfig, getBlobClientConfigFromEnv } from './config.js';
|
|
13
15
|
import type { BlobClientInterface, GetBlobSidecarOptions } from './interface.js';
|
|
14
16
|
|
|
@@ -21,6 +23,15 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
21
23
|
protected readonly fileStoreUploadClient: FileStoreBlobClient | undefined;
|
|
22
24
|
|
|
23
25
|
private disabled = false;
|
|
26
|
+
private healthcheckUploadIntervalId?: NodeJS.Timeout;
|
|
27
|
+
|
|
28
|
+
/** Cached beacon genesis time (seconds since Unix epoch). Fetched once at startup. */
|
|
29
|
+
private beaconGenesisTime?: bigint;
|
|
30
|
+
/** Cached beacon slot duration in seconds. Fetched once at startup. */
|
|
31
|
+
private beaconSecondsPerSlot?: number;
|
|
32
|
+
|
|
33
|
+
/** Indexes of consensus hosts that serve blob sidecars (supernodes). Populated by testSources(). */
|
|
34
|
+
private superNodeHostIndexes?: Set<number>;
|
|
24
35
|
|
|
25
36
|
constructor(
|
|
26
37
|
config?: BlobClientConfig,
|
|
@@ -87,44 +98,75 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
87
98
|
const archiveUrl = this.archiveClient?.getBaseUrl();
|
|
88
99
|
this.log.info(`Testing configured blob sources`, { l1ConsensusHostUrls, archiveUrl });
|
|
89
100
|
|
|
90
|
-
let
|
|
101
|
+
let consensusSuperNodes = 0;
|
|
102
|
+
let consensusNonSuperNodes = 0;
|
|
103
|
+
let archiveSources = 0;
|
|
104
|
+
let blobSinks = 0;
|
|
105
|
+
|
|
106
|
+
const detectedSuperNodes = new Set<number>();
|
|
91
107
|
|
|
92
108
|
if (l1ConsensusHostUrls && l1ConsensusHostUrls.length > 0) {
|
|
93
109
|
for (let l1ConsensusHostIndex = 0; l1ConsensusHostIndex < l1ConsensusHostUrls.length; l1ConsensusHostIndex++) {
|
|
94
110
|
const l1ConsensusHostUrl = l1ConsensusHostUrls[l1ConsensusHostIndex];
|
|
95
111
|
try {
|
|
96
112
|
const { url, ...options } = getBeaconNodeFetchOptions(
|
|
97
|
-
`${l1ConsensusHostUrl}/eth/v1/beacon/headers`,
|
|
113
|
+
`${l1ConsensusHostUrl}/eth/v1/beacon/headers/head`,
|
|
98
114
|
this.config,
|
|
99
115
|
l1ConsensusHostIndex,
|
|
100
116
|
);
|
|
101
117
|
const res = await this.fetch(url, options);
|
|
102
|
-
if (res.ok) {
|
|
103
|
-
this.log.info(`L1 consensus host is reachable`, { l1ConsensusHostUrl });
|
|
104
|
-
successfulSourceCount++;
|
|
105
|
-
} else {
|
|
118
|
+
if (!res.ok) {
|
|
106
119
|
this.log.error(`Failure reaching L1 consensus host: ${res.statusText} (${res.status})`, {
|
|
107
120
|
l1ConsensusHostUrl,
|
|
108
121
|
});
|
|
122
|
+
continue;
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
this.log.info(`L1 consensus host is reachable`, { l1ConsensusHostUrl });
|
|
126
|
+
|
|
127
|
+
// Check if the host serves blob sidecars (supernode/semi-supernode).
|
|
128
|
+
// Post-Fusaka (PeerDAS), non-supernode beacon nodes no longer serve the
|
|
129
|
+
// blob sidecar endpoint. A 200 response (even with an empty data array
|
|
130
|
+
// for a slot with no blobs) means the node supports serving blob sidecars.
|
|
131
|
+
const body = await res.json();
|
|
132
|
+
const headSlot = body?.data?.header?.message?.slot;
|
|
133
|
+
if (headSlot) {
|
|
134
|
+
const { url: blobUrl, ...blobOptions } = getBeaconNodeFetchOptions(
|
|
135
|
+
`${l1ConsensusHostUrl}/eth/v1/beacon/blobs/${headSlot}`,
|
|
136
|
+
this.config,
|
|
137
|
+
l1ConsensusHostIndex,
|
|
138
|
+
);
|
|
139
|
+
const blobRes = await this.fetch(blobUrl, blobOptions);
|
|
140
|
+
if (blobRes.ok) {
|
|
141
|
+
this.log.info(`L1 consensus host serves blob sidecars (supernode)`, { l1ConsensusHostUrl });
|
|
142
|
+
detectedSuperNodes.add(l1ConsensusHostIndex);
|
|
143
|
+
consensusSuperNodes++;
|
|
144
|
+
} else {
|
|
145
|
+
this.log.info(`L1 consensus host does not serve blob sidecars, skipping for blob fetching`, {
|
|
146
|
+
l1ConsensusHostUrl,
|
|
147
|
+
});
|
|
148
|
+
consensusNonSuperNodes++;
|
|
149
|
+
}
|
|
150
|
+
} else {
|
|
151
|
+
this.log.info(`L1 consensus host is reachable but could not determine head slot`, { l1ConsensusHostUrl });
|
|
152
|
+
consensusNonSuperNodes++;
|
|
109
153
|
}
|
|
110
154
|
} catch (err) {
|
|
111
155
|
this.log.error(`Error reaching L1 consensus host`, err, { l1ConsensusHostUrl });
|
|
112
156
|
}
|
|
113
157
|
}
|
|
114
|
-
} else {
|
|
115
|
-
this.log.warn('No L1 consensus host urls configured');
|
|
116
158
|
}
|
|
117
159
|
|
|
160
|
+
this.superNodeHostIndexes = detectedSuperNodes;
|
|
161
|
+
|
|
118
162
|
if (this.archiveClient) {
|
|
119
163
|
try {
|
|
120
164
|
const latest = await this.archiveClient.getLatestBlock();
|
|
121
165
|
this.log.info(`Archive client is reachable and synced to L1 block ${latest.number}`, { latest, archiveUrl });
|
|
122
|
-
|
|
166
|
+
archiveSources++;
|
|
123
167
|
} catch (err) {
|
|
124
168
|
this.log.error(`Error reaching archive client`, err, { archiveUrl });
|
|
125
169
|
}
|
|
126
|
-
} else {
|
|
127
|
-
this.log.warn('No archive client configured');
|
|
128
170
|
}
|
|
129
171
|
|
|
130
172
|
if (this.fileStoreClients.length > 0) {
|
|
@@ -133,7 +175,7 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
133
175
|
const accessible = await fileStoreClient.testConnection();
|
|
134
176
|
if (accessible) {
|
|
135
177
|
this.log.info(`FileStore is reachable`, { url: fileStoreClient.getBaseUrl() });
|
|
136
|
-
|
|
178
|
+
blobSinks++;
|
|
137
179
|
} else {
|
|
138
180
|
this.log.warn(`FileStore is not accessible`, { url: fileStoreClient.getBaseUrl() });
|
|
139
181
|
}
|
|
@@ -143,12 +185,24 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
143
185
|
}
|
|
144
186
|
}
|
|
145
187
|
|
|
188
|
+
// Emit a single summary after validating all sources
|
|
189
|
+
const successfulSourceCount = consensusSuperNodes + archiveSources + blobSinks;
|
|
190
|
+
|
|
191
|
+
let summary = `Blob client running with consensusSuperNodes=${consensusSuperNodes} archiveSources=${archiveSources} blobSinks=${blobSinks}`;
|
|
192
|
+
if (consensusNonSuperNodes > 0) {
|
|
193
|
+
summary += `. ${consensusNonSuperNodes} consensus client(s) ignored because they are not running in supernode or semi-supernode mode`;
|
|
194
|
+
}
|
|
195
|
+
|
|
146
196
|
if (successfulSourceCount === 0) {
|
|
147
197
|
if (this.config.blobAllowEmptySources) {
|
|
148
|
-
this.log.warn(
|
|
198
|
+
this.log.warn(summary);
|
|
149
199
|
} else {
|
|
150
|
-
throw new Error(
|
|
200
|
+
throw new Error(summary);
|
|
151
201
|
}
|
|
202
|
+
} else if (consensusSuperNodes === 0) {
|
|
203
|
+
this.log.warn(summary);
|
|
204
|
+
} else {
|
|
205
|
+
this.log.info(summary);
|
|
152
206
|
}
|
|
153
207
|
}
|
|
154
208
|
|
|
@@ -174,18 +228,15 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
174
228
|
}
|
|
175
229
|
|
|
176
230
|
/**
|
|
177
|
-
* Get the blob sidecar
|
|
231
|
+
* Get the blob sidecar.
|
|
178
232
|
*
|
|
179
|
-
*
|
|
180
|
-
*
|
|
181
|
-
*
|
|
182
|
-
* Source ordering depends on sync state:
|
|
183
|
-
* - Historical sync: blob client → FileStore → L1 consensus → Archive
|
|
184
|
-
* - Near tip sync: blob client → FileStore → L1 consensus → FileStore (with retries) → Archive (eg blobscan)
|
|
233
|
+
* Alternates between two primary sources (consensus and filestore) in a retry loop,
|
|
234
|
+
* then falls back to archive if blobs are still missing. The order of the primary
|
|
235
|
+
* sources is configurable via `blobPreferFilestores`.
|
|
185
236
|
*
|
|
186
237
|
* @param blockHash - The block hash
|
|
187
238
|
* @param blobHashes - The blob hashes to fetch
|
|
188
|
-
* @param opts - Options
|
|
239
|
+
* @param opts - Options for slot resolution
|
|
189
240
|
* @returns The blobs
|
|
190
241
|
*/
|
|
191
242
|
public async getBlobSidecar(
|
|
@@ -198,12 +249,11 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
198
249
|
return [];
|
|
199
250
|
}
|
|
200
251
|
|
|
201
|
-
const isHistoricalSync = opts?.isHistoricalSync ?? false;
|
|
202
252
|
// Accumulate blobs across sources, preserving order and handling duplicates
|
|
203
253
|
// resultBlobs[i] will contain the blob for blobHashes[i], or undefined if not yet found
|
|
204
254
|
const resultBlobs: (Blob | undefined)[] = new Array(blobHashes.length).fill(undefined);
|
|
205
255
|
|
|
206
|
-
// Helper to get
|
|
256
|
+
// Helper to get missing blob hashes that we still need to fetch
|
|
207
257
|
const getMissingBlobHashes = (): Buffer[] =>
|
|
208
258
|
blobHashes
|
|
209
259
|
.map((bh, i) => (resultBlobs[i] === undefined ? bh : undefined))
|
|
@@ -213,8 +263,8 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
213
263
|
const getFilledBlobs = (): Blob[] => resultBlobs.filter((b): b is Blob => b !== undefined);
|
|
214
264
|
|
|
215
265
|
// Helper to fill in results from fetched blobs
|
|
216
|
-
const fillResults = (fetchedBlobs: BlobJson[]): Blob[] => {
|
|
217
|
-
const blobs = processFetchedBlobs(fetchedBlobs, blobHashes, this.log);
|
|
266
|
+
const fillResults = async (fetchedBlobs: BlobJson[]): Promise<Blob[]> => {
|
|
267
|
+
const blobs = await processFetchedBlobs(fetchedBlobs, blobHashes, this.log);
|
|
218
268
|
// Fill in any missing positions with matching blobs
|
|
219
269
|
for (let i = 0; i < blobHashes.length; i++) {
|
|
220
270
|
if (resultBlobs[i] === undefined) {
|
|
@@ -232,85 +282,66 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
232
282
|
return blobs;
|
|
233
283
|
};
|
|
234
284
|
|
|
235
|
-
const { l1ConsensusHostUrls } = this.config;
|
|
236
|
-
|
|
237
285
|
const ctx = { blockHash, blobHashes: blobHashes.map(bufferToHex) };
|
|
238
286
|
|
|
239
|
-
//
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
287
|
+
// Lazily resolve the slot number — only resolved when consensus hosts are actually tried.
|
|
288
|
+
let slotNumber: number | undefined;
|
|
289
|
+
let slotResolved = false;
|
|
290
|
+
const getSlotNumber = async (): Promise<number | undefined> => {
|
|
291
|
+
if (!slotResolved) {
|
|
292
|
+
slotNumber = await this.resolveSlotNumber(blockHash, opts);
|
|
293
|
+
slotResolved = true;
|
|
244
294
|
}
|
|
245
|
-
|
|
295
|
+
return slotNumber;
|
|
296
|
+
};
|
|
246
297
|
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
const consensusCtx = { l1ConsensusHostUrls, ...ctx };
|
|
251
|
-
this.log.trace(`Attempting to get slot number for block hash`, consensusCtx);
|
|
252
|
-
const slotNumber = await this.getSlotNumber(blockHash);
|
|
253
|
-
this.log.debug(`Got slot number ${slotNumber} from consensus host for querying blobs`, consensusCtx);
|
|
254
|
-
|
|
255
|
-
if (slotNumber) {
|
|
256
|
-
let l1ConsensusHostUrl: string;
|
|
257
|
-
for (let l1ConsensusHostIndex = 0; l1ConsensusHostIndex < l1ConsensusHostUrls.length; l1ConsensusHostIndex++) {
|
|
258
|
-
const missingHashes = getMissingBlobHashes();
|
|
259
|
-
if (missingHashes.length === 0) {
|
|
260
|
-
break;
|
|
261
|
-
}
|
|
298
|
+
// Build the two source-try functions. The order depends on the config.
|
|
299
|
+
const tryConsensus = () => this.tryConsensusHosts(getSlotNumber, getMissingBlobHashes, fillResults, ctx);
|
|
300
|
+
const tryFilestores = () => this.tryFileStores(getMissingBlobHashes, fillResults, ctx);
|
|
262
301
|
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
slotNumber,
|
|
266
|
-
l1ConsensusHostUrl,
|
|
267
|
-
...ctx,
|
|
268
|
-
});
|
|
269
|
-
const blobs = await this.getBlobsFromHost(l1ConsensusHostUrl, slotNumber, l1ConsensusHostIndex);
|
|
270
|
-
const result = fillResults(blobs);
|
|
271
|
-
this.log.debug(
|
|
272
|
-
`Got ${blobs.length} blobs from consensus host (total: ${result.length}/${blobHashes.length})`,
|
|
273
|
-
{ slotNumber, l1ConsensusHostUrl, ...ctx },
|
|
274
|
-
);
|
|
275
|
-
if (result.length === blobHashes.length) {
|
|
276
|
-
return returnWithCallback(result);
|
|
277
|
-
}
|
|
278
|
-
}
|
|
279
|
-
}
|
|
280
|
-
}
|
|
302
|
+
const preferFilestores = this.config.blobPreferFilestores ?? false;
|
|
303
|
+
const [trySourceA, trySourceB] = preferFilestores ? [tryFilestores, tryConsensus] : [tryConsensus, tryFilestores];
|
|
281
304
|
|
|
282
|
-
//
|
|
283
|
-
//
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
305
|
+
// Historical sync: blobs should already exist, use shorter backoff for transient errors.
|
|
306
|
+
// Near-tip sync: blobs may still be uploading, use longer backoff for eventual consistency.
|
|
307
|
+
const isHistoricalSync = opts?.isHistoricalSync ?? false;
|
|
308
|
+
const backoff = isHistoricalSync ? [1, 1] : [1, 1, 1, 2, 2];
|
|
309
|
+
|
|
310
|
+
// Retry loop: alternate between the two primary sources with backoff.
|
|
311
|
+
try {
|
|
312
|
+
await retry(
|
|
313
|
+
async () => {
|
|
314
|
+
if (getMissingBlobHashes().length > 0) {
|
|
315
|
+
await trySourceA();
|
|
316
|
+
}
|
|
317
|
+
if (getMissingBlobHashes().length > 0) {
|
|
318
|
+
await trySourceB();
|
|
319
|
+
}
|
|
320
|
+
if (getMissingBlobHashes().length > 0) {
|
|
321
|
+
throw new Error('Still missing blobs after trying all primary sources');
|
|
322
|
+
}
|
|
323
|
+
},
|
|
324
|
+
'blob retrieval',
|
|
325
|
+
makeBackoff(backoff),
|
|
326
|
+
this.log,
|
|
327
|
+
true, // failSilently — expected during eventual consistency
|
|
328
|
+
);
|
|
329
|
+
return returnWithCallback(getFilledBlobs());
|
|
330
|
+
} catch {
|
|
331
|
+
// Exhausted retries, continue to archive fallback
|
|
302
332
|
}
|
|
303
333
|
|
|
304
|
-
|
|
305
|
-
|
|
334
|
+
// Archive fallback
|
|
335
|
+
const missingAfterPrimary = getMissingBlobHashes();
|
|
336
|
+
if (missingAfterPrimary.length > 0 && this.archiveClient) {
|
|
306
337
|
const archiveCtx = { archiveUrl: this.archiveClient.getBaseUrl(), ...ctx };
|
|
307
|
-
this.log.trace(`Attempting to get ${
|
|
338
|
+
this.log.trace(`Attempting to get ${missingAfterPrimary.length} blobs from archive`, archiveCtx);
|
|
308
339
|
const allBlobs = await this.archiveClient.getBlobsFromBlock(blockHash);
|
|
309
340
|
if (!allBlobs) {
|
|
310
341
|
this.log.debug('No blobs found from archive client', archiveCtx);
|
|
311
342
|
} else {
|
|
312
343
|
this.log.trace(`Got ${allBlobs.length} blobs from archive client before filtering`, archiveCtx);
|
|
313
|
-
const result = fillResults(allBlobs);
|
|
344
|
+
const result = await fillResults(allBlobs);
|
|
314
345
|
this.log.debug(
|
|
315
346
|
`Got ${allBlobs.length} blobs from archive client (total: ${result.length}/${blobHashes.length})`,
|
|
316
347
|
archiveCtx,
|
|
@@ -326,7 +357,7 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
326
357
|
this.log.warn(
|
|
327
358
|
`Failed to fetch all blobs for ${blockHash} from all blob sources (got ${result.length}/${blobHashes.length})`,
|
|
328
359
|
{
|
|
329
|
-
l1ConsensusHostUrls,
|
|
360
|
+
l1ConsensusHostUrls: this.config.l1ConsensusHostUrls,
|
|
330
361
|
archiveUrl: this.archiveClient?.getBaseUrl(),
|
|
331
362
|
fileStoreUrls: this.fileStoreClients.map(c => c.getBaseUrl()),
|
|
332
363
|
},
|
|
@@ -335,6 +366,71 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
335
366
|
return returnWithCallback(result);
|
|
336
367
|
}
|
|
337
368
|
|
|
369
|
+
/** Resolves the beacon slot number for the given block hash. Returns undefined if no consensus hosts. */
|
|
370
|
+
private resolveSlotNumber(
|
|
371
|
+
blockHash: `0x${string}`,
|
|
372
|
+
opts?: GetBlobSidecarOptions,
|
|
373
|
+
): Promise<number | undefined> | undefined {
|
|
374
|
+
const { l1ConsensusHostUrls } = this.config;
|
|
375
|
+
if (!l1ConsensusHostUrls || l1ConsensusHostUrls.length === 0) {
|
|
376
|
+
return undefined;
|
|
377
|
+
}
|
|
378
|
+
// If no supernodes, no point resolving the slot
|
|
379
|
+
if (this.superNodeHostIndexes && this.superNodeHostIndexes.size === 0) {
|
|
380
|
+
return undefined;
|
|
381
|
+
}
|
|
382
|
+
return this.getSlotNumber(blockHash, opts?.parentBeaconBlockRoot, opts?.l1BlockTimestamp);
|
|
383
|
+
}
|
|
384
|
+
|
|
385
|
+
/**
|
|
386
|
+
* Try all supernode consensus hosts for blob sidecars.
|
|
387
|
+
* Skips hosts that were detected as non-supernodes during testSources().
|
|
388
|
+
*/
|
|
389
|
+
private async tryConsensusHosts(
|
|
390
|
+
getSlotNumber: () => Promise<number | undefined>,
|
|
391
|
+
getMissingBlobHashes: () => Buffer[],
|
|
392
|
+
fillResults: (blobs: BlobJson[]) => Promise<Blob[]>,
|
|
393
|
+
ctx: { blockHash: string; blobHashes: string[] },
|
|
394
|
+
): Promise<void> {
|
|
395
|
+
const { l1ConsensusHostUrls } = this.config;
|
|
396
|
+
if (!l1ConsensusHostUrls || l1ConsensusHostUrls.length === 0) {
|
|
397
|
+
return;
|
|
398
|
+
}
|
|
399
|
+
|
|
400
|
+
const slotNumber = await getSlotNumber();
|
|
401
|
+
if (!slotNumber) {
|
|
402
|
+
return;
|
|
403
|
+
}
|
|
404
|
+
|
|
405
|
+
for (let l1ConsensusHostIndex = 0; l1ConsensusHostIndex < l1ConsensusHostUrls.length; l1ConsensusHostIndex++) {
|
|
406
|
+
const missingHashes = getMissingBlobHashes();
|
|
407
|
+
if (missingHashes.length === 0) {
|
|
408
|
+
break;
|
|
409
|
+
}
|
|
410
|
+
|
|
411
|
+
// Skip non-supernode hosts if we've already detected supernodes
|
|
412
|
+
if (this.superNodeHostIndexes && !this.superNodeHostIndexes.has(l1ConsensusHostIndex)) {
|
|
413
|
+
this.log.trace(`Skipping non-supernode consensus host`, {
|
|
414
|
+
l1ConsensusHostUrl: l1ConsensusHostUrls[l1ConsensusHostIndex],
|
|
415
|
+
});
|
|
416
|
+
continue;
|
|
417
|
+
}
|
|
418
|
+
|
|
419
|
+
const l1ConsensusHostUrl = l1ConsensusHostUrls[l1ConsensusHostIndex];
|
|
420
|
+
this.log.trace(`Attempting to get ${missingHashes.length} blobs from consensus host`, {
|
|
421
|
+
slotNumber,
|
|
422
|
+
l1ConsensusHostUrl,
|
|
423
|
+
...ctx,
|
|
424
|
+
});
|
|
425
|
+
const blobs = await this.getBlobsFromHost(l1ConsensusHostUrl, slotNumber, l1ConsensusHostIndex, missingHashes);
|
|
426
|
+
const result = await fillResults(blobs);
|
|
427
|
+
this.log.debug(
|
|
428
|
+
`Got ${blobs.length} blobs from consensus host (total: ${result.length}/${ctx.blobHashes.length})`,
|
|
429
|
+
{ slotNumber, l1ConsensusHostUrl, ...ctx },
|
|
430
|
+
);
|
|
431
|
+
}
|
|
432
|
+
}
|
|
433
|
+
|
|
338
434
|
/**
|
|
339
435
|
* Try all filestores once (shuffled for load distribution).
|
|
340
436
|
* @param getMissingBlobHashes - Function to get remaining blob hashes to fetch
|
|
@@ -343,7 +439,7 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
343
439
|
*/
|
|
344
440
|
private async tryFileStores(
|
|
345
441
|
getMissingBlobHashes: () => Buffer[],
|
|
346
|
-
fillResults: (blobs: BlobJson[]) => Blob[]
|
|
442
|
+
fillResults: (blobs: BlobJson[]) => Promise<Blob[]>,
|
|
347
443
|
ctx: { blockHash: string; blobHashes: string[] },
|
|
348
444
|
): Promise<void> {
|
|
349
445
|
// Shuffle clients for load distribution
|
|
@@ -364,7 +460,7 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
364
460
|
});
|
|
365
461
|
const blobs = await client.getBlobsByHashes(blobHashStrings);
|
|
366
462
|
if (blobs.length > 0) {
|
|
367
|
-
const result = fillResults(blobs);
|
|
463
|
+
const result = await fillResults(blobs);
|
|
368
464
|
this.log.debug(
|
|
369
465
|
`Got ${blobs.length} blobs from filestore (total: ${result.length}/${ctx.blobHashes.length})`,
|
|
370
466
|
{
|
|
@@ -385,19 +481,20 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
385
481
|
blobHashes: Buffer[] = [],
|
|
386
482
|
l1ConsensusHostIndex?: number,
|
|
387
483
|
): Promise<Blob[]> {
|
|
388
|
-
const blobs = await this.getBlobsFromHost(hostUrl, blockHashOrSlot, l1ConsensusHostIndex);
|
|
389
|
-
return processFetchedBlobs(blobs, blobHashes, this.log).filter((b): b is Blob => b !== undefined);
|
|
484
|
+
const blobs = await this.getBlobsFromHost(hostUrl, blockHashOrSlot, l1ConsensusHostIndex, blobHashes);
|
|
485
|
+
return (await processFetchedBlobs(blobs, blobHashes, this.log)).filter((b): b is Blob => b !== undefined);
|
|
390
486
|
}
|
|
391
487
|
|
|
392
488
|
public async getBlobsFromHost(
|
|
393
489
|
hostUrl: string,
|
|
394
490
|
blockHashOrSlot: string | number,
|
|
395
491
|
l1ConsensusHostIndex?: number,
|
|
492
|
+
blobHashes?: Buffer[],
|
|
396
493
|
): Promise<BlobJson[]> {
|
|
397
494
|
try {
|
|
398
|
-
let res = await this.fetchBlobSidecars(hostUrl, blockHashOrSlot, l1ConsensusHostIndex);
|
|
495
|
+
let res = await this.fetchBlobSidecars(hostUrl, blockHashOrSlot, l1ConsensusHostIndex, blobHashes);
|
|
399
496
|
if (res.ok) {
|
|
400
|
-
return parseBlobJsonsFromResponse(await res.json(), this.log);
|
|
497
|
+
return await parseBlobJsonsFromResponse(await res.json(), this.log);
|
|
401
498
|
}
|
|
402
499
|
|
|
403
500
|
if (res.status === 404 && typeof blockHashOrSlot === 'number') {
|
|
@@ -412,9 +509,9 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
412
509
|
let currentSlot = blockHashOrSlot + 1;
|
|
413
510
|
while (res.status === 404 && maxRetries > 0 && latestSlot !== undefined && currentSlot <= latestSlot) {
|
|
414
511
|
this.log.debug(`Trying slot ${currentSlot}`);
|
|
415
|
-
res = await this.fetchBlobSidecars(hostUrl, currentSlot, l1ConsensusHostIndex);
|
|
512
|
+
res = await this.fetchBlobSidecars(hostUrl, currentSlot, l1ConsensusHostIndex, blobHashes);
|
|
416
513
|
if (res.ok) {
|
|
417
|
-
return parseBlobJsonsFromResponse(await res.json(), this.log);
|
|
514
|
+
return await parseBlobJsonsFromResponse(await res.json(), this.log);
|
|
418
515
|
}
|
|
419
516
|
currentSlot++;
|
|
420
517
|
maxRetries--;
|
|
@@ -437,19 +534,29 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
437
534
|
hostUrl: string,
|
|
438
535
|
blockHashOrSlot: string | number,
|
|
439
536
|
l1ConsensusHostIndex?: number,
|
|
537
|
+
blobHashes?: Buffer[],
|
|
440
538
|
): Promise<Response> {
|
|
441
|
-
|
|
539
|
+
let baseUrl = `${hostUrl}/eth/v1/beacon/blobs/${blockHashOrSlot}`;
|
|
540
|
+
|
|
541
|
+
if (blobHashes && blobHashes.length > 0) {
|
|
542
|
+
const params = new URLSearchParams();
|
|
543
|
+
for (const hash of blobHashes) {
|
|
544
|
+
params.append('versioned_hashes', `0x${hash.toString('hex')}`);
|
|
545
|
+
}
|
|
546
|
+
baseUrl += `?${params.toString()}`;
|
|
547
|
+
}
|
|
442
548
|
|
|
443
|
-
const { url, ...options } = getBeaconNodeFetchOptions(baseUrl, this.config, l1ConsensusHostIndex);
|
|
444
|
-
this.log.debug(`Fetching blob sidecar for ${blockHashOrSlot}`, { url, ...options });
|
|
445
|
-
|
|
549
|
+
const { url, logSafeUrl, ...options } = getBeaconNodeFetchOptions(baseUrl, this.config, l1ConsensusHostIndex);
|
|
550
|
+
this.log.debug(`Fetching blob sidecar for ${blockHashOrSlot}`, { url: logSafeUrl, ...options });
|
|
551
|
+
// No retry here — this is called inside the main retry loop in getBlobSidecar
|
|
552
|
+
return fetch(url, options);
|
|
446
553
|
}
|
|
447
554
|
|
|
448
555
|
private async getLatestSlotNumber(hostUrl: string, l1ConsensusHostIndex?: number): Promise<number | undefined> {
|
|
449
556
|
try {
|
|
450
557
|
const baseUrl = `${hostUrl}/eth/v1/beacon/headers/head`;
|
|
451
|
-
const { url, ...options } = getBeaconNodeFetchOptions(baseUrl, this.config, l1ConsensusHostIndex);
|
|
452
|
-
this.log.debug(`Fetching latest slot number`, { url, ...options });
|
|
558
|
+
const { url, logSafeUrl, ...options } = getBeaconNodeFetchOptions(baseUrl, this.config, l1ConsensusHostIndex);
|
|
559
|
+
this.log.debug(`Fetching latest slot number`, { url: logSafeUrl, ...options });
|
|
453
560
|
const res = await this.fetch(url, options);
|
|
454
561
|
if (res.ok) {
|
|
455
562
|
const body = await res.json();
|
|
@@ -480,34 +587,50 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
480
587
|
* @param blockHash - The block hash
|
|
481
588
|
* @returns The slot number
|
|
482
589
|
*/
|
|
483
|
-
private async getSlotNumber(
|
|
590
|
+
private async getSlotNumber(
|
|
591
|
+
blockHash: `0x${string}`,
|
|
592
|
+
parentBeaconBlockRoot?: string,
|
|
593
|
+
l1BlockTimestamp?: bigint,
|
|
594
|
+
): Promise<number | undefined> {
|
|
484
595
|
const { l1ConsensusHostUrls, l1RpcUrls } = this.config;
|
|
485
596
|
if (!l1ConsensusHostUrls || l1ConsensusHostUrls.length === 0) {
|
|
486
597
|
this.log.debug('No consensus host url configured');
|
|
487
598
|
return undefined;
|
|
488
599
|
}
|
|
489
600
|
|
|
490
|
-
if (
|
|
491
|
-
|
|
492
|
-
|
|
601
|
+
// Primary path: compute slot from timestamp if genesis config is cached (no network call needed)
|
|
602
|
+
if (
|
|
603
|
+
l1BlockTimestamp !== undefined &&
|
|
604
|
+
this.beaconGenesisTime !== undefined &&
|
|
605
|
+
this.beaconSecondsPerSlot !== undefined
|
|
606
|
+
) {
|
|
607
|
+
const slot = Number((l1BlockTimestamp - this.beaconGenesisTime) / BigInt(this.beaconSecondsPerSlot));
|
|
608
|
+
this.log.debug(`Computed slot ${slot} from L1 block timestamp`, { l1BlockTimestamp });
|
|
609
|
+
return slot;
|
|
493
610
|
}
|
|
494
611
|
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
612
|
+
if (!parentBeaconBlockRoot) {
|
|
613
|
+
// parentBeaconBlockRoot not provided by caller — fetch it from the execution RPC
|
|
614
|
+
if (!l1RpcUrls || l1RpcUrls.length === 0) {
|
|
615
|
+
this.log.debug('No execution host url configured');
|
|
616
|
+
return undefined;
|
|
617
|
+
}
|
|
618
|
+
|
|
619
|
+
const client = createPublicClient({
|
|
620
|
+
transport: makeL1HttpTransport(l1RpcUrls, { timeout: this.config.l1HttpTimeoutMS }),
|
|
504
621
|
});
|
|
622
|
+
try {
|
|
623
|
+
const res: RpcBlock = await client.request({
|
|
624
|
+
method: 'eth_getBlockByHash',
|
|
625
|
+
params: [blockHash, /*tx flag*/ false],
|
|
626
|
+
});
|
|
505
627
|
|
|
506
|
-
|
|
507
|
-
|
|
628
|
+
if (res.parentBeaconBlockRoot) {
|
|
629
|
+
parentBeaconBlockRoot = res.parentBeaconBlockRoot;
|
|
630
|
+
}
|
|
631
|
+
} catch (err) {
|
|
632
|
+
this.log.error(`Error getting parent beacon block root`, err);
|
|
508
633
|
}
|
|
509
|
-
} catch (err) {
|
|
510
|
-
this.log.error(`Error getting parent beacon block root`, err);
|
|
511
634
|
}
|
|
512
635
|
|
|
513
636
|
if (!parentBeaconBlockRoot) {
|
|
@@ -545,12 +668,105 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
545
668
|
public getArchiveClient(): BlobArchiveClient | undefined {
|
|
546
669
|
return this.archiveClient;
|
|
547
670
|
}
|
|
671
|
+
|
|
672
|
+
/** Returns true if this client can upload blobs to filestore. */
|
|
673
|
+
public canUpload(): boolean {
|
|
674
|
+
return this.fileStoreUploadClient !== undefined;
|
|
675
|
+
}
|
|
676
|
+
|
|
677
|
+
/**
|
|
678
|
+
* Start the blob client.
|
|
679
|
+
* Fetches and caches beacon genesis config for timestamp-based slot resolution,
|
|
680
|
+
* then uploads the initial healthcheck file (awaited) and starts periodic uploads.
|
|
681
|
+
*/
|
|
682
|
+
public async start(): Promise<void> {
|
|
683
|
+
await this.fetchBeaconConfig();
|
|
684
|
+
|
|
685
|
+
if (!this.fileStoreUploadClient) {
|
|
686
|
+
return;
|
|
687
|
+
}
|
|
688
|
+
|
|
689
|
+
await this.fileStoreUploadClient.uploadHealthcheck();
|
|
690
|
+
this.log.debug('Initial healthcheck file uploaded');
|
|
691
|
+
|
|
692
|
+
this.startPeriodicHealthcheckUpload();
|
|
693
|
+
}
|
|
694
|
+
|
|
695
|
+
/**
|
|
696
|
+
* Start periodic healthcheck upload to the file store to ensure it remains available even if accidentally deleted.
|
|
697
|
+
*/
|
|
698
|
+
private startPeriodicHealthcheckUpload(): void {
|
|
699
|
+
const intervalMs =
|
|
700
|
+
(this.config.blobHealthcheckUploadIntervalMinutes ?? DEFAULT_HEALTHCHECK_UPLOAD_INTERVAL_MINUTES) * 60 * 1000;
|
|
701
|
+
|
|
702
|
+
this.healthcheckUploadIntervalId = setInterval(() => {
|
|
703
|
+
void this.fileStoreUploadClient!.uploadHealthcheck().catch(err => {
|
|
704
|
+
this.log.warn('Failed to upload periodic healthcheck file', err);
|
|
705
|
+
});
|
|
706
|
+
}, intervalMs);
|
|
707
|
+
}
|
|
708
|
+
|
|
709
|
+
/**
|
|
710
|
+
* Fetches and caches beacon genesis time and slot duration from the first available consensus host.
|
|
711
|
+
* These static values enable timestamp-based slot resolution, eliminating the per-fetch headers call.
|
|
712
|
+
* Logs a warning and leaves fields undefined if all hosts fail, callers fall back gracefully.
|
|
713
|
+
*/
|
|
714
|
+
private async fetchBeaconConfig(): Promise<void> {
|
|
715
|
+
const { l1ConsensusHostUrls } = this.config;
|
|
716
|
+
if (!l1ConsensusHostUrls || l1ConsensusHostUrls.length === 0) {
|
|
717
|
+
return;
|
|
718
|
+
}
|
|
719
|
+
|
|
720
|
+
for (let i = 0; i < l1ConsensusHostUrls.length; i++) {
|
|
721
|
+
try {
|
|
722
|
+
const { url: genesisUrl, ...genesisOptions } = getBeaconNodeFetchOptions(
|
|
723
|
+
`${l1ConsensusHostUrls[i]}/eth/v1/config/genesis`,
|
|
724
|
+
this.config,
|
|
725
|
+
i,
|
|
726
|
+
);
|
|
727
|
+
const { url: specUrl, ...specOptions } = getBeaconNodeFetchOptions(
|
|
728
|
+
`${l1ConsensusHostUrls[i]}/eth/v1/config/spec`,
|
|
729
|
+
this.config,
|
|
730
|
+
i,
|
|
731
|
+
);
|
|
732
|
+
|
|
733
|
+
const [genesisRes, specRes] = await Promise.all([
|
|
734
|
+
this.fetch(genesisUrl, genesisOptions),
|
|
735
|
+
this.fetch(specUrl, specOptions),
|
|
736
|
+
]);
|
|
737
|
+
|
|
738
|
+
if (genesisRes.ok && specRes.ok) {
|
|
739
|
+
const genesis = await genesisRes.json();
|
|
740
|
+
const spec = await specRes.json();
|
|
741
|
+
this.beaconGenesisTime = BigInt(genesis.data.genesisTime);
|
|
742
|
+
this.beaconSecondsPerSlot = parseInt(spec.data.secondsPerSlot);
|
|
743
|
+
this.log.debug(`Fetched beacon genesis config`, {
|
|
744
|
+
genesisTime: this.beaconGenesisTime,
|
|
745
|
+
secondsPerSlot: this.beaconSecondsPerSlot,
|
|
746
|
+
});
|
|
747
|
+
return;
|
|
748
|
+
}
|
|
749
|
+
} catch (err) {
|
|
750
|
+
this.log.warn(`Failed to fetch beacon config from host ${l1ConsensusHostUrls[i]}`, err);
|
|
751
|
+
}
|
|
752
|
+
}
|
|
753
|
+
this.log.warn('Could not fetch beacon genesis config from any consensus host — will use headers call fallback');
|
|
754
|
+
}
|
|
755
|
+
|
|
756
|
+
/**
|
|
757
|
+
* Stop the blob client, clearing any periodic tasks.
|
|
758
|
+
*/
|
|
759
|
+
public stop(): void {
|
|
760
|
+
if (this.healthcheckUploadIntervalId) {
|
|
761
|
+
clearInterval(this.healthcheckUploadIntervalId);
|
|
762
|
+
this.healthcheckUploadIntervalId = undefined;
|
|
763
|
+
}
|
|
764
|
+
}
|
|
548
765
|
}
|
|
549
766
|
|
|
550
|
-
function parseBlobJsonsFromResponse(response: any, logger: Logger): BlobJson[] {
|
|
767
|
+
async function parseBlobJsonsFromResponse(response: any, logger: Logger): Promise<BlobJson[]> {
|
|
551
768
|
try {
|
|
552
|
-
|
|
553
|
-
return blobs;
|
|
769
|
+
return await Promise.all((response.data as string[]).map(parseBlobJson));
|
|
554
770
|
} catch (err) {
|
|
555
771
|
logger.error(`Error parsing blob json from response`, err);
|
|
556
772
|
return [];
|
|
@@ -561,16 +777,19 @@ function parseBlobJsonsFromResponse(response: any, logger: Logger): BlobJson[] {
|
|
|
561
777
|
// https://ethereum.github.io/beacon-APIs/?urls.primaryName=dev#/Beacon/getBlobSidecars
|
|
562
778
|
// Here we attempt to parse the response data to Buffer, and check the lengths (via Blob's constructor), to avoid
|
|
563
779
|
// throwing an error down the line when calling Blob.fromJson().
|
|
564
|
-
function parseBlobJson(
|
|
565
|
-
const blobBuffer = Buffer.from(
|
|
566
|
-
const
|
|
567
|
-
const blob = new Blob(blobBuffer, commitmentBuffer);
|
|
780
|
+
async function parseBlobJson(rawHex: string): Promise<BlobJson> {
|
|
781
|
+
const blobBuffer = Buffer.from(rawHex.slice(2), 'hex');
|
|
782
|
+
const blob = await Blob.fromBlobBuffer(blobBuffer);
|
|
568
783
|
return blob.toJSON();
|
|
569
784
|
}
|
|
570
785
|
|
|
571
786
|
// Returns an array that maps each blob hash to the corresponding blob, or undefined if the blob is not found
|
|
572
787
|
// or the data does not match the commitment.
|
|
573
|
-
function processFetchedBlobs(
|
|
788
|
+
async function processFetchedBlobs(
|
|
789
|
+
blobs: BlobJson[],
|
|
790
|
+
blobHashes: Buffer[],
|
|
791
|
+
logger: Logger,
|
|
792
|
+
): Promise<(Blob | undefined)[]> {
|
|
574
793
|
const requestedBlobHashes = new Set<string>(blobHashes.map(bufferToHex));
|
|
575
794
|
const hashToBlob = new Map<string, Blob>();
|
|
576
795
|
for (const blobJson of blobs) {
|
|
@@ -580,7 +799,7 @@ function processFetchedBlobs(blobs: BlobJson[], blobHashes: Buffer[], logger: Lo
|
|
|
580
799
|
}
|
|
581
800
|
|
|
582
801
|
try {
|
|
583
|
-
const blob = Blob.fromJson(blobJson);
|
|
802
|
+
const blob = await Blob.fromJson(blobJson);
|
|
584
803
|
hashToBlob.set(hashHex, blob);
|
|
585
804
|
} catch (err) {
|
|
586
805
|
// If the above throws, it's likely that the blob commitment does not match the hash of the blob data.
|
|
@@ -600,12 +819,16 @@ function getBeaconNodeFetchOptions(url: string, config: BlobClientConfig, l1Cons
|
|
|
600
819
|
l1ConsensusHostApiKeyHeaders[l1ConsensusHostIndex];
|
|
601
820
|
|
|
602
821
|
let formattedUrl = url;
|
|
822
|
+
let logSafeUrl = url;
|
|
603
823
|
if (l1ConsensusHostApiKey && l1ConsensusHostApiKey.getValue() !== '' && !l1ConsensusHostApiKeyHeader) {
|
|
604
|
-
|
|
824
|
+
const separator = formattedUrl.includes('?') ? '&' : '?';
|
|
825
|
+
formattedUrl += `${separator}key=${l1ConsensusHostApiKey.getValue()}`;
|
|
826
|
+
logSafeUrl += `${separator}key=[REDACTED]`;
|
|
605
827
|
}
|
|
606
828
|
|
|
607
829
|
return {
|
|
608
830
|
url: formattedUrl,
|
|
831
|
+
logSafeUrl,
|
|
609
832
|
...(l1ConsensusHostApiKey &&
|
|
610
833
|
l1ConsensusHostApiKeyHeader && {
|
|
611
834
|
headers: {
|