@aztec/blob-client 0.0.1-commit.f2ce05ee → 0.0.1-commit.f5d02921e
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dest/blobstore/blob_store_test_suite.js +9 -9
- package/dest/client/config.d.ts +7 -1
- package/dest/client/config.d.ts.map +1 -1
- package/dest/client/config.js +16 -1
- package/dest/client/factory.d.ts +1 -1
- package/dest/client/factory.d.ts.map +1 -1
- package/dest/client/factory.js +7 -1
- package/dest/client/http.d.ts +19 -11
- package/dest/client/http.d.ts.map +1 -1
- package/dest/client/http.js +271 -140
- package/dest/client/interface.d.ts +13 -4
- package/dest/client/interface.d.ts.map +1 -1
- package/dest/client/tests.js +3 -3
- package/dest/filestore/factory.d.ts +5 -4
- package/dest/filestore/factory.d.ts.map +1 -1
- package/dest/filestore/factory.js +4 -4
- package/package.json +7 -7
- package/src/blobstore/blob_store_test_suite.ts +9 -9
- package/src/client/config.ts +25 -0
- package/src/client/factory.ts +8 -1
- package/src/client/http.ts +312 -135
- package/src/client/interface.ts +12 -3
- package/src/client/tests.ts +2 -2
- package/src/filestore/factory.ts +7 -2
package/dest/client/http.js
CHANGED
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
import { Blob, computeEthVersionedBlobHash } from '@aztec/blob-lib';
|
|
2
|
+
import { makeL1HttpTransport } from '@aztec/ethereum/client';
|
|
2
3
|
import { shuffle } from '@aztec/foundation/array';
|
|
3
4
|
import { createLogger } from '@aztec/foundation/log';
|
|
4
5
|
import { makeBackoff, retry } from '@aztec/foundation/retry';
|
|
5
6
|
import { bufferToHex, hexToBuffer } from '@aztec/foundation/string';
|
|
6
|
-
import { createPublicClient
|
|
7
|
+
import { createPublicClient } from 'viem';
|
|
7
8
|
import { createBlobArchiveClient } from '../archive/factory.js';
|
|
8
9
|
import { DEFAULT_HEALTHCHECK_UPLOAD_INTERVAL_MINUTES } from '../filestore/healthcheck.js';
|
|
9
10
|
import { getBlobClientConfigFromEnv } from './config.js';
|
|
@@ -17,6 +18,9 @@ export class HttpBlobClient {
|
|
|
17
18
|
fileStoreUploadClient;
|
|
18
19
|
disabled;
|
|
19
20
|
healthcheckUploadIntervalId;
|
|
21
|
+
/** Cached beacon genesis time (seconds since Unix epoch). Fetched once at startup. */ beaconGenesisTime;
|
|
22
|
+
/** Cached beacon slot duration in seconds. Fetched once at startup. */ beaconSecondsPerSlot;
|
|
23
|
+
/** Indexes of consensus hosts that serve blob sidecars (supernodes). Populated by testSources(). */ superNodeHostIndexes;
|
|
20
24
|
constructor(config, opts = {}){
|
|
21
25
|
this.opts = opts;
|
|
22
26
|
this.disabled = false;
|
|
@@ -66,22 +70,52 @@ export class HttpBlobClient {
|
|
|
66
70
|
l1ConsensusHostUrls,
|
|
67
71
|
archiveUrl
|
|
68
72
|
});
|
|
69
|
-
let
|
|
73
|
+
let consensusSuperNodes = 0;
|
|
74
|
+
let consensusNonSuperNodes = 0;
|
|
75
|
+
let archiveSources = 0;
|
|
76
|
+
let blobSinks = 0;
|
|
77
|
+
const detectedSuperNodes = new Set();
|
|
70
78
|
if (l1ConsensusHostUrls && l1ConsensusHostUrls.length > 0) {
|
|
71
79
|
for(let l1ConsensusHostIndex = 0; l1ConsensusHostIndex < l1ConsensusHostUrls.length; l1ConsensusHostIndex++){
|
|
72
80
|
const l1ConsensusHostUrl = l1ConsensusHostUrls[l1ConsensusHostIndex];
|
|
73
81
|
try {
|
|
74
|
-
const { url, ...options } = getBeaconNodeFetchOptions(`${l1ConsensusHostUrl}/eth/v1/beacon/headers`, this.config, l1ConsensusHostIndex);
|
|
82
|
+
const { url, ...options } = getBeaconNodeFetchOptions(`${l1ConsensusHostUrl}/eth/v1/beacon/headers/head`, this.config, l1ConsensusHostIndex);
|
|
75
83
|
const res = await this.fetch(url, options);
|
|
76
|
-
if (res.ok) {
|
|
77
|
-
this.log.
|
|
84
|
+
if (!res.ok) {
|
|
85
|
+
this.log.error(`Failure reaching L1 consensus host: ${res.statusText} (${res.status})`, {
|
|
78
86
|
l1ConsensusHostUrl
|
|
79
87
|
});
|
|
80
|
-
|
|
88
|
+
continue;
|
|
89
|
+
}
|
|
90
|
+
this.log.info(`L1 consensus host is reachable`, {
|
|
91
|
+
l1ConsensusHostUrl
|
|
92
|
+
});
|
|
93
|
+
// Check if the host serves blob sidecars (supernode/semi-supernode).
|
|
94
|
+
// Post-Fusaka (PeerDAS), non-supernode beacon nodes no longer serve the
|
|
95
|
+
// blob sidecar endpoint. A 200 response (even with an empty data array
|
|
96
|
+
// for a slot with no blobs) means the node supports serving blob sidecars.
|
|
97
|
+
const body = await res.json();
|
|
98
|
+
const headSlot = body?.data?.header?.message?.slot;
|
|
99
|
+
if (headSlot) {
|
|
100
|
+
const { url: blobUrl, ...blobOptions } = getBeaconNodeFetchOptions(`${l1ConsensusHostUrl}/eth/v1/beacon/blobs/${headSlot}`, this.config, l1ConsensusHostIndex);
|
|
101
|
+
const blobRes = await this.fetch(blobUrl, blobOptions);
|
|
102
|
+
if (blobRes.ok) {
|
|
103
|
+
this.log.info(`L1 consensus host serves blob sidecars (supernode)`, {
|
|
104
|
+
l1ConsensusHostUrl
|
|
105
|
+
});
|
|
106
|
+
detectedSuperNodes.add(l1ConsensusHostIndex);
|
|
107
|
+
consensusSuperNodes++;
|
|
108
|
+
} else {
|
|
109
|
+
this.log.info(`L1 consensus host does not serve blob sidecars, skipping for blob fetching`, {
|
|
110
|
+
l1ConsensusHostUrl
|
|
111
|
+
});
|
|
112
|
+
consensusNonSuperNodes++;
|
|
113
|
+
}
|
|
81
114
|
} else {
|
|
82
|
-
this.log.
|
|
115
|
+
this.log.info(`L1 consensus host is reachable but could not determine head slot`, {
|
|
83
116
|
l1ConsensusHostUrl
|
|
84
117
|
});
|
|
118
|
+
consensusNonSuperNodes++;
|
|
85
119
|
}
|
|
86
120
|
} catch (err) {
|
|
87
121
|
this.log.error(`Error reaching L1 consensus host`, err, {
|
|
@@ -89,9 +123,8 @@ export class HttpBlobClient {
|
|
|
89
123
|
});
|
|
90
124
|
}
|
|
91
125
|
}
|
|
92
|
-
} else {
|
|
93
|
-
this.log.warn('No L1 consensus host urls configured');
|
|
94
126
|
}
|
|
127
|
+
this.superNodeHostIndexes = detectedSuperNodes;
|
|
95
128
|
if (this.archiveClient) {
|
|
96
129
|
try {
|
|
97
130
|
const latest = await this.archiveClient.getLatestBlock();
|
|
@@ -99,14 +132,12 @@ export class HttpBlobClient {
|
|
|
99
132
|
latest,
|
|
100
133
|
archiveUrl
|
|
101
134
|
});
|
|
102
|
-
|
|
135
|
+
archiveSources++;
|
|
103
136
|
} catch (err) {
|
|
104
137
|
this.log.error(`Error reaching archive client`, err, {
|
|
105
138
|
archiveUrl
|
|
106
139
|
});
|
|
107
140
|
}
|
|
108
|
-
} else {
|
|
109
|
-
this.log.warn('No archive client configured');
|
|
110
141
|
}
|
|
111
142
|
if (this.fileStoreClients.length > 0) {
|
|
112
143
|
for (const fileStoreClient of this.fileStoreClients){
|
|
@@ -116,7 +147,7 @@ export class HttpBlobClient {
|
|
|
116
147
|
this.log.info(`FileStore is reachable`, {
|
|
117
148
|
url: fileStoreClient.getBaseUrl()
|
|
118
149
|
});
|
|
119
|
-
|
|
150
|
+
blobSinks++;
|
|
120
151
|
} else {
|
|
121
152
|
this.log.warn(`FileStore is not accessible`, {
|
|
122
153
|
url: fileStoreClient.getBaseUrl()
|
|
@@ -129,12 +160,22 @@ export class HttpBlobClient {
|
|
|
129
160
|
}
|
|
130
161
|
}
|
|
131
162
|
}
|
|
163
|
+
// Emit a single summary after validating all sources
|
|
164
|
+
const successfulSourceCount = consensusSuperNodes + archiveSources + blobSinks;
|
|
165
|
+
let summary = `Blob client running with consensusSuperNodes=${consensusSuperNodes} archiveSources=${archiveSources} blobSinks=${blobSinks}`;
|
|
166
|
+
if (consensusNonSuperNodes > 0) {
|
|
167
|
+
summary += `. ${consensusNonSuperNodes} consensus client(s) ignored because they are not running in supernode or semi-supernode mode`;
|
|
168
|
+
}
|
|
132
169
|
if (successfulSourceCount === 0) {
|
|
133
170
|
if (this.config.blobAllowEmptySources) {
|
|
134
|
-
this.log.warn(
|
|
171
|
+
this.log.warn(summary);
|
|
135
172
|
} else {
|
|
136
|
-
throw new Error(
|
|
173
|
+
throw new Error(summary);
|
|
137
174
|
}
|
|
175
|
+
} else if (consensusSuperNodes === 0) {
|
|
176
|
+
this.log.warn(summary);
|
|
177
|
+
} else {
|
|
178
|
+
this.log.info(summary);
|
|
138
179
|
}
|
|
139
180
|
}
|
|
140
181
|
async sendBlobsToFilestore(blobs) {
|
|
@@ -156,35 +197,31 @@ export class HttpBlobClient {
|
|
|
156
197
|
}
|
|
157
198
|
}
|
|
158
199
|
/**
|
|
159
|
-
* Get the blob sidecar
|
|
200
|
+
* Get the blob sidecar.
|
|
160
201
|
*
|
|
161
|
-
*
|
|
162
|
-
*
|
|
163
|
-
*
|
|
164
|
-
* Source ordering depends on sync state:
|
|
165
|
-
* - Historical sync: blob client → FileStore → L1 consensus → Archive
|
|
166
|
-
* - Near tip sync: blob client → FileStore → L1 consensus → FileStore (with retries) → Archive (eg blobscan)
|
|
202
|
+
* Alternates between two primary sources (consensus and filestore) in a retry loop,
|
|
203
|
+
* then falls back to archive if blobs are still missing. The order of the primary
|
|
204
|
+
* sources is configurable via `blobPreferFilestores`.
|
|
167
205
|
*
|
|
168
206
|
* @param blockHash - The block hash
|
|
169
207
|
* @param blobHashes - The blob hashes to fetch
|
|
170
|
-
* @param opts - Options
|
|
208
|
+
* @param opts - Options for slot resolution
|
|
171
209
|
* @returns The blobs
|
|
172
210
|
*/ async getBlobSidecar(blockHash, blobHashes, opts) {
|
|
173
211
|
if (this.disabled) {
|
|
174
212
|
this.log.warn('Blob storage is disabled, returning empty blob sidecar');
|
|
175
213
|
return [];
|
|
176
214
|
}
|
|
177
|
-
const isHistoricalSync = opts?.isHistoricalSync ?? false;
|
|
178
215
|
// Accumulate blobs across sources, preserving order and handling duplicates
|
|
179
216
|
// resultBlobs[i] will contain the blob for blobHashes[i], or undefined if not yet found
|
|
180
217
|
const resultBlobs = new Array(blobHashes.length).fill(undefined);
|
|
181
|
-
// Helper to get
|
|
218
|
+
// Helper to get missing blob hashes that we still need to fetch
|
|
182
219
|
const getMissingBlobHashes = ()=>blobHashes.map((bh, i)=>resultBlobs[i] === undefined ? bh : undefined).filter((bh)=>bh !== undefined);
|
|
183
220
|
// Return the result, ignoring any undefined ones
|
|
184
221
|
const getFilledBlobs = ()=>resultBlobs.filter((b)=>b !== undefined);
|
|
185
222
|
// Helper to fill in results from fetched blobs
|
|
186
|
-
const fillResults = (fetchedBlobs)=>{
|
|
187
|
-
const blobs = processFetchedBlobs(fetchedBlobs, blobHashes, this.log);
|
|
223
|
+
const fillResults = async (fetchedBlobs)=>{
|
|
224
|
+
const blobs = await processFetchedBlobs(fetchedBlobs, blobHashes, this.log);
|
|
188
225
|
// Fill in any missing positions with matching blobs
|
|
189
226
|
for(let i = 0; i < blobHashes.length; i++){
|
|
190
227
|
if (resultBlobs[i] === undefined) {
|
|
@@ -200,86 +237,75 @@ export class HttpBlobClient {
|
|
|
200
237
|
}
|
|
201
238
|
return blobs;
|
|
202
239
|
};
|
|
203
|
-
const { l1ConsensusHostUrls } = this.config;
|
|
204
240
|
const ctx = {
|
|
205
241
|
blockHash,
|
|
206
242
|
blobHashes: blobHashes.map(bufferToHex)
|
|
207
243
|
};
|
|
208
|
-
//
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
244
|
+
// Lazily resolve the slot number — only resolved when consensus hosts are actually tried.
|
|
245
|
+
let slotNumber;
|
|
246
|
+
let slotResolved = false;
|
|
247
|
+
const getSlotNumber = async ()=>{
|
|
248
|
+
if (!slotResolved) {
|
|
249
|
+
slotNumber = await this.resolveSlotNumber(blockHash, opts);
|
|
250
|
+
slotResolved = true;
|
|
213
251
|
}
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
if (result.length === blobHashes.length) {
|
|
246
|
-
return returnWithCallback(result);
|
|
247
|
-
}
|
|
252
|
+
return slotNumber;
|
|
253
|
+
};
|
|
254
|
+
// Build the two source-try functions. The order depends on the config.
|
|
255
|
+
const tryConsensus = ()=>this.tryConsensusHosts(getSlotNumber, getMissingBlobHashes, fillResults, ctx);
|
|
256
|
+
const tryFilestores = ()=>this.tryFileStores(getMissingBlobHashes, fillResults, ctx);
|
|
257
|
+
const preferFilestores = this.config.blobPreferFilestores ?? false;
|
|
258
|
+
const [trySourceA, trySourceB] = preferFilestores ? [
|
|
259
|
+
tryFilestores,
|
|
260
|
+
tryConsensus
|
|
261
|
+
] : [
|
|
262
|
+
tryConsensus,
|
|
263
|
+
tryFilestores
|
|
264
|
+
];
|
|
265
|
+
// Historical sync: blobs should already exist, use shorter backoff for transient errors.
|
|
266
|
+
// Near-tip sync: blobs may still be uploading, use longer backoff for eventual consistency.
|
|
267
|
+
const isHistoricalSync = opts?.isHistoricalSync ?? false;
|
|
268
|
+
const backoff = isHistoricalSync ? [
|
|
269
|
+
1,
|
|
270
|
+
1
|
|
271
|
+
] : [
|
|
272
|
+
1,
|
|
273
|
+
1,
|
|
274
|
+
1,
|
|
275
|
+
2,
|
|
276
|
+
2
|
|
277
|
+
];
|
|
278
|
+
// Retry loop: alternate between the two primary sources with backoff.
|
|
279
|
+
try {
|
|
280
|
+
await retry(async ()=>{
|
|
281
|
+
if (getMissingBlobHashes().length > 0) {
|
|
282
|
+
await trySourceA();
|
|
248
283
|
}
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
}
|
|
260
|
-
}, 'filestore blob retrieval', makeBackoff([
|
|
261
|
-
1,
|
|
262
|
-
1,
|
|
263
|
-
2
|
|
264
|
-
]), this.log, true);
|
|
265
|
-
return returnWithCallback(getFilledBlobs());
|
|
266
|
-
} catch {
|
|
267
|
-
// Exhausted retries, continue to archive fallback
|
|
268
|
-
}
|
|
284
|
+
if (getMissingBlobHashes().length > 0) {
|
|
285
|
+
await trySourceB();
|
|
286
|
+
}
|
|
287
|
+
if (getMissingBlobHashes().length > 0) {
|
|
288
|
+
throw new Error('Still missing blobs after trying all primary sources');
|
|
289
|
+
}
|
|
290
|
+
}, 'blob retrieval', makeBackoff(backoff), this.log, true);
|
|
291
|
+
return returnWithCallback(getFilledBlobs());
|
|
292
|
+
} catch {
|
|
293
|
+
// Exhausted retries, continue to archive fallback
|
|
269
294
|
}
|
|
270
|
-
|
|
271
|
-
|
|
295
|
+
// Archive fallback
|
|
296
|
+
const missingAfterPrimary = getMissingBlobHashes();
|
|
297
|
+
if (missingAfterPrimary.length > 0 && this.archiveClient) {
|
|
272
298
|
const archiveCtx = {
|
|
273
299
|
archiveUrl: this.archiveClient.getBaseUrl(),
|
|
274
300
|
...ctx
|
|
275
301
|
};
|
|
276
|
-
this.log.trace(`Attempting to get ${
|
|
302
|
+
this.log.trace(`Attempting to get ${missingAfterPrimary.length} blobs from archive`, archiveCtx);
|
|
277
303
|
const allBlobs = await this.archiveClient.getBlobsFromBlock(blockHash);
|
|
278
304
|
if (!allBlobs) {
|
|
279
305
|
this.log.debug('No blobs found from archive client', archiveCtx);
|
|
280
306
|
} else {
|
|
281
307
|
this.log.trace(`Got ${allBlobs.length} blobs from archive client before filtering`, archiveCtx);
|
|
282
|
-
const result = fillResults(allBlobs);
|
|
308
|
+
const result = await fillResults(allBlobs);
|
|
283
309
|
this.log.debug(`Got ${allBlobs.length} blobs from archive client (total: ${result.length}/${blobHashes.length})`, archiveCtx);
|
|
284
310
|
if (result.length === blobHashes.length) {
|
|
285
311
|
return returnWithCallback(result);
|
|
@@ -289,13 +315,63 @@ export class HttpBlobClient {
|
|
|
289
315
|
const result = getFilledBlobs();
|
|
290
316
|
if (result.length < blobHashes.length) {
|
|
291
317
|
this.log.warn(`Failed to fetch all blobs for ${blockHash} from all blob sources (got ${result.length}/${blobHashes.length})`, {
|
|
292
|
-
l1ConsensusHostUrls,
|
|
318
|
+
l1ConsensusHostUrls: this.config.l1ConsensusHostUrls,
|
|
293
319
|
archiveUrl: this.archiveClient?.getBaseUrl(),
|
|
294
320
|
fileStoreUrls: this.fileStoreClients.map((c)=>c.getBaseUrl())
|
|
295
321
|
});
|
|
296
322
|
}
|
|
297
323
|
return returnWithCallback(result);
|
|
298
324
|
}
|
|
325
|
+
/** Resolves the beacon slot number for the given block hash. Returns undefined if no consensus hosts. */ resolveSlotNumber(blockHash, opts) {
|
|
326
|
+
const { l1ConsensusHostUrls } = this.config;
|
|
327
|
+
if (!l1ConsensusHostUrls || l1ConsensusHostUrls.length === 0) {
|
|
328
|
+
return undefined;
|
|
329
|
+
}
|
|
330
|
+
// If no supernodes, no point resolving the slot
|
|
331
|
+
if (this.superNodeHostIndexes && this.superNodeHostIndexes.size === 0) {
|
|
332
|
+
return undefined;
|
|
333
|
+
}
|
|
334
|
+
return this.getSlotNumber(blockHash, opts?.parentBeaconBlockRoot, opts?.l1BlockTimestamp);
|
|
335
|
+
}
|
|
336
|
+
/**
|
|
337
|
+
* Try all supernode consensus hosts for blob sidecars.
|
|
338
|
+
* Skips hosts that were detected as non-supernodes during testSources().
|
|
339
|
+
*/ async tryConsensusHosts(getSlotNumber, getMissingBlobHashes, fillResults, ctx) {
|
|
340
|
+
const { l1ConsensusHostUrls } = this.config;
|
|
341
|
+
if (!l1ConsensusHostUrls || l1ConsensusHostUrls.length === 0) {
|
|
342
|
+
return;
|
|
343
|
+
}
|
|
344
|
+
const slotNumber = await getSlotNumber();
|
|
345
|
+
if (!slotNumber) {
|
|
346
|
+
return;
|
|
347
|
+
}
|
|
348
|
+
for(let l1ConsensusHostIndex = 0; l1ConsensusHostIndex < l1ConsensusHostUrls.length; l1ConsensusHostIndex++){
|
|
349
|
+
const missingHashes = getMissingBlobHashes();
|
|
350
|
+
if (missingHashes.length === 0) {
|
|
351
|
+
break;
|
|
352
|
+
}
|
|
353
|
+
// Skip non-supernode hosts if we've already detected supernodes
|
|
354
|
+
if (this.superNodeHostIndexes && !this.superNodeHostIndexes.has(l1ConsensusHostIndex)) {
|
|
355
|
+
this.log.trace(`Skipping non-supernode consensus host`, {
|
|
356
|
+
l1ConsensusHostUrl: l1ConsensusHostUrls[l1ConsensusHostIndex]
|
|
357
|
+
});
|
|
358
|
+
continue;
|
|
359
|
+
}
|
|
360
|
+
const l1ConsensusHostUrl = l1ConsensusHostUrls[l1ConsensusHostIndex];
|
|
361
|
+
this.log.trace(`Attempting to get ${missingHashes.length} blobs from consensus host`, {
|
|
362
|
+
slotNumber,
|
|
363
|
+
l1ConsensusHostUrl,
|
|
364
|
+
...ctx
|
|
365
|
+
});
|
|
366
|
+
const blobs = await this.getBlobsFromHost(l1ConsensusHostUrl, slotNumber, l1ConsensusHostIndex, missingHashes);
|
|
367
|
+
const result = await fillResults(blobs);
|
|
368
|
+
this.log.debug(`Got ${blobs.length} blobs from consensus host (total: ${result.length}/${ctx.blobHashes.length})`, {
|
|
369
|
+
slotNumber,
|
|
370
|
+
l1ConsensusHostUrl,
|
|
371
|
+
...ctx
|
|
372
|
+
});
|
|
373
|
+
}
|
|
374
|
+
}
|
|
299
375
|
/**
|
|
300
376
|
* Try all filestores once (shuffled for load distribution).
|
|
301
377
|
* @param getMissingBlobHashes - Function to get remaining blob hashes to fetch
|
|
@@ -320,7 +396,7 @@ export class HttpBlobClient {
|
|
|
320
396
|
});
|
|
321
397
|
const blobs = await client.getBlobsByHashes(blobHashStrings);
|
|
322
398
|
if (blobs.length > 0) {
|
|
323
|
-
const result = fillResults(blobs);
|
|
399
|
+
const result = await fillResults(blobs);
|
|
324
400
|
this.log.debug(`Got ${blobs.length} blobs from filestore (total: ${result.length}/${ctx.blobHashes.length})`, {
|
|
325
401
|
url: client.getBaseUrl(),
|
|
326
402
|
...ctx
|
|
@@ -334,14 +410,14 @@ export class HttpBlobClient {
|
|
|
334
410
|
}
|
|
335
411
|
}
|
|
336
412
|
async getBlobSidecarFrom(hostUrl, blockHashOrSlot, blobHashes = [], l1ConsensusHostIndex) {
|
|
337
|
-
const blobs = await this.getBlobsFromHost(hostUrl, blockHashOrSlot, l1ConsensusHostIndex);
|
|
338
|
-
return processFetchedBlobs(blobs, blobHashes, this.log).filter((b)=>b !== undefined);
|
|
413
|
+
const blobs = await this.getBlobsFromHost(hostUrl, blockHashOrSlot, l1ConsensusHostIndex, blobHashes);
|
|
414
|
+
return (await processFetchedBlobs(blobs, blobHashes, this.log)).filter((b)=>b !== undefined);
|
|
339
415
|
}
|
|
340
|
-
async getBlobsFromHost(hostUrl, blockHashOrSlot, l1ConsensusHostIndex) {
|
|
416
|
+
async getBlobsFromHost(hostUrl, blockHashOrSlot, l1ConsensusHostIndex, blobHashes) {
|
|
341
417
|
try {
|
|
342
|
-
let res = await this.fetchBlobSidecars(hostUrl, blockHashOrSlot, l1ConsensusHostIndex);
|
|
418
|
+
let res = await this.fetchBlobSidecars(hostUrl, blockHashOrSlot, l1ConsensusHostIndex, blobHashes);
|
|
343
419
|
if (res.ok) {
|
|
344
|
-
return parseBlobJsonsFromResponse(await res.json(), this.log);
|
|
420
|
+
return await parseBlobJsonsFromResponse(await res.json(), this.log);
|
|
345
421
|
}
|
|
346
422
|
if (res.status === 404 && typeof blockHashOrSlot === 'number') {
|
|
347
423
|
const latestSlot = await this.getLatestSlotNumber(hostUrl, l1ConsensusHostIndex);
|
|
@@ -354,9 +430,9 @@ export class HttpBlobClient {
|
|
|
354
430
|
let currentSlot = blockHashOrSlot + 1;
|
|
355
431
|
while(res.status === 404 && maxRetries > 0 && latestSlot !== undefined && currentSlot <= latestSlot){
|
|
356
432
|
this.log.debug(`Trying slot ${currentSlot}`);
|
|
357
|
-
res = await this.fetchBlobSidecars(hostUrl, currentSlot, l1ConsensusHostIndex);
|
|
433
|
+
res = await this.fetchBlobSidecars(hostUrl, currentSlot, l1ConsensusHostIndex, blobHashes);
|
|
358
434
|
if (res.ok) {
|
|
359
|
-
return parseBlobJsonsFromResponse(await res.json(), this.log);
|
|
435
|
+
return await parseBlobJsonsFromResponse(await res.json(), this.log);
|
|
360
436
|
}
|
|
361
437
|
currentSlot++;
|
|
362
438
|
maxRetries--;
|
|
@@ -373,21 +449,29 @@ export class HttpBlobClient {
|
|
|
373
449
|
return [];
|
|
374
450
|
}
|
|
375
451
|
}
|
|
376
|
-
fetchBlobSidecars(hostUrl, blockHashOrSlot, l1ConsensusHostIndex) {
|
|
377
|
-
|
|
378
|
-
|
|
452
|
+
fetchBlobSidecars(hostUrl, blockHashOrSlot, l1ConsensusHostIndex, blobHashes) {
|
|
453
|
+
let baseUrl = `${hostUrl}/eth/v1/beacon/blobs/${blockHashOrSlot}`;
|
|
454
|
+
if (blobHashes && blobHashes.length > 0) {
|
|
455
|
+
const params = new URLSearchParams();
|
|
456
|
+
for (const hash of blobHashes){
|
|
457
|
+
params.append('versioned_hashes', `0x${hash.toString('hex')}`);
|
|
458
|
+
}
|
|
459
|
+
baseUrl += `?${params.toString()}`;
|
|
460
|
+
}
|
|
461
|
+
const { url, logSafeUrl, ...options } = getBeaconNodeFetchOptions(baseUrl, this.config, l1ConsensusHostIndex);
|
|
379
462
|
this.log.debug(`Fetching blob sidecar for ${blockHashOrSlot}`, {
|
|
380
|
-
url,
|
|
463
|
+
url: logSafeUrl,
|
|
381
464
|
...options
|
|
382
465
|
});
|
|
383
|
-
|
|
466
|
+
// No retry here — this is called inside the main retry loop in getBlobSidecar
|
|
467
|
+
return fetch(url, options);
|
|
384
468
|
}
|
|
385
469
|
async getLatestSlotNumber(hostUrl, l1ConsensusHostIndex) {
|
|
386
470
|
try {
|
|
387
471
|
const baseUrl = `${hostUrl}/eth/v1/beacon/headers/head`;
|
|
388
|
-
const { url, ...options } = getBeaconNodeFetchOptions(baseUrl, this.config, l1ConsensusHostIndex);
|
|
472
|
+
const { url, logSafeUrl, ...options } = getBeaconNodeFetchOptions(baseUrl, this.config, l1ConsensusHostIndex);
|
|
389
473
|
this.log.debug(`Fetching latest slot number`, {
|
|
390
|
-
url,
|
|
474
|
+
url: logSafeUrl,
|
|
391
475
|
...options
|
|
392
476
|
});
|
|
393
477
|
const res = await this.fetch(url, options);
|
|
@@ -420,36 +504,45 @@ export class HttpBlobClient {
|
|
|
420
504
|
*
|
|
421
505
|
* @param blockHash - The block hash
|
|
422
506
|
* @returns The slot number
|
|
423
|
-
*/ async getSlotNumber(blockHash) {
|
|
507
|
+
*/ async getSlotNumber(blockHash, parentBeaconBlockRoot, l1BlockTimestamp) {
|
|
424
508
|
const { l1ConsensusHostUrls, l1RpcUrls } = this.config;
|
|
425
509
|
if (!l1ConsensusHostUrls || l1ConsensusHostUrls.length === 0) {
|
|
426
510
|
this.log.debug('No consensus host url configured');
|
|
427
511
|
return undefined;
|
|
428
512
|
}
|
|
429
|
-
if (
|
|
430
|
-
|
|
431
|
-
|
|
513
|
+
// Primary path: compute slot from timestamp if genesis config is cached (no network call needed)
|
|
514
|
+
if (l1BlockTimestamp !== undefined && this.beaconGenesisTime !== undefined && this.beaconSecondsPerSlot !== undefined) {
|
|
515
|
+
const slot = Number((l1BlockTimestamp - this.beaconGenesisTime) / BigInt(this.beaconSecondsPerSlot));
|
|
516
|
+
this.log.debug(`Computed slot ${slot} from L1 block timestamp`, {
|
|
517
|
+
l1BlockTimestamp
|
|
518
|
+
});
|
|
519
|
+
return slot;
|
|
432
520
|
}
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
params: [
|
|
444
|
-
blockHash,
|
|
445
|
-
/*tx flag*/ false
|
|
446
|
-
]
|
|
521
|
+
if (!parentBeaconBlockRoot) {
|
|
522
|
+
// parentBeaconBlockRoot not provided by caller — fetch it from the execution RPC
|
|
523
|
+
if (!l1RpcUrls || l1RpcUrls.length === 0) {
|
|
524
|
+
this.log.debug('No execution host url configured');
|
|
525
|
+
return undefined;
|
|
526
|
+
}
|
|
527
|
+
const client = createPublicClient({
|
|
528
|
+
transport: makeL1HttpTransport(l1RpcUrls, {
|
|
529
|
+
timeout: this.config.l1HttpTimeoutMS
|
|
530
|
+
})
|
|
447
531
|
});
|
|
448
|
-
|
|
449
|
-
|
|
532
|
+
try {
|
|
533
|
+
const res = await client.request({
|
|
534
|
+
method: 'eth_getBlockByHash',
|
|
535
|
+
params: [
|
|
536
|
+
blockHash,
|
|
537
|
+
/*tx flag*/ false
|
|
538
|
+
]
|
|
539
|
+
});
|
|
540
|
+
if (res.parentBeaconBlockRoot) {
|
|
541
|
+
parentBeaconBlockRoot = res.parentBeaconBlockRoot;
|
|
542
|
+
}
|
|
543
|
+
} catch (err) {
|
|
544
|
+
this.log.error(`Error getting parent beacon block root`, err);
|
|
450
545
|
}
|
|
451
|
-
} catch (err) {
|
|
452
|
-
this.log.error(`Error getting parent beacon block root`, err);
|
|
453
546
|
}
|
|
454
547
|
if (!parentBeaconBlockRoot) {
|
|
455
548
|
this.log.error(`No parent beacon block root found for block ${blockHash}`);
|
|
@@ -481,8 +574,10 @@ export class HttpBlobClient {
|
|
|
481
574
|
}
|
|
482
575
|
/**
|
|
483
576
|
* Start the blob client.
|
|
484
|
-
*
|
|
577
|
+
* Fetches and caches beacon genesis config for timestamp-based slot resolution,
|
|
578
|
+
* then uploads the initial healthcheck file (awaited) and starts periodic uploads.
|
|
485
579
|
*/ async start() {
|
|
580
|
+
await this.fetchBeaconConfig();
|
|
486
581
|
if (!this.fileStoreUploadClient) {
|
|
487
582
|
return;
|
|
488
583
|
}
|
|
@@ -501,6 +596,40 @@ export class HttpBlobClient {
|
|
|
501
596
|
}, intervalMs);
|
|
502
597
|
}
|
|
503
598
|
/**
|
|
599
|
+
* Fetches and caches beacon genesis time and slot duration from the first available consensus host.
|
|
600
|
+
* These static values enable timestamp-based slot resolution, eliminating the per-fetch headers call.
|
|
601
|
+
* Logs a warning and leaves fields undefined if all hosts fail, callers fall back gracefully.
|
|
602
|
+
*/ async fetchBeaconConfig() {
|
|
603
|
+
const { l1ConsensusHostUrls } = this.config;
|
|
604
|
+
if (!l1ConsensusHostUrls || l1ConsensusHostUrls.length === 0) {
|
|
605
|
+
return;
|
|
606
|
+
}
|
|
607
|
+
for(let i = 0; i < l1ConsensusHostUrls.length; i++){
|
|
608
|
+
try {
|
|
609
|
+
const { url: genesisUrl, ...genesisOptions } = getBeaconNodeFetchOptions(`${l1ConsensusHostUrls[i]}/eth/v1/config/genesis`, this.config, i);
|
|
610
|
+
const { url: specUrl, ...specOptions } = getBeaconNodeFetchOptions(`${l1ConsensusHostUrls[i]}/eth/v1/config/spec`, this.config, i);
|
|
611
|
+
const [genesisRes, specRes] = await Promise.all([
|
|
612
|
+
this.fetch(genesisUrl, genesisOptions),
|
|
613
|
+
this.fetch(specUrl, specOptions)
|
|
614
|
+
]);
|
|
615
|
+
if (genesisRes.ok && specRes.ok) {
|
|
616
|
+
const genesis = await genesisRes.json();
|
|
617
|
+
const spec = await specRes.json();
|
|
618
|
+
this.beaconGenesisTime = BigInt(genesis.data.genesisTime);
|
|
619
|
+
this.beaconSecondsPerSlot = parseInt(spec.data.secondsPerSlot);
|
|
620
|
+
this.log.debug(`Fetched beacon genesis config`, {
|
|
621
|
+
genesisTime: this.beaconGenesisTime,
|
|
622
|
+
secondsPerSlot: this.beaconSecondsPerSlot
|
|
623
|
+
});
|
|
624
|
+
return;
|
|
625
|
+
}
|
|
626
|
+
} catch (err) {
|
|
627
|
+
this.log.warn(`Failed to fetch beacon config from host ${l1ConsensusHostUrls[i]}`, err);
|
|
628
|
+
}
|
|
629
|
+
}
|
|
630
|
+
this.log.warn('Could not fetch beacon genesis config from any consensus host — will use headers call fallback');
|
|
631
|
+
}
|
|
632
|
+
/**
|
|
504
633
|
* Stop the blob client, clearing any periodic tasks.
|
|
505
634
|
*/ stop() {
|
|
506
635
|
if (this.healthcheckUploadIntervalId) {
|
|
@@ -509,10 +638,9 @@ export class HttpBlobClient {
|
|
|
509
638
|
}
|
|
510
639
|
}
|
|
511
640
|
}
|
|
512
|
-
function parseBlobJsonsFromResponse(response, logger) {
|
|
641
|
+
async function parseBlobJsonsFromResponse(response, logger) {
|
|
513
642
|
try {
|
|
514
|
-
|
|
515
|
-
return blobs;
|
|
643
|
+
return await Promise.all(response.data.map(parseBlobJson));
|
|
516
644
|
} catch (err) {
|
|
517
645
|
logger.error(`Error parsing blob json from response`, err);
|
|
518
646
|
return [];
|
|
@@ -522,15 +650,14 @@ function parseBlobJsonsFromResponse(response, logger) {
|
|
|
522
650
|
// https://ethereum.github.io/beacon-APIs/?urls.primaryName=dev#/Beacon/getBlobSidecars
|
|
523
651
|
// Here we attempt to parse the response data to Buffer, and check the lengths (via Blob's constructor), to avoid
|
|
524
652
|
// throwing an error down the line when calling Blob.fromJson().
|
|
525
|
-
function parseBlobJson(
|
|
526
|
-
const blobBuffer = Buffer.from(
|
|
527
|
-
const
|
|
528
|
-
const blob = new Blob(blobBuffer, commitmentBuffer);
|
|
653
|
+
async function parseBlobJson(rawHex) {
|
|
654
|
+
const blobBuffer = Buffer.from(rawHex.slice(2), 'hex');
|
|
655
|
+
const blob = await Blob.fromBlobBuffer(blobBuffer);
|
|
529
656
|
return blob.toJSON();
|
|
530
657
|
}
|
|
531
658
|
// Returns an array that maps each blob hash to the corresponding blob, or undefined if the blob is not found
|
|
532
659
|
// or the data does not match the commitment.
|
|
533
|
-
function processFetchedBlobs(blobs, blobHashes, logger) {
|
|
660
|
+
async function processFetchedBlobs(blobs, blobHashes, logger) {
|
|
534
661
|
const requestedBlobHashes = new Set(blobHashes.map(bufferToHex));
|
|
535
662
|
const hashToBlob = new Map();
|
|
536
663
|
for (const blobJson of blobs){
|
|
@@ -539,7 +666,7 @@ function processFetchedBlobs(blobs, blobHashes, logger) {
|
|
|
539
666
|
continue;
|
|
540
667
|
}
|
|
541
668
|
try {
|
|
542
|
-
const blob = Blob.fromJson(blobJson);
|
|
669
|
+
const blob = await Blob.fromJson(blobJson);
|
|
543
670
|
hashToBlob.set(hashHex, blob);
|
|
544
671
|
} catch (err) {
|
|
545
672
|
// If the above throws, it's likely that the blob commitment does not match the hash of the blob data.
|
|
@@ -553,11 +680,15 @@ function getBeaconNodeFetchOptions(url, config, l1ConsensusHostIndex) {
|
|
|
553
680
|
const l1ConsensusHostApiKey = l1ConsensusHostIndex !== undefined && l1ConsensusHostApiKeys && l1ConsensusHostApiKeys[l1ConsensusHostIndex];
|
|
554
681
|
const l1ConsensusHostApiKeyHeader = l1ConsensusHostIndex !== undefined && l1ConsensusHostApiKeyHeaders && l1ConsensusHostApiKeyHeaders[l1ConsensusHostIndex];
|
|
555
682
|
let formattedUrl = url;
|
|
683
|
+
let logSafeUrl = url;
|
|
556
684
|
if (l1ConsensusHostApiKey && l1ConsensusHostApiKey.getValue() !== '' && !l1ConsensusHostApiKeyHeader) {
|
|
557
|
-
|
|
685
|
+
const separator = formattedUrl.includes('?') ? '&' : '?';
|
|
686
|
+
formattedUrl += `${separator}key=${l1ConsensusHostApiKey.getValue()}`;
|
|
687
|
+
logSafeUrl += `${separator}key=[REDACTED]`;
|
|
558
688
|
}
|
|
559
689
|
return {
|
|
560
690
|
url: formattedUrl,
|
|
691
|
+
logSafeUrl,
|
|
561
692
|
...l1ConsensusHostApiKey && l1ConsensusHostApiKeyHeader && {
|
|
562
693
|
headers: {
|
|
563
694
|
[l1ConsensusHostApiKeyHeader]: l1ConsensusHostApiKey.getValue()
|