@aztec/blob-client 0.0.1-commit.3469e52 → 0.0.1-commit.35158ae7e
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dest/archive/instrumentation.d.ts +1 -1
- package/dest/archive/instrumentation.d.ts.map +1 -1
- package/dest/archive/instrumentation.js +13 -4
- package/dest/blobstore/blob_store_test_suite.js +9 -9
- package/dest/client/config.d.ts +3 -1
- package/dest/client/config.d.ts.map +1 -1
- package/dest/client/config.js +6 -1
- package/dest/client/http.d.ts +9 -3
- package/dest/client/http.d.ts.map +1 -1
- package/dest/client/http.js +150 -63
- package/dest/client/interface.d.ts +12 -1
- package/dest/client/interface.d.ts.map +1 -1
- package/dest/client/tests.js +3 -3
- package/package.json +7 -7
- package/src/archive/instrumentation.ts +22 -4
- package/src/blobstore/blob_store_test_suite.ts +9 -9
- package/src/client/config.ts +9 -0
- package/src/client/http.ts +182 -57
- package/src/client/interface.ts +11 -0
- package/src/client/tests.ts +2 -2
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"interface.d.ts","sourceRoot":"","sources":["../../src/client/interface.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAC;AAE5C;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC;;;;;OAKG;IACH,gBAAgB,CAAC,EAAE,OAAO,CAAC;
|
|
1
|
+
{"version":3,"file":"interface.d.ts","sourceRoot":"","sources":["../../src/client/interface.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAC;AAE5C;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC;;;;;OAKG;IACH,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B;;;OAGG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B;;;;OAIG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAED,MAAM,WAAW,mBAAmB;IAClC,0EAA0E;IAC1E,oBAAoB,CAAC,KAAK,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IACtD,qEAAqE;IACrE,cAAc,CAAC,OAAO,EAAE,MAAM,EAAE,UAAU,CAAC,EAAE,MAAM,EAAE,EAAE,IAAI,CAAC,EAAE,qBAAqB,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;IACtG,6EAA6E;IAC7E,KAAK,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACxB,oFAAoF;IACpF,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAC7B,0DAA0D;IAC1D,IAAI,CAAC,IAAI,IAAI,CAAC;IACd,iEAAiE;IACjE,SAAS,IAAI,OAAO,CAAC;CACtB"}
|
package/dest/client/tests.js
CHANGED
|
@@ -17,7 +17,7 @@ import { makeRandomBlob } from '@aztec/blob-lib/testing';
|
|
|
17
17
|
await cleanup();
|
|
18
18
|
});
|
|
19
19
|
it('should send and retrieve blobs by hash', async ()=>{
|
|
20
|
-
const blob = makeRandomBlob(5);
|
|
20
|
+
const blob = await makeRandomBlob(5);
|
|
21
21
|
const blobHash = blob.getEthVersionedBlobHash();
|
|
22
22
|
await client.sendBlobsToFilestore([
|
|
23
23
|
blob
|
|
@@ -29,9 +29,9 @@ import { makeRandomBlob } from '@aztec/blob-lib/testing';
|
|
|
29
29
|
expect(retrievedBlobs[0]).toEqual(blob);
|
|
30
30
|
});
|
|
31
31
|
it('should handle multiple blobs', async ()=>{
|
|
32
|
-
const blobs = Array.from({
|
|
32
|
+
const blobs = await Promise.all(Array.from({
|
|
33
33
|
length: 3
|
|
34
|
-
}, ()=>makeRandomBlob(7));
|
|
34
|
+
}, ()=>makeRandomBlob(7)));
|
|
35
35
|
const blobHashes = blobs.map((blob)=>blob.getEthVersionedBlobHash());
|
|
36
36
|
await client.sendBlobsToFilestore(blobs);
|
|
37
37
|
const retrievedBlobs = await client.getBlobSidecar(blockId, blobHashes);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@aztec/blob-client",
|
|
3
|
-
"version": "0.0.1-commit.
|
|
3
|
+
"version": "0.0.1-commit.35158ae7e",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"bin": "./dest/client/bin/index.js",
|
|
6
6
|
"exports": {
|
|
@@ -56,12 +56,12 @@
|
|
|
56
56
|
]
|
|
57
57
|
},
|
|
58
58
|
"dependencies": {
|
|
59
|
-
"@aztec/blob-lib": "0.0.1-commit.
|
|
60
|
-
"@aztec/ethereum": "0.0.1-commit.
|
|
61
|
-
"@aztec/foundation": "0.0.1-commit.
|
|
62
|
-
"@aztec/kv-store": "0.0.1-commit.
|
|
63
|
-
"@aztec/stdlib": "0.0.1-commit.
|
|
64
|
-
"@aztec/telemetry-client": "0.0.1-commit.
|
|
59
|
+
"@aztec/blob-lib": "0.0.1-commit.35158ae7e",
|
|
60
|
+
"@aztec/ethereum": "0.0.1-commit.35158ae7e",
|
|
61
|
+
"@aztec/foundation": "0.0.1-commit.35158ae7e",
|
|
62
|
+
"@aztec/kv-store": "0.0.1-commit.35158ae7e",
|
|
63
|
+
"@aztec/stdlib": "0.0.1-commit.35158ae7e",
|
|
64
|
+
"@aztec/telemetry-client": "0.0.1-commit.35158ae7e",
|
|
65
65
|
"express": "^4.21.2",
|
|
66
66
|
"snappy": "^7.2.2",
|
|
67
67
|
"source-map-support": "^0.5.21",
|
|
@@ -1,4 +1,10 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import {
|
|
2
|
+
Attributes,
|
|
3
|
+
Metrics,
|
|
4
|
+
type TelemetryClient,
|
|
5
|
+
type UpDownCounter,
|
|
6
|
+
createUpDownCounterWithDefault,
|
|
7
|
+
} from '@aztec/telemetry-client';
|
|
2
8
|
|
|
3
9
|
export class BlobArchiveClientInstrumentation {
|
|
4
10
|
private blockRequestCounter: UpDownCounter;
|
|
@@ -11,11 +17,23 @@ export class BlobArchiveClientInstrumentation {
|
|
|
11
17
|
name: string,
|
|
12
18
|
) {
|
|
13
19
|
const meter = client.getMeter(name);
|
|
14
|
-
|
|
20
|
+
const requestAttrs = {
|
|
21
|
+
[Attributes.HTTP_RESPONSE_STATUS_CODE]: [200, 404],
|
|
22
|
+
[Attributes.HTTP_REQUEST_HOST]: [httpHost],
|
|
23
|
+
};
|
|
24
|
+
this.blockRequestCounter = createUpDownCounterWithDefault(
|
|
25
|
+
meter,
|
|
26
|
+
Metrics.BLOB_SINK_ARCHIVE_BLOCK_REQUEST_COUNT,
|
|
27
|
+
requestAttrs,
|
|
28
|
+
);
|
|
15
29
|
|
|
16
|
-
this.blobRequestCounter =
|
|
30
|
+
this.blobRequestCounter = createUpDownCounterWithDefault(
|
|
31
|
+
meter,
|
|
32
|
+
Metrics.BLOB_SINK_ARCHIVE_BLOB_REQUEST_COUNT,
|
|
33
|
+
requestAttrs,
|
|
34
|
+
);
|
|
17
35
|
|
|
18
|
-
this.retrievedBlobs = meter
|
|
36
|
+
this.retrievedBlobs = createUpDownCounterWithDefault(meter, Metrics.BLOB_SINK_ARCHIVE_BLOB_COUNT);
|
|
19
37
|
}
|
|
20
38
|
|
|
21
39
|
incRequest(type: 'blocks' | 'blobs', status: number) {
|
|
@@ -13,7 +13,7 @@ export function describeBlobStore(getBlobStore: () => Promise<BlobStore>) {
|
|
|
13
13
|
it('should store and retrieve a blob by hash', async () => {
|
|
14
14
|
// Create a test blob with random fields
|
|
15
15
|
const testFields = [Fr.random(), Fr.random(), Fr.random()];
|
|
16
|
-
const blob = Blob.fromFields(testFields);
|
|
16
|
+
const blob = await Blob.fromFields(testFields);
|
|
17
17
|
const blobHash = blob.getEthVersionedBlobHash();
|
|
18
18
|
|
|
19
19
|
// Store the blob
|
|
@@ -29,8 +29,8 @@ export function describeBlobStore(getBlobStore: () => Promise<BlobStore>) {
|
|
|
29
29
|
|
|
30
30
|
it('should handle multiple blobs stored and retrieved by their hashes', async () => {
|
|
31
31
|
// Create two different blobs
|
|
32
|
-
const blob1 = Blob.fromFields([Fr.random(), Fr.random()]);
|
|
33
|
-
const blob2 = Blob.fromFields([Fr.random(), Fr.random(), Fr.random()]);
|
|
32
|
+
const blob1 = await Blob.fromFields([Fr.random(), Fr.random()]);
|
|
33
|
+
const blob2 = await Blob.fromFields([Fr.random(), Fr.random(), Fr.random()]);
|
|
34
34
|
|
|
35
35
|
const blobHash1 = blob1.getEthVersionedBlobHash();
|
|
36
36
|
const blobHash2 = blob2.getEthVersionedBlobHash();
|
|
@@ -57,9 +57,9 @@ export function describeBlobStore(getBlobStore: () => Promise<BlobStore>) {
|
|
|
57
57
|
|
|
58
58
|
it('should handle retrieving subset of stored blobs', async () => {
|
|
59
59
|
// Store multiple blobs
|
|
60
|
-
const blob1 = Blob.fromFields([Fr.random()]);
|
|
61
|
-
const blob2 = Blob.fromFields([Fr.random()]);
|
|
62
|
-
const blob3 = Blob.fromFields([Fr.random()]);
|
|
60
|
+
const blob1 = await Blob.fromFields([Fr.random()]);
|
|
61
|
+
const blob2 = await Blob.fromFields([Fr.random()]);
|
|
62
|
+
const blob3 = await Blob.fromFields([Fr.random()]);
|
|
63
63
|
|
|
64
64
|
await blobStore.addBlobs([blob1, blob2, blob3]);
|
|
65
65
|
|
|
@@ -75,7 +75,7 @@ export function describeBlobStore(getBlobStore: () => Promise<BlobStore>) {
|
|
|
75
75
|
});
|
|
76
76
|
|
|
77
77
|
it('should handle duplicate blob hashes in request', async () => {
|
|
78
|
-
const blob = Blob.fromFields([Fr.random()]);
|
|
78
|
+
const blob = await Blob.fromFields([Fr.random()]);
|
|
79
79
|
const blobHash = blob.getEthVersionedBlobHash();
|
|
80
80
|
|
|
81
81
|
await blobStore.addBlobs([blob]);
|
|
@@ -91,8 +91,8 @@ export function describeBlobStore(getBlobStore: () => Promise<BlobStore>) {
|
|
|
91
91
|
it('should overwrite blob when storing with same hash', async () => {
|
|
92
92
|
// Create two blobs that will have the same hash (same content)
|
|
93
93
|
const fields = [Fr.random(), Fr.random()];
|
|
94
|
-
const blob1 = Blob.fromFields(fields);
|
|
95
|
-
const blob2 = Blob.fromFields(fields);
|
|
94
|
+
const blob1 = await Blob.fromFields(fields);
|
|
95
|
+
const blob2 = await Blob.fromFields(fields);
|
|
96
96
|
|
|
97
97
|
const blobHash = blob1.getEthVersionedBlobHash();
|
|
98
98
|
|
package/src/client/config.ts
CHANGED
|
@@ -3,6 +3,7 @@ import {
|
|
|
3
3
|
SecretValue,
|
|
4
4
|
booleanConfigHelper,
|
|
5
5
|
getConfigFromMappings,
|
|
6
|
+
optionalNumberConfigHelper,
|
|
6
7
|
} from '@aztec/foundation/config';
|
|
7
8
|
|
|
8
9
|
import { type BlobArchiveApiConfig, blobArchiveApiConfigMappings } from '../archive/config.js';
|
|
@@ -55,6 +56,9 @@ export interface BlobClientConfig extends BlobArchiveApiConfig {
|
|
|
55
56
|
* Interval in minutes for uploading healthcheck file to file store (default: 60 = 1 hour)
|
|
56
57
|
*/
|
|
57
58
|
blobHealthcheckUploadIntervalMinutes?: number;
|
|
59
|
+
|
|
60
|
+
/** Timeout for HTTP requests to the L1 RPC node in ms. */
|
|
61
|
+
l1HttpTimeoutMS?: number;
|
|
58
62
|
}
|
|
59
63
|
|
|
60
64
|
export const blobClientConfigMapping: ConfigMappingsType<BlobClientConfig> = {
|
|
@@ -108,6 +112,11 @@ export const blobClientConfigMapping: ConfigMappingsType<BlobClientConfig> = {
|
|
|
108
112
|
description: 'Interval in minutes for uploading healthcheck file to file store (default: 60 = 1 hour)',
|
|
109
113
|
parseEnv: (val: string | undefined) => (val ? +val : undefined),
|
|
110
114
|
},
|
|
115
|
+
l1HttpTimeoutMS: {
|
|
116
|
+
env: 'ETHEREUM_HTTP_TIMEOUT_MS',
|
|
117
|
+
description: 'Timeout for HTTP requests to the L1 RPC node in ms.',
|
|
118
|
+
...optionalNumberConfigHelper(),
|
|
119
|
+
},
|
|
111
120
|
...blobArchiveApiConfigMappings,
|
|
112
121
|
};
|
|
113
122
|
|
package/src/client/http.ts
CHANGED
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
import { Blob, type BlobJson, computeEthVersionedBlobHash } from '@aztec/blob-lib';
|
|
2
|
+
import { makeL1HttpTransport } from '@aztec/ethereum/client';
|
|
2
3
|
import { shuffle } from '@aztec/foundation/array';
|
|
3
4
|
import { type Logger, createLogger } from '@aztec/foundation/log';
|
|
4
5
|
import { makeBackoff, retry } from '@aztec/foundation/retry';
|
|
5
6
|
import { bufferToHex, hexToBuffer } from '@aztec/foundation/string';
|
|
6
7
|
|
|
7
|
-
import { type RpcBlock, createPublicClient
|
|
8
|
+
import { type RpcBlock, createPublicClient } from 'viem';
|
|
8
9
|
|
|
9
10
|
import { createBlobArchiveClient } from '../archive/factory.js';
|
|
10
11
|
import type { BlobArchiveClient } from '../archive/interface.js';
|
|
@@ -24,6 +25,11 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
24
25
|
private disabled = false;
|
|
25
26
|
private healthcheckUploadIntervalId?: NodeJS.Timeout;
|
|
26
27
|
|
|
28
|
+
/** Cached beacon genesis time (seconds since Unix epoch). Fetched once at startup. */
|
|
29
|
+
private beaconGenesisTime?: bigint;
|
|
30
|
+
/** Cached beacon slot duration in seconds. Fetched once at startup. */
|
|
31
|
+
private beaconSecondsPerSlot?: number;
|
|
32
|
+
|
|
27
33
|
constructor(
|
|
28
34
|
config?: BlobClientConfig,
|
|
29
35
|
private readonly opts: {
|
|
@@ -89,44 +95,68 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
89
95
|
const archiveUrl = this.archiveClient?.getBaseUrl();
|
|
90
96
|
this.log.info(`Testing configured blob sources`, { l1ConsensusHostUrls, archiveUrl });
|
|
91
97
|
|
|
92
|
-
let
|
|
98
|
+
let consensusSuperNodes = 0;
|
|
99
|
+
let consensusNonSuperNodes = 0;
|
|
100
|
+
let archiveSources = 0;
|
|
101
|
+
let blobSinks = 0;
|
|
93
102
|
|
|
94
103
|
if (l1ConsensusHostUrls && l1ConsensusHostUrls.length > 0) {
|
|
95
104
|
for (let l1ConsensusHostIndex = 0; l1ConsensusHostIndex < l1ConsensusHostUrls.length; l1ConsensusHostIndex++) {
|
|
96
105
|
const l1ConsensusHostUrl = l1ConsensusHostUrls[l1ConsensusHostIndex];
|
|
97
106
|
try {
|
|
98
107
|
const { url, ...options } = getBeaconNodeFetchOptions(
|
|
99
|
-
`${l1ConsensusHostUrl}/eth/v1/beacon/headers`,
|
|
108
|
+
`${l1ConsensusHostUrl}/eth/v1/beacon/headers/head`,
|
|
100
109
|
this.config,
|
|
101
110
|
l1ConsensusHostIndex,
|
|
102
111
|
);
|
|
103
112
|
const res = await this.fetch(url, options);
|
|
104
|
-
if (res.ok) {
|
|
105
|
-
this.log.info(`L1 consensus host is reachable`, { l1ConsensusHostUrl });
|
|
106
|
-
successfulSourceCount++;
|
|
107
|
-
} else {
|
|
113
|
+
if (!res.ok) {
|
|
108
114
|
this.log.error(`Failure reaching L1 consensus host: ${res.statusText} (${res.status})`, {
|
|
109
115
|
l1ConsensusHostUrl,
|
|
110
116
|
});
|
|
117
|
+
continue;
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
this.log.info(`L1 consensus host is reachable`, { l1ConsensusHostUrl });
|
|
121
|
+
|
|
122
|
+
// Check if the host serves blob sidecars (supernode/semi-supernode).
|
|
123
|
+
// Post-Fusaka (PeerDAS), non-supernode beacon nodes no longer serve the
|
|
124
|
+
// blob sidecar endpoint. A 200 response (even with an empty data array
|
|
125
|
+
// for a slot with no blobs) means the node supports serving blob sidecars.
|
|
126
|
+
const body = await res.json();
|
|
127
|
+
const headSlot = body?.data?.header?.message?.slot;
|
|
128
|
+
if (headSlot) {
|
|
129
|
+
const { url: blobUrl, ...blobOptions } = getBeaconNodeFetchOptions(
|
|
130
|
+
`${l1ConsensusHostUrl}/eth/v1/beacon/blobs/${headSlot}`,
|
|
131
|
+
this.config,
|
|
132
|
+
l1ConsensusHostIndex,
|
|
133
|
+
);
|
|
134
|
+
const blobRes = await this.fetch(blobUrl, blobOptions);
|
|
135
|
+
if (blobRes.ok) {
|
|
136
|
+
this.log.info(`L1 consensus host serves blob sidecars (supernode)`, { l1ConsensusHostUrl });
|
|
137
|
+
consensusSuperNodes++;
|
|
138
|
+
} else {
|
|
139
|
+
this.log.info(`L1 consensus host does not serve blob sidecars`, { l1ConsensusHostUrl });
|
|
140
|
+
consensusNonSuperNodes++;
|
|
141
|
+
}
|
|
142
|
+
} else {
|
|
143
|
+
this.log.info(`L1 consensus host is reachable but could not determine head slot`, { l1ConsensusHostUrl });
|
|
144
|
+
consensusNonSuperNodes++;
|
|
111
145
|
}
|
|
112
146
|
} catch (err) {
|
|
113
147
|
this.log.error(`Error reaching L1 consensus host`, err, { l1ConsensusHostUrl });
|
|
114
148
|
}
|
|
115
149
|
}
|
|
116
|
-
} else {
|
|
117
|
-
this.log.warn('No L1 consensus host urls configured');
|
|
118
150
|
}
|
|
119
151
|
|
|
120
152
|
if (this.archiveClient) {
|
|
121
153
|
try {
|
|
122
154
|
const latest = await this.archiveClient.getLatestBlock();
|
|
123
155
|
this.log.info(`Archive client is reachable and synced to L1 block ${latest.number}`, { latest, archiveUrl });
|
|
124
|
-
|
|
156
|
+
archiveSources++;
|
|
125
157
|
} catch (err) {
|
|
126
158
|
this.log.error(`Error reaching archive client`, err, { archiveUrl });
|
|
127
159
|
}
|
|
128
|
-
} else {
|
|
129
|
-
this.log.warn('No archive client configured');
|
|
130
160
|
}
|
|
131
161
|
|
|
132
162
|
if (this.fileStoreClients.length > 0) {
|
|
@@ -135,7 +165,7 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
135
165
|
const accessible = await fileStoreClient.testConnection();
|
|
136
166
|
if (accessible) {
|
|
137
167
|
this.log.info(`FileStore is reachable`, { url: fileStoreClient.getBaseUrl() });
|
|
138
|
-
|
|
168
|
+
blobSinks++;
|
|
139
169
|
} else {
|
|
140
170
|
this.log.warn(`FileStore is not accessible`, { url: fileStoreClient.getBaseUrl() });
|
|
141
171
|
}
|
|
@@ -145,12 +175,24 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
145
175
|
}
|
|
146
176
|
}
|
|
147
177
|
|
|
178
|
+
// Emit a single summary after validating all sources
|
|
179
|
+
const successfulSourceCount = consensusSuperNodes + archiveSources + blobSinks;
|
|
180
|
+
|
|
181
|
+
let summary = `Blob client running with consensusSuperNodes=${consensusSuperNodes} archiveSources=${archiveSources} blobSinks=${blobSinks}`;
|
|
182
|
+
if (consensusNonSuperNodes > 0) {
|
|
183
|
+
summary += `. ${consensusNonSuperNodes} consensus client(s) ignored because they are not running in supernode or semi-supernode mode`;
|
|
184
|
+
}
|
|
185
|
+
|
|
148
186
|
if (successfulSourceCount === 0) {
|
|
149
187
|
if (this.config.blobAllowEmptySources) {
|
|
150
|
-
this.log.warn(
|
|
188
|
+
this.log.warn(summary);
|
|
151
189
|
} else {
|
|
152
|
-
throw new Error(
|
|
190
|
+
throw new Error(summary);
|
|
153
191
|
}
|
|
192
|
+
} else if (consensusSuperNodes === 0) {
|
|
193
|
+
this.log.warn(summary);
|
|
194
|
+
} else {
|
|
195
|
+
this.log.info(summary);
|
|
154
196
|
}
|
|
155
197
|
}
|
|
156
198
|
|
|
@@ -215,8 +257,8 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
215
257
|
const getFilledBlobs = (): Blob[] => resultBlobs.filter((b): b is Blob => b !== undefined);
|
|
216
258
|
|
|
217
259
|
// Helper to fill in results from fetched blobs
|
|
218
|
-
const fillResults = (fetchedBlobs: BlobJson[]): Blob[] => {
|
|
219
|
-
const blobs = processFetchedBlobs(fetchedBlobs, blobHashes, this.log);
|
|
260
|
+
const fillResults = async (fetchedBlobs: BlobJson[]): Promise<Blob[]> => {
|
|
261
|
+
const blobs = await processFetchedBlobs(fetchedBlobs, blobHashes, this.log);
|
|
220
262
|
// Fill in any missing positions with matching blobs
|
|
221
263
|
for (let i = 0; i < blobHashes.length; i++) {
|
|
222
264
|
if (resultBlobs[i] === undefined) {
|
|
@@ -251,7 +293,7 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
251
293
|
// The beacon api can query by slot number, so we get that first
|
|
252
294
|
const consensusCtx = { l1ConsensusHostUrls, ...ctx };
|
|
253
295
|
this.log.trace(`Attempting to get slot number for block hash`, consensusCtx);
|
|
254
|
-
const slotNumber = await this.getSlotNumber(blockHash);
|
|
296
|
+
const slotNumber = await this.getSlotNumber(blockHash, opts?.parentBeaconBlockRoot, opts?.l1BlockTimestamp);
|
|
255
297
|
this.log.debug(`Got slot number ${slotNumber} from consensus host for querying blobs`, consensusCtx);
|
|
256
298
|
|
|
257
299
|
if (slotNumber) {
|
|
@@ -268,8 +310,13 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
268
310
|
l1ConsensusHostUrl,
|
|
269
311
|
...ctx,
|
|
270
312
|
});
|
|
271
|
-
const blobs = await this.getBlobsFromHost(
|
|
272
|
-
|
|
313
|
+
const blobs = await this.getBlobsFromHost(
|
|
314
|
+
l1ConsensusHostUrl,
|
|
315
|
+
slotNumber,
|
|
316
|
+
l1ConsensusHostIndex,
|
|
317
|
+
getMissingBlobHashes(),
|
|
318
|
+
);
|
|
319
|
+
const result = await fillResults(blobs);
|
|
273
320
|
this.log.debug(
|
|
274
321
|
`Got ${blobs.length} blobs from consensus host (total: ${result.length}/${blobHashes.length})`,
|
|
275
322
|
{ slotNumber, l1ConsensusHostUrl, ...ctx },
|
|
@@ -312,7 +359,7 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
312
359
|
this.log.debug('No blobs found from archive client', archiveCtx);
|
|
313
360
|
} else {
|
|
314
361
|
this.log.trace(`Got ${allBlobs.length} blobs from archive client before filtering`, archiveCtx);
|
|
315
|
-
const result = fillResults(allBlobs);
|
|
362
|
+
const result = await fillResults(allBlobs);
|
|
316
363
|
this.log.debug(
|
|
317
364
|
`Got ${allBlobs.length} blobs from archive client (total: ${result.length}/${blobHashes.length})`,
|
|
318
365
|
archiveCtx,
|
|
@@ -345,7 +392,7 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
345
392
|
*/
|
|
346
393
|
private async tryFileStores(
|
|
347
394
|
getMissingBlobHashes: () => Buffer[],
|
|
348
|
-
fillResults: (blobs: BlobJson[]) => Blob[]
|
|
395
|
+
fillResults: (blobs: BlobJson[]) => Promise<Blob[]>,
|
|
349
396
|
ctx: { blockHash: string; blobHashes: string[] },
|
|
350
397
|
): Promise<void> {
|
|
351
398
|
// Shuffle clients for load distribution
|
|
@@ -366,7 +413,7 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
366
413
|
});
|
|
367
414
|
const blobs = await client.getBlobsByHashes(blobHashStrings);
|
|
368
415
|
if (blobs.length > 0) {
|
|
369
|
-
const result = fillResults(blobs);
|
|
416
|
+
const result = await fillResults(blobs);
|
|
370
417
|
this.log.debug(
|
|
371
418
|
`Got ${blobs.length} blobs from filestore (total: ${result.length}/${ctx.blobHashes.length})`,
|
|
372
419
|
{
|
|
@@ -387,19 +434,20 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
387
434
|
blobHashes: Buffer[] = [],
|
|
388
435
|
l1ConsensusHostIndex?: number,
|
|
389
436
|
): Promise<Blob[]> {
|
|
390
|
-
const blobs = await this.getBlobsFromHost(hostUrl, blockHashOrSlot, l1ConsensusHostIndex);
|
|
391
|
-
return processFetchedBlobs(blobs, blobHashes, this.log).filter((b): b is Blob => b !== undefined);
|
|
437
|
+
const blobs = await this.getBlobsFromHost(hostUrl, blockHashOrSlot, l1ConsensusHostIndex, blobHashes);
|
|
438
|
+
return (await processFetchedBlobs(blobs, blobHashes, this.log)).filter((b): b is Blob => b !== undefined);
|
|
392
439
|
}
|
|
393
440
|
|
|
394
441
|
public async getBlobsFromHost(
|
|
395
442
|
hostUrl: string,
|
|
396
443
|
blockHashOrSlot: string | number,
|
|
397
444
|
l1ConsensusHostIndex?: number,
|
|
445
|
+
blobHashes?: Buffer[],
|
|
398
446
|
): Promise<BlobJson[]> {
|
|
399
447
|
try {
|
|
400
|
-
let res = await this.fetchBlobSidecars(hostUrl, blockHashOrSlot, l1ConsensusHostIndex);
|
|
448
|
+
let res = await this.fetchBlobSidecars(hostUrl, blockHashOrSlot, l1ConsensusHostIndex, blobHashes);
|
|
401
449
|
if (res.ok) {
|
|
402
|
-
return parseBlobJsonsFromResponse(await res.json(), this.log);
|
|
450
|
+
return await parseBlobJsonsFromResponse(await res.json(), this.log);
|
|
403
451
|
}
|
|
404
452
|
|
|
405
453
|
if (res.status === 404 && typeof blockHashOrSlot === 'number') {
|
|
@@ -414,9 +462,9 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
414
462
|
let currentSlot = blockHashOrSlot + 1;
|
|
415
463
|
while (res.status === 404 && maxRetries > 0 && latestSlot !== undefined && currentSlot <= latestSlot) {
|
|
416
464
|
this.log.debug(`Trying slot ${currentSlot}`);
|
|
417
|
-
res = await this.fetchBlobSidecars(hostUrl, currentSlot, l1ConsensusHostIndex);
|
|
465
|
+
res = await this.fetchBlobSidecars(hostUrl, currentSlot, l1ConsensusHostIndex, blobHashes);
|
|
418
466
|
if (res.ok) {
|
|
419
|
-
return parseBlobJsonsFromResponse(await res.json(), this.log);
|
|
467
|
+
return await parseBlobJsonsFromResponse(await res.json(), this.log);
|
|
420
468
|
}
|
|
421
469
|
currentSlot++;
|
|
422
470
|
maxRetries--;
|
|
@@ -439,8 +487,17 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
439
487
|
hostUrl: string,
|
|
440
488
|
blockHashOrSlot: string | number,
|
|
441
489
|
l1ConsensusHostIndex?: number,
|
|
490
|
+
blobHashes?: Buffer[],
|
|
442
491
|
): Promise<Response> {
|
|
443
|
-
|
|
492
|
+
let baseUrl = `${hostUrl}/eth/v1/beacon/blobs/${blockHashOrSlot}`;
|
|
493
|
+
|
|
494
|
+
if (blobHashes && blobHashes.length > 0) {
|
|
495
|
+
const params = new URLSearchParams();
|
|
496
|
+
for (const hash of blobHashes) {
|
|
497
|
+
params.append('versioned_hashes', `0x${hash.toString('hex')}`);
|
|
498
|
+
}
|
|
499
|
+
baseUrl += `?${params.toString()}`;
|
|
500
|
+
}
|
|
444
501
|
|
|
445
502
|
const { url, ...options } = getBeaconNodeFetchOptions(baseUrl, this.config, l1ConsensusHostIndex);
|
|
446
503
|
this.log.debug(`Fetching blob sidecar for ${blockHashOrSlot}`, { url, ...options });
|
|
@@ -482,34 +539,50 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
482
539
|
* @param blockHash - The block hash
|
|
483
540
|
* @returns The slot number
|
|
484
541
|
*/
|
|
485
|
-
private async getSlotNumber(
|
|
542
|
+
private async getSlotNumber(
|
|
543
|
+
blockHash: `0x${string}`,
|
|
544
|
+
parentBeaconBlockRoot?: string,
|
|
545
|
+
l1BlockTimestamp?: bigint,
|
|
546
|
+
): Promise<number | undefined> {
|
|
486
547
|
const { l1ConsensusHostUrls, l1RpcUrls } = this.config;
|
|
487
548
|
if (!l1ConsensusHostUrls || l1ConsensusHostUrls.length === 0) {
|
|
488
549
|
this.log.debug('No consensus host url configured');
|
|
489
550
|
return undefined;
|
|
490
551
|
}
|
|
491
552
|
|
|
492
|
-
if (
|
|
493
|
-
|
|
494
|
-
|
|
553
|
+
// Primary path: compute slot from timestamp if genesis config is cached (no network call needed)
|
|
554
|
+
if (
|
|
555
|
+
l1BlockTimestamp !== undefined &&
|
|
556
|
+
this.beaconGenesisTime !== undefined &&
|
|
557
|
+
this.beaconSecondsPerSlot !== undefined
|
|
558
|
+
) {
|
|
559
|
+
const slot = Number((l1BlockTimestamp - this.beaconGenesisTime) / BigInt(this.beaconSecondsPerSlot));
|
|
560
|
+
this.log.debug(`Computed slot ${slot} from L1 block timestamp`, { l1BlockTimestamp });
|
|
561
|
+
return slot;
|
|
495
562
|
}
|
|
496
563
|
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
564
|
+
if (!parentBeaconBlockRoot) {
|
|
565
|
+
// parentBeaconBlockRoot not provided by caller — fetch it from the execution RPC
|
|
566
|
+
if (!l1RpcUrls || l1RpcUrls.length === 0) {
|
|
567
|
+
this.log.debug('No execution host url configured');
|
|
568
|
+
return undefined;
|
|
569
|
+
}
|
|
570
|
+
|
|
571
|
+
const client = createPublicClient({
|
|
572
|
+
transport: makeL1HttpTransport(l1RpcUrls, { timeout: this.config.l1HttpTimeoutMS }),
|
|
506
573
|
});
|
|
574
|
+
try {
|
|
575
|
+
const res: RpcBlock = await client.request({
|
|
576
|
+
method: 'eth_getBlockByHash',
|
|
577
|
+
params: [blockHash, /*tx flag*/ false],
|
|
578
|
+
});
|
|
507
579
|
|
|
508
|
-
|
|
509
|
-
|
|
580
|
+
if (res.parentBeaconBlockRoot) {
|
|
581
|
+
parentBeaconBlockRoot = res.parentBeaconBlockRoot;
|
|
582
|
+
}
|
|
583
|
+
} catch (err) {
|
|
584
|
+
this.log.error(`Error getting parent beacon block root`, err);
|
|
510
585
|
}
|
|
511
|
-
} catch (err) {
|
|
512
|
-
this.log.error(`Error getting parent beacon block root`, err);
|
|
513
586
|
}
|
|
514
587
|
|
|
515
588
|
if (!parentBeaconBlockRoot) {
|
|
@@ -555,9 +628,12 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
555
628
|
|
|
556
629
|
/**
|
|
557
630
|
* Start the blob client.
|
|
558
|
-
*
|
|
631
|
+
* Fetches and caches beacon genesis config for timestamp-based slot resolution,
|
|
632
|
+
* then uploads the initial healthcheck file (awaited) and starts periodic uploads.
|
|
559
633
|
*/
|
|
560
634
|
public async start(): Promise<void> {
|
|
635
|
+
await this.fetchBeaconConfig();
|
|
636
|
+
|
|
561
637
|
if (!this.fileStoreUploadClient) {
|
|
562
638
|
return;
|
|
563
639
|
}
|
|
@@ -582,6 +658,53 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
582
658
|
}, intervalMs);
|
|
583
659
|
}
|
|
584
660
|
|
|
661
|
+
/**
|
|
662
|
+
* Fetches and caches beacon genesis time and slot duration from the first available consensus host.
|
|
663
|
+
* These static values enable timestamp-based slot resolution, eliminating the per-fetch headers call.
|
|
664
|
+
* Logs a warning and leaves fields undefined if all hosts fail, callers fall back gracefully.
|
|
665
|
+
*/
|
|
666
|
+
private async fetchBeaconConfig(): Promise<void> {
|
|
667
|
+
const { l1ConsensusHostUrls } = this.config;
|
|
668
|
+
if (!l1ConsensusHostUrls || l1ConsensusHostUrls.length === 0) {
|
|
669
|
+
return;
|
|
670
|
+
}
|
|
671
|
+
|
|
672
|
+
for (let i = 0; i < l1ConsensusHostUrls.length; i++) {
|
|
673
|
+
try {
|
|
674
|
+
const { url: genesisUrl, ...genesisOptions } = getBeaconNodeFetchOptions(
|
|
675
|
+
`${l1ConsensusHostUrls[i]}/eth/v1/config/genesis`,
|
|
676
|
+
this.config,
|
|
677
|
+
i,
|
|
678
|
+
);
|
|
679
|
+
const { url: specUrl, ...specOptions } = getBeaconNodeFetchOptions(
|
|
680
|
+
`${l1ConsensusHostUrls[i]}/eth/v1/config/spec`,
|
|
681
|
+
this.config,
|
|
682
|
+
i,
|
|
683
|
+
);
|
|
684
|
+
|
|
685
|
+
const [genesisRes, specRes] = await Promise.all([
|
|
686
|
+
this.fetch(genesisUrl, genesisOptions),
|
|
687
|
+
this.fetch(specUrl, specOptions),
|
|
688
|
+
]);
|
|
689
|
+
|
|
690
|
+
if (genesisRes.ok && specRes.ok) {
|
|
691
|
+
const genesis = await genesisRes.json();
|
|
692
|
+
const spec = await specRes.json();
|
|
693
|
+
this.beaconGenesisTime = BigInt(genesis.data.genesisTime);
|
|
694
|
+
this.beaconSecondsPerSlot = parseInt(spec.data.secondsPerSlot);
|
|
695
|
+
this.log.debug(`Fetched beacon genesis config`, {
|
|
696
|
+
genesisTime: this.beaconGenesisTime,
|
|
697
|
+
secondsPerSlot: this.beaconSecondsPerSlot,
|
|
698
|
+
});
|
|
699
|
+
return;
|
|
700
|
+
}
|
|
701
|
+
} catch (err) {
|
|
702
|
+
this.log.warn(`Failed to fetch beacon config from host ${l1ConsensusHostUrls[i]}`, err);
|
|
703
|
+
}
|
|
704
|
+
}
|
|
705
|
+
this.log.warn('Could not fetch beacon genesis config from any consensus host — will use headers call fallback');
|
|
706
|
+
}
|
|
707
|
+
|
|
585
708
|
/**
|
|
586
709
|
* Stop the blob client, clearing any periodic tasks.
|
|
587
710
|
*/
|
|
@@ -593,10 +716,9 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
593
716
|
}
|
|
594
717
|
}
|
|
595
718
|
|
|
596
|
-
function parseBlobJsonsFromResponse(response: any, logger: Logger): BlobJson[] {
|
|
719
|
+
async function parseBlobJsonsFromResponse(response: any, logger: Logger): Promise<BlobJson[]> {
|
|
597
720
|
try {
|
|
598
|
-
|
|
599
|
-
return blobs;
|
|
721
|
+
return await Promise.all((response.data as string[]).map(parseBlobJson));
|
|
600
722
|
} catch (err) {
|
|
601
723
|
logger.error(`Error parsing blob json from response`, err);
|
|
602
724
|
return [];
|
|
@@ -607,16 +729,19 @@ function parseBlobJsonsFromResponse(response: any, logger: Logger): BlobJson[] {
|
|
|
607
729
|
// https://ethereum.github.io/beacon-APIs/?urls.primaryName=dev#/Beacon/getBlobSidecars
|
|
608
730
|
// Here we attempt to parse the response data to Buffer, and check the lengths (via Blob's constructor), to avoid
|
|
609
731
|
// throwing an error down the line when calling Blob.fromJson().
|
|
610
|
-
function parseBlobJson(
|
|
611
|
-
const blobBuffer = Buffer.from(
|
|
612
|
-
const
|
|
613
|
-
const blob = new Blob(blobBuffer, commitmentBuffer);
|
|
732
|
+
async function parseBlobJson(rawHex: string): Promise<BlobJson> {
|
|
733
|
+
const blobBuffer = Buffer.from(rawHex.slice(2), 'hex');
|
|
734
|
+
const blob = await Blob.fromBlobBuffer(blobBuffer);
|
|
614
735
|
return blob.toJSON();
|
|
615
736
|
}
|
|
616
737
|
|
|
617
738
|
// Returns an array that maps each blob hash to the corresponding blob, or undefined if the blob is not found
|
|
618
739
|
// or the data does not match the commitment.
|
|
619
|
-
function processFetchedBlobs(
|
|
740
|
+
async function processFetchedBlobs(
|
|
741
|
+
blobs: BlobJson[],
|
|
742
|
+
blobHashes: Buffer[],
|
|
743
|
+
logger: Logger,
|
|
744
|
+
): Promise<(Blob | undefined)[]> {
|
|
620
745
|
const requestedBlobHashes = new Set<string>(blobHashes.map(bufferToHex));
|
|
621
746
|
const hashToBlob = new Map<string, Blob>();
|
|
622
747
|
for (const blobJson of blobs) {
|
|
@@ -626,7 +751,7 @@ function processFetchedBlobs(blobs: BlobJson[], blobHashes: Buffer[], logger: Lo
|
|
|
626
751
|
}
|
|
627
752
|
|
|
628
753
|
try {
|
|
629
|
-
const blob = Blob.fromJson(blobJson);
|
|
754
|
+
const blob = await Blob.fromJson(blobJson);
|
|
630
755
|
hashToBlob.set(hashHex, blob);
|
|
631
756
|
} catch (err) {
|
|
632
757
|
// If the above throws, it's likely that the blob commitment does not match the hash of the blob data.
|
package/src/client/interface.ts
CHANGED
|
@@ -11,6 +11,17 @@ export interface GetBlobSidecarOptions {
|
|
|
11
11
|
* - Near tip: FileStore first with no retries (data should exist), L1 consensus second (freshest data), then FileStore with retries, then archive (eg. blobscan)
|
|
12
12
|
*/
|
|
13
13
|
isHistoricalSync?: boolean;
|
|
14
|
+
/**
|
|
15
|
+
* The parent beacon block root for the L1 block containing the blobs.
|
|
16
|
+
* If provided, skips the eth_getBlockByHash execution RPC call inside getSlotNumber.
|
|
17
|
+
*/
|
|
18
|
+
parentBeaconBlockRoot?: string;
|
|
19
|
+
/**
|
|
20
|
+
* The timestamp of the L1 execution block containing the blobs.
|
|
21
|
+
* When provided alongside a cached beacon genesis config (fetched at startup), allows computing
|
|
22
|
+
* the beacon slot directly via timestamp math, skipping the beacon headers network call entirely.
|
|
23
|
+
*/
|
|
24
|
+
l1BlockTimestamp?: bigint;
|
|
14
25
|
}
|
|
15
26
|
|
|
16
27
|
export interface BlobClientInterface {
|
package/src/client/tests.ts
CHANGED
|
@@ -28,7 +28,7 @@ export function runBlobClientTests(
|
|
|
28
28
|
});
|
|
29
29
|
|
|
30
30
|
it('should send and retrieve blobs by hash', async () => {
|
|
31
|
-
const blob = makeRandomBlob(5);
|
|
31
|
+
const blob = await makeRandomBlob(5);
|
|
32
32
|
const blobHash = blob.getEthVersionedBlobHash();
|
|
33
33
|
|
|
34
34
|
await client.sendBlobsToFilestore([blob]);
|
|
@@ -39,7 +39,7 @@ export function runBlobClientTests(
|
|
|
39
39
|
});
|
|
40
40
|
|
|
41
41
|
it('should handle multiple blobs', async () => {
|
|
42
|
-
const blobs = Array.from({ length: 3 }, () => makeRandomBlob(7));
|
|
42
|
+
const blobs = await Promise.all(Array.from({ length: 3 }, () => makeRandomBlob(7)));
|
|
43
43
|
const blobHashes = blobs.map(blob => blob.getEthVersionedBlobHash());
|
|
44
44
|
|
|
45
45
|
await client.sendBlobsToFilestore(blobs);
|