@aztec/blob-client 0.0.1-commit.03f7ef2 → 0.0.1-commit.0658669b3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -1
- package/dest/archive/config.js +1 -1
- package/dest/archive/instrumentation.d.ts +1 -1
- package/dest/archive/instrumentation.d.ts.map +1 -1
- package/dest/archive/instrumentation.js +13 -13
- package/dest/blobstore/blob_store_test_suite.js +9 -9
- package/dest/client/config.d.ts +5 -1
- package/dest/client/config.d.ts.map +1 -1
- package/dest/client/config.js +5 -0
- package/dest/client/factory.d.ts +3 -2
- package/dest/client/factory.d.ts.map +1 -1
- package/dest/client/factory.js +5 -2
- package/dest/client/http.d.ts +17 -1
- package/dest/client/http.d.ts.map +1 -1
- package/dest/client/http.js +42 -8
- package/dest/client/interface.d.ts +7 -1
- package/dest/client/interface.d.ts.map +1 -1
- package/dest/client/local.d.ts +3 -1
- package/dest/client/local.d.ts.map +1 -1
- package/dest/client/local.js +3 -0
- package/dest/client/tests.js +3 -3
- package/dest/filestore/filestore_blob_client.d.ts +14 -2
- package/dest/filestore/filestore_blob_client.d.ts.map +1 -1
- package/dest/filestore/filestore_blob_client.js +29 -5
- package/dest/filestore/healthcheck.d.ts +5 -0
- package/dest/filestore/healthcheck.d.ts.map +1 -0
- package/dest/filestore/healthcheck.js +3 -0
- package/package.json +8 -8
- package/src/archive/config.ts +1 -1
- package/src/archive/instrumentation.ts +22 -13
- package/src/blobstore/blob_store_test_suite.ts +9 -9
- package/src/client/config.ts +10 -0
- package/src/client/factory.ts +7 -2
- package/src/client/http.ts +59 -9
- package/src/client/interface.ts +6 -0
- package/src/client/local.ts +5 -0
- package/src/client/tests.ts +2 -2
- package/src/filestore/filestore_blob_client.ts +33 -5
- package/src/filestore/healthcheck.ts +5 -0
package/README.md
CHANGED
|
@@ -31,9 +31,18 @@ URL for uploading blobs to a file store.
|
|
|
31
31
|
**L1 Consensus Host URLs** (`L1_CONSENSUS_HOST_URLS`):
|
|
32
32
|
Beacon node URLs for fetching recent blobs directly from L1.
|
|
33
33
|
|
|
34
|
-
**Archive API URL** (`
|
|
34
|
+
**Archive API URL** (`BLOB_ARCHIVE_API_URL`):
|
|
35
35
|
Blobscan or similar archive API for historical blob data.
|
|
36
36
|
|
|
37
|
+
### File Store Connectivity Testing
|
|
38
|
+
|
|
39
|
+
All file stores (S3, GCS, HTTP, local) test connectivity by checking if a well-known healthcheck file (`.healthcheck`) exists. This approach was chosen because:
|
|
40
|
+
|
|
41
|
+
1. **HTTP compatibility**: For HTTP-based file stores, requesting a known file is the only reliable way to verify connectivity
|
|
42
|
+
2. **Uniform behavior**: Using the same healthcheck mechanism across all store types ensures consistent behavior and simplifies testing
|
|
43
|
+
|
|
44
|
+
When uploading is enabled, the sequencer uploads the healthcheck file on startup and then periodically re-uploads it (by default every 60 minutes) to ensure it remains available. This guards against accidental deletion, storage pruning, or other failures that might remove the file.
|
|
45
|
+
|
|
37
46
|
### Example Usage
|
|
38
47
|
|
|
39
48
|
```typescript
|
package/dest/archive/config.js
CHANGED
|
@@ -2,7 +2,7 @@ import { l1ReaderConfigMappings } from '@aztec/ethereum/l1-reader';
|
|
|
2
2
|
import { pickConfigMappings } from '@aztec/foundation/config';
|
|
3
3
|
export const blobArchiveApiConfigMappings = {
|
|
4
4
|
archiveApiUrl: {
|
|
5
|
-
env: '
|
|
5
|
+
env: 'BLOB_ARCHIVE_API_URL',
|
|
6
6
|
description: 'The URL of the archive API'
|
|
7
7
|
},
|
|
8
8
|
...pickConfigMappings(l1ReaderConfigMappings, [
|
|
@@ -8,4 +8,4 @@ export declare class BlobArchiveClientInstrumentation {
|
|
|
8
8
|
incRequest(type: 'blocks' | 'blobs', status: number): void;
|
|
9
9
|
incRetrievedBlobs(count: number): void;
|
|
10
10
|
}
|
|
11
|
-
//# sourceMappingURL=data:application/json;base64,
|
|
11
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiaW5zdHJ1bWVudGF0aW9uLmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi9zcmMvYXJjaGl2ZS9pbnN0cnVtZW50YXRpb24udHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsT0FBTyxFQUdMLEtBQUssZUFBZSxFQUdyQixNQUFNLHlCQUF5QixDQUFDO0FBRWpDLHFCQUFhLGdDQUFnQztJQU96QyxPQUFPLENBQUMsUUFBUTtJQU5sQixPQUFPLENBQUMsbUJBQW1CLENBQWdCO0lBQzNDLE9BQU8sQ0FBQyxrQkFBa0IsQ0FBZ0I7SUFDMUMsT0FBTyxDQUFDLGNBQWMsQ0FBZ0I7SUFFdEMsWUFDRSxNQUFNLEVBQUUsZUFBZSxFQUNmLFFBQVEsRUFBRSxNQUFNLEVBQ3hCLElBQUksRUFBRSxNQUFNLEVBb0JiO0lBRUQsVUFBVSxDQUFDLElBQUksRUFBRSxRQUFRLEdBQUcsT0FBTyxFQUFFLE1BQU0sRUFBRSxNQUFNLFFBTWxEO0lBRUQsaUJBQWlCLENBQUMsS0FBSyxFQUFFLE1BQU0sUUFFOUI7Q0FDRiJ9
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"instrumentation.d.ts","sourceRoot":"","sources":["../../src/archive/instrumentation.ts"],"names":[],"mappings":"AAAA,OAAO,
|
|
1
|
+
{"version":3,"file":"instrumentation.d.ts","sourceRoot":"","sources":["../../src/archive/instrumentation.ts"],"names":[],"mappings":"AAAA,OAAO,EAGL,KAAK,eAAe,EAGrB,MAAM,yBAAyB,CAAC;AAEjC,qBAAa,gCAAgC;IAOzC,OAAO,CAAC,QAAQ;IANlB,OAAO,CAAC,mBAAmB,CAAgB;IAC3C,OAAO,CAAC,kBAAkB,CAAgB;IAC1C,OAAO,CAAC,cAAc,CAAgB;IAEtC,YACE,MAAM,EAAE,eAAe,EACf,QAAQ,EAAE,MAAM,EACxB,IAAI,EAAE,MAAM,EAoBb;IAED,UAAU,CAAC,IAAI,EAAE,QAAQ,GAAG,OAAO,EAAE,MAAM,EAAE,MAAM,QAMlD;IAED,iBAAiB,CAAC,KAAK,EAAE,MAAM,QAE9B;CACF"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { Attributes, Metrics,
|
|
1
|
+
import { Attributes, Metrics, createUpDownCounterWithDefault } from '@aztec/telemetry-client';
|
|
2
2
|
export class BlobArchiveClientInstrumentation {
|
|
3
3
|
httpHost;
|
|
4
4
|
blockRequestCounter;
|
|
@@ -7,18 +7,18 @@ export class BlobArchiveClientInstrumentation {
|
|
|
7
7
|
constructor(client, httpHost, name){
|
|
8
8
|
this.httpHost = httpHost;
|
|
9
9
|
const meter = client.getMeter(name);
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
10
|
+
const requestAttrs = {
|
|
11
|
+
[Attributes.HTTP_RESPONSE_STATUS_CODE]: [
|
|
12
|
+
200,
|
|
13
|
+
404
|
|
14
|
+
],
|
|
15
|
+
[Attributes.HTTP_REQUEST_HOST]: [
|
|
16
|
+
httpHost
|
|
17
|
+
]
|
|
18
|
+
};
|
|
19
|
+
this.blockRequestCounter = createUpDownCounterWithDefault(meter, Metrics.BLOB_SINK_ARCHIVE_BLOCK_REQUEST_COUNT, requestAttrs);
|
|
20
|
+
this.blobRequestCounter = createUpDownCounterWithDefault(meter, Metrics.BLOB_SINK_ARCHIVE_BLOB_REQUEST_COUNT, requestAttrs);
|
|
21
|
+
this.retrievedBlobs = createUpDownCounterWithDefault(meter, Metrics.BLOB_SINK_ARCHIVE_BLOB_COUNT);
|
|
22
22
|
}
|
|
23
23
|
incRequest(type, status) {
|
|
24
24
|
const counter = type === 'blocks' ? this.blockRequestCounter : this.blobRequestCounter;
|
|
@@ -12,7 +12,7 @@ export function describeBlobStore(getBlobStore) {
|
|
|
12
12
|
Fr.random(),
|
|
13
13
|
Fr.random()
|
|
14
14
|
];
|
|
15
|
-
const blob = Blob.fromFields(testFields);
|
|
15
|
+
const blob = await Blob.fromFields(testFields);
|
|
16
16
|
const blobHash = blob.getEthVersionedBlobHash();
|
|
17
17
|
// Store the blob
|
|
18
18
|
await blobStore.addBlobs([
|
|
@@ -28,11 +28,11 @@ export function describeBlobStore(getBlobStore) {
|
|
|
28
28
|
});
|
|
29
29
|
it('should handle multiple blobs stored and retrieved by their hashes', async ()=>{
|
|
30
30
|
// Create two different blobs
|
|
31
|
-
const blob1 = Blob.fromFields([
|
|
31
|
+
const blob1 = await Blob.fromFields([
|
|
32
32
|
Fr.random(),
|
|
33
33
|
Fr.random()
|
|
34
34
|
]);
|
|
35
|
-
const blob2 = Blob.fromFields([
|
|
35
|
+
const blob2 = await Blob.fromFields([
|
|
36
36
|
Fr.random(),
|
|
37
37
|
Fr.random(),
|
|
38
38
|
Fr.random()
|
|
@@ -64,13 +64,13 @@ export function describeBlobStore(getBlobStore) {
|
|
|
64
64
|
});
|
|
65
65
|
it('should handle retrieving subset of stored blobs', async ()=>{
|
|
66
66
|
// Store multiple blobs
|
|
67
|
-
const blob1 = Blob.fromFields([
|
|
67
|
+
const blob1 = await Blob.fromFields([
|
|
68
68
|
Fr.random()
|
|
69
69
|
]);
|
|
70
|
-
const blob2 = Blob.fromFields([
|
|
70
|
+
const blob2 = await Blob.fromFields([
|
|
71
71
|
Fr.random()
|
|
72
72
|
]);
|
|
73
|
-
const blob3 = Blob.fromFields([
|
|
73
|
+
const blob3 = await Blob.fromFields([
|
|
74
74
|
Fr.random()
|
|
75
75
|
]);
|
|
76
76
|
await blobStore.addBlobs([
|
|
@@ -90,7 +90,7 @@ export function describeBlobStore(getBlobStore) {
|
|
|
90
90
|
expect(retrievedBlobs[1]).toEqual(blob3);
|
|
91
91
|
});
|
|
92
92
|
it('should handle duplicate blob hashes in request', async ()=>{
|
|
93
|
-
const blob = Blob.fromFields([
|
|
93
|
+
const blob = await Blob.fromFields([
|
|
94
94
|
Fr.random()
|
|
95
95
|
]);
|
|
96
96
|
const blobHash = blob.getEthVersionedBlobHash();
|
|
@@ -112,8 +112,8 @@ export function describeBlobStore(getBlobStore) {
|
|
|
112
112
|
Fr.random(),
|
|
113
113
|
Fr.random()
|
|
114
114
|
];
|
|
115
|
-
const blob1 = Blob.fromFields(fields);
|
|
116
|
-
const blob2 = Blob.fromFields(fields);
|
|
115
|
+
const blob1 = await Blob.fromFields(fields);
|
|
116
|
+
const blob2 = await Blob.fromFields(fields);
|
|
117
117
|
const blobHash = blob1.getEthVersionedBlobHash();
|
|
118
118
|
// Store first blob
|
|
119
119
|
await blobStore.addBlobs([
|
package/dest/client/config.d.ts
CHANGED
|
@@ -36,6 +36,10 @@ export interface BlobClientConfig extends BlobArchiveApiConfig {
|
|
|
36
36
|
* URL for uploading blobs to filestore (s3://, gs://, file://)
|
|
37
37
|
*/
|
|
38
38
|
blobFileStoreUploadUrl?: string;
|
|
39
|
+
/**
|
|
40
|
+
* Interval in minutes for uploading healthcheck file to file store (default: 60 = 1 hour)
|
|
41
|
+
*/
|
|
42
|
+
blobHealthcheckUploadIntervalMinutes?: number;
|
|
39
43
|
}
|
|
40
44
|
export declare const blobClientConfigMapping: ConfigMappingsType<BlobClientConfig>;
|
|
41
45
|
/**
|
|
@@ -47,4 +51,4 @@ export declare function getBlobClientConfigFromEnv(): BlobClientConfig;
|
|
|
47
51
|
* Returns whether the given blob client config has any remote sources defined.
|
|
48
52
|
*/
|
|
49
53
|
export declare function hasRemoteBlobSources(config?: BlobClientConfig): boolean;
|
|
50
|
-
//# sourceMappingURL=data:application/json;base64,
|
|
54
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiY29uZmlnLmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi9zcmMvY2xpZW50L2NvbmZpZy50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSxPQUFPLEVBQ0wsS0FBSyxrQkFBa0IsRUFDdkIsV0FBVyxFQUdaLE1BQU0sMEJBQTBCLENBQUM7QUFFbEMsT0FBTyxFQUFFLEtBQUssb0JBQW9CLEVBQWdDLE1BQU0sc0JBQXNCLENBQUM7QUFFL0Y7O0dBRUc7QUFDSCxNQUFNLFdBQVcsZ0JBQWlCLFNBQVEsb0JBQW9CO0lBQzVEOztPQUVHO0lBQ0gsU0FBUyxDQUFDLEVBQUUsTUFBTSxFQUFFLENBQUM7SUFFckI7O09BRUc7SUFDSCxtQkFBbUIsQ0FBQyxFQUFFLE1BQU0sRUFBRSxDQUFDO0lBRS9COztPQUVHO0lBQ0gsc0JBQXNCLENBQUMsRUFBRSxXQUFXLENBQUMsTUFBTSxDQUFDLEVBQUUsQ0FBQztJQUUvQzs7T0FFRztJQUNILDRCQUE0QixDQUFDLEVBQUUsTUFBTSxFQUFFLENBQUM7SUFFeEM7O09BRUc7SUFDSCxpQkFBaUIsQ0FBQyxFQUFFLE1BQU0sQ0FBQztJQUUzQjs7T0FFRztJQUNILHFCQUFxQixDQUFDLEVBQUUsT0FBTyxDQUFDO0lBRWhDOztPQUVHO0lBQ0gsaUJBQWlCLENBQUMsRUFBRSxNQUFNLEVBQUUsQ0FBQztJQUU3Qjs7T0FFRztJQUNILHNCQUFzQixDQUFDLEVBQUUsTUFBTSxDQUFDO0lBRWhDOztPQUVHO0lBQ0gsb0NBQW9DLENBQUMsRUFBRSxNQUFNLENBQUM7Q0FDL0M7QUFFRCxlQUFPLE1BQU0sdUJBQXVCLEVBQUUsa0JBQWtCLENBQUMsZ0JBQWdCLENBb0R4RSxDQUFDO0FBRUY7OztHQUdHO0FBQ0gsd0JBQWdCLDBCQUEwQixJQUFJLGdCQUFnQixDQUU3RDtBQUVEOztHQUVHO0FBQ0gsd0JBQWdCLG9CQUFvQixDQUFDLE1BQU0sR0FBRSxnQkFBcUIsR0FBRyxPQUFPLENBRTNFIn0=
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../src/client/config.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,kBAAkB,EACvB,WAAW,EAGZ,MAAM,0BAA0B,CAAC;AAElC,OAAO,EAAE,KAAK,oBAAoB,EAAgC,MAAM,sBAAsB,CAAC;AAE/F;;GAEG;AACH,MAAM,WAAW,gBAAiB,SAAQ,oBAAoB;IAC5D;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IAErB;;OAEG;IACH,mBAAmB,CAAC,EAAE,MAAM,EAAE,CAAC;IAE/B;;OAEG;IACH,sBAAsB,CAAC,EAAE,WAAW,CAAC,MAAM,CAAC,EAAE,CAAC;IAE/C;;OAEG;IACH,4BAA4B,CAAC,EAAE,MAAM,EAAE,CAAC;IAExC;;OAEG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAE3B;;OAEG;IACH,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAEhC;;OAEG;IACH,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;IAE7B;;OAEG;IACH,sBAAsB,CAAC,EAAE,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../src/client/config.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,kBAAkB,EACvB,WAAW,EAGZ,MAAM,0BAA0B,CAAC;AAElC,OAAO,EAAE,KAAK,oBAAoB,EAAgC,MAAM,sBAAsB,CAAC;AAE/F;;GAEG;AACH,MAAM,WAAW,gBAAiB,SAAQ,oBAAoB;IAC5D;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IAErB;;OAEG;IACH,mBAAmB,CAAC,EAAE,MAAM,EAAE,CAAC;IAE/B;;OAEG;IACH,sBAAsB,CAAC,EAAE,WAAW,CAAC,MAAM,CAAC,EAAE,CAAC;IAE/C;;OAEG;IACH,4BAA4B,CAAC,EAAE,MAAM,EAAE,CAAC;IAExC;;OAEG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAE3B;;OAEG;IACH,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAEhC;;OAEG;IACH,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;IAE7B;;OAEG;IACH,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAEhC;;OAEG;IACH,oCAAoC,CAAC,EAAE,MAAM,CAAC;CAC/C;AAED,eAAO,MAAM,uBAAuB,EAAE,kBAAkB,CAAC,gBAAgB,CAoDxE,CAAC;AAEF;;;GAGG;AACH,wBAAgB,0BAA0B,IAAI,gBAAgB,CAE7D;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAAC,MAAM,GAAE,gBAAqB,GAAG,OAAO,CAE3E"}
|
package/dest/client/config.js
CHANGED
|
@@ -40,6 +40,11 @@ export const blobClientConfigMapping = {
|
|
|
40
40
|
env: 'BLOB_FILE_STORE_UPLOAD_URL',
|
|
41
41
|
description: 'URL for uploading blobs to filestore (s3://, gs://, file://)'
|
|
42
42
|
},
|
|
43
|
+
blobHealthcheckUploadIntervalMinutes: {
|
|
44
|
+
env: 'BLOB_HEALTHCHECK_UPLOAD_INTERVAL_MINUTES',
|
|
45
|
+
description: 'Interval in minutes for uploading healthcheck file to file store (default: 60 = 1 hour)',
|
|
46
|
+
parseEnv: (val)=>val ? +val : undefined
|
|
47
|
+
},
|
|
43
48
|
...blobArchiveApiConfigMappings
|
|
44
49
|
};
|
|
45
50
|
/**
|
package/dest/client/factory.d.ts
CHANGED
|
@@ -30,10 +30,11 @@ export interface BlobClientWithFileStoresConfig extends BlobClientConfig {
|
|
|
30
30
|
* 2. Creating read-only FileStore clients
|
|
31
31
|
* 3. Creating a writable FileStore client for uploads
|
|
32
32
|
* 4. Creating the BlobClient with these dependencies
|
|
33
|
+
* 5. Starting the client (uploads initial healthcheck file if upload client is configured)
|
|
33
34
|
*
|
|
34
35
|
* @param config - Configuration containing blob client settings and chain metadata
|
|
35
36
|
* @param logger - Optional logger for the blob client
|
|
36
|
-
* @returns A BlobClientInterface configured with file store support
|
|
37
|
+
* @returns A BlobClientInterface configured with file store support, already started
|
|
37
38
|
*/
|
|
38
39
|
export declare function createBlobClientWithFileStores(config: BlobClientWithFileStoresConfig, logger?: Logger): Promise<BlobClientInterface>;
|
|
39
|
-
//# sourceMappingURL=data:application/json;base64,
|
|
40
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiZmFjdG9yeS5kLnRzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vLi4vc3JjL2NsaWVudC9mYWN0b3J5LnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLE9BQU8sRUFBRSxLQUFLLE1BQU0sRUFBZ0IsTUFBTSx1QkFBdUIsQ0FBQztBQVFsRSxPQUFPLEtBQUssRUFBRSxtQkFBbUIsRUFBRSxNQUFNLHVDQUF1QyxDQUFDO0FBQ2pGLE9BQU8sRUFBRSxLQUFLLGdCQUFnQixFQUF3QixNQUFNLGFBQWEsQ0FBQztBQUUxRSxPQUFPLEtBQUssRUFBRSxtQkFBbUIsRUFBRSxNQUFNLGdCQUFnQixDQUFDO0FBRzFELE1BQU0sV0FBVyxvQkFBb0I7SUFDbkMsTUFBTSxDQUFDLEVBQUUsTUFBTSxDQUFDO0lBQ2hCLDBDQUEwQztJQUMxQyxnQkFBZ0IsQ0FBQyxFQUFFLG1CQUFtQixFQUFFLENBQUM7SUFDekMsMkNBQTJDO0lBQzNDLHFCQUFxQixDQUFDLEVBQUUsbUJBQW1CLENBQUM7Q0FDN0M7QUFFRCx3QkFBZ0IsZ0JBQWdCLENBQUMsTUFBTSxDQUFDLEVBQUUsZ0JBQWdCLEVBQUUsSUFBSSxDQUFDLEVBQUUsb0JBQW9CLEdBQUcsbUJBQW1CLENBbUI1RztBQUVEOzs7R0FHRztBQUNILE1BQU0sV0FBVyw4QkFBK0IsU0FBUSxnQkFBZ0I7SUFDdEUsU0FBUyxFQUFFLE1BQU0sQ0FBQztJQUNsQixhQUFhLEVBQUUsTUFBTSxDQUFDO0lBQ3RCLFdBQVcsRUFBRTtRQUFFLGFBQWEsRUFBRTtZQUFFLFFBQVEsSUFBSSxNQUFNLENBQUE7U0FBRSxDQUFBO0tBQUUsQ0FBQztDQUN4RDtBQUVEOzs7Ozs7Ozs7Ozs7R0FZRztBQUNILHdCQUFzQiw4QkFBOEIsQ0FDbEQsTUFBTSxFQUFFLDhCQUE4QixFQUN0QyxNQUFNLENBQUMsRUFBRSxNQUFNLEdBQ2QsT0FBTyxDQUFDLG1CQUFtQixDQUFDLENBdUI5QiJ9
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"factory.d.ts","sourceRoot":"","sources":["../../src/client/factory.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,MAAM,EAAgB,MAAM,uBAAuB,CAAC;AAQlE,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,uCAAuC,CAAC;AACjF,OAAO,EAAE,KAAK,gBAAgB,EAAwB,MAAM,aAAa,CAAC;AAE1E,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAG1D,MAAM,WAAW,oBAAoB;IACnC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,0CAA0C;IAC1C,gBAAgB,CAAC,EAAE,mBAAmB,EAAE,CAAC;IACzC,2CAA2C;IAC3C,qBAAqB,CAAC,EAAE,mBAAmB,CAAC;CAC7C;AAED,wBAAgB,gBAAgB,CAAC,MAAM,CAAC,EAAE,gBAAgB,EAAE,IAAI,CAAC,EAAE,oBAAoB,GAAG,mBAAmB,CAmB5G;AAED;;;GAGG;AACH,MAAM,WAAW,8BAA+B,SAAQ,gBAAgB;IACtE,SAAS,EAAE,MAAM,CAAC;IAClB,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE;QAAE,aAAa,EAAE;YAAE,QAAQ,IAAI,MAAM,CAAA;SAAE,CAAA;KAAE,CAAC;CACxD;AAED
|
|
1
|
+
{"version":3,"file":"factory.d.ts","sourceRoot":"","sources":["../../src/client/factory.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,MAAM,EAAgB,MAAM,uBAAuB,CAAC;AAQlE,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,uCAAuC,CAAC;AACjF,OAAO,EAAE,KAAK,gBAAgB,EAAwB,MAAM,aAAa,CAAC;AAE1E,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAG1D,MAAM,WAAW,oBAAoB;IACnC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,0CAA0C;IAC1C,gBAAgB,CAAC,EAAE,mBAAmB,EAAE,CAAC;IACzC,2CAA2C;IAC3C,qBAAqB,CAAC,EAAE,mBAAmB,CAAC;CAC7C;AAED,wBAAgB,gBAAgB,CAAC,MAAM,CAAC,EAAE,gBAAgB,EAAE,IAAI,CAAC,EAAE,oBAAoB,GAAG,mBAAmB,CAmB5G;AAED;;;GAGG;AACH,MAAM,WAAW,8BAA+B,SAAQ,gBAAgB;IACtE,SAAS,EAAE,MAAM,CAAC;IAClB,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE;QAAE,aAAa,EAAE;YAAE,QAAQ,IAAI,MAAM,CAAA;SAAE,CAAA;KAAE,CAAC;CACxD;AAED;;;;;;;;;;;;GAYG;AACH,wBAAsB,8BAA8B,CAClD,MAAM,EAAE,8BAA8B,EACtC,MAAM,CAAC,EAAE,MAAM,GACd,OAAO,CAAC,mBAAmB,CAAC,CAuB9B"}
|
package/dest/client/factory.js
CHANGED
|
@@ -30,10 +30,11 @@ export function createBlobClient(config, deps) {
|
|
|
30
30
|
* 2. Creating read-only FileStore clients
|
|
31
31
|
* 3. Creating a writable FileStore client for uploads
|
|
32
32
|
* 4. Creating the BlobClient with these dependencies
|
|
33
|
+
* 5. Starting the client (uploads initial healthcheck file if upload client is configured)
|
|
33
34
|
*
|
|
34
35
|
* @param config - Configuration containing blob client settings and chain metadata
|
|
35
36
|
* @param logger - Optional logger for the blob client
|
|
36
|
-
* @returns A BlobClientInterface configured with file store support
|
|
37
|
+
* @returns A BlobClientInterface configured with file store support, already started
|
|
37
38
|
*/ export async function createBlobClientWithFileStores(config, logger) {
|
|
38
39
|
const log = logger ?? createLogger('blob-client');
|
|
39
40
|
const fileStoreMetadata = {
|
|
@@ -45,9 +46,11 @@ export function createBlobClient(config, deps) {
|
|
|
45
46
|
createReadOnlyFileStoreBlobClients(config.blobFileStoreUrls, fileStoreMetadata, log),
|
|
46
47
|
createWritableFileStoreBlobClient(config.blobFileStoreUploadUrl, fileStoreMetadata, log)
|
|
47
48
|
]);
|
|
48
|
-
|
|
49
|
+
const client = createBlobClient(config, {
|
|
49
50
|
logger: log,
|
|
50
51
|
fileStoreClients,
|
|
51
52
|
fileStoreUploadClient
|
|
52
53
|
});
|
|
54
|
+
await client.start?.();
|
|
55
|
+
return client;
|
|
53
56
|
}
|
package/dest/client/http.d.ts
CHANGED
|
@@ -13,6 +13,7 @@ export declare class HttpBlobClient implements BlobClientInterface {
|
|
|
13
13
|
protected readonly fileStoreClients: FileStoreBlobClient[];
|
|
14
14
|
protected readonly fileStoreUploadClient: FileStoreBlobClient | undefined;
|
|
15
15
|
private disabled;
|
|
16
|
+
private healthcheckUploadIntervalId?;
|
|
16
17
|
constructor(config?: BlobClientConfig, opts?: {
|
|
17
18
|
logger?: Logger;
|
|
18
19
|
archiveClient?: BlobArchiveClient;
|
|
@@ -59,5 +60,20 @@ export declare class HttpBlobClient implements BlobClientInterface {
|
|
|
59
60
|
private getSlotNumber;
|
|
60
61
|
/** @internal - exposed for testing */
|
|
61
62
|
getArchiveClient(): BlobArchiveClient | undefined;
|
|
63
|
+
/** Returns true if this client can upload blobs to filestore. */
|
|
64
|
+
canUpload(): boolean;
|
|
65
|
+
/**
|
|
66
|
+
* Start the blob client.
|
|
67
|
+
* Uploads the initial healthcheck file (awaited) and starts periodic uploads.
|
|
68
|
+
*/
|
|
69
|
+
start(): Promise<void>;
|
|
70
|
+
/**
|
|
71
|
+
* Start periodic healthcheck upload to the file store to ensure it remains available even if accidentally deleted.
|
|
72
|
+
*/
|
|
73
|
+
private startPeriodicHealthcheckUpload;
|
|
74
|
+
/**
|
|
75
|
+
* Stop the blob client, clearing any periodic tasks.
|
|
76
|
+
*/
|
|
77
|
+
stop(): void;
|
|
62
78
|
}
|
|
63
|
-
//# sourceMappingURL=data:application/json;base64,
|
|
79
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiaHR0cC5kLnRzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vLi4vc3JjL2NsaWVudC9odHRwLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLE9BQU8sRUFBRSxJQUFJLEVBQUUsS0FBSyxRQUFRLEVBQStCLE1BQU0saUJBQWlCLENBQUM7QUFFbkYsT0FBTyxFQUFFLEtBQUssTUFBTSxFQUFnQixNQUFNLHVCQUF1QixDQUFDO0FBT2xFLE9BQU8sS0FBSyxFQUFFLGlCQUFpQixFQUFFLE1BQU0seUJBQXlCLENBQUM7QUFDakUsT0FBTyxLQUFLLEVBQUUsbUJBQW1CLEVBQUUsTUFBTSx1Q0FBdUMsQ0FBQztBQUVqRixPQUFPLEVBQUUsS0FBSyxnQkFBZ0IsRUFBOEIsTUFBTSxhQUFhLENBQUM7QUFDaEYsT0FBTyxLQUFLLEVBQUUsbUJBQW1CLEVBQUUscUJBQXFCLEVBQUUsTUFBTSxnQkFBZ0IsQ0FBQztBQUVqRixxQkFBYSxjQUFlLFlBQVcsbUJBQW1CO0lBYXRELE9BQU8sQ0FBQyxRQUFRLENBQUMsSUFBSTtJQVp2QixTQUFTLENBQUMsUUFBUSxDQUFDLEdBQUcsRUFBRSxNQUFNLENBQUM7SUFDL0IsU0FBUyxDQUFDLFFBQVEsQ0FBQyxNQUFNLEVBQUUsZ0JBQWdCLENBQUM7SUFDNUMsU0FBUyxDQUFDLFFBQVEsQ0FBQyxhQUFhLEVBQUUsaUJBQWlCLEdBQUcsU0FBUyxDQUFDO0lBQ2hFLFNBQVMsQ0FBQyxRQUFRLENBQUMsS0FBSyxFQUFFLE9BQU8sS0FBSyxDQUFDO0lBQ3ZDLFNBQVMsQ0FBQyxRQUFRLENBQUMsZ0JBQWdCLEVBQUUsbUJBQW1CLEVBQUUsQ0FBQztJQUMzRCxTQUFTLENBQUMsUUFBUSxDQUFDLHFCQUFxQixFQUFFLG1CQUFtQixHQUFHLFNBQVMsQ0FBQztJQUUxRSxPQUFPLENBQUMsUUFBUSxDQUFTO0lBQ3pCLE9BQU8sQ0FBQywyQkFBMkIsQ0FBQyxDQUFpQjtJQUVyRCxZQUNFLE1BQU0sQ0FBQyxFQUFFLGdCQUFnQixFQUNSLElBQUksR0FBRTtRQUNyQixNQUFNLENBQUMsRUFBRSxNQUFNLENBQUM7UUFDaEIsYUFBYSxDQUFDLEVBQUUsaUJBQWlCLENBQUM7UUFDbEMsZ0JBQWdCLENBQUMsRUFBRSxtQkFBbUIsRUFBRSxDQUFDO1FBQ3pDLHFCQUFxQixDQUFDLEVBQUUsbUJBQW1CLENBQUM7UUFDNUMseUVBQXlFO1FBQ3pFLGNBQWMsQ0FBQyxFQUFFLENBQUMsS0FBSyxFQUFFLElBQUksRUFBRSxLQUFLLElBQUksQ0FBQztLQUNyQyxFQXdCUDtJQUVEOzs7T0FHRztJQUNILE9BQU8sQ0FBQyxzQkFBc0I7SUFVOUI7Ozs7O09BS0c7SUFDSSxXQUFXLENBQUMsS0FBSyxFQUFFLE9BQU8sR0FBRyxJQUFJLENBR3ZDO0lBRVksV0FBVyxrQkFvRXZCO0lBRVksb0JBQW9CLENBQUMsS0FBSyxFQUFFLElBQUksRUFBRSxHQUFHLE9BQU8sQ0FBQyxPQUFPLENBQUMsQ0FtQmpFO0lBRUQ7Ozs7Ozs7Ozs7Ozs7O09BY0c7SUFDVSxjQUFjLENBQ3pCLFNBQVMsRUFBRSxLQUFLLE1BQU0sRUFBRSxFQUN4QixVQUFVLEVBQUUsTUFBTSxFQUFFLEVBQ3BCLElBQUksQ0FBQyxFQUFFLHFCQUFxQixHQUMzQixPQUFPLENBQUMsSUFBSSxFQUFFLENBQUMsQ0E2SWpCO1lBUWEsYUFBYTtJQXNDZCxrQkFBa0IsQ0FDN0IsT0FBTyxFQUFFLE1BQU0sRUFDZixlQUFlLEVBQUUsTUFBTSxHQUFHLE1BQU0sRUFDaEMsVUFBVSxHQUFFLE1BQU0sRUFBTyxFQUN6QixvQkFBb0IsQ0FBQyxFQUFFLE1BQU0sR0FDNUIsT0FBTyxDQUFDLElBQUksRUFBRSxDQUFDLENBR2pCO0lBRVksZ0JBQWdCLENBQzNCLE9BQU8sRUFBRSxNQUFNLEVBQ2YsZUFBZSxFQUFFLE1BQU0sR0FBRyxNQUFNLEVBQ2hDLG9CQUFvQixDQUFDLEVBQUUsTUFBTSxHQUM1QixPQUFPLENBQUMsUUFBUSxFQUFFLENBQUMsQ0FzQ3JCO0lBRUQsT0FBTyxDQUFDLGlCQUFpQjtZQVlYLG1CQUFtQjtZQW1DbkIsYUFBYTtJQTZEM0Isc0NBQXNDO0lBQy9CLGdCQUFnQixJQUFJLGlCQUFpQixHQUFHLFNBQVMsQ0FFdkQ7SUFFRCxpRUFBaUU7SUFDMUQsU0FBUyxJQUFJLE9BQU8sQ0FFMUI7SUFFRDs7O09BR0c7SUFDVSxLQUFLLElBQUksT0FBTyxDQUFDLElBQUksQ0FBQyxDQVNsQztJQUVEOztPQUVHO0lBQ0gsT0FBTyxDQUFDLDhCQUE4QjtJQVd0Qzs7T0FFRztJQUNJLElBQUksSUFBSSxJQUFJLENBS2xCO0NBQ0YifQ==
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"http.d.ts","sourceRoot":"","sources":["../../src/client/http.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,KAAK,QAAQ,EAA+B,MAAM,iBAAiB,CAAC;AAEnF,OAAO,EAAE,KAAK,MAAM,EAAgB,MAAM,uBAAuB,CAAC;AAOlE,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,yBAAyB,CAAC;AACjE,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,uCAAuC,CAAC;
|
|
1
|
+
{"version":3,"file":"http.d.ts","sourceRoot":"","sources":["../../src/client/http.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,KAAK,QAAQ,EAA+B,MAAM,iBAAiB,CAAC;AAEnF,OAAO,EAAE,KAAK,MAAM,EAAgB,MAAM,uBAAuB,CAAC;AAOlE,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,yBAAyB,CAAC;AACjE,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,uCAAuC,CAAC;AAEjF,OAAO,EAAE,KAAK,gBAAgB,EAA8B,MAAM,aAAa,CAAC;AAChF,OAAO,KAAK,EAAE,mBAAmB,EAAE,qBAAqB,EAAE,MAAM,gBAAgB,CAAC;AAEjF,qBAAa,cAAe,YAAW,mBAAmB;IAatD,OAAO,CAAC,QAAQ,CAAC,IAAI;IAZvB,SAAS,CAAC,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IAC/B,SAAS,CAAC,QAAQ,CAAC,MAAM,EAAE,gBAAgB,CAAC;IAC5C,SAAS,CAAC,QAAQ,CAAC,aAAa,EAAE,iBAAiB,GAAG,SAAS,CAAC;IAChE,SAAS,CAAC,QAAQ,CAAC,KAAK,EAAE,OAAO,KAAK,CAAC;IACvC,SAAS,CAAC,QAAQ,CAAC,gBAAgB,EAAE,mBAAmB,EAAE,CAAC;IAC3D,SAAS,CAAC,QAAQ,CAAC,qBAAqB,EAAE,mBAAmB,GAAG,SAAS,CAAC;IAE1E,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,2BAA2B,CAAC,CAAiB;IAErD,YACE,MAAM,CAAC,EAAE,gBAAgB,EACR,IAAI,GAAE;QACrB,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,aAAa,CAAC,EAAE,iBAAiB,CAAC;QAClC,gBAAgB,CAAC,EAAE,mBAAmB,EAAE,CAAC;QACzC,qBAAqB,CAAC,EAAE,mBAAmB,CAAC;QAC5C,yEAAyE;QACzE,cAAc,CAAC,EAAE,CAAC,KAAK,EAAE,IAAI,EAAE,KAAK,IAAI,CAAC;KACrC,EAwBP;IAED;;;OAGG;IACH,OAAO,CAAC,sBAAsB;IAU9B;;;;;OAKG;IACI,WAAW,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI,CAGvC;IAEY,WAAW,kBAoEvB;IAEY,oBAAoB,CAAC,KAAK,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC,CAmBjE;IAED;;;;;;;;;;;;;;OAcG;IACU,cAAc,CACzB,SAAS,EAAE,KAAK,MAAM,EAAE,EACxB,UAAU,EAAE,MAAM,EAAE,EACpB,IAAI,CAAC,EAAE,qBAAqB,GAC3B,OAAO,CAAC,IAAI,EAAE,CAAC,CA6IjB;YAQa,aAAa;IAsCd,kBAAkB,CAC7B,OAAO,EAAE,MAAM,EACf,eAAe,EAAE,MAAM,GAAG,MAAM,EAChC,UAAU,GAAE,MAAM,EAAO,EACzB,oBAAoB,CAAC,EAAE,MAAM,GAC5B,OAAO,CAAC,IAAI,EAAE,CAAC,CAGjB;IAEY,gBAAgB,CAC3B,OAAO,EAAE,MAAM,EACf,eAAe,EAAE,MAAM,GAAG,MAAM,EAChC,oBAAoB,CAAC,EAAE,MAAM,GAC5B,OAAO,CAAC,QAAQ,EAAE,CAAC,CAsCrB;IAED,OAAO,CAAC,iBAAiB;YAYX,mBAAmB;YAmCnB,aAAa;IA6D3B,sCAAsC;IAC/B,gBAAgB,IAAI,iBAAiB,GAAG,SAAS,CAEvD;IAED,iEAAiE;IAC1D,SAAS,IAAI,OAAO,CAE1B;IAED;;;OAGG;IACU,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CASlC;IAED;;OAEG;IACH,OAAO,CAAC,8BAA8B;IAWtC;;OAEG;IACI,IAAI,IAAI,IAAI,CAKlB;CACF"}
|
package/dest/client/http.js
CHANGED
|
@@ -5,6 +5,7 @@ import { makeBackoff, retry } from '@aztec/foundation/retry';
|
|
|
5
5
|
import { bufferToHex, hexToBuffer } from '@aztec/foundation/string';
|
|
6
6
|
import { createPublicClient, fallback, http } from 'viem';
|
|
7
7
|
import { createBlobArchiveClient } from '../archive/factory.js';
|
|
8
|
+
import { DEFAULT_HEALTHCHECK_UPLOAD_INTERVAL_MINUTES } from '../filestore/healthcheck.js';
|
|
8
9
|
import { getBlobClientConfigFromEnv } from './config.js';
|
|
9
10
|
export class HttpBlobClient {
|
|
10
11
|
opts;
|
|
@@ -15,6 +16,7 @@ export class HttpBlobClient {
|
|
|
15
16
|
fileStoreClients;
|
|
16
17
|
fileStoreUploadClient;
|
|
17
18
|
disabled;
|
|
19
|
+
healthcheckUploadIntervalId;
|
|
18
20
|
constructor(config, opts = {}){
|
|
19
21
|
this.opts = opts;
|
|
20
22
|
this.disabled = false;
|
|
@@ -181,8 +183,8 @@ export class HttpBlobClient {
|
|
|
181
183
|
// Return the result, ignoring any undefined ones
|
|
182
184
|
const getFilledBlobs = ()=>resultBlobs.filter((b)=>b !== undefined);
|
|
183
185
|
// Helper to fill in results from fetched blobs
|
|
184
|
-
const fillResults = (fetchedBlobs)=>{
|
|
185
|
-
const blobs = processFetchedBlobs(fetchedBlobs, blobHashes, this.log);
|
|
186
|
+
const fillResults = async (fetchedBlobs)=>{
|
|
187
|
+
const blobs = await processFetchedBlobs(fetchedBlobs, blobHashes, this.log);
|
|
186
188
|
// Fill in any missing positions with matching blobs
|
|
187
189
|
for(let i = 0; i < blobHashes.length; i++){
|
|
188
190
|
if (resultBlobs[i] === undefined) {
|
|
@@ -234,7 +236,7 @@ export class HttpBlobClient {
|
|
|
234
236
|
...ctx
|
|
235
237
|
});
|
|
236
238
|
const blobs = await this.getBlobsFromHost(l1ConsensusHostUrl, slotNumber, l1ConsensusHostIndex);
|
|
237
|
-
const result = fillResults(blobs);
|
|
239
|
+
const result = await fillResults(blobs);
|
|
238
240
|
this.log.debug(`Got ${blobs.length} blobs from consensus host (total: ${result.length}/${blobHashes.length})`, {
|
|
239
241
|
slotNumber,
|
|
240
242
|
l1ConsensusHostUrl,
|
|
@@ -277,7 +279,7 @@ export class HttpBlobClient {
|
|
|
277
279
|
this.log.debug('No blobs found from archive client', archiveCtx);
|
|
278
280
|
} else {
|
|
279
281
|
this.log.trace(`Got ${allBlobs.length} blobs from archive client before filtering`, archiveCtx);
|
|
280
|
-
const result = fillResults(allBlobs);
|
|
282
|
+
const result = await fillResults(allBlobs);
|
|
281
283
|
this.log.debug(`Got ${allBlobs.length} blobs from archive client (total: ${result.length}/${blobHashes.length})`, archiveCtx);
|
|
282
284
|
if (result.length === blobHashes.length) {
|
|
283
285
|
return returnWithCallback(result);
|
|
@@ -318,7 +320,7 @@ export class HttpBlobClient {
|
|
|
318
320
|
});
|
|
319
321
|
const blobs = await client.getBlobsByHashes(blobHashStrings);
|
|
320
322
|
if (blobs.length > 0) {
|
|
321
|
-
const result = fillResults(blobs);
|
|
323
|
+
const result = await fillResults(blobs);
|
|
322
324
|
this.log.debug(`Got ${blobs.length} blobs from filestore (total: ${result.length}/${ctx.blobHashes.length})`, {
|
|
323
325
|
url: client.getBaseUrl(),
|
|
324
326
|
...ctx
|
|
@@ -333,7 +335,7 @@ export class HttpBlobClient {
|
|
|
333
335
|
}
|
|
334
336
|
async getBlobSidecarFrom(hostUrl, blockHashOrSlot, blobHashes = [], l1ConsensusHostIndex) {
|
|
335
337
|
const blobs = await this.getBlobsFromHost(hostUrl, blockHashOrSlot, l1ConsensusHostIndex);
|
|
336
|
-
return processFetchedBlobs(blobs, blobHashes, this.log).filter((b)=>b !== undefined);
|
|
338
|
+
return (await processFetchedBlobs(blobs, blobHashes, this.log)).filter((b)=>b !== undefined);
|
|
337
339
|
}
|
|
338
340
|
async getBlobsFromHost(hostUrl, blockHashOrSlot, l1ConsensusHostIndex) {
|
|
339
341
|
try {
|
|
@@ -474,6 +476,38 @@ export class HttpBlobClient {
|
|
|
474
476
|
/** @internal - exposed for testing */ getArchiveClient() {
|
|
475
477
|
return this.archiveClient;
|
|
476
478
|
}
|
|
479
|
+
/** Returns true if this client can upload blobs to filestore. */ canUpload() {
|
|
480
|
+
return this.fileStoreUploadClient !== undefined;
|
|
481
|
+
}
|
|
482
|
+
/**
|
|
483
|
+
* Start the blob client.
|
|
484
|
+
* Uploads the initial healthcheck file (awaited) and starts periodic uploads.
|
|
485
|
+
*/ async start() {
|
|
486
|
+
if (!this.fileStoreUploadClient) {
|
|
487
|
+
return;
|
|
488
|
+
}
|
|
489
|
+
await this.fileStoreUploadClient.uploadHealthcheck();
|
|
490
|
+
this.log.debug('Initial healthcheck file uploaded');
|
|
491
|
+
this.startPeriodicHealthcheckUpload();
|
|
492
|
+
}
|
|
493
|
+
/**
|
|
494
|
+
* Start periodic healthcheck upload to the file store to ensure it remains available even if accidentally deleted.
|
|
495
|
+
*/ startPeriodicHealthcheckUpload() {
|
|
496
|
+
const intervalMs = (this.config.blobHealthcheckUploadIntervalMinutes ?? DEFAULT_HEALTHCHECK_UPLOAD_INTERVAL_MINUTES) * 60 * 1000;
|
|
497
|
+
this.healthcheckUploadIntervalId = setInterval(()=>{
|
|
498
|
+
void this.fileStoreUploadClient.uploadHealthcheck().catch((err)=>{
|
|
499
|
+
this.log.warn('Failed to upload periodic healthcheck file', err);
|
|
500
|
+
});
|
|
501
|
+
}, intervalMs);
|
|
502
|
+
}
|
|
503
|
+
/**
|
|
504
|
+
* Stop the blob client, clearing any periodic tasks.
|
|
505
|
+
*/ stop() {
|
|
506
|
+
if (this.healthcheckUploadIntervalId) {
|
|
507
|
+
clearInterval(this.healthcheckUploadIntervalId);
|
|
508
|
+
this.healthcheckUploadIntervalId = undefined;
|
|
509
|
+
}
|
|
510
|
+
}
|
|
477
511
|
}
|
|
478
512
|
function parseBlobJsonsFromResponse(response, logger) {
|
|
479
513
|
try {
|
|
@@ -496,7 +530,7 @@ function parseBlobJson(data) {
|
|
|
496
530
|
}
|
|
497
531
|
// Returns an array that maps each blob hash to the corresponding blob, or undefined if the blob is not found
|
|
498
532
|
// or the data does not match the commitment.
|
|
499
|
-
function processFetchedBlobs(blobs, blobHashes, logger) {
|
|
533
|
+
async function processFetchedBlobs(blobs, blobHashes, logger) {
|
|
500
534
|
const requestedBlobHashes = new Set(blobHashes.map(bufferToHex));
|
|
501
535
|
const hashToBlob = new Map();
|
|
502
536
|
for (const blobJson of blobs){
|
|
@@ -505,7 +539,7 @@ function processFetchedBlobs(blobs, blobHashes, logger) {
|
|
|
505
539
|
continue;
|
|
506
540
|
}
|
|
507
541
|
try {
|
|
508
|
-
const blob = Blob.fromJson(blobJson);
|
|
542
|
+
const blob = await Blob.fromJson(blobJson);
|
|
509
543
|
hashToBlob.set(hashHex, blob);
|
|
510
544
|
} catch (err) {
|
|
511
545
|
// If the above throws, it's likely that the blob commitment does not match the hash of the blob data.
|
|
@@ -16,7 +16,13 @@ export interface BlobClientInterface {
|
|
|
16
16
|
sendBlobsToFilestore(blobs: Blob[]): Promise<boolean>;
|
|
17
17
|
/** Fetches the given blob sidecars by block hash and blob hashes. */
|
|
18
18
|
getBlobSidecar(blockId: string, blobHashes?: Buffer[], opts?: GetBlobSidecarOptions): Promise<Blob[]>;
|
|
19
|
+
/** Starts the blob client (e.g., uploads healthcheck file if not exists). */
|
|
20
|
+
start?(): Promise<void>;
|
|
19
21
|
/** Tests all configured blob sources and logs whether they are reachable or not. */
|
|
20
22
|
testSources(): Promise<void>;
|
|
23
|
+
/** Stops the blob client, clearing any periodic tasks. */
|
|
24
|
+
stop?(): void;
|
|
25
|
+
/** Returns true if this client can upload blobs to filestore. */
|
|
26
|
+
canUpload(): boolean;
|
|
21
27
|
}
|
|
22
|
-
//# sourceMappingURL=data:application/json;base64,
|
|
28
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiaW50ZXJmYWNlLmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi9zcmMvY2xpZW50L2ludGVyZmFjZS50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSxPQUFPLEtBQUssRUFBRSxJQUFJLEVBQUUsTUFBTSxpQkFBaUIsQ0FBQztBQUU1Qzs7R0FFRztBQUNILE1BQU0sV0FBVyxxQkFBcUI7SUFDcEM7Ozs7O09BS0c7SUFDSCxnQkFBZ0IsQ0FBQyxFQUFFLE9BQU8sQ0FBQztDQUM1QjtBQUVELE1BQU0sV0FBVyxtQkFBbUI7SUFDbEMsMEVBQTBFO0lBQzFFLG9CQUFvQixDQUFDLEtBQUssRUFBRSxJQUFJLEVBQUUsR0FBRyxPQUFPLENBQUMsT0FBTyxDQUFDLENBQUM7SUFDdEQscUVBQXFFO0lBQ3JFLGNBQWMsQ0FBQyxPQUFPLEVBQUUsTUFBTSxFQUFFLFVBQVUsQ0FBQyxFQUFFLE1BQU0sRUFBRSxFQUFFLElBQUksQ0FBQyxFQUFFLHFCQUFxQixHQUFHLE9BQU8sQ0FBQyxJQUFJLEVBQUUsQ0FBQyxDQUFDO0lBQ3RHLDZFQUE2RTtJQUM3RSxLQUFLLENBQUMsSUFBSSxPQUFPLENBQUMsSUFBSSxDQUFDLENBQUM7SUFDeEIsb0ZBQW9GO0lBQ3BGLFdBQVcsSUFBSSxPQUFPLENBQUMsSUFBSSxDQUFDLENBQUM7SUFDN0IsMERBQTBEO0lBQzFELElBQUksQ0FBQyxJQUFJLElBQUksQ0FBQztJQUNkLGlFQUFpRTtJQUNqRSxTQUFTLElBQUksT0FBTyxDQUFDO0NBQ3RCIn0=
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"interface.d.ts","sourceRoot":"","sources":["../../src/client/interface.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAC;AAE5C;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC;;;;;OAKG;IACH,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED,MAAM,WAAW,mBAAmB;IAClC,0EAA0E;IAC1E,oBAAoB,CAAC,KAAK,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IACtD,qEAAqE;IACrE,cAAc,CAAC,OAAO,EAAE,MAAM,EAAE,UAAU,CAAC,EAAE,MAAM,EAAE,EAAE,IAAI,CAAC,EAAE,qBAAqB,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;IACtG,oFAAoF;IACpF,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"interface.d.ts","sourceRoot":"","sources":["../../src/client/interface.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAC;AAE5C;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC;;;;;OAKG;IACH,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED,MAAM,WAAW,mBAAmB;IAClC,0EAA0E;IAC1E,oBAAoB,CAAC,KAAK,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IACtD,qEAAqE;IACrE,cAAc,CAAC,OAAO,EAAE,MAAM,EAAE,UAAU,CAAC,EAAE,MAAM,EAAE,EAAE,IAAI,CAAC,EAAE,qBAAqB,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;IACtG,6EAA6E;IAC7E,KAAK,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACxB,oFAAoF;IACpF,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAC7B,0DAA0D;IAC1D,IAAI,CAAC,IAAI,IAAI,CAAC;IACd,iEAAiE;IACjE,SAAS,IAAI,OAAO,CAAC;CACtB"}
|
package/dest/client/local.d.ts
CHANGED
|
@@ -7,5 +7,7 @@ export declare class LocalBlobClient implements BlobClientInterface {
|
|
|
7
7
|
testSources(): Promise<void>;
|
|
8
8
|
sendBlobsToFilestore(blobs: Blob[]): Promise<boolean>;
|
|
9
9
|
getBlobSidecar(_blockId: string, blobHashes: Buffer[], _opts?: GetBlobSidecarOptions): Promise<Blob[]>;
|
|
10
|
+
/** Returns true if this client can upload blobs. Always true for local client. */
|
|
11
|
+
canUpload(): boolean;
|
|
10
12
|
}
|
|
11
|
-
//# sourceMappingURL=data:application/json;base64,
|
|
13
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoibG9jYWwuZC50cyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uL3NyYy9jbGllbnQvbG9jYWwudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsT0FBTyxLQUFLLEVBQUUsSUFBSSxFQUFFLE1BQU0saUJBQWlCLENBQUM7QUFFNUMsT0FBTyxLQUFLLEVBQUUsU0FBUyxFQUFFLE1BQU0sdUJBQXVCLENBQUM7QUFDdkQsT0FBTyxLQUFLLEVBQUUsbUJBQW1CLEVBQUUscUJBQXFCLEVBQUUsTUFBTSxnQkFBZ0IsQ0FBQztBQUVqRixxQkFBYSxlQUFnQixZQUFXLG1CQUFtQjtJQUN6RCxPQUFPLENBQUMsUUFBUSxDQUFDLFNBQVMsQ0FBWTtJQUV0QyxZQUFZLFNBQVMsRUFBRSxTQUFTLEVBRS9CO0lBRU0sV0FBVyxJQUFJLE9BQU8sQ0FBQyxJQUFJLENBQUMsQ0FFbEM7SUFFWSxvQkFBb0IsQ0FBQyxLQUFLLEVBQUUsSUFBSSxFQUFFLEdBQUcsT0FBTyxDQUFDLE9BQU8sQ0FBQyxDQUdqRTtJQUVNLGNBQWMsQ0FBQyxRQUFRLEVBQUUsTUFBTSxFQUFFLFVBQVUsRUFBRSxNQUFNLEVBQUUsRUFBRSxLQUFLLENBQUMsRUFBRSxxQkFBcUIsR0FBRyxPQUFPLENBQUMsSUFBSSxFQUFFLENBQUMsQ0FFNUc7SUFFRCxrRkFBa0Y7SUFDM0UsU0FBUyxJQUFJLE9BQU8sQ0FFMUI7Q0FDRiJ9
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"local.d.ts","sourceRoot":"","sources":["../../src/client/local.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAC;AAE5C,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAC;AACvD,OAAO,KAAK,EAAE,mBAAmB,EAAE,qBAAqB,EAAE,MAAM,gBAAgB,CAAC;AAEjF,qBAAa,eAAgB,YAAW,mBAAmB;IACzD,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAY;IAEtC,YAAY,SAAS,EAAE,SAAS,EAE/B;IAEM,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC,CAElC;IAEY,oBAAoB,CAAC,KAAK,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC,CAGjE;IAEM,cAAc,CAAC,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,EAAE,KAAK,CAAC,EAAE,qBAAqB,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,CAE5G;CACF"}
|
|
1
|
+
{"version":3,"file":"local.d.ts","sourceRoot":"","sources":["../../src/client/local.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAC;AAE5C,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAC;AACvD,OAAO,KAAK,EAAE,mBAAmB,EAAE,qBAAqB,EAAE,MAAM,gBAAgB,CAAC;AAEjF,qBAAa,eAAgB,YAAW,mBAAmB;IACzD,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAY;IAEtC,YAAY,SAAS,EAAE,SAAS,EAE/B;IAEM,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC,CAElC;IAEY,oBAAoB,CAAC,KAAK,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC,CAGjE;IAEM,cAAc,CAAC,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,EAAE,KAAK,CAAC,EAAE,qBAAqB,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,CAE5G;IAED,kFAAkF;IAC3E,SAAS,IAAI,OAAO,CAE1B;CACF"}
|
package/dest/client/local.js
CHANGED
|
@@ -13,4 +13,7 @@ export class LocalBlobClient {
|
|
|
13
13
|
getBlobSidecar(_blockId, blobHashes, _opts) {
|
|
14
14
|
return this.blobStore.getBlobsByHashes(blobHashes);
|
|
15
15
|
}
|
|
16
|
+
/** Returns true if this client can upload blobs. Always true for local client. */ canUpload() {
|
|
17
|
+
return true;
|
|
18
|
+
}
|
|
16
19
|
}
|
package/dest/client/tests.js
CHANGED
|
@@ -17,7 +17,7 @@ import { makeRandomBlob } from '@aztec/blob-lib/testing';
|
|
|
17
17
|
await cleanup();
|
|
18
18
|
});
|
|
19
19
|
it('should send and retrieve blobs by hash', async ()=>{
|
|
20
|
-
const blob = makeRandomBlob(5);
|
|
20
|
+
const blob = await makeRandomBlob(5);
|
|
21
21
|
const blobHash = blob.getEthVersionedBlobHash();
|
|
22
22
|
await client.sendBlobsToFilestore([
|
|
23
23
|
blob
|
|
@@ -29,9 +29,9 @@ import { makeRandomBlob } from '@aztec/blob-lib/testing';
|
|
|
29
29
|
expect(retrievedBlobs[0]).toEqual(blob);
|
|
30
30
|
});
|
|
31
31
|
it('should handle multiple blobs', async ()=>{
|
|
32
|
-
const blobs = Array.from({
|
|
32
|
+
const blobs = await Promise.all(Array.from({
|
|
33
33
|
length: 3
|
|
34
|
-
}, ()=>makeRandomBlob(7));
|
|
34
|
+
}, ()=>makeRandomBlob(7)));
|
|
35
35
|
const blobHashes = blobs.map((blob)=>blob.getEthVersionedBlobHash());
|
|
36
36
|
await client.sendBlobsToFilestore(blobs);
|
|
37
37
|
const retrievedBlobs = await client.getBlobSidecar(blockId, blobHashes);
|
|
@@ -15,6 +15,11 @@ export declare class FileStoreBlobClient {
|
|
|
15
15
|
* Format: basePath/blobs/{versionedBlobHash}.data
|
|
16
16
|
*/
|
|
17
17
|
private blobPath;
|
|
18
|
+
/**
|
|
19
|
+
* Get the path for the healthcheck file.
|
|
20
|
+
* Format: basePath/.healthcheck
|
|
21
|
+
*/
|
|
22
|
+
private healthcheckPath;
|
|
18
23
|
/**
|
|
19
24
|
* Fetch blobs by their versioned hashes.
|
|
20
25
|
* @param blobHashes - Array of versioned blob hashes (0x-prefixed hex strings)
|
|
@@ -44,12 +49,19 @@ export declare class FileStoreBlobClient {
|
|
|
44
49
|
*/
|
|
45
50
|
getBaseUrl(): string;
|
|
46
51
|
/**
|
|
47
|
-
* Test if the filestore connection is working.
|
|
52
|
+
* Test if the filestore connection is working by checking for healthcheck file.
|
|
53
|
+
* The healthcheck file is uploaded periodically by writable clients via HttpBlobClient.start().
|
|
54
|
+
* This provides a uniform connection test across all store types (S3/GCS/Local/HTTP).
|
|
48
55
|
*/
|
|
49
56
|
testConnection(): Promise<boolean>;
|
|
57
|
+
/**
|
|
58
|
+
* Upload the healthcheck file if it doesn't already exist.
|
|
59
|
+
* This enables read-only clients (HTTP) to verify connectivity.
|
|
60
|
+
*/
|
|
61
|
+
uploadHealthcheck(): Promise<void>;
|
|
50
62
|
/**
|
|
51
63
|
* Check if the store supports write operations.
|
|
52
64
|
*/
|
|
53
65
|
private isWritable;
|
|
54
66
|
}
|
|
55
|
-
//# sourceMappingURL=data:application/json;base64,
|
|
67
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiZmlsZXN0b3JlX2Jsb2JfY2xpZW50LmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi9zcmMvZmlsZXN0b3JlL2ZpbGVzdG9yZV9ibG9iX2NsaWVudC50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSxPQUFPLEVBQUUsSUFBSSxFQUFFLEtBQUssUUFBUSxFQUErQixNQUFNLGlCQUFpQixDQUFDO0FBQ25GLE9BQU8sRUFBRSxLQUFLLE1BQU0sRUFBZ0IsTUFBTSx1QkFBdUIsQ0FBQztBQUNsRSxPQUFPLEtBQUssRUFBRSxTQUFTLEVBQUUsaUJBQWlCLEVBQUUsTUFBTSwwQkFBMEIsQ0FBQztBQUs3RTs7O0dBR0c7QUFDSCxxQkFBYSxtQkFBbUI7SUFJNUIsT0FBTyxDQUFDLFFBQVEsQ0FBQyxLQUFLO0lBQ3RCLE9BQU8sQ0FBQyxRQUFRLENBQUMsUUFBUTtJQUozQixPQUFPLENBQUMsUUFBUSxDQUFDLEdBQUcsQ0FBUztJQUU3QixZQUNtQixLQUFLLEVBQUUsaUJBQWlCLEdBQUcsU0FBUyxFQUNwQyxRQUFRLEVBQUUsTUFBTSxFQUNqQyxNQUFNLENBQUMsRUFBRSxNQUFNLEVBR2hCO0lBRUQ7OztPQUdHO0lBQ0gsT0FBTyxDQUFDLFFBQVE7SUFJaEI7OztPQUdHO0lBQ0gsT0FBTyxDQUFDLGVBQWU7SUFJdkI7Ozs7T0FJRztJQUNHLGdCQUFnQixDQUFDLFVBQVUsRUFBRSxNQUFNLEVBQUUsR0FBRyxPQUFPLENBQUMsUUFBUSxFQUFFLENBQUMsQ0FtQmhFO0lBRUQ7OztPQUdHO0lBQ0gsTUFBTSxDQUFDLGlCQUFpQixFQUFFLE1BQU0sR0FBRyxPQUFPLENBQUMsT0FBTyxDQUFDLENBRWxEO0lBRUQ7Ozs7O09BS0c7SUFDRyxRQUFRLENBQUMsSUFBSSxFQUFFLElBQUksRUFBRSxZQUFZLFVBQU8sR0FBRyxPQUFPLENBQUMsSUFBSSxDQUFDLENBa0I3RDtJQUVEOzs7O09BSUc7SUFDRyxTQUFTLENBQUMsS0FBSyxFQUFFLElBQUksRUFBRSxFQUFFLFlBQVksVUFBTyxHQUFHLE9BQU8sQ0FBQyxJQUFJLENBQUMsQ0FFakU7SUFFRDs7T0FFRztJQUNILFVBQVUsSUFBSSxNQUFNLENBRW5CO0lBRUQ7Ozs7T0FJRztJQUNHLGNBQWMsSUFBSSxPQUFPLENBQUMsT0FBTyxDQUFDLENBT3ZDO0lBRUQ7OztPQUdHO0lBQ0csaUJBQWlCLElBQUksT0FBTyxDQUFDLElBQUksQ0FBQyxDQVF2QztJQUVEOztPQUVHO0lBQ0gsT0FBTyxDQUFDLFVBQVU7Q0FHbkIifQ==
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"filestore_blob_client.d.ts","sourceRoot":"","sources":["../../src/filestore/filestore_blob_client.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,KAAK,QAAQ,EAA+B,MAAM,iBAAiB,CAAC;AACnF,OAAO,EAAE,KAAK,MAAM,EAAgB,MAAM,uBAAuB,CAAC;AAClE,OAAO,KAAK,EAAE,SAAS,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;
|
|
1
|
+
{"version":3,"file":"filestore_blob_client.d.ts","sourceRoot":"","sources":["../../src/filestore/filestore_blob_client.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,KAAK,QAAQ,EAA+B,MAAM,iBAAiB,CAAC;AACnF,OAAO,EAAE,KAAK,MAAM,EAAgB,MAAM,uBAAuB,CAAC;AAClE,OAAO,KAAK,EAAE,SAAS,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAK7E;;;GAGG;AACH,qBAAa,mBAAmB;IAI5B,OAAO,CAAC,QAAQ,CAAC,KAAK;IACtB,OAAO,CAAC,QAAQ,CAAC,QAAQ;IAJ3B,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAS;IAE7B,YACmB,KAAK,EAAE,iBAAiB,GAAG,SAAS,EACpC,QAAQ,EAAE,MAAM,EACjC,MAAM,CAAC,EAAE,MAAM,EAGhB;IAED;;;OAGG;IACH,OAAO,CAAC,QAAQ;IAIhB;;;OAGG;IACH,OAAO,CAAC,eAAe;IAIvB;;;;OAIG;IACG,gBAAgB,CAAC,UAAU,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,CAmBhE;IAED;;;OAGG;IACH,MAAM,CAAC,iBAAiB,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAElD;IAED;;;;;OAKG;IACG,QAAQ,CAAC,IAAI,EAAE,IAAI,EAAE,YAAY,UAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAkB7D;IAED;;;;OAIG;IACG,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,YAAY,UAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAEjE;IAED;;OAEG;IACH,UAAU,IAAI,MAAM,CAEnB;IAED;;;;OAIG;IACG,cAAc,IAAI,OAAO,CAAC,OAAO,CAAC,CAOvC;IAED;;;OAGG;IACG,iBAAiB,IAAI,OAAO,CAAC,IAAI,CAAC,CAQvC;IAED;;OAEG;IACH,OAAO,CAAC,UAAU;CAGnB"}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { computeEthVersionedBlobHash } from '@aztec/blob-lib';
|
|
2
2
|
import { createLogger } from '@aztec/foundation/log';
|
|
3
3
|
import { inboundTransform, outboundTransform } from '../encoding/index.js';
|
|
4
|
+
import { HEALTHCHECK_CONTENT, HEALTHCHECK_FILENAME } from './healthcheck.js';
|
|
4
5
|
/**
|
|
5
6
|
* A blob client that uses a FileStore (S3/GCS/local) as the data source.
|
|
6
7
|
* Blobs are stored as JSON files keyed by their versioned blob hash.
|
|
@@ -20,6 +21,12 @@ import { inboundTransform, outboundTransform } from '../encoding/index.js';
|
|
|
20
21
|
return `${this.basePath}/blobs/${versionedBlobHash}.data`;
|
|
21
22
|
}
|
|
22
23
|
/**
|
|
24
|
+
* Get the path for the healthcheck file.
|
|
25
|
+
* Format: basePath/.healthcheck
|
|
26
|
+
*/ healthcheckPath() {
|
|
27
|
+
return `${this.basePath}/${HEALTHCHECK_FILENAME}`;
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
23
30
|
* Fetch blobs by their versioned hashes.
|
|
24
31
|
* @param blobHashes - Array of versioned blob hashes (0x-prefixed hex strings)
|
|
25
32
|
* @returns Array of BlobJson objects for found blobs
|
|
@@ -77,11 +84,28 @@ import { inboundTransform, outboundTransform } from '../encoding/index.js';
|
|
|
77
84
|
return this.basePath;
|
|
78
85
|
}
|
|
79
86
|
/**
|
|
80
|
-
* Test if the filestore connection is working.
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
87
|
+
* Test if the filestore connection is working by checking for healthcheck file.
|
|
88
|
+
* The healthcheck file is uploaded periodically by writable clients via HttpBlobClient.start().
|
|
89
|
+
* This provides a uniform connection test across all store types (S3/GCS/Local/HTTP).
|
|
90
|
+
*/ async testConnection() {
|
|
91
|
+
try {
|
|
92
|
+
return await this.store.exists(this.healthcheckPath());
|
|
93
|
+
} catch (err) {
|
|
94
|
+
this.log.warn(`Connection test failed: ${err?.message ?? String(err)}`);
|
|
95
|
+
return false;
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
/**
|
|
99
|
+
* Upload the healthcheck file if it doesn't already exist.
|
|
100
|
+
* This enables read-only clients (HTTP) to verify connectivity.
|
|
101
|
+
*/ async uploadHealthcheck() {
|
|
102
|
+
if (!this.isWritable()) {
|
|
103
|
+
this.log.trace('Cannot upload healthcheck: store is read-only');
|
|
104
|
+
return;
|
|
105
|
+
}
|
|
106
|
+
const path = this.healthcheckPath();
|
|
107
|
+
await this.store.save(path, Buffer.from(HEALTHCHECK_CONTENT));
|
|
108
|
+
this.log.debug(`Uploaded healthcheck file to ${path}`);
|
|
85
109
|
}
|
|
86
110
|
/**
|
|
87
111
|
* Check if the store supports write operations.
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
/** Constants for healthcheck file used to test file store connectivity. */
|
|
2
|
+
export declare const HEALTHCHECK_FILENAME = ".healthcheck";
|
|
3
|
+
export declare const HEALTHCHECK_CONTENT = "ok";
|
|
4
|
+
export declare const DEFAULT_HEALTHCHECK_UPLOAD_INTERVAL_MINUTES = 60;
|
|
5
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiaGVhbHRoY2hlY2suZC50cyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uL3NyYy9maWxlc3RvcmUvaGVhbHRoY2hlY2sudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsMkVBQTJFO0FBRTNFLGVBQU8sTUFBTSxvQkFBb0IsaUJBQWlCLENBQUM7QUFDbkQsZUFBTyxNQUFNLG1CQUFtQixPQUFPLENBQUM7QUFDeEMsZUFBTyxNQUFNLDJDQUEyQyxLQUFLLENBQUMifQ==
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"healthcheck.d.ts","sourceRoot":"","sources":["../../src/filestore/healthcheck.ts"],"names":[],"mappings":"AAAA,2EAA2E;AAE3E,eAAO,MAAM,oBAAoB,iBAAiB,CAAC;AACnD,eAAO,MAAM,mBAAmB,OAAO,CAAC;AACxC,eAAO,MAAM,2CAA2C,KAAK,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@aztec/blob-client",
|
|
3
|
-
"version": "0.0.1-commit.
|
|
3
|
+
"version": "0.0.1-commit.0658669b3",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"bin": "./dest/client/bin/index.js",
|
|
6
6
|
"exports": {
|
|
@@ -56,12 +56,12 @@
|
|
|
56
56
|
]
|
|
57
57
|
},
|
|
58
58
|
"dependencies": {
|
|
59
|
-
"@aztec/blob-lib": "0.0.1-commit.
|
|
60
|
-
"@aztec/ethereum": "0.0.1-commit.
|
|
61
|
-
"@aztec/foundation": "0.0.1-commit.
|
|
62
|
-
"@aztec/kv-store": "0.0.1-commit.
|
|
63
|
-
"@aztec/stdlib": "0.0.1-commit.
|
|
64
|
-
"@aztec/telemetry-client": "0.0.1-commit.
|
|
59
|
+
"@aztec/blob-lib": "0.0.1-commit.0658669b3",
|
|
60
|
+
"@aztec/ethereum": "0.0.1-commit.0658669b3",
|
|
61
|
+
"@aztec/foundation": "0.0.1-commit.0658669b3",
|
|
62
|
+
"@aztec/kv-store": "0.0.1-commit.0658669b3",
|
|
63
|
+
"@aztec/stdlib": "0.0.1-commit.0658669b3",
|
|
64
|
+
"@aztec/telemetry-client": "0.0.1-commit.0658669b3",
|
|
65
65
|
"express": "^4.21.2",
|
|
66
66
|
"snappy": "^7.2.2",
|
|
67
67
|
"source-map-support": "^0.5.21",
|
|
@@ -75,7 +75,7 @@
|
|
|
75
75
|
"@types/node": "^22.15.17",
|
|
76
76
|
"@types/source-map-support": "^0.5.10",
|
|
77
77
|
"@types/supertest": "^6.0.2",
|
|
78
|
-
"@typescript/native-preview": "7.0.0-dev.
|
|
78
|
+
"@typescript/native-preview": "7.0.0-dev.20260113.1",
|
|
79
79
|
"jest": "^30.0.0",
|
|
80
80
|
"jest-mock-extended": "^4.0.0",
|
|
81
81
|
"supertest": "^7.0.0",
|
package/src/archive/config.ts
CHANGED
|
@@ -7,7 +7,7 @@ export type BlobArchiveApiConfig = {
|
|
|
7
7
|
|
|
8
8
|
export const blobArchiveApiConfigMappings: ConfigMappingsType<BlobArchiveApiConfig> = {
|
|
9
9
|
archiveApiUrl: {
|
|
10
|
-
env: '
|
|
10
|
+
env: 'BLOB_ARCHIVE_API_URL',
|
|
11
11
|
description: 'The URL of the archive API',
|
|
12
12
|
},
|
|
13
13
|
...pickConfigMappings(l1ReaderConfigMappings, ['l1ChainId']),
|
|
@@ -1,4 +1,10 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import {
|
|
2
|
+
Attributes,
|
|
3
|
+
Metrics,
|
|
4
|
+
type TelemetryClient,
|
|
5
|
+
type UpDownCounter,
|
|
6
|
+
createUpDownCounterWithDefault,
|
|
7
|
+
} from '@aztec/telemetry-client';
|
|
2
8
|
|
|
3
9
|
export class BlobArchiveClientInstrumentation {
|
|
4
10
|
private blockRequestCounter: UpDownCounter;
|
|
@@ -11,20 +17,23 @@ export class BlobArchiveClientInstrumentation {
|
|
|
11
17
|
name: string,
|
|
12
18
|
) {
|
|
13
19
|
const meter = client.getMeter(name);
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
}
|
|
20
|
+
const requestAttrs = {
|
|
21
|
+
[Attributes.HTTP_RESPONSE_STATUS_CODE]: [200, 404],
|
|
22
|
+
[Attributes.HTTP_REQUEST_HOST]: [httpHost],
|
|
23
|
+
};
|
|
24
|
+
this.blockRequestCounter = createUpDownCounterWithDefault(
|
|
25
|
+
meter,
|
|
26
|
+
Metrics.BLOB_SINK_ARCHIVE_BLOCK_REQUEST_COUNT,
|
|
27
|
+
requestAttrs,
|
|
28
|
+
);
|
|
18
29
|
|
|
19
|
-
this.blobRequestCounter =
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
30
|
+
this.blobRequestCounter = createUpDownCounterWithDefault(
|
|
31
|
+
meter,
|
|
32
|
+
Metrics.BLOB_SINK_ARCHIVE_BLOB_REQUEST_COUNT,
|
|
33
|
+
requestAttrs,
|
|
34
|
+
);
|
|
23
35
|
|
|
24
|
-
this.retrievedBlobs = meter
|
|
25
|
-
description: 'Number of blobs retrieved from the blob archive',
|
|
26
|
-
valueType: ValueType.INT,
|
|
27
|
-
});
|
|
36
|
+
this.retrievedBlobs = createUpDownCounterWithDefault(meter, Metrics.BLOB_SINK_ARCHIVE_BLOB_COUNT);
|
|
28
37
|
}
|
|
29
38
|
|
|
30
39
|
incRequest(type: 'blocks' | 'blobs', status: number) {
|
|
@@ -13,7 +13,7 @@ export function describeBlobStore(getBlobStore: () => Promise<BlobStore>) {
|
|
|
13
13
|
it('should store and retrieve a blob by hash', async () => {
|
|
14
14
|
// Create a test blob with random fields
|
|
15
15
|
const testFields = [Fr.random(), Fr.random(), Fr.random()];
|
|
16
|
-
const blob = Blob.fromFields(testFields);
|
|
16
|
+
const blob = await Blob.fromFields(testFields);
|
|
17
17
|
const blobHash = blob.getEthVersionedBlobHash();
|
|
18
18
|
|
|
19
19
|
// Store the blob
|
|
@@ -29,8 +29,8 @@ export function describeBlobStore(getBlobStore: () => Promise<BlobStore>) {
|
|
|
29
29
|
|
|
30
30
|
it('should handle multiple blobs stored and retrieved by their hashes', async () => {
|
|
31
31
|
// Create two different blobs
|
|
32
|
-
const blob1 = Blob.fromFields([Fr.random(), Fr.random()]);
|
|
33
|
-
const blob2 = Blob.fromFields([Fr.random(), Fr.random(), Fr.random()]);
|
|
32
|
+
const blob1 = await Blob.fromFields([Fr.random(), Fr.random()]);
|
|
33
|
+
const blob2 = await Blob.fromFields([Fr.random(), Fr.random(), Fr.random()]);
|
|
34
34
|
|
|
35
35
|
const blobHash1 = blob1.getEthVersionedBlobHash();
|
|
36
36
|
const blobHash2 = blob2.getEthVersionedBlobHash();
|
|
@@ -57,9 +57,9 @@ export function describeBlobStore(getBlobStore: () => Promise<BlobStore>) {
|
|
|
57
57
|
|
|
58
58
|
it('should handle retrieving subset of stored blobs', async () => {
|
|
59
59
|
// Store multiple blobs
|
|
60
|
-
const blob1 = Blob.fromFields([Fr.random()]);
|
|
61
|
-
const blob2 = Blob.fromFields([Fr.random()]);
|
|
62
|
-
const blob3 = Blob.fromFields([Fr.random()]);
|
|
60
|
+
const blob1 = await Blob.fromFields([Fr.random()]);
|
|
61
|
+
const blob2 = await Blob.fromFields([Fr.random()]);
|
|
62
|
+
const blob3 = await Blob.fromFields([Fr.random()]);
|
|
63
63
|
|
|
64
64
|
await blobStore.addBlobs([blob1, blob2, blob3]);
|
|
65
65
|
|
|
@@ -75,7 +75,7 @@ export function describeBlobStore(getBlobStore: () => Promise<BlobStore>) {
|
|
|
75
75
|
});
|
|
76
76
|
|
|
77
77
|
it('should handle duplicate blob hashes in request', async () => {
|
|
78
|
-
const blob = Blob.fromFields([Fr.random()]);
|
|
78
|
+
const blob = await Blob.fromFields([Fr.random()]);
|
|
79
79
|
const blobHash = blob.getEthVersionedBlobHash();
|
|
80
80
|
|
|
81
81
|
await blobStore.addBlobs([blob]);
|
|
@@ -91,8 +91,8 @@ export function describeBlobStore(getBlobStore: () => Promise<BlobStore>) {
|
|
|
91
91
|
it('should overwrite blob when storing with same hash', async () => {
|
|
92
92
|
// Create two blobs that will have the same hash (same content)
|
|
93
93
|
const fields = [Fr.random(), Fr.random()];
|
|
94
|
-
const blob1 = Blob.fromFields(fields);
|
|
95
|
-
const blob2 = Blob.fromFields(fields);
|
|
94
|
+
const blob1 = await Blob.fromFields(fields);
|
|
95
|
+
const blob2 = await Blob.fromFields(fields);
|
|
96
96
|
|
|
97
97
|
const blobHash = blob1.getEthVersionedBlobHash();
|
|
98
98
|
|
package/src/client/config.ts
CHANGED
|
@@ -50,6 +50,11 @@ export interface BlobClientConfig extends BlobArchiveApiConfig {
|
|
|
50
50
|
* URL for uploading blobs to filestore (s3://, gs://, file://)
|
|
51
51
|
*/
|
|
52
52
|
blobFileStoreUploadUrl?: string;
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* Interval in minutes for uploading healthcheck file to file store (default: 60 = 1 hour)
|
|
56
|
+
*/
|
|
57
|
+
blobHealthcheckUploadIntervalMinutes?: number;
|
|
53
58
|
}
|
|
54
59
|
|
|
55
60
|
export const blobClientConfigMapping: ConfigMappingsType<BlobClientConfig> = {
|
|
@@ -98,6 +103,11 @@ export const blobClientConfigMapping: ConfigMappingsType<BlobClientConfig> = {
|
|
|
98
103
|
env: 'BLOB_FILE_STORE_UPLOAD_URL',
|
|
99
104
|
description: 'URL for uploading blobs to filestore (s3://, gs://, file://)',
|
|
100
105
|
},
|
|
106
|
+
blobHealthcheckUploadIntervalMinutes: {
|
|
107
|
+
env: 'BLOB_HEALTHCHECK_UPLOAD_INTERVAL_MINUTES',
|
|
108
|
+
description: 'Interval in minutes for uploading healthcheck file to file store (default: 60 = 1 hour)',
|
|
109
|
+
parseEnv: (val: string | undefined) => (val ? +val : undefined),
|
|
110
|
+
},
|
|
101
111
|
...blobArchiveApiConfigMappings,
|
|
102
112
|
};
|
|
103
113
|
|
package/src/client/factory.ts
CHANGED
|
@@ -58,10 +58,11 @@ export interface BlobClientWithFileStoresConfig extends BlobClientConfig {
|
|
|
58
58
|
* 2. Creating read-only FileStore clients
|
|
59
59
|
* 3. Creating a writable FileStore client for uploads
|
|
60
60
|
* 4. Creating the BlobClient with these dependencies
|
|
61
|
+
* 5. Starting the client (uploads initial healthcheck file if upload client is configured)
|
|
61
62
|
*
|
|
62
63
|
* @param config - Configuration containing blob client settings and chain metadata
|
|
63
64
|
* @param logger - Optional logger for the blob client
|
|
64
|
-
* @returns A BlobClientInterface configured with file store support
|
|
65
|
+
* @returns A BlobClientInterface configured with file store support, already started
|
|
65
66
|
*/
|
|
66
67
|
export async function createBlobClientWithFileStores(
|
|
67
68
|
config: BlobClientWithFileStoresConfig,
|
|
@@ -80,9 +81,13 @@ export async function createBlobClientWithFileStores(
|
|
|
80
81
|
createWritableFileStoreBlobClient(config.blobFileStoreUploadUrl, fileStoreMetadata, log),
|
|
81
82
|
]);
|
|
82
83
|
|
|
83
|
-
|
|
84
|
+
const client = createBlobClient(config, {
|
|
84
85
|
logger: log,
|
|
85
86
|
fileStoreClients,
|
|
86
87
|
fileStoreUploadClient,
|
|
87
88
|
});
|
|
89
|
+
|
|
90
|
+
await client.start?.();
|
|
91
|
+
|
|
92
|
+
return client;
|
|
88
93
|
}
|
package/src/client/http.ts
CHANGED
|
@@ -9,6 +9,7 @@ import { type RpcBlock, createPublicClient, fallback, http } from 'viem';
|
|
|
9
9
|
import { createBlobArchiveClient } from '../archive/factory.js';
|
|
10
10
|
import type { BlobArchiveClient } from '../archive/interface.js';
|
|
11
11
|
import type { FileStoreBlobClient } from '../filestore/filestore_blob_client.js';
|
|
12
|
+
import { DEFAULT_HEALTHCHECK_UPLOAD_INTERVAL_MINUTES } from '../filestore/healthcheck.js';
|
|
12
13
|
import { type BlobClientConfig, getBlobClientConfigFromEnv } from './config.js';
|
|
13
14
|
import type { BlobClientInterface, GetBlobSidecarOptions } from './interface.js';
|
|
14
15
|
|
|
@@ -21,6 +22,7 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
21
22
|
protected readonly fileStoreUploadClient: FileStoreBlobClient | undefined;
|
|
22
23
|
|
|
23
24
|
private disabled = false;
|
|
25
|
+
private healthcheckUploadIntervalId?: NodeJS.Timeout;
|
|
24
26
|
|
|
25
27
|
constructor(
|
|
26
28
|
config?: BlobClientConfig,
|
|
@@ -213,8 +215,8 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
213
215
|
const getFilledBlobs = (): Blob[] => resultBlobs.filter((b): b is Blob => b !== undefined);
|
|
214
216
|
|
|
215
217
|
// Helper to fill in results from fetched blobs
|
|
216
|
-
const fillResults = (fetchedBlobs: BlobJson[]): Blob[] => {
|
|
217
|
-
const blobs = processFetchedBlobs(fetchedBlobs, blobHashes, this.log);
|
|
218
|
+
const fillResults = async (fetchedBlobs: BlobJson[]): Promise<Blob[]> => {
|
|
219
|
+
const blobs = await processFetchedBlobs(fetchedBlobs, blobHashes, this.log);
|
|
218
220
|
// Fill in any missing positions with matching blobs
|
|
219
221
|
for (let i = 0; i < blobHashes.length; i++) {
|
|
220
222
|
if (resultBlobs[i] === undefined) {
|
|
@@ -267,7 +269,7 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
267
269
|
...ctx,
|
|
268
270
|
});
|
|
269
271
|
const blobs = await this.getBlobsFromHost(l1ConsensusHostUrl, slotNumber, l1ConsensusHostIndex);
|
|
270
|
-
const result = fillResults(blobs);
|
|
272
|
+
const result = await fillResults(blobs);
|
|
271
273
|
this.log.debug(
|
|
272
274
|
`Got ${blobs.length} blobs from consensus host (total: ${result.length}/${blobHashes.length})`,
|
|
273
275
|
{ slotNumber, l1ConsensusHostUrl, ...ctx },
|
|
@@ -310,7 +312,7 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
310
312
|
this.log.debug('No blobs found from archive client', archiveCtx);
|
|
311
313
|
} else {
|
|
312
314
|
this.log.trace(`Got ${allBlobs.length} blobs from archive client before filtering`, archiveCtx);
|
|
313
|
-
const result = fillResults(allBlobs);
|
|
315
|
+
const result = await fillResults(allBlobs);
|
|
314
316
|
this.log.debug(
|
|
315
317
|
`Got ${allBlobs.length} blobs from archive client (total: ${result.length}/${blobHashes.length})`,
|
|
316
318
|
archiveCtx,
|
|
@@ -343,7 +345,7 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
343
345
|
*/
|
|
344
346
|
private async tryFileStores(
|
|
345
347
|
getMissingBlobHashes: () => Buffer[],
|
|
346
|
-
fillResults: (blobs: BlobJson[]) => Blob[]
|
|
348
|
+
fillResults: (blobs: BlobJson[]) => Promise<Blob[]>,
|
|
347
349
|
ctx: { blockHash: string; blobHashes: string[] },
|
|
348
350
|
): Promise<void> {
|
|
349
351
|
// Shuffle clients for load distribution
|
|
@@ -364,7 +366,7 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
364
366
|
});
|
|
365
367
|
const blobs = await client.getBlobsByHashes(blobHashStrings);
|
|
366
368
|
if (blobs.length > 0) {
|
|
367
|
-
const result = fillResults(blobs);
|
|
369
|
+
const result = await fillResults(blobs);
|
|
368
370
|
this.log.debug(
|
|
369
371
|
`Got ${blobs.length} blobs from filestore (total: ${result.length}/${ctx.blobHashes.length})`,
|
|
370
372
|
{
|
|
@@ -386,7 +388,7 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
386
388
|
l1ConsensusHostIndex?: number,
|
|
387
389
|
): Promise<Blob[]> {
|
|
388
390
|
const blobs = await this.getBlobsFromHost(hostUrl, blockHashOrSlot, l1ConsensusHostIndex);
|
|
389
|
-
return processFetchedBlobs(blobs, blobHashes, this.log).filter((b): b is Blob => b !== undefined);
|
|
391
|
+
return (await processFetchedBlobs(blobs, blobHashes, this.log)).filter((b): b is Blob => b !== undefined);
|
|
390
392
|
}
|
|
391
393
|
|
|
392
394
|
public async getBlobsFromHost(
|
|
@@ -545,6 +547,50 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
545
547
|
public getArchiveClient(): BlobArchiveClient | undefined {
|
|
546
548
|
return this.archiveClient;
|
|
547
549
|
}
|
|
550
|
+
|
|
551
|
+
/** Returns true if this client can upload blobs to filestore. */
|
|
552
|
+
public canUpload(): boolean {
|
|
553
|
+
return this.fileStoreUploadClient !== undefined;
|
|
554
|
+
}
|
|
555
|
+
|
|
556
|
+
/**
|
|
557
|
+
* Start the blob client.
|
|
558
|
+
* Uploads the initial healthcheck file (awaited) and starts periodic uploads.
|
|
559
|
+
*/
|
|
560
|
+
public async start(): Promise<void> {
|
|
561
|
+
if (!this.fileStoreUploadClient) {
|
|
562
|
+
return;
|
|
563
|
+
}
|
|
564
|
+
|
|
565
|
+
await this.fileStoreUploadClient.uploadHealthcheck();
|
|
566
|
+
this.log.debug('Initial healthcheck file uploaded');
|
|
567
|
+
|
|
568
|
+
this.startPeriodicHealthcheckUpload();
|
|
569
|
+
}
|
|
570
|
+
|
|
571
|
+
/**
|
|
572
|
+
* Start periodic healthcheck upload to the file store to ensure it remains available even if accidentally deleted.
|
|
573
|
+
*/
|
|
574
|
+
private startPeriodicHealthcheckUpload(): void {
|
|
575
|
+
const intervalMs =
|
|
576
|
+
(this.config.blobHealthcheckUploadIntervalMinutes ?? DEFAULT_HEALTHCHECK_UPLOAD_INTERVAL_MINUTES) * 60 * 1000;
|
|
577
|
+
|
|
578
|
+
this.healthcheckUploadIntervalId = setInterval(() => {
|
|
579
|
+
void this.fileStoreUploadClient!.uploadHealthcheck().catch(err => {
|
|
580
|
+
this.log.warn('Failed to upload periodic healthcheck file', err);
|
|
581
|
+
});
|
|
582
|
+
}, intervalMs);
|
|
583
|
+
}
|
|
584
|
+
|
|
585
|
+
/**
|
|
586
|
+
* Stop the blob client, clearing any periodic tasks.
|
|
587
|
+
*/
|
|
588
|
+
public stop(): void {
|
|
589
|
+
if (this.healthcheckUploadIntervalId) {
|
|
590
|
+
clearInterval(this.healthcheckUploadIntervalId);
|
|
591
|
+
this.healthcheckUploadIntervalId = undefined;
|
|
592
|
+
}
|
|
593
|
+
}
|
|
548
594
|
}
|
|
549
595
|
|
|
550
596
|
function parseBlobJsonsFromResponse(response: any, logger: Logger): BlobJson[] {
|
|
@@ -570,7 +616,11 @@ function parseBlobJson(data: any): BlobJson {
|
|
|
570
616
|
|
|
571
617
|
// Returns an array that maps each blob hash to the corresponding blob, or undefined if the blob is not found
|
|
572
618
|
// or the data does not match the commitment.
|
|
573
|
-
function processFetchedBlobs(
|
|
619
|
+
async function processFetchedBlobs(
|
|
620
|
+
blobs: BlobJson[],
|
|
621
|
+
blobHashes: Buffer[],
|
|
622
|
+
logger: Logger,
|
|
623
|
+
): Promise<(Blob | undefined)[]> {
|
|
574
624
|
const requestedBlobHashes = new Set<string>(blobHashes.map(bufferToHex));
|
|
575
625
|
const hashToBlob = new Map<string, Blob>();
|
|
576
626
|
for (const blobJson of blobs) {
|
|
@@ -580,7 +630,7 @@ function processFetchedBlobs(blobs: BlobJson[], blobHashes: Buffer[], logger: Lo
|
|
|
580
630
|
}
|
|
581
631
|
|
|
582
632
|
try {
|
|
583
|
-
const blob = Blob.fromJson(blobJson);
|
|
633
|
+
const blob = await Blob.fromJson(blobJson);
|
|
584
634
|
hashToBlob.set(hashHex, blob);
|
|
585
635
|
} catch (err) {
|
|
586
636
|
// If the above throws, it's likely that the blob commitment does not match the hash of the blob data.
|
package/src/client/interface.ts
CHANGED
|
@@ -18,6 +18,12 @@ export interface BlobClientInterface {
|
|
|
18
18
|
sendBlobsToFilestore(blobs: Blob[]): Promise<boolean>;
|
|
19
19
|
/** Fetches the given blob sidecars by block hash and blob hashes. */
|
|
20
20
|
getBlobSidecar(blockId: string, blobHashes?: Buffer[], opts?: GetBlobSidecarOptions): Promise<Blob[]>;
|
|
21
|
+
/** Starts the blob client (e.g., uploads healthcheck file if not exists). */
|
|
22
|
+
start?(): Promise<void>;
|
|
21
23
|
/** Tests all configured blob sources and logs whether they are reachable or not. */
|
|
22
24
|
testSources(): Promise<void>;
|
|
25
|
+
/** Stops the blob client, clearing any periodic tasks. */
|
|
26
|
+
stop?(): void;
|
|
27
|
+
/** Returns true if this client can upload blobs to filestore. */
|
|
28
|
+
canUpload(): boolean;
|
|
23
29
|
}
|
package/src/client/local.ts
CHANGED
|
@@ -22,4 +22,9 @@ export class LocalBlobClient implements BlobClientInterface {
|
|
|
22
22
|
public getBlobSidecar(_blockId: string, blobHashes: Buffer[], _opts?: GetBlobSidecarOptions): Promise<Blob[]> {
|
|
23
23
|
return this.blobStore.getBlobsByHashes(blobHashes);
|
|
24
24
|
}
|
|
25
|
+
|
|
26
|
+
/** Returns true if this client can upload blobs. Always true for local client. */
|
|
27
|
+
public canUpload(): boolean {
|
|
28
|
+
return true;
|
|
29
|
+
}
|
|
25
30
|
}
|
package/src/client/tests.ts
CHANGED
|
@@ -28,7 +28,7 @@ export function runBlobClientTests(
|
|
|
28
28
|
});
|
|
29
29
|
|
|
30
30
|
it('should send and retrieve blobs by hash', async () => {
|
|
31
|
-
const blob = makeRandomBlob(5);
|
|
31
|
+
const blob = await makeRandomBlob(5);
|
|
32
32
|
const blobHash = blob.getEthVersionedBlobHash();
|
|
33
33
|
|
|
34
34
|
await client.sendBlobsToFilestore([blob]);
|
|
@@ -39,7 +39,7 @@ export function runBlobClientTests(
|
|
|
39
39
|
});
|
|
40
40
|
|
|
41
41
|
it('should handle multiple blobs', async () => {
|
|
42
|
-
const blobs = Array.from({ length: 3 }, () => makeRandomBlob(7));
|
|
42
|
+
const blobs = await Promise.all(Array.from({ length: 3 }, () => makeRandomBlob(7)));
|
|
43
43
|
const blobHashes = blobs.map(blob => blob.getEthVersionedBlobHash());
|
|
44
44
|
|
|
45
45
|
await client.sendBlobsToFilestore(blobs);
|
|
@@ -3,6 +3,7 @@ import { type Logger, createLogger } from '@aztec/foundation/log';
|
|
|
3
3
|
import type { FileStore, ReadOnlyFileStore } from '@aztec/stdlib/file-store';
|
|
4
4
|
|
|
5
5
|
import { inboundTransform, outboundTransform } from '../encoding/index.js';
|
|
6
|
+
import { HEALTHCHECK_CONTENT, HEALTHCHECK_FILENAME } from './healthcheck.js';
|
|
6
7
|
|
|
7
8
|
/**
|
|
8
9
|
* A blob client that uses a FileStore (S3/GCS/local) as the data source.
|
|
@@ -27,6 +28,14 @@ export class FileStoreBlobClient {
|
|
|
27
28
|
return `${this.basePath}/blobs/${versionedBlobHash}.data`;
|
|
28
29
|
}
|
|
29
30
|
|
|
31
|
+
/**
|
|
32
|
+
* Get the path for the healthcheck file.
|
|
33
|
+
* Format: basePath/.healthcheck
|
|
34
|
+
*/
|
|
35
|
+
private healthcheckPath(): string {
|
|
36
|
+
return `${this.basePath}/${HEALTHCHECK_FILENAME}`;
|
|
37
|
+
}
|
|
38
|
+
|
|
30
39
|
/**
|
|
31
40
|
* Fetch blobs by their versioned hashes.
|
|
32
41
|
* @param blobHashes - Array of versioned blob hashes (0x-prefixed hex strings)
|
|
@@ -104,12 +113,31 @@ export class FileStoreBlobClient {
|
|
|
104
113
|
}
|
|
105
114
|
|
|
106
115
|
/**
|
|
107
|
-
* Test if the filestore connection is working.
|
|
116
|
+
* Test if the filestore connection is working by checking for healthcheck file.
|
|
117
|
+
* The healthcheck file is uploaded periodically by writable clients via HttpBlobClient.start().
|
|
118
|
+
* This provides a uniform connection test across all store types (S3/GCS/Local/HTTP).
|
|
119
|
+
*/
|
|
120
|
+
async testConnection(): Promise<boolean> {
|
|
121
|
+
try {
|
|
122
|
+
return await this.store.exists(this.healthcheckPath());
|
|
123
|
+
} catch (err: any) {
|
|
124
|
+
this.log.warn(`Connection test failed: ${err?.message ?? String(err)}`);
|
|
125
|
+
return false;
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
/**
|
|
130
|
+
* Upload the healthcheck file if it doesn't already exist.
|
|
131
|
+
* This enables read-only clients (HTTP) to verify connectivity.
|
|
108
132
|
*/
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
133
|
+
async uploadHealthcheck(): Promise<void> {
|
|
134
|
+
if (!this.isWritable()) {
|
|
135
|
+
this.log.trace('Cannot upload healthcheck: store is read-only');
|
|
136
|
+
return;
|
|
137
|
+
}
|
|
138
|
+
const path = this.healthcheckPath();
|
|
139
|
+
await (this.store as FileStore).save(path, Buffer.from(HEALTHCHECK_CONTENT));
|
|
140
|
+
this.log.debug(`Uploaded healthcheck file to ${path}`);
|
|
113
141
|
}
|
|
114
142
|
|
|
115
143
|
/**
|