@aztec/blob-client 3.0.0-nightly.20251223 → 3.0.0-nightly.20251224
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dest/archive/blobscan_archive_client.d.ts +1 -2
- package/dest/archive/blobscan_archive_client.d.ts.map +1 -1
- package/dest/archive/blobscan_archive_client.js +1 -4
- package/dest/blobstore/blob_store_test_suite.d.ts +1 -1
- package/dest/blobstore/blob_store_test_suite.d.ts.map +1 -1
- package/dest/blobstore/blob_store_test_suite.js +18 -49
- package/dest/blobstore/interface.d.ts +4 -4
- package/dest/blobstore/interface.d.ts.map +1 -1
- package/dest/blobstore/memory_blob_store.d.ts +4 -4
- package/dest/blobstore/memory_blob_store.d.ts.map +1 -1
- package/dest/blobstore/memory_blob_store.js +3 -3
- package/dest/client/http.d.ts +7 -7
- package/dest/client/http.d.ts.map +1 -1
- package/dest/client/http.js +21 -24
- package/dest/client/interface.d.ts +3 -4
- package/dest/client/interface.d.ts.map +1 -1
- package/dest/client/local.d.ts +2 -3
- package/dest/client/local.d.ts.map +1 -1
- package/dest/client/local.js +2 -4
- package/dest/client/tests.d.ts +1 -1
- package/dest/client/tests.d.ts.map +1 -1
- package/dest/client/tests.js +2 -16
- package/dest/filestore/filestore_blob_client.d.ts +3 -4
- package/dest/filestore/filestore_blob_client.d.ts.map +1 -1
- package/dest/filestore/filestore_blob_client.js +4 -12
- package/package.json +7 -8
- package/src/archive/blobscan_archive_client.ts +8 -10
- package/src/blobstore/blob_store_test_suite.ts +15 -42
- package/src/blobstore/interface.ts +3 -3
- package/src/blobstore/memory_blob_store.ts +6 -6
- package/src/client/http.ts +32 -36
- package/src/client/interface.ts +2 -9
- package/src/client/local.ts +2 -9
- package/src/client/tests.ts +2 -18
- package/src/filestore/filestore_blob_client.ts +5 -13
- package/dest/types/api.d.ts +0 -65
- package/dest/types/api.d.ts.map +0 -1
- package/dest/types/api.js +0 -22
- package/dest/types/blob_with_index.d.ts +0 -25
- package/dest/types/blob_with_index.d.ts.map +0 -1
- package/dest/types/blob_with_index.js +0 -43
- package/dest/types/index.d.ts +0 -2
- package/dest/types/index.d.ts.map +0 -1
- package/dest/types/index.js +0 -1
- package/src/types/api.ts +0 -50
- package/src/types/blob_with_index.ts +0 -48
- package/src/types/index.ts +0 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"filestore_blob_client.d.ts","sourceRoot":"","sources":["../../src/filestore/filestore_blob_client.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,KAAK,QAAQ,EAA+B,MAAM,iBAAiB,CAAC;AACnF,OAAO,EAAE,KAAK,MAAM,EAAgB,MAAM,uBAAuB,CAAC;AAClE,OAAO,KAAK,EAAE,SAAS,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;
|
|
1
|
+
{"version":3,"file":"filestore_blob_client.d.ts","sourceRoot":"","sources":["../../src/filestore/filestore_blob_client.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,KAAK,QAAQ,EAA+B,MAAM,iBAAiB,CAAC;AACnF,OAAO,EAAE,KAAK,MAAM,EAAgB,MAAM,uBAAuB,CAAC;AAClE,OAAO,KAAK,EAAE,SAAS,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAI7E;;;GAGG;AACH,qBAAa,mBAAmB;IAI5B,OAAO,CAAC,QAAQ,CAAC,KAAK;IACtB,OAAO,CAAC,QAAQ,CAAC,QAAQ;IAJ3B,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAS;IAE7B,YACmB,KAAK,EAAE,iBAAiB,GAAG,SAAS,EACpC,QAAQ,EAAE,MAAM,EACjC,MAAM,CAAC,EAAE,MAAM,EAGhB;IAED;;;OAGG;IACH,OAAO,CAAC,QAAQ;IAIhB;;;;OAIG;IACG,gBAAgB,CAAC,UAAU,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,CAmBhE;IAED;;;OAGG;IACH,MAAM,CAAC,iBAAiB,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAElD;IAED;;;;;OAKG;IACG,QAAQ,CAAC,IAAI,EAAE,IAAI,EAAE,YAAY,UAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAkB7D;IAED;;;;OAIG;IACG,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,YAAY,UAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAEjE;IAED;;OAEG;IACH,UAAU,IAAI,MAAM,CAEnB;IAED;;OAEG;IACH,cAAc,IAAI,OAAO,CAAC,OAAO,CAAC,CAIjC;IAED;;OAEG;IACH,OAAO,CAAC,UAAU;CAGnB"}
|
|
@@ -33,11 +33,7 @@ import { inboundTransform, outboundTransform } from '../encoding/index.js';
|
|
|
33
33
|
}
|
|
34
34
|
const data = await this.store.read(path);
|
|
35
35
|
const json = JSON.parse(inboundTransform(data).toString());
|
|
36
|
-
|
|
37
|
-
blobs.push({
|
|
38
|
-
...json,
|
|
39
|
-
index: '-1'
|
|
40
|
-
});
|
|
36
|
+
blobs.push(json);
|
|
41
37
|
} catch (err) {
|
|
42
38
|
this.log.warn(`Failed to read blob ${blobHashes[i]} from filestore`, err);
|
|
43
39
|
}
|
|
@@ -64,20 +60,16 @@ import { inboundTransform, outboundTransform } from '../encoding/index.js';
|
|
|
64
60
|
this.log.trace(`Blob ${versionedHash} already exists, skipping`);
|
|
65
61
|
return;
|
|
66
62
|
}
|
|
67
|
-
|
|
68
|
-
const json = blob.toJson(-1);
|
|
63
|
+
const json = blob.toJSON();
|
|
69
64
|
await this.store.save(this.blobPath(versionedHash), outboundTransform(Buffer.from(JSON.stringify(json))));
|
|
70
65
|
this.log.debug(`Saved blob ${versionedHash} to filestore`);
|
|
71
66
|
}
|
|
72
67
|
/**
|
|
73
68
|
* Save multiple blobs to the store in parallel.
|
|
74
|
-
* @param blobs - The blobs to save
|
|
69
|
+
* @param blobs - The blobs to save
|
|
75
70
|
* @param skipIfExists - Skip saving if blob already exists (default: true)
|
|
76
71
|
*/ async saveBlobs(blobs, skipIfExists = true) {
|
|
77
|
-
await Promise.all(blobs.map((
|
|
78
|
-
const blob = 'blob' in b ? b.blob : b;
|
|
79
|
-
return this.saveBlob(blob, skipIfExists);
|
|
80
|
-
}));
|
|
72
|
+
await Promise.all(blobs.map((blob)=>this.saveBlob(blob, skipIfExists)));
|
|
81
73
|
}
|
|
82
74
|
/**
|
|
83
75
|
* Get the base URL/path of the filestore.
|
package/package.json
CHANGED
|
@@ -1,13 +1,12 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@aztec/blob-client",
|
|
3
|
-
"version": "3.0.0-nightly.
|
|
3
|
+
"version": "3.0.0-nightly.20251224",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"bin": "./dest/client/bin/index.js",
|
|
6
6
|
"exports": {
|
|
7
7
|
"./client": "./dest/client/index.js",
|
|
8
8
|
"./client/config": "./dest/client/config.js",
|
|
9
9
|
"./encoding": "./dest/encoding/index.js",
|
|
10
|
-
"./types": "./dest/types/index.js",
|
|
11
10
|
"./filestore": "./dest/filestore/index.js"
|
|
12
11
|
},
|
|
13
12
|
"inherits": [
|
|
@@ -57,12 +56,12 @@
|
|
|
57
56
|
]
|
|
58
57
|
},
|
|
59
58
|
"dependencies": {
|
|
60
|
-
"@aztec/blob-lib": "3.0.0-nightly.
|
|
61
|
-
"@aztec/ethereum": "3.0.0-nightly.
|
|
62
|
-
"@aztec/foundation": "3.0.0-nightly.
|
|
63
|
-
"@aztec/kv-store": "3.0.0-nightly.
|
|
64
|
-
"@aztec/stdlib": "3.0.0-nightly.
|
|
65
|
-
"@aztec/telemetry-client": "3.0.0-nightly.
|
|
59
|
+
"@aztec/blob-lib": "3.0.0-nightly.20251224",
|
|
60
|
+
"@aztec/ethereum": "3.0.0-nightly.20251224",
|
|
61
|
+
"@aztec/foundation": "3.0.0-nightly.20251224",
|
|
62
|
+
"@aztec/kv-store": "3.0.0-nightly.20251224",
|
|
63
|
+
"@aztec/stdlib": "3.0.0-nightly.20251224",
|
|
64
|
+
"@aztec/telemetry-client": "3.0.0-nightly.20251224",
|
|
66
65
|
"express": "^4.21.2",
|
|
67
66
|
"snappy": "^7.2.2",
|
|
68
67
|
"source-map-support": "^0.5.21",
|
|
@@ -25,22 +25,20 @@ export const BlobscanBlockResponseSchema = zodFor<BlobJson[]>()(
|
|
|
25
25
|
commitment: z.string(),
|
|
26
26
|
proof: z.string(),
|
|
27
27
|
size: z.number().int(),
|
|
28
|
-
index: z.number().int().optional(), //
|
|
28
|
+
index: z.number().int().optional(), // Unused, kept for schema compatibility with blobscan API
|
|
29
29
|
}),
|
|
30
30
|
),
|
|
31
31
|
}),
|
|
32
32
|
),
|
|
33
33
|
})
|
|
34
34
|
.transform(data =>
|
|
35
|
-
data.transactions
|
|
36
|
-
.
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
)
|
|
43
|
-
.map((blob, index) => ({ ...blob, index: index.toString() })),
|
|
35
|
+
data.transactions.flatMap(tx =>
|
|
36
|
+
tx.blobs.map(blob => ({
|
|
37
|
+
blob: blob.data,
|
|
38
|
+
// eslint-disable-next-line camelcase
|
|
39
|
+
kzg_commitment: blob.commitment,
|
|
40
|
+
})),
|
|
41
|
+
),
|
|
44
42
|
),
|
|
45
43
|
);
|
|
46
44
|
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import { Blob } from '@aztec/blob-lib';
|
|
2
2
|
import { Fr } from '@aztec/foundation/curves/bn254';
|
|
3
3
|
|
|
4
|
-
import { BlobWithIndex } from '../types/index.js';
|
|
5
4
|
import type { BlobStore } from './interface.js';
|
|
6
5
|
|
|
7
6
|
export function describeBlobStore(getBlobStore: () => Promise<BlobStore>) {
|
|
@@ -15,39 +14,36 @@ export function describeBlobStore(getBlobStore: () => Promise<BlobStore>) {
|
|
|
15
14
|
// Create a test blob with random fields
|
|
16
15
|
const testFields = [Fr.random(), Fr.random(), Fr.random()];
|
|
17
16
|
const blob = Blob.fromFields(testFields);
|
|
18
|
-
const blobWithIndex = new BlobWithIndex(blob, 0);
|
|
19
17
|
const blobHash = blob.getEthVersionedBlobHash();
|
|
20
18
|
|
|
21
19
|
// Store the blob
|
|
22
|
-
await blobStore.addBlobs([
|
|
20
|
+
await blobStore.addBlobs([blob]);
|
|
23
21
|
|
|
24
22
|
// Retrieve the blob by hash
|
|
25
23
|
const retrievedBlobs = await blobStore.getBlobsByHashes([blobHash]);
|
|
26
24
|
|
|
27
25
|
// Verify the blob was retrieved and matches
|
|
28
26
|
expect(retrievedBlobs.length).toBe(1);
|
|
29
|
-
expect(retrievedBlobs[0]
|
|
27
|
+
expect(retrievedBlobs[0]).toEqual(blob);
|
|
30
28
|
});
|
|
31
29
|
|
|
32
30
|
it('should handle multiple blobs stored and retrieved by their hashes', async () => {
|
|
33
31
|
// Create two different blobs
|
|
34
32
|
const blob1 = Blob.fromFields([Fr.random(), Fr.random()]);
|
|
35
33
|
const blob2 = Blob.fromFields([Fr.random(), Fr.random(), Fr.random()]);
|
|
36
|
-
const blobWithIndex1 = new BlobWithIndex(blob1, 0);
|
|
37
|
-
const blobWithIndex2 = new BlobWithIndex(blob2, 1);
|
|
38
34
|
|
|
39
35
|
const blobHash1 = blob1.getEthVersionedBlobHash();
|
|
40
36
|
const blobHash2 = blob2.getEthVersionedBlobHash();
|
|
41
37
|
|
|
42
38
|
// Store both blobs
|
|
43
|
-
await blobStore.addBlobs([
|
|
39
|
+
await blobStore.addBlobs([blob1, blob2]);
|
|
44
40
|
|
|
45
41
|
// Retrieve and verify both blobs
|
|
46
42
|
const retrievedBlobs = await blobStore.getBlobsByHashes([blobHash1, blobHash2]);
|
|
47
43
|
|
|
48
44
|
expect(retrievedBlobs.length).toBe(2);
|
|
49
|
-
expect(retrievedBlobs[0]
|
|
50
|
-
expect(retrievedBlobs[1]
|
|
45
|
+
expect(retrievedBlobs[0]).toEqual(blob1);
|
|
46
|
+
expect(retrievedBlobs[1]).toEqual(blob2);
|
|
51
47
|
});
|
|
52
48
|
|
|
53
49
|
it('should return empty array for non-existent blob hash', async () => {
|
|
@@ -59,31 +55,13 @@ export function describeBlobStore(getBlobStore: () => Promise<BlobStore>) {
|
|
|
59
55
|
expect(retrievedBlobs).toEqual([]);
|
|
60
56
|
});
|
|
61
57
|
|
|
62
|
-
it('should handle storing blobs with different indices', async () => {
|
|
63
|
-
// Create blobs with different indices
|
|
64
|
-
const blob1 = Blob.fromFields([Fr.random()]);
|
|
65
|
-
const blob2 = Blob.fromFields([Fr.random()]);
|
|
66
|
-
const blobWithIndex1 = new BlobWithIndex(blob1, 0);
|
|
67
|
-
const blobWithIndex2 = new BlobWithIndex(blob2, 1);
|
|
68
|
-
|
|
69
|
-
await blobStore.addBlobs([blobWithIndex1, blobWithIndex2]);
|
|
70
|
-
|
|
71
|
-
const blobHash1 = blob1.getEthVersionedBlobHash();
|
|
72
|
-
const blobHash2 = blob2.getEthVersionedBlobHash();
|
|
73
|
-
|
|
74
|
-
const retrievedBlobs = await blobStore.getBlobsByHashes([blobHash1, blobHash2]);
|
|
75
|
-
|
|
76
|
-
expect(retrievedBlobs[0].index).toBe(0);
|
|
77
|
-
expect(retrievedBlobs[1].index).toBe(1);
|
|
78
|
-
});
|
|
79
|
-
|
|
80
58
|
it('should handle retrieving subset of stored blobs', async () => {
|
|
81
59
|
// Store multiple blobs
|
|
82
60
|
const blob1 = Blob.fromFields([Fr.random()]);
|
|
83
61
|
const blob2 = Blob.fromFields([Fr.random()]);
|
|
84
62
|
const blob3 = Blob.fromFields([Fr.random()]);
|
|
85
63
|
|
|
86
|
-
await blobStore.addBlobs([
|
|
64
|
+
await blobStore.addBlobs([blob1, blob2, blob3]);
|
|
87
65
|
|
|
88
66
|
// Retrieve only some of them
|
|
89
67
|
const blobHash1 = blob1.getEthVersionedBlobHash();
|
|
@@ -92,23 +70,22 @@ export function describeBlobStore(getBlobStore: () => Promise<BlobStore>) {
|
|
|
92
70
|
const retrievedBlobs = await blobStore.getBlobsByHashes([blobHash1, blobHash3]);
|
|
93
71
|
|
|
94
72
|
expect(retrievedBlobs.length).toBe(2);
|
|
95
|
-
expect(retrievedBlobs[0]
|
|
96
|
-
expect(retrievedBlobs[1]
|
|
73
|
+
expect(retrievedBlobs[0]).toEqual(blob1);
|
|
74
|
+
expect(retrievedBlobs[1]).toEqual(blob3);
|
|
97
75
|
});
|
|
98
76
|
|
|
99
77
|
it('should handle duplicate blob hashes in request', async () => {
|
|
100
78
|
const blob = Blob.fromFields([Fr.random()]);
|
|
101
|
-
const blobWithIndex = new BlobWithIndex(blob, 0);
|
|
102
79
|
const blobHash = blob.getEthVersionedBlobHash();
|
|
103
80
|
|
|
104
|
-
await blobStore.addBlobs([
|
|
81
|
+
await blobStore.addBlobs([blob]);
|
|
105
82
|
|
|
106
83
|
// Request the same blob hash multiple times
|
|
107
84
|
const retrievedBlobs = await blobStore.getBlobsByHashes([blobHash, blobHash]);
|
|
108
85
|
|
|
109
86
|
// Implementation may return duplicates or deduplicate - both are valid
|
|
110
87
|
expect(retrievedBlobs.length).toBeGreaterThanOrEqual(1);
|
|
111
|
-
expect(retrievedBlobs[0]
|
|
88
|
+
expect(retrievedBlobs[0]).toEqual(blob);
|
|
112
89
|
});
|
|
113
90
|
|
|
114
91
|
it('should overwrite blob when storing with same hash', async () => {
|
|
@@ -117,21 +94,17 @@ export function describeBlobStore(getBlobStore: () => Promise<BlobStore>) {
|
|
|
117
94
|
const blob1 = Blob.fromFields(fields);
|
|
118
95
|
const blob2 = Blob.fromFields(fields);
|
|
119
96
|
|
|
120
|
-
// Store with different indices
|
|
121
|
-
const blobWithIndex1 = new BlobWithIndex(blob1, 0);
|
|
122
|
-
const blobWithIndex2 = new BlobWithIndex(blob2, 5);
|
|
123
|
-
|
|
124
97
|
const blobHash = blob1.getEthVersionedBlobHash();
|
|
125
98
|
|
|
126
99
|
// Store first blob
|
|
127
|
-
await blobStore.addBlobs([
|
|
100
|
+
await blobStore.addBlobs([blob1]);
|
|
128
101
|
|
|
129
|
-
// Overwrite with second blob (same hash
|
|
130
|
-
await blobStore.addBlobs([
|
|
102
|
+
// Overwrite with second blob (same hash)
|
|
103
|
+
await blobStore.addBlobs([blob2]);
|
|
131
104
|
|
|
132
|
-
// Retrieve and verify it
|
|
105
|
+
// Retrieve and verify it exists
|
|
133
106
|
const retrievedBlobs = await blobStore.getBlobsByHashes([blobHash]);
|
|
134
107
|
expect(retrievedBlobs.length).toBe(1);
|
|
135
|
-
expect(retrievedBlobs[0]
|
|
108
|
+
expect(retrievedBlobs[0]).toEqual(blob1); // Same content
|
|
136
109
|
});
|
|
137
110
|
}
|
|
@@ -1,12 +1,12 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { Blob } from '@aztec/blob-lib';
|
|
2
2
|
|
|
3
3
|
export interface BlobStore {
|
|
4
4
|
/**
|
|
5
5
|
* Get blobs by their hashes
|
|
6
6
|
*/
|
|
7
|
-
getBlobsByHashes: (blobHashes: Buffer[]) => Promise<
|
|
7
|
+
getBlobsByHashes: (blobHashes: Buffer[]) => Promise<Blob[]>;
|
|
8
8
|
/**
|
|
9
9
|
* Add blobs to the store, indexed by their hashes
|
|
10
10
|
*/
|
|
11
|
-
addBlobs: (blobs:
|
|
11
|
+
addBlobs: (blobs: Blob[]) => Promise<void>;
|
|
12
12
|
}
|
|
@@ -1,28 +1,28 @@
|
|
|
1
|
+
import { Blob } from '@aztec/blob-lib';
|
|
1
2
|
import { bufferToHex } from '@aztec/foundation/string';
|
|
2
3
|
|
|
3
|
-
import { BlobWithIndex } from '../types/index.js';
|
|
4
4
|
import type { BlobStore } from './interface.js';
|
|
5
5
|
|
|
6
6
|
export class MemoryBlobStore implements BlobStore {
|
|
7
7
|
private blobs: Map<string, Buffer> = new Map();
|
|
8
8
|
|
|
9
|
-
public getBlobsByHashes(blobHashes: Buffer[]): Promise<
|
|
10
|
-
const results:
|
|
9
|
+
public getBlobsByHashes(blobHashes: Buffer[]): Promise<Blob[]> {
|
|
10
|
+
const results: Blob[] = [];
|
|
11
11
|
|
|
12
12
|
for (const blobHash of blobHashes) {
|
|
13
13
|
const key = bufferToHex(blobHash);
|
|
14
14
|
const blobBuffer = this.blobs.get(key);
|
|
15
15
|
if (blobBuffer) {
|
|
16
|
-
results.push(
|
|
16
|
+
results.push(Blob.fromBuffer(blobBuffer));
|
|
17
17
|
}
|
|
18
18
|
}
|
|
19
19
|
|
|
20
20
|
return Promise.resolve(results);
|
|
21
21
|
}
|
|
22
22
|
|
|
23
|
-
public addBlobs(blobs:
|
|
23
|
+
public addBlobs(blobs: Blob[]): Promise<void> {
|
|
24
24
|
for (const blob of blobs) {
|
|
25
|
-
const blobHash = blob.
|
|
25
|
+
const blobHash = blob.getEthVersionedBlobHash();
|
|
26
26
|
const key = bufferToHex(blobHash);
|
|
27
27
|
this.blobs.set(key, blob.toBuffer());
|
|
28
28
|
}
|
package/src/client/http.ts
CHANGED
|
@@ -9,7 +9,6 @@ import { type RpcBlock, createPublicClient, fallback, http } from 'viem';
|
|
|
9
9
|
import { createBlobArchiveClient } from '../archive/factory.js';
|
|
10
10
|
import type { BlobArchiveClient } from '../archive/interface.js';
|
|
11
11
|
import type { FileStoreBlobClient } from '../filestore/filestore_blob_client.js';
|
|
12
|
-
import { BlobWithIndex } from '../types/blob_with_index.js';
|
|
13
12
|
import { type BlobClientConfig, getBlobClientConfigFromEnv } from './config.js';
|
|
14
13
|
import type { BlobClientInterface, GetBlobSidecarOptions } from './interface.js';
|
|
15
14
|
|
|
@@ -31,7 +30,7 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
31
30
|
fileStoreClients?: FileStoreBlobClient[];
|
|
32
31
|
fileStoreUploadClient?: FileStoreBlobClient;
|
|
33
32
|
/** Callback fired when blobs are successfully fetched from any source */
|
|
34
|
-
onBlobsFetched?: (blobs:
|
|
33
|
+
onBlobsFetched?: (blobs: Blob[]) => void;
|
|
35
34
|
} = {},
|
|
36
35
|
) {
|
|
37
36
|
this.config = config ?? getBlobClientConfigFromEnv();
|
|
@@ -62,7 +61,7 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
62
61
|
* Upload fetched blobs to filestore (fire-and-forget).
|
|
63
62
|
* Called automatically when blobs are fetched from any source.
|
|
64
63
|
*/
|
|
65
|
-
private uploadBlobsToFileStore(blobs:
|
|
64
|
+
private uploadBlobsToFileStore(blobs: Blob[]): void {
|
|
66
65
|
if (!this.fileStoreUploadClient) {
|
|
67
66
|
return;
|
|
68
67
|
}
|
|
@@ -186,16 +185,14 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
186
185
|
*
|
|
187
186
|
* @param blockHash - The block hash
|
|
188
187
|
* @param blobHashes - The blob hashes to fetch
|
|
189
|
-
* @param indices - The indices of the blobs to get
|
|
190
188
|
* @param opts - Options including isHistoricalSync flag
|
|
191
189
|
* @returns The blobs
|
|
192
190
|
*/
|
|
193
191
|
public async getBlobSidecar(
|
|
194
192
|
blockHash: `0x${string}`,
|
|
195
193
|
blobHashes: Buffer[],
|
|
196
|
-
indices?: number[],
|
|
197
194
|
opts?: GetBlobSidecarOptions,
|
|
198
|
-
): Promise<
|
|
195
|
+
): Promise<Blob[]> {
|
|
199
196
|
if (this.disabled) {
|
|
200
197
|
this.log.warn('Blob storage is disabled, returning empty blob sidecar');
|
|
201
198
|
return [];
|
|
@@ -204,7 +201,7 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
204
201
|
const isHistoricalSync = opts?.isHistoricalSync ?? false;
|
|
205
202
|
// Accumulate blobs across sources, preserving order and handling duplicates
|
|
206
203
|
// resultBlobs[i] will contain the blob for blobHashes[i], or undefined if not yet found
|
|
207
|
-
const resultBlobs: (
|
|
204
|
+
const resultBlobs: (Blob | undefined)[] = new Array(blobHashes.length).fill(undefined);
|
|
208
205
|
|
|
209
206
|
// Helper to get missing blob hashes that we still need to fetch
|
|
210
207
|
const getMissingBlobHashes = (): Buffer[] =>
|
|
@@ -213,10 +210,10 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
213
210
|
.filter((bh): bh is Buffer => bh !== undefined);
|
|
214
211
|
|
|
215
212
|
// Return the result, ignoring any undefined ones
|
|
216
|
-
const getFilledBlobs = ():
|
|
213
|
+
const getFilledBlobs = (): Blob[] => resultBlobs.filter((b): b is Blob => b !== undefined);
|
|
217
214
|
|
|
218
215
|
// Helper to fill in results from fetched blobs
|
|
219
|
-
const fillResults = (fetchedBlobs: BlobJson[]):
|
|
216
|
+
const fillResults = (fetchedBlobs: BlobJson[]): Blob[] => {
|
|
220
217
|
const blobs = processFetchedBlobs(fetchedBlobs, blobHashes, this.log);
|
|
221
218
|
// Fill in any missing positions with matching blobs
|
|
222
219
|
for (let i = 0; i < blobHashes.length; i++) {
|
|
@@ -228,7 +225,7 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
228
225
|
};
|
|
229
226
|
|
|
230
227
|
// Fire callback when returning blobs (fire-and-forget)
|
|
231
|
-
const returnWithCallback = (blobs:
|
|
228
|
+
const returnWithCallback = (blobs: Blob[]): Blob[] => {
|
|
232
229
|
if (blobs.length > 0 && this.opts.onBlobsFetched) {
|
|
233
230
|
void Promise.resolve().then(() => this.opts.onBlobsFetched!(blobs));
|
|
234
231
|
}
|
|
@@ -237,7 +234,7 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
237
234
|
|
|
238
235
|
const { l1ConsensusHostUrls } = this.config;
|
|
239
236
|
|
|
240
|
-
const ctx = { blockHash, blobHashes: blobHashes.map(bufferToHex)
|
|
237
|
+
const ctx = { blockHash, blobHashes: blobHashes.map(bufferToHex) };
|
|
241
238
|
|
|
242
239
|
// Try filestore (quick, no retries) - useful for both historical and near-tip sync
|
|
243
240
|
if (this.fileStoreClients.length > 0 && getMissingBlobHashes().length > 0) {
|
|
@@ -269,7 +266,7 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
269
266
|
l1ConsensusHostUrl,
|
|
270
267
|
...ctx,
|
|
271
268
|
});
|
|
272
|
-
const blobs = await this.getBlobsFromHost(l1ConsensusHostUrl, slotNumber,
|
|
269
|
+
const blobs = await this.getBlobsFromHost(l1ConsensusHostUrl, slotNumber, l1ConsensusHostIndex);
|
|
273
270
|
const result = fillResults(blobs);
|
|
274
271
|
this.log.debug(
|
|
275
272
|
`Got ${blobs.length} blobs from consensus host (total: ${result.length}/${blobHashes.length})`,
|
|
@@ -346,8 +343,8 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
346
343
|
*/
|
|
347
344
|
private async tryFileStores(
|
|
348
345
|
getMissingBlobHashes: () => Buffer[],
|
|
349
|
-
fillResults: (blobs: BlobJson[]) =>
|
|
350
|
-
ctx: { blockHash: string; blobHashes: string[]
|
|
346
|
+
fillResults: (blobs: BlobJson[]) => Blob[],
|
|
347
|
+
ctx: { blockHash: string; blobHashes: string[] },
|
|
351
348
|
): Promise<void> {
|
|
352
349
|
// Shuffle clients for load distribution
|
|
353
350
|
const shuffledClients = [...this.fileStoreClients];
|
|
@@ -386,21 +383,19 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
386
383
|
hostUrl: string,
|
|
387
384
|
blockHashOrSlot: string | number,
|
|
388
385
|
blobHashes: Buffer[] = [],
|
|
389
|
-
indices: number[] = [],
|
|
390
386
|
l1ConsensusHostIndex?: number,
|
|
391
|
-
): Promise<
|
|
392
|
-
const blobs = await this.getBlobsFromHost(hostUrl, blockHashOrSlot,
|
|
393
|
-
return processFetchedBlobs(blobs, blobHashes, this.log).filter((b): b is
|
|
387
|
+
): Promise<Blob[]> {
|
|
388
|
+
const blobs = await this.getBlobsFromHost(hostUrl, blockHashOrSlot, l1ConsensusHostIndex);
|
|
389
|
+
return processFetchedBlobs(blobs, blobHashes, this.log).filter((b): b is Blob => b !== undefined);
|
|
394
390
|
}
|
|
395
391
|
|
|
396
392
|
public async getBlobsFromHost(
|
|
397
393
|
hostUrl: string,
|
|
398
394
|
blockHashOrSlot: string | number,
|
|
399
|
-
indices: number[] = [],
|
|
400
395
|
l1ConsensusHostIndex?: number,
|
|
401
396
|
): Promise<BlobJson[]> {
|
|
402
397
|
try {
|
|
403
|
-
let res = await this.fetchBlobSidecars(hostUrl, blockHashOrSlot,
|
|
398
|
+
let res = await this.fetchBlobSidecars(hostUrl, blockHashOrSlot, l1ConsensusHostIndex);
|
|
404
399
|
if (res.ok) {
|
|
405
400
|
return parseBlobJsonsFromResponse(await res.json(), this.log);
|
|
406
401
|
}
|
|
@@ -416,8 +411,8 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
416
411
|
let maxRetries = 10;
|
|
417
412
|
let currentSlot = blockHashOrSlot + 1;
|
|
418
413
|
while (res.status === 404 && maxRetries > 0 && latestSlot !== undefined && currentSlot <= latestSlot) {
|
|
419
|
-
this.log.debug(`Trying slot ${currentSlot}
|
|
420
|
-
res = await this.fetchBlobSidecars(hostUrl, currentSlot,
|
|
414
|
+
this.log.debug(`Trying slot ${currentSlot}`);
|
|
415
|
+
res = await this.fetchBlobSidecars(hostUrl, currentSlot, l1ConsensusHostIndex);
|
|
421
416
|
if (res.ok) {
|
|
422
417
|
return parseBlobJsonsFromResponse(await res.json(), this.log);
|
|
423
418
|
}
|
|
@@ -441,13 +436,9 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
441
436
|
private fetchBlobSidecars(
|
|
442
437
|
hostUrl: string,
|
|
443
438
|
blockHashOrSlot: string | number,
|
|
444
|
-
indices: number[],
|
|
445
439
|
l1ConsensusHostIndex?: number,
|
|
446
440
|
): Promise<Response> {
|
|
447
|
-
|
|
448
|
-
if (indices.length > 0) {
|
|
449
|
-
baseUrl += `?indices=${indices.join(',')}`;
|
|
450
|
-
}
|
|
441
|
+
const baseUrl = `${hostUrl}/eth/v1/beacon/blob_sidecars/${blockHashOrSlot}`;
|
|
451
442
|
|
|
452
443
|
const { url, ...options } = getBeaconNodeFetchOptions(baseUrl, this.config, l1ConsensusHostIndex);
|
|
453
444
|
this.log.debug(`Fetching blob sidecar for ${blockHashOrSlot}`, { url, ...options });
|
|
@@ -549,6 +540,11 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
549
540
|
|
|
550
541
|
return undefined;
|
|
551
542
|
}
|
|
543
|
+
|
|
544
|
+
/** @internal - exposed for testing */
|
|
545
|
+
public getArchiveClient(): BlobArchiveClient | undefined {
|
|
546
|
+
return this.archiveClient;
|
|
547
|
+
}
|
|
552
548
|
}
|
|
553
549
|
|
|
554
550
|
function parseBlobJsonsFromResponse(response: any, logger: Logger): BlobJson[] {
|
|
@@ -564,28 +560,28 @@ function parseBlobJsonsFromResponse(response: any, logger: Logger): BlobJson[] {
|
|
|
564
560
|
// Blobs will be in this form when requested from the blob client, or from the beacon chain via `getBlobSidecars`:
|
|
565
561
|
// https://ethereum.github.io/beacon-APIs/?urls.primaryName=dev#/Beacon/getBlobSidecars
|
|
566
562
|
// Here we attempt to parse the response data to Buffer, and check the lengths (via Blob's constructor), to avoid
|
|
567
|
-
// throwing an error down the line when calling
|
|
563
|
+
// throwing an error down the line when calling Blob.fromJson().
|
|
568
564
|
function parseBlobJson(data: any): BlobJson {
|
|
569
565
|
const blobBuffer = Buffer.from(data.blob.slice(2), 'hex');
|
|
570
566
|
const commitmentBuffer = Buffer.from(data.kzg_commitment.slice(2), 'hex');
|
|
571
567
|
const blob = new Blob(blobBuffer, commitmentBuffer);
|
|
572
|
-
return blob.
|
|
568
|
+
return blob.toJSON();
|
|
573
569
|
}
|
|
574
570
|
|
|
575
|
-
// Returns an array that maps each blob hash to the corresponding blob
|
|
571
|
+
// Returns an array that maps each blob hash to the corresponding blob, or undefined if the blob is not found
|
|
576
572
|
// or the data does not match the commitment.
|
|
577
|
-
function processFetchedBlobs(blobs: BlobJson[], blobHashes: Buffer[], logger: Logger): (
|
|
573
|
+
function processFetchedBlobs(blobs: BlobJson[], blobHashes: Buffer[], logger: Logger): (Blob | undefined)[] {
|
|
578
574
|
const requestedBlobHashes = new Set<string>(blobHashes.map(bufferToHex));
|
|
579
|
-
const hashToBlob = new Map<string,
|
|
580
|
-
for (const
|
|
581
|
-
const hashHex = bufferToHex(computeEthVersionedBlobHash(hexToBuffer(
|
|
575
|
+
const hashToBlob = new Map<string, Blob>();
|
|
576
|
+
for (const blobJson of blobs) {
|
|
577
|
+
const hashHex = bufferToHex(computeEthVersionedBlobHash(hexToBuffer(blobJson.kzg_commitment)));
|
|
582
578
|
if (!requestedBlobHashes.has(hashHex) || hashToBlob.has(hashHex)) {
|
|
583
579
|
continue;
|
|
584
580
|
}
|
|
585
581
|
|
|
586
582
|
try {
|
|
587
|
-
const
|
|
588
|
-
hashToBlob.set(hashHex,
|
|
583
|
+
const blob = Blob.fromJson(blobJson);
|
|
584
|
+
hashToBlob.set(hashHex, blob);
|
|
589
585
|
} catch (err) {
|
|
590
586
|
// If the above throws, it's likely that the blob commitment does not match the hash of the blob data.
|
|
591
587
|
logger.error(`Error converting blob from json`, err);
|
package/src/client/interface.ts
CHANGED
|
@@ -1,7 +1,5 @@
|
|
|
1
1
|
import type { Blob } from '@aztec/blob-lib';
|
|
2
2
|
|
|
3
|
-
import type { BlobWithIndex } from '../types/blob_with_index.js';
|
|
4
|
-
|
|
5
3
|
/**
|
|
6
4
|
* Options for getBlobSidecar method.
|
|
7
5
|
*/
|
|
@@ -18,13 +16,8 @@ export interface GetBlobSidecarOptions {
|
|
|
18
16
|
export interface BlobClientInterface {
|
|
19
17
|
/** Sends the given blobs to the filestore, to be indexed by blob hash. */
|
|
20
18
|
sendBlobsToFilestore(blobs: Blob[]): Promise<boolean>;
|
|
21
|
-
/** Fetches the given blob sidecars by block
|
|
22
|
-
getBlobSidecar(
|
|
23
|
-
blockId: string,
|
|
24
|
-
blobHashes?: Buffer[],
|
|
25
|
-
indices?: number[],
|
|
26
|
-
opts?: GetBlobSidecarOptions,
|
|
27
|
-
): Promise<BlobWithIndex[]>;
|
|
19
|
+
/** Fetches the given blob sidecars by block hash and blob hashes. */
|
|
20
|
+
getBlobSidecar(blockId: string, blobHashes?: Buffer[], opts?: GetBlobSidecarOptions): Promise<Blob[]>;
|
|
28
21
|
/** Tests all configured blob sources and logs whether they are reachable or not. */
|
|
29
22
|
testSources(): Promise<void>;
|
|
30
23
|
}
|
package/src/client/local.ts
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import type { Blob } from '@aztec/blob-lib';
|
|
2
2
|
|
|
3
3
|
import type { BlobStore } from '../blobstore/index.js';
|
|
4
|
-
import { BlobWithIndex } from '../types/blob_with_index.js';
|
|
5
4
|
import type { BlobClientInterface, GetBlobSidecarOptions } from './interface.js';
|
|
6
5
|
|
|
7
6
|
export class LocalBlobClient implements BlobClientInterface {
|
|
@@ -16,17 +15,11 @@ export class LocalBlobClient implements BlobClientInterface {
|
|
|
16
15
|
}
|
|
17
16
|
|
|
18
17
|
public async sendBlobsToFilestore(blobs: Blob[]): Promise<boolean> {
|
|
19
|
-
|
|
20
|
-
await this.blobStore.addBlobs(blobsWithIndex);
|
|
18
|
+
await this.blobStore.addBlobs(blobs);
|
|
21
19
|
return true;
|
|
22
20
|
}
|
|
23
21
|
|
|
24
|
-
public getBlobSidecar(
|
|
25
|
-
_blockId: string,
|
|
26
|
-
blobHashes: Buffer[],
|
|
27
|
-
_indices?: number[],
|
|
28
|
-
_opts?: GetBlobSidecarOptions,
|
|
29
|
-
): Promise<BlobWithIndex[]> {
|
|
22
|
+
public getBlobSidecar(_blockId: string, blobHashes: Buffer[], _opts?: GetBlobSidecarOptions): Promise<Blob[]> {
|
|
30
23
|
return this.blobStore.getBlobsByHashes(blobHashes);
|
|
31
24
|
}
|
|
32
25
|
}
|
package/src/client/tests.ts
CHANGED
|
@@ -35,7 +35,7 @@ export function runBlobClientTests(
|
|
|
35
35
|
|
|
36
36
|
const retrievedBlobs = await client.getBlobSidecar(blockId, [blobHash]);
|
|
37
37
|
expect(retrievedBlobs).toHaveLength(1);
|
|
38
|
-
expect(retrievedBlobs[0]
|
|
38
|
+
expect(retrievedBlobs[0]).toEqual(blob);
|
|
39
39
|
});
|
|
40
40
|
|
|
41
41
|
it('should handle multiple blobs', async () => {
|
|
@@ -48,7 +48,7 @@ export function runBlobClientTests(
|
|
|
48
48
|
expect(retrievedBlobs.length).toBe(3);
|
|
49
49
|
|
|
50
50
|
for (let i = 0; i < blobs.length; i++) {
|
|
51
|
-
expect(retrievedBlobs[i]
|
|
51
|
+
expect(retrievedBlobs[i]).toEqual(blobs[i]);
|
|
52
52
|
}
|
|
53
53
|
});
|
|
54
54
|
|
|
@@ -59,20 +59,4 @@ export function runBlobClientTests(
|
|
|
59
59
|
const retrievedBlobs = await client.getBlobSidecar(blockId, [nonExistentHash]);
|
|
60
60
|
expect(retrievedBlobs).toEqual([]);
|
|
61
61
|
});
|
|
62
|
-
|
|
63
|
-
it('should preserve blob indices', async () => {
|
|
64
|
-
const blobs = Array.from({ length: 3 }, () => makeRandomBlob(7));
|
|
65
|
-
const blobHashes = blobs.map(blob => blob.getEthVersionedBlobHash());
|
|
66
|
-
|
|
67
|
-
await client.sendBlobsToFilestore(blobs);
|
|
68
|
-
|
|
69
|
-
const retrievedBlobs = await client.getBlobSidecar(blockId, blobHashes);
|
|
70
|
-
expect(retrievedBlobs.length).toBe(blobs.length);
|
|
71
|
-
|
|
72
|
-
// Indices should be assigned sequentially based on the order they were sent
|
|
73
|
-
for (let i = 0; i < blobs.length; i++) {
|
|
74
|
-
expect(retrievedBlobs[i].blob).toEqual(blobs[i]);
|
|
75
|
-
expect(retrievedBlobs[i].index).toBe(i);
|
|
76
|
-
}
|
|
77
|
-
});
|
|
78
62
|
}
|
|
@@ -3,7 +3,6 @@ import { type Logger, createLogger } from '@aztec/foundation/log';
|
|
|
3
3
|
import type { FileStore, ReadOnlyFileStore } from '@aztec/stdlib/file-store';
|
|
4
4
|
|
|
5
5
|
import { inboundTransform, outboundTransform } from '../encoding/index.js';
|
|
6
|
-
import { BlobWithIndex } from '../types/blob_with_index.js';
|
|
7
6
|
|
|
8
7
|
/**
|
|
9
8
|
* A blob client that uses a FileStore (S3/GCS/local) as the data source.
|
|
@@ -45,8 +44,7 @@ export class FileStoreBlobClient {
|
|
|
45
44
|
|
|
46
45
|
const data = await this.store.read(path);
|
|
47
46
|
const json = JSON.parse(inboundTransform(data).toString()) as BlobJson;
|
|
48
|
-
|
|
49
|
-
blobs.push({ ...json, index: '-1' });
|
|
47
|
+
blobs.push(json);
|
|
50
48
|
} catch (err) {
|
|
51
49
|
this.log.warn(`Failed to read blob ${blobHashes[i]} from filestore`, err);
|
|
52
50
|
}
|
|
@@ -81,8 +79,7 @@ export class FileStoreBlobClient {
|
|
|
81
79
|
return;
|
|
82
80
|
}
|
|
83
81
|
|
|
84
|
-
|
|
85
|
-
const json = blob.toJson(-1);
|
|
82
|
+
const json = blob.toJSON();
|
|
86
83
|
await (this.store as FileStore).save(
|
|
87
84
|
this.blobPath(versionedHash),
|
|
88
85
|
outboundTransform(Buffer.from(JSON.stringify(json))),
|
|
@@ -92,16 +89,11 @@ export class FileStoreBlobClient {
|
|
|
92
89
|
|
|
93
90
|
/**
|
|
94
91
|
* Save multiple blobs to the store in parallel.
|
|
95
|
-
* @param blobs - The blobs to save
|
|
92
|
+
* @param blobs - The blobs to save
|
|
96
93
|
* @param skipIfExists - Skip saving if blob already exists (default: true)
|
|
97
94
|
*/
|
|
98
|
-
async saveBlobs(blobs: Blob[]
|
|
99
|
-
await Promise.all(
|
|
100
|
-
blobs.map(b => {
|
|
101
|
-
const blob = 'blob' in b ? b.blob : b;
|
|
102
|
-
return this.saveBlob(blob, skipIfExists);
|
|
103
|
-
}),
|
|
104
|
-
);
|
|
95
|
+
async saveBlobs(blobs: Blob[], skipIfExists = true): Promise<void> {
|
|
96
|
+
await Promise.all(blobs.map(blob => this.saveBlob(blob, skipIfExists)));
|
|
105
97
|
}
|
|
106
98
|
|
|
107
99
|
/**
|