@aztec/blob-client 0.0.1-commit.9ee6fcc6 → 0.0.1-commit.9ef841308
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dest/client/config.d.ts +5 -1
- package/dest/client/config.d.ts.map +1 -1
- package/dest/client/config.js +10 -0
- package/dest/client/factory.d.ts +1 -1
- package/dest/client/factory.d.ts.map +1 -1
- package/dest/client/factory.js +7 -1
- package/dest/client/http.d.ts +11 -9
- package/dest/client/http.d.ts.map +1 -1
- package/dest/client/http.js +163 -89
- package/dest/client/interface.d.ts +2 -4
- package/dest/client/interface.d.ts.map +1 -1
- package/dest/filestore/factory.d.ts +5 -4
- package/dest/filestore/factory.d.ts.map +1 -1
- package/dest/filestore/factory.js +4 -4
- package/package.json +7 -7
- package/src/client/config.ts +16 -0
- package/src/client/factory.ts +8 -1
- package/src/client/http.ts +179 -95
- package/src/client/interface.ts +1 -3
- package/src/filestore/factory.ts +7 -2
package/dest/client/config.d.ts
CHANGED
|
@@ -42,6 +42,10 @@ export interface BlobClientConfig extends BlobArchiveApiConfig {
|
|
|
42
42
|
blobHealthcheckUploadIntervalMinutes?: number;
|
|
43
43
|
/** Timeout for HTTP requests to the L1 RPC node in ms. */
|
|
44
44
|
l1HttpTimeoutMS?: number;
|
|
45
|
+
/** Whether to prefer filestores over consensus clients when fetching blobs. Default: false (consensus first). */
|
|
46
|
+
blobPreferFilestores?: boolean;
|
|
47
|
+
/** Timeout in ms for HTTP requests to the blob file store. Default: 10000 (10s). */
|
|
48
|
+
blobFileStoreTimeoutMs?: number;
|
|
45
49
|
}
|
|
46
50
|
export declare const blobClientConfigMapping: ConfigMappingsType<BlobClientConfig>;
|
|
47
51
|
/**
|
|
@@ -53,4 +57,4 @@ export declare function getBlobClientConfigFromEnv(): BlobClientConfig;
|
|
|
53
57
|
* Returns whether the given blob client config has any remote sources defined.
|
|
54
58
|
*/
|
|
55
59
|
export declare function hasRemoteBlobSources(config?: BlobClientConfig): boolean;
|
|
56
|
-
//# sourceMappingURL=data:application/json;base64,
|
|
60
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiY29uZmlnLmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi9zcmMvY2xpZW50L2NvbmZpZy50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSxPQUFPLEVBQ0wsS0FBSyxrQkFBa0IsRUFDdkIsV0FBVyxFQUlaLE1BQU0sMEJBQTBCLENBQUM7QUFFbEMsT0FBTyxFQUFFLEtBQUssb0JBQW9CLEVBQWdDLE1BQU0sc0JBQXNCLENBQUM7QUFFL0Y7O0dBRUc7QUFDSCxNQUFNLFdBQVcsZ0JBQWlCLFNBQVEsb0JBQW9CO0lBQzVEOztPQUVHO0lBQ0gsU0FBUyxDQUFDLEVBQUUsTUFBTSxFQUFFLENBQUM7SUFFckI7O09BRUc7SUFDSCxtQkFBbUIsQ0FBQyxFQUFFLE1BQU0sRUFBRSxDQUFDO0lBRS9COztPQUVHO0lBQ0gsc0JBQXNCLENBQUMsRUFBRSxXQUFXLENBQUMsTUFBTSxDQUFDLEVBQUUsQ0FBQztJQUUvQzs7T0FFRztJQUNILDRCQUE0QixDQUFDLEVBQUUsTUFBTSxFQUFFLENBQUM7SUFFeEM7O09BRUc7SUFDSCxpQkFBaUIsQ0FBQyxFQUFFLE1BQU0sQ0FBQztJQUUzQjs7T0FFRztJQUNILHFCQUFxQixDQUFDLEVBQUUsT0FBTyxDQUFDO0lBRWhDOztPQUVHO0lBQ0gsaUJBQWlCLENBQUMsRUFBRSxNQUFNLEVBQUUsQ0FBQztJQUU3Qjs7T0FFRztJQUNILHNCQUFzQixDQUFDLEVBQUUsTUFBTSxDQUFDO0lBRWhDOztPQUVHO0lBQ0gsb0NBQW9DLENBQUMsRUFBRSxNQUFNLENBQUM7SUFFOUMsMERBQTBEO0lBQzFELGVBQWUsQ0FBQyxFQUFFLE1BQU0sQ0FBQztJQUV6QixpSEFBaUg7SUFDakgsb0JBQW9CLENBQUMsRUFBRSxPQUFPLENBQUM7SUFFL0Isb0ZBQW9GO0lBQ3BGLHNCQUFzQixDQUFDLEVBQUUsTUFBTSxDQUFDO0NBQ2pDO0FBRUQsZUFBTyxNQUFNLHVCQUF1QixFQUFFLGtCQUFrQixDQUFDLGdCQUFnQixDQW1FeEUsQ0FBQztBQUVGOzs7R0FHRztBQUNILHdCQUFnQiwwQkFBMEIsSUFBSSxnQkFBZ0IsQ0FFN0Q7QUFFRDs7R0FFRztBQUNILHdCQUFnQixvQkFBb0IsQ0FBQyxNQUFNLEdBQUUsZ0JBQXFCLEdBQUcsT0FBTyxDQUUzRSJ9
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../src/client/config.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,kBAAkB,EACvB,WAAW,EAIZ,MAAM,0BAA0B,CAAC;AAElC,OAAO,EAAE,KAAK,oBAAoB,EAAgC,MAAM,sBAAsB,CAAC;AAE/F;;GAEG;AACH,MAAM,WAAW,gBAAiB,SAAQ,oBAAoB;IAC5D;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IAErB;;OAEG;IACH,mBAAmB,CAAC,EAAE,MAAM,EAAE,CAAC;IAE/B;;OAEG;IACH,sBAAsB,CAAC,EAAE,WAAW,CAAC,MAAM,CAAC,EAAE,CAAC;IAE/C;;OAEG;IACH,4BAA4B,CAAC,EAAE,MAAM,EAAE,CAAC;IAExC;;OAEG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAE3B;;OAEG;IACH,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAEhC;;OAEG;IACH,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;IAE7B;;OAEG;IACH,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAEhC;;OAEG;IACH,oCAAoC,CAAC,EAAE,MAAM,CAAC;IAE9C,0DAA0D;IAC1D,eAAe,CAAC,EAAE,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../src/client/config.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,kBAAkB,EACvB,WAAW,EAIZ,MAAM,0BAA0B,CAAC;AAElC,OAAO,EAAE,KAAK,oBAAoB,EAAgC,MAAM,sBAAsB,CAAC;AAE/F;;GAEG;AACH,MAAM,WAAW,gBAAiB,SAAQ,oBAAoB;IAC5D;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IAErB;;OAEG;IACH,mBAAmB,CAAC,EAAE,MAAM,EAAE,CAAC;IAE/B;;OAEG;IACH,sBAAsB,CAAC,EAAE,WAAW,CAAC,MAAM,CAAC,EAAE,CAAC;IAE/C;;OAEG;IACH,4BAA4B,CAAC,EAAE,MAAM,EAAE,CAAC;IAExC;;OAEG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAE3B;;OAEG;IACH,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAEhC;;OAEG;IACH,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;IAE7B;;OAEG;IACH,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAEhC;;OAEG;IACH,oCAAoC,CAAC,EAAE,MAAM,CAAC;IAE9C,0DAA0D;IAC1D,eAAe,CAAC,EAAE,MAAM,CAAC;IAEzB,iHAAiH;IACjH,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAE/B,oFAAoF;IACpF,sBAAsB,CAAC,EAAE,MAAM,CAAC;CACjC;AAED,eAAO,MAAM,uBAAuB,EAAE,kBAAkB,CAAC,gBAAgB,CAmExE,CAAC;AAEF;;;GAGG;AACH,wBAAgB,0BAA0B,IAAI,gBAAgB,CAE7D;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAAC,MAAM,GAAE,gBAAqB,GAAG,OAAO,CAE3E"}
|
package/dest/client/config.js
CHANGED
|
@@ -50,6 +50,16 @@ export const blobClientConfigMapping = {
|
|
|
50
50
|
description: 'Timeout for HTTP requests to the L1 RPC node in ms.',
|
|
51
51
|
...optionalNumberConfigHelper()
|
|
52
52
|
},
|
|
53
|
+
blobPreferFilestores: {
|
|
54
|
+
env: 'BLOB_PREFER_FILESTORES',
|
|
55
|
+
description: 'Whether to prefer filestores over consensus clients when fetching blobs. Default: false.',
|
|
56
|
+
...booleanConfigHelper(false)
|
|
57
|
+
},
|
|
58
|
+
blobFileStoreTimeoutMs: {
|
|
59
|
+
env: 'BLOB_FILE_STORE_TIMEOUT_MS',
|
|
60
|
+
description: 'Timeout in ms for HTTP requests to the blob file store. Default: 10000 (10s).',
|
|
61
|
+
...optionalNumberConfigHelper()
|
|
62
|
+
},
|
|
53
63
|
...blobArchiveApiConfigMappings
|
|
54
64
|
};
|
|
55
65
|
/**
|
package/dest/client/factory.d.ts
CHANGED
|
@@ -37,4 +37,4 @@ export interface BlobClientWithFileStoresConfig extends BlobClientConfig {
|
|
|
37
37
|
* @returns A BlobClientInterface configured with file store support, already started
|
|
38
38
|
*/
|
|
39
39
|
export declare function createBlobClientWithFileStores(config: BlobClientWithFileStoresConfig, logger?: Logger): Promise<BlobClientInterface>;
|
|
40
|
-
//# sourceMappingURL=data:application/json;base64,
|
|
40
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiZmFjdG9yeS5kLnRzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vLi4vc3JjL2NsaWVudC9mYWN0b3J5LnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLE9BQU8sRUFBRSxLQUFLLE1BQU0sRUFBZ0IsTUFBTSx1QkFBdUIsQ0FBQztBQVFsRSxPQUFPLEtBQUssRUFBRSxtQkFBbUIsRUFBRSxNQUFNLHVDQUF1QyxDQUFDO0FBQ2pGLE9BQU8sRUFBRSxLQUFLLGdCQUFnQixFQUF3QixNQUFNLGFBQWEsQ0FBQztBQUUxRSxPQUFPLEtBQUssRUFBRSxtQkFBbUIsRUFBRSxNQUFNLGdCQUFnQixDQUFDO0FBRzFELE1BQU0sV0FBVyxvQkFBb0I7SUFDbkMsTUFBTSxDQUFDLEVBQUUsTUFBTSxDQUFDO0lBQ2hCLDBDQUEwQztJQUMxQyxnQkFBZ0IsQ0FBQyxFQUFFLG1CQUFtQixFQUFFLENBQUM7SUFDekMsMkNBQTJDO0lBQzNDLHFCQUFxQixDQUFDLEVBQUUsbUJBQW1CLENBQUM7Q0FDN0M7QUFFRCx3QkFBZ0IsZ0JBQWdCLENBQUMsTUFBTSxDQUFDLEVBQUUsZ0JBQWdCLEVBQUUsSUFBSSxDQUFDLEVBQUUsb0JBQW9CLEdBQUcsbUJBQW1CLENBbUI1RztBQUVEOzs7R0FHRztBQUNILE1BQU0sV0FBVyw4QkFBK0IsU0FBUSxnQkFBZ0I7SUFDdEUsU0FBUyxFQUFFLE1BQU0sQ0FBQztJQUNsQixhQUFhLEVBQUUsTUFBTSxDQUFDO0lBQ3RCLFdBQVcsRUFBRTtRQUFFLGFBQWEsRUFBRTtZQUFFLFFBQVEsSUFBSSxNQUFNLENBQUE7U0FBRSxDQUFBO0tBQUUsQ0FBQztDQUN4RDtBQUVEOzs7Ozs7Ozs7Ozs7R0FZRztBQUNILHdCQUFzQiw4QkFBOEIsQ0FDbEQsTUFBTSxFQUFFLDhCQUE4QixFQUN0QyxNQUFNLENBQUMsRUFBRSxNQUFNLEdBQ2QsT0FBTyxDQUFDLG1CQUFtQixDQUFDLENBOEI5QiJ9
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"factory.d.ts","sourceRoot":"","sources":["../../src/client/factory.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,MAAM,EAAgB,MAAM,uBAAuB,CAAC;AAQlE,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,uCAAuC,CAAC;AACjF,OAAO,EAAE,KAAK,gBAAgB,EAAwB,MAAM,aAAa,CAAC;AAE1E,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAG1D,MAAM,WAAW,oBAAoB;IACnC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,0CAA0C;IAC1C,gBAAgB,CAAC,EAAE,mBAAmB,EAAE,CAAC;IACzC,2CAA2C;IAC3C,qBAAqB,CAAC,EAAE,mBAAmB,CAAC;CAC7C;AAED,wBAAgB,gBAAgB,CAAC,MAAM,CAAC,EAAE,gBAAgB,EAAE,IAAI,CAAC,EAAE,oBAAoB,GAAG,mBAAmB,CAmB5G;AAED;;;GAGG;AACH,MAAM,WAAW,8BAA+B,SAAQ,gBAAgB;IACtE,SAAS,EAAE,MAAM,CAAC;IAClB,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE;QAAE,aAAa,EAAE;YAAE,QAAQ,IAAI,MAAM,CAAA;SAAE,CAAA;KAAE,CAAC;CACxD;AAED;;;;;;;;;;;;GAYG;AACH,wBAAsB,8BAA8B,CAClD,MAAM,EAAE,8BAA8B,EACtC,MAAM,CAAC,EAAE,MAAM,GACd,OAAO,CAAC,mBAAmB,CAAC,
|
|
1
|
+
{"version":3,"file":"factory.d.ts","sourceRoot":"","sources":["../../src/client/factory.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,MAAM,EAAgB,MAAM,uBAAuB,CAAC;AAQlE,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,uCAAuC,CAAC;AACjF,OAAO,EAAE,KAAK,gBAAgB,EAAwB,MAAM,aAAa,CAAC;AAE1E,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAG1D,MAAM,WAAW,oBAAoB;IACnC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,0CAA0C;IAC1C,gBAAgB,CAAC,EAAE,mBAAmB,EAAE,CAAC;IACzC,2CAA2C;IAC3C,qBAAqB,CAAC,EAAE,mBAAmB,CAAC;CAC7C;AAED,wBAAgB,gBAAgB,CAAC,MAAM,CAAC,EAAE,gBAAgB,EAAE,IAAI,CAAC,EAAE,oBAAoB,GAAG,mBAAmB,CAmB5G;AAED;;;GAGG;AACH,MAAM,WAAW,8BAA+B,SAAQ,gBAAgB;IACtE,SAAS,EAAE,MAAM,CAAC;IAClB,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE;QAAE,aAAa,EAAE;YAAE,QAAQ,IAAI,MAAM,CAAA;SAAE,CAAA;KAAE,CAAC;CACxD;AAED;;;;;;;;;;;;GAYG;AACH,wBAAsB,8BAA8B,CAClD,MAAM,EAAE,8BAA8B,EACtC,MAAM,CAAC,EAAE,MAAM,GACd,OAAO,CAAC,mBAAmB,CAAC,CA8B9B"}
|
package/dest/client/factory.js
CHANGED
|
@@ -42,8 +42,14 @@ export function createBlobClient(config, deps) {
|
|
|
42
42
|
rollupVersion: config.rollupVersion,
|
|
43
43
|
rollupAddress: config.l1Contracts.rollupAddress.toString()
|
|
44
44
|
};
|
|
45
|
+
// Disable internal retries for blob file stores — retry logic is handled by HttpBlobClient.
|
|
46
|
+
// Set a configurable timeout (default 10s) to avoid hanging on slow stores.
|
|
47
|
+
const httpOptions = {
|
|
48
|
+
retryBackoff: [],
|
|
49
|
+
timeoutMs: config.blobFileStoreTimeoutMs ?? 10_000
|
|
50
|
+
};
|
|
45
51
|
const [fileStoreClients, fileStoreUploadClient] = await Promise.all([
|
|
46
|
-
createReadOnlyFileStoreBlobClients(config.blobFileStoreUrls, fileStoreMetadata, log),
|
|
52
|
+
createReadOnlyFileStoreBlobClients(config.blobFileStoreUrls, fileStoreMetadata, log, httpOptions),
|
|
47
53
|
createWritableFileStoreBlobClient(config.blobFileStoreUploadUrl, fileStoreMetadata, log)
|
|
48
54
|
]);
|
|
49
55
|
const client = createBlobClient(config, {
|
package/dest/client/http.d.ts
CHANGED
|
@@ -18,6 +18,8 @@ export declare class HttpBlobClient implements BlobClientInterface {
|
|
|
18
18
|
private beaconGenesisTime?;
|
|
19
19
|
/** Cached beacon slot duration in seconds. Fetched once at startup. */
|
|
20
20
|
private beaconSecondsPerSlot?;
|
|
21
|
+
/** Indexes of consensus hosts that serve blob sidecars (supernodes). Populated by testSources(). */
|
|
22
|
+
private superNodeHostIndexes?;
|
|
21
23
|
constructor(config?: BlobClientConfig, opts?: {
|
|
22
24
|
logger?: Logger;
|
|
23
25
|
archiveClient?: BlobArchiveClient;
|
|
@@ -41,21 +43,21 @@ export declare class HttpBlobClient implements BlobClientInterface {
|
|
|
41
43
|
testSources(): Promise<void>;
|
|
42
44
|
sendBlobsToFilestore(blobs: Blob[]): Promise<boolean>;
|
|
43
45
|
/**
|
|
44
|
-
* Get the blob sidecar
|
|
46
|
+
* Get the blob sidecar.
|
|
45
47
|
*
|
|
46
|
-
*
|
|
47
|
-
*
|
|
48
|
-
*
|
|
49
|
-
* Source ordering depends on sync state:
|
|
50
|
-
* - Historical sync: blob client → FileStore → L1 consensus → Archive
|
|
51
|
-
* - Near tip sync: blob client → FileStore → L1 consensus → FileStore (with retries) → Archive (eg blobscan)
|
|
48
|
+
* Alternates between two primary sources (consensus and filestore) in a retry loop,
|
|
49
|
+
* then falls back to archive if blobs are still missing. The order of the primary
|
|
50
|
+
* sources is configurable via `blobPreferFilestores`.
|
|
52
51
|
*
|
|
53
52
|
* @param blockHash - The block hash
|
|
54
53
|
* @param blobHashes - The blob hashes to fetch
|
|
55
|
-
* @param opts - Options
|
|
54
|
+
* @param opts - Options for slot resolution
|
|
56
55
|
* @returns The blobs
|
|
57
56
|
*/
|
|
58
57
|
getBlobSidecar(blockHash: `0x${string}`, blobHashes: Buffer[], opts?: GetBlobSidecarOptions): Promise<Blob[]>;
|
|
58
|
+
/** Resolves the beacon slot number for the given block hash. Returns undefined if no consensus hosts. */
|
|
59
|
+
private resolveSlotNumber;
|
|
60
|
+
private tryConsensusHosts;
|
|
59
61
|
private tryFileStores;
|
|
60
62
|
getBlobSidecarFrom(hostUrl: string, blockHashOrSlot: string | number, blobHashes?: Buffer[], l1ConsensusHostIndex?: number): Promise<Blob[]>;
|
|
61
63
|
getBlobsFromHost(hostUrl: string, blockHashOrSlot: string | number, l1ConsensusHostIndex?: number, blobHashes?: Buffer[]): Promise<BlobJson[]>;
|
|
@@ -82,4 +84,4 @@ export declare class HttpBlobClient implements BlobClientInterface {
|
|
|
82
84
|
*/
|
|
83
85
|
stop(): void;
|
|
84
86
|
}
|
|
85
|
-
//# sourceMappingURL=data:application/json;base64,
|
|
87
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiaHR0cC5kLnRzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vLi4vc3JjL2NsaWVudC9odHRwLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLE9BQU8sRUFBRSxJQUFJLEVBQUUsS0FBSyxRQUFRLEVBQStCLE1BQU0saUJBQWlCLENBQUM7QUFHbkYsT0FBTyxFQUFFLEtBQUssTUFBTSxFQUFnQixNQUFNLHVCQUF1QixDQUFDO0FBT2xFLE9BQU8sS0FBSyxFQUFFLGlCQUFpQixFQUFFLE1BQU0seUJBQXlCLENBQUM7QUFDakUsT0FBTyxLQUFLLEVBQUUsbUJBQW1CLEVBQUUsTUFBTSx1Q0FBdUMsQ0FBQztBQUVqRixPQUFPLEVBQUUsS0FBSyxnQkFBZ0IsRUFBOEIsTUFBTSxhQUFhLENBQUM7QUFDaEYsT0FBTyxLQUFLLEVBQUUsbUJBQW1CLEVBQUUscUJBQXFCLEVBQUUsTUFBTSxnQkFBZ0IsQ0FBQztBQUVqRixxQkFBYSxjQUFlLFlBQVcsbUJBQW1CO0lBcUJ0RCxPQUFPLENBQUMsUUFBUSxDQUFDLElBQUk7SUFwQnZCLFNBQVMsQ0FBQyxRQUFRLENBQUMsR0FBRyxFQUFFLE1BQU0sQ0FBQztJQUMvQixTQUFTLENBQUMsUUFBUSxDQUFDLE1BQU0sRUFBRSxnQkFBZ0IsQ0FBQztJQUM1QyxTQUFTLENBQUMsUUFBUSxDQUFDLGFBQWEsRUFBRSxpQkFBaUIsR0FBRyxTQUFTLENBQUM7SUFDaEUsU0FBUyxDQUFDLFFBQVEsQ0FBQyxLQUFLLEVBQUUsT0FBTyxLQUFLLENBQUM7SUFDdkMsU0FBUyxDQUFDLFFBQVEsQ0FBQyxnQkFBZ0IsRUFBRSxtQkFBbUIsRUFBRSxDQUFDO0lBQzNELFNBQVMsQ0FBQyxRQUFRLENBQUMscUJBQXFCLEVBQUUsbUJBQW1CLEdBQUcsU0FBUyxDQUFDO0lBRTFFLE9BQU8sQ0FBQyxRQUFRLENBQVM7SUFDekIsT0FBTyxDQUFDLDJCQUEyQixDQUFDLENBQWlCO0lBRXJELHNGQUFzRjtJQUN0RixPQUFPLENBQUMsaUJBQWlCLENBQUMsQ0FBUztJQUNuQyx1RUFBdUU7SUFDdkUsT0FBTyxDQUFDLG9CQUFvQixDQUFDLENBQVM7SUFFdEMsb0dBQW9HO0lBQ3BHLE9BQU8sQ0FBQyxvQkFBb0IsQ0FBQyxDQUFjO0lBRTNDLFlBQ0UsTUFBTSxDQUFDLEVBQUUsZ0JBQWdCLEVBQ1IsSUFBSSxHQUFFO1FBQ3JCLE1BQU0sQ0FBQyxFQUFFLE1BQU0sQ0FBQztRQUNoQixhQUFhLENBQUMsRUFBRSxpQkFBaUIsQ0FBQztRQUNsQyxnQkFBZ0IsQ0FBQyxFQUFFLG1CQUFtQixFQUFFLENBQUM7UUFDekMscUJBQXFCLENBQUMsRUFBRSxtQkFBbUIsQ0FBQztRQUM1Qyx5RUFBeUU7UUFDekUsY0FBYyxDQUFDLEVBQUUsQ0FBQyxLQUFLLEVBQUUsSUFBSSxFQUFFLEtBQUssSUFBSSxDQUFDO0tBQ3JDLEVBd0JQO0lBRUQ7OztPQUdHO0lBQ0gsT0FBTyxDQUFDLHNCQUFzQjtJQVU5Qjs7Ozs7T0FLRztJQUNJLFdBQVcsQ0FBQyxLQUFLLEVBQUUsT0FBTyxHQUFHLElBQUksQ0FHdkM7SUFFWSxXQUFXLGtCQStHdkI7SUFFWSxvQkFBb0IsQ0FBQyxLQUFLLEVBQUUsSUFBSSxFQUFFLEdBQUcsT0FBTyxDQUFDLE9BQU8sQ0FBQyxDQW1CakU7SUFFRDs7Ozs7Ozs7Ozs7T0FXRztJQUNVLGNBQWMsQ0FDekIsU0FBUyxFQUFFLEtBQUssTUFBTSxFQUFFLEVBQ3hCLFVBQVUsRUFBRSxNQUFNLEVBQUUsRUFDcEIsSUFBSSxDQUFDLEVBQUUscUJBQXFCLEdBQzNCLE9BQU8sQ0FBQyxJQUFJLEVBQUUsQ0FBQyxDQXlIakI7SUFFRCx5R0FBeUc7SUFDekcsT0FBTyxDQUFDLGlCQUFpQjtZQW1CWCxpQkFBaUI7WUFtRGpCLGFBQWE7SUFzQ2Qsa0JBQWtCLENBQzdCLE9BQU8sRUFBRSxNQUFNLEVBQ2YsZUFBZSxFQUFFLE1BQU0sR0FBRyxNQUFNLEVBQ2hDLFVBQVUsR0FBRSxNQUFNLEVBQU8sRUFDekIsb0JBQW9CLENBQUMsRUFBRSxNQUFNLEdBQzVCLE9BQU8sQ0FBQyxJQUFJLEVBQUUsQ0FBQyxDQUdqQjtJQUVZLGdCQUFnQixDQUMzQixPQUFPLEVBQUUsTUFBTSxFQUNmLGVBQWUsRUFBRSxNQUFNLEdBQUcsTUFBTSxFQUNoQyxvQkFBb0IsQ0FBQyxFQUFFLE1BQU0sRUFDN0IsVUFBVSxDQUFDLEVBQUUsTUFBTSxFQUFFLEdBQ3BCLE9BQU8sQ0FBQyxRQUFRLEVBQUUsQ0FBQyxDQXNDckI7SUFFRCxPQUFPLENBQUMsaUJBQWlCO1lBc0JYLG1CQUFtQjtZQW1DbkIsYUFBYTtJQTZFM0Isc0NBQXNDO0lBQy9CLGdCQUFnQixJQUFJLGlCQUFpQixHQUFHLFNBQVMsQ0FFdkQ7SUFFRCxpRUFBaUU7SUFDMUQsU0FBUyxJQUFJLE9BQU8sQ0FFMUI7SUFFRDs7OztPQUlHO0lBQ1UsS0FBSyxJQUFJLE9BQU8sQ0FBQyxJQUFJLENBQUMsQ0FXbEM7SUFFRDs7T0FFRztJQUNILE9BQU8sQ0FBQyw4QkFBOEI7WUFnQnhCLGlCQUFpQjtJQTBDL0I7O09BRUc7SUFDSSxJQUFJLElBQUksSUFBSSxDQUtsQjtDQUNGIn0=
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"http.d.ts","sourceRoot":"","sources":["../../src/client/http.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,KAAK,QAAQ,EAA+B,MAAM,iBAAiB,CAAC;AAGnF,OAAO,EAAE,KAAK,MAAM,EAAgB,MAAM,uBAAuB,CAAC;AAOlE,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,yBAAyB,CAAC;AACjE,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,uCAAuC,CAAC;AAEjF,OAAO,EAAE,KAAK,gBAAgB,EAA8B,MAAM,aAAa,CAAC;AAChF,OAAO,KAAK,EAAE,mBAAmB,EAAE,qBAAqB,EAAE,MAAM,gBAAgB,CAAC;AAEjF,qBAAa,cAAe,YAAW,mBAAmB;
|
|
1
|
+
{"version":3,"file":"http.d.ts","sourceRoot":"","sources":["../../src/client/http.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,KAAK,QAAQ,EAA+B,MAAM,iBAAiB,CAAC;AAGnF,OAAO,EAAE,KAAK,MAAM,EAAgB,MAAM,uBAAuB,CAAC;AAOlE,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,yBAAyB,CAAC;AACjE,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,uCAAuC,CAAC;AAEjF,OAAO,EAAE,KAAK,gBAAgB,EAA8B,MAAM,aAAa,CAAC;AAChF,OAAO,KAAK,EAAE,mBAAmB,EAAE,qBAAqB,EAAE,MAAM,gBAAgB,CAAC;AAEjF,qBAAa,cAAe,YAAW,mBAAmB;IAqBtD,OAAO,CAAC,QAAQ,CAAC,IAAI;IApBvB,SAAS,CAAC,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IAC/B,SAAS,CAAC,QAAQ,CAAC,MAAM,EAAE,gBAAgB,CAAC;IAC5C,SAAS,CAAC,QAAQ,CAAC,aAAa,EAAE,iBAAiB,GAAG,SAAS,CAAC;IAChE,SAAS,CAAC,QAAQ,CAAC,KAAK,EAAE,OAAO,KAAK,CAAC;IACvC,SAAS,CAAC,QAAQ,CAAC,gBAAgB,EAAE,mBAAmB,EAAE,CAAC;IAC3D,SAAS,CAAC,QAAQ,CAAC,qBAAqB,EAAE,mBAAmB,GAAG,SAAS,CAAC;IAE1E,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,2BAA2B,CAAC,CAAiB;IAErD,sFAAsF;IACtF,OAAO,CAAC,iBAAiB,CAAC,CAAS;IACnC,uEAAuE;IACvE,OAAO,CAAC,oBAAoB,CAAC,CAAS;IAEtC,oGAAoG;IACpG,OAAO,CAAC,oBAAoB,CAAC,CAAc;IAE3C,YACE,MAAM,CAAC,EAAE,gBAAgB,EACR,IAAI,GAAE;QACrB,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,aAAa,CAAC,EAAE,iBAAiB,CAAC;QAClC,gBAAgB,CAAC,EAAE,mBAAmB,EAAE,CAAC;QACzC,qBAAqB,CAAC,EAAE,mBAAmB,CAAC;QAC5C,yEAAyE;QACzE,cAAc,CAAC,EAAE,CAAC,KAAK,EAAE,IAAI,EAAE,KAAK,IAAI,CAAC;KACrC,EAwBP;IAED;;;OAGG;IACH,OAAO,CAAC,sBAAsB;IAU9B;;;;;OAKG;IACI,WAAW,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI,CAGvC;IAEY,WAAW,kBA+GvB;IAEY,oBAAoB,CAAC,KAAK,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC,CAmBjE;IAED;;;;;;;;;;;OAWG;IACU,cAAc,CACzB,SAAS,EAAE,KAAK,MAAM,EAAE,EACxB,UAAU,EAAE,MAAM,EAAE,EACpB,IAAI,CAAC,EAAE,qBAAqB,GAC3B,OAAO,CAAC,IAAI,EAAE,CAAC,CAyHjB;IAED,yGAAyG;IACzG,OAAO,CAAC,iBAAiB;YAmBX,iBAAiB;YAmDjB,aAAa;IAsCd,kBAAkB,CAC7B,OAAO,EAAE,MAAM,EACf,eAAe,EAAE,MAAM,GAAG,MAAM,EAChC,UAAU,GAAE,MAAM,EAAO,EACzB,oBAAoB,CAAC,EAAE,MAAM,GAC5B,OAAO,CAAC,IAAI,EAAE,CAAC,CAGjB;IAEY,gBAAgB,CAC3B,OAAO,EAAE,MAAM,EACf,eAAe,EAAE,MAAM,GAAG,MAAM,EAChC,oBAAoB,CAAC,EAAE,MAAM,EAC7B,UAAU,CAAC,EAAE,MAAM,EAAE,GACpB,OAAO,CAAC,QAAQ,EAAE,CAAC,CAsCrB;IAED,OAAO,CAAC,iBAAiB;YAsBX,mBAAmB;YAmCnB,aAAa;IA6E3B,sCAAsC;IAC/B,gBAAgB,IAAI,iBAAiB,GAAG,SAAS,CAEvD;IAED,iEAAiE;IAC1D,SAAS,IAAI,OAAO,CAE1B;IAED;;;;OAIG;IACU,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAWlC;IAED;;OAEG;IACH,OAAO,CAAC,8BAA8B;YAgBxB,iBAAiB;IA0C/B;;OAEG;IACI,IAAI,IAAI,IAAI,CAKlB;CACF"}
|
package/dest/client/http.js
CHANGED
|
@@ -20,6 +20,7 @@ export class HttpBlobClient {
|
|
|
20
20
|
healthcheckUploadIntervalId;
|
|
21
21
|
/** Cached beacon genesis time (seconds since Unix epoch). Fetched once at startup. */ beaconGenesisTime;
|
|
22
22
|
/** Cached beacon slot duration in seconds. Fetched once at startup. */ beaconSecondsPerSlot;
|
|
23
|
+
/** Indexes of consensus hosts that serve blob sidecars (supernodes). Populated by testSources(). */ superNodeHostIndexes;
|
|
23
24
|
constructor(config, opts = {}){
|
|
24
25
|
this.opts = opts;
|
|
25
26
|
this.disabled = false;
|
|
@@ -69,22 +70,52 @@ export class HttpBlobClient {
|
|
|
69
70
|
l1ConsensusHostUrls,
|
|
70
71
|
archiveUrl
|
|
71
72
|
});
|
|
72
|
-
let
|
|
73
|
+
let consensusSuperNodes = 0;
|
|
74
|
+
let consensusNonSuperNodes = 0;
|
|
75
|
+
let archiveSources = 0;
|
|
76
|
+
let blobSinks = 0;
|
|
77
|
+
const detectedSuperNodes = new Set();
|
|
73
78
|
if (l1ConsensusHostUrls && l1ConsensusHostUrls.length > 0) {
|
|
74
79
|
for(let l1ConsensusHostIndex = 0; l1ConsensusHostIndex < l1ConsensusHostUrls.length; l1ConsensusHostIndex++){
|
|
75
80
|
const l1ConsensusHostUrl = l1ConsensusHostUrls[l1ConsensusHostIndex];
|
|
76
81
|
try {
|
|
77
|
-
const { url, ...options } = getBeaconNodeFetchOptions(`${l1ConsensusHostUrl}/eth/v1/beacon/headers`, this.config, l1ConsensusHostIndex);
|
|
82
|
+
const { url, ...options } = getBeaconNodeFetchOptions(`${l1ConsensusHostUrl}/eth/v1/beacon/headers/head`, this.config, l1ConsensusHostIndex);
|
|
78
83
|
const res = await this.fetch(url, options);
|
|
79
|
-
if (res.ok) {
|
|
80
|
-
this.log.
|
|
84
|
+
if (!res.ok) {
|
|
85
|
+
this.log.error(`Failure reaching L1 consensus host: ${res.statusText} (${res.status})`, {
|
|
81
86
|
l1ConsensusHostUrl
|
|
82
87
|
});
|
|
83
|
-
|
|
88
|
+
continue;
|
|
89
|
+
}
|
|
90
|
+
this.log.info(`L1 consensus host is reachable`, {
|
|
91
|
+
l1ConsensusHostUrl
|
|
92
|
+
});
|
|
93
|
+
// Check if the host serves blob sidecars (supernode/semi-supernode).
|
|
94
|
+
// Post-Fusaka (PeerDAS), non-supernode beacon nodes no longer serve the
|
|
95
|
+
// blob sidecar endpoint. A 200 response (even with an empty data array
|
|
96
|
+
// for a slot with no blobs) means the node supports serving blob sidecars.
|
|
97
|
+
const body = await res.json();
|
|
98
|
+
const headSlot = body?.data?.header?.message?.slot;
|
|
99
|
+
if (headSlot) {
|
|
100
|
+
const { url: blobUrl, ...blobOptions } = getBeaconNodeFetchOptions(`${l1ConsensusHostUrl}/eth/v1/beacon/blobs/${headSlot}`, this.config, l1ConsensusHostIndex);
|
|
101
|
+
const blobRes = await this.fetch(blobUrl, blobOptions);
|
|
102
|
+
if (blobRes.ok) {
|
|
103
|
+
this.log.info(`L1 consensus host serves blob sidecars (supernode)`, {
|
|
104
|
+
l1ConsensusHostUrl
|
|
105
|
+
});
|
|
106
|
+
detectedSuperNodes.add(l1ConsensusHostIndex);
|
|
107
|
+
consensusSuperNodes++;
|
|
108
|
+
} else {
|
|
109
|
+
this.log.info(`L1 consensus host does not serve blob sidecars, skipping for blob fetching`, {
|
|
110
|
+
l1ConsensusHostUrl
|
|
111
|
+
});
|
|
112
|
+
consensusNonSuperNodes++;
|
|
113
|
+
}
|
|
84
114
|
} else {
|
|
85
|
-
this.log.
|
|
115
|
+
this.log.info(`L1 consensus host is reachable but could not determine head slot`, {
|
|
86
116
|
l1ConsensusHostUrl
|
|
87
117
|
});
|
|
118
|
+
consensusNonSuperNodes++;
|
|
88
119
|
}
|
|
89
120
|
} catch (err) {
|
|
90
121
|
this.log.error(`Error reaching L1 consensus host`, err, {
|
|
@@ -92,9 +123,8 @@ export class HttpBlobClient {
|
|
|
92
123
|
});
|
|
93
124
|
}
|
|
94
125
|
}
|
|
95
|
-
} else {
|
|
96
|
-
this.log.warn('No L1 consensus host urls configured');
|
|
97
126
|
}
|
|
127
|
+
this.superNodeHostIndexes = detectedSuperNodes;
|
|
98
128
|
if (this.archiveClient) {
|
|
99
129
|
try {
|
|
100
130
|
const latest = await this.archiveClient.getLatestBlock();
|
|
@@ -102,14 +132,12 @@ export class HttpBlobClient {
|
|
|
102
132
|
latest,
|
|
103
133
|
archiveUrl
|
|
104
134
|
});
|
|
105
|
-
|
|
135
|
+
archiveSources++;
|
|
106
136
|
} catch (err) {
|
|
107
137
|
this.log.error(`Error reaching archive client`, err, {
|
|
108
138
|
archiveUrl
|
|
109
139
|
});
|
|
110
140
|
}
|
|
111
|
-
} else {
|
|
112
|
-
this.log.warn('No archive client configured');
|
|
113
141
|
}
|
|
114
142
|
if (this.fileStoreClients.length > 0) {
|
|
115
143
|
for (const fileStoreClient of this.fileStoreClients){
|
|
@@ -119,7 +147,7 @@ export class HttpBlobClient {
|
|
|
119
147
|
this.log.info(`FileStore is reachable`, {
|
|
120
148
|
url: fileStoreClient.getBaseUrl()
|
|
121
149
|
});
|
|
122
|
-
|
|
150
|
+
blobSinks++;
|
|
123
151
|
} else {
|
|
124
152
|
this.log.warn(`FileStore is not accessible`, {
|
|
125
153
|
url: fileStoreClient.getBaseUrl()
|
|
@@ -132,12 +160,22 @@ export class HttpBlobClient {
|
|
|
132
160
|
}
|
|
133
161
|
}
|
|
134
162
|
}
|
|
163
|
+
// Emit a single summary after validating all sources
|
|
164
|
+
const successfulSourceCount = consensusSuperNodes + archiveSources + blobSinks;
|
|
165
|
+
let summary = `Blob client running with consensusSuperNodes=${consensusSuperNodes} archiveSources=${archiveSources} blobSinks=${blobSinks}`;
|
|
166
|
+
if (consensusNonSuperNodes > 0) {
|
|
167
|
+
summary += `. ${consensusNonSuperNodes} consensus client(s) ignored because they are not running in supernode or semi-supernode mode`;
|
|
168
|
+
}
|
|
135
169
|
if (successfulSourceCount === 0) {
|
|
136
170
|
if (this.config.blobAllowEmptySources) {
|
|
137
|
-
this.log.warn(
|
|
171
|
+
this.log.warn(summary);
|
|
138
172
|
} else {
|
|
139
|
-
throw new Error(
|
|
173
|
+
throw new Error(summary);
|
|
140
174
|
}
|
|
175
|
+
} else if (consensusSuperNodes === 0) {
|
|
176
|
+
this.log.warn(summary);
|
|
177
|
+
} else {
|
|
178
|
+
this.log.info(summary);
|
|
141
179
|
}
|
|
142
180
|
}
|
|
143
181
|
async sendBlobsToFilestore(blobs) {
|
|
@@ -159,29 +197,25 @@ export class HttpBlobClient {
|
|
|
159
197
|
}
|
|
160
198
|
}
|
|
161
199
|
/**
|
|
162
|
-
* Get the blob sidecar
|
|
163
|
-
*
|
|
164
|
-
* If requesting from the blob client, we send the blobkHash
|
|
165
|
-
* If requesting from the beacon node, we send the slot number
|
|
200
|
+
* Get the blob sidecar.
|
|
166
201
|
*
|
|
167
|
-
*
|
|
168
|
-
*
|
|
169
|
-
*
|
|
202
|
+
* Alternates between two primary sources (consensus and filestore) in a retry loop,
|
|
203
|
+
* then falls back to archive if blobs are still missing. The order of the primary
|
|
204
|
+
* sources is configurable via `blobPreferFilestores`.
|
|
170
205
|
*
|
|
171
206
|
* @param blockHash - The block hash
|
|
172
207
|
* @param blobHashes - The blob hashes to fetch
|
|
173
|
-
* @param opts - Options
|
|
208
|
+
* @param opts - Options for slot resolution
|
|
174
209
|
* @returns The blobs
|
|
175
210
|
*/ async getBlobSidecar(blockHash, blobHashes, opts) {
|
|
176
211
|
if (this.disabled) {
|
|
177
212
|
this.log.warn('Blob storage is disabled, returning empty blob sidecar');
|
|
178
213
|
return [];
|
|
179
214
|
}
|
|
180
|
-
const isHistoricalSync = opts?.isHistoricalSync ?? false;
|
|
181
215
|
// Accumulate blobs across sources, preserving order and handling duplicates
|
|
182
216
|
// resultBlobs[i] will contain the blob for blobHashes[i], or undefined if not yet found
|
|
183
217
|
const resultBlobs = new Array(blobHashes.length).fill(undefined);
|
|
184
|
-
// Helper to get
|
|
218
|
+
// Helper to get missing blob hashes that we still need to fetch
|
|
185
219
|
const getMissingBlobHashes = ()=>blobHashes.map((bh, i)=>resultBlobs[i] === undefined ? bh : undefined).filter((bh)=>bh !== undefined);
|
|
186
220
|
// Return the result, ignoring any undefined ones
|
|
187
221
|
const getFilledBlobs = ()=>resultBlobs.filter((b)=>b !== undefined);
|
|
@@ -203,80 +237,69 @@ export class HttpBlobClient {
|
|
|
203
237
|
}
|
|
204
238
|
return blobs;
|
|
205
239
|
};
|
|
206
|
-
const { l1ConsensusHostUrls } = this.config;
|
|
207
240
|
const ctx = {
|
|
208
241
|
blockHash,
|
|
209
242
|
blobHashes: blobHashes.map(bufferToHex)
|
|
210
243
|
};
|
|
211
|
-
//
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
244
|
+
// Lazily resolve the slot number — only resolved when consensus hosts are actually tried.
|
|
245
|
+
let slotNumber;
|
|
246
|
+
let slotResolved = false;
|
|
247
|
+
const getSlotNumber = async ()=>{
|
|
248
|
+
if (!slotResolved) {
|
|
249
|
+
slotNumber = await this.resolveSlotNumber(blockHash, opts);
|
|
250
|
+
slotResolved = true;
|
|
216
251
|
}
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
if (result.length === blobHashes.length) {
|
|
249
|
-
return returnWithCallback(result);
|
|
250
|
-
}
|
|
252
|
+
return slotNumber;
|
|
253
|
+
};
|
|
254
|
+
// Build the two source-try functions. The order depends on the config.
|
|
255
|
+
const tryConsensus = ()=>this.tryConsensusHosts(getSlotNumber, getMissingBlobHashes, fillResults, ctx);
|
|
256
|
+
const tryFilestores = ()=>this.tryFileStores(getMissingBlobHashes, fillResults, ctx);
|
|
257
|
+
const preferFilestores = this.config.blobPreferFilestores ?? false;
|
|
258
|
+
const [trySourceA, trySourceB] = preferFilestores ? [
|
|
259
|
+
tryFilestores,
|
|
260
|
+
tryConsensus
|
|
261
|
+
] : [
|
|
262
|
+
tryConsensus,
|
|
263
|
+
tryFilestores
|
|
264
|
+
];
|
|
265
|
+
// Historical sync: blobs should already exist, use shorter backoff for transient errors.
|
|
266
|
+
// Near-tip sync: blobs may still be uploading, use longer backoff for eventual consistency.
|
|
267
|
+
const isHistoricalSync = opts?.isHistoricalSync ?? false;
|
|
268
|
+
const backoff = isHistoricalSync ? [
|
|
269
|
+
1,
|
|
270
|
+
1
|
|
271
|
+
] : [
|
|
272
|
+
1,
|
|
273
|
+
1,
|
|
274
|
+
1,
|
|
275
|
+
2,
|
|
276
|
+
2
|
|
277
|
+
];
|
|
278
|
+
// Retry loop: alternate between the two primary sources with backoff.
|
|
279
|
+
try {
|
|
280
|
+
await retry(async ()=>{
|
|
281
|
+
if (getMissingBlobHashes().length > 0) {
|
|
282
|
+
await trySourceA();
|
|
251
283
|
}
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
}
|
|
263
|
-
}, 'filestore blob retrieval', makeBackoff([
|
|
264
|
-
1,
|
|
265
|
-
1,
|
|
266
|
-
2
|
|
267
|
-
]), this.log, true);
|
|
268
|
-
return returnWithCallback(getFilledBlobs());
|
|
269
|
-
} catch {
|
|
270
|
-
// Exhausted retries, continue to archive fallback
|
|
271
|
-
}
|
|
284
|
+
if (getMissingBlobHashes().length > 0) {
|
|
285
|
+
await trySourceB();
|
|
286
|
+
}
|
|
287
|
+
if (getMissingBlobHashes().length > 0) {
|
|
288
|
+
throw new Error('Still missing blobs after trying all primary sources');
|
|
289
|
+
}
|
|
290
|
+
}, 'blob retrieval', makeBackoff(backoff), this.log, true);
|
|
291
|
+
return returnWithCallback(getFilledBlobs());
|
|
292
|
+
} catch {
|
|
293
|
+
// Exhausted retries, continue to archive fallback
|
|
272
294
|
}
|
|
273
|
-
|
|
274
|
-
|
|
295
|
+
// Archive fallback
|
|
296
|
+
const missingAfterPrimary = getMissingBlobHashes();
|
|
297
|
+
if (missingAfterPrimary.length > 0 && this.archiveClient) {
|
|
275
298
|
const archiveCtx = {
|
|
276
299
|
archiveUrl: this.archiveClient.getBaseUrl(),
|
|
277
300
|
...ctx
|
|
278
301
|
};
|
|
279
|
-
this.log.trace(`Attempting to get ${
|
|
302
|
+
this.log.trace(`Attempting to get ${missingAfterPrimary.length} blobs from archive`, archiveCtx);
|
|
280
303
|
const allBlobs = await this.archiveClient.getBlobsFromBlock(blockHash);
|
|
281
304
|
if (!allBlobs) {
|
|
282
305
|
this.log.debug('No blobs found from archive client', archiveCtx);
|
|
@@ -292,13 +315,63 @@ export class HttpBlobClient {
|
|
|
292
315
|
const result = getFilledBlobs();
|
|
293
316
|
if (result.length < blobHashes.length) {
|
|
294
317
|
this.log.warn(`Failed to fetch all blobs for ${blockHash} from all blob sources (got ${result.length}/${blobHashes.length})`, {
|
|
295
|
-
l1ConsensusHostUrls,
|
|
318
|
+
l1ConsensusHostUrls: this.config.l1ConsensusHostUrls,
|
|
296
319
|
archiveUrl: this.archiveClient?.getBaseUrl(),
|
|
297
320
|
fileStoreUrls: this.fileStoreClients.map((c)=>c.getBaseUrl())
|
|
298
321
|
});
|
|
299
322
|
}
|
|
300
323
|
return returnWithCallback(result);
|
|
301
324
|
}
|
|
325
|
+
/** Resolves the beacon slot number for the given block hash. Returns undefined if no consensus hosts. */ resolveSlotNumber(blockHash, opts) {
|
|
326
|
+
const { l1ConsensusHostUrls } = this.config;
|
|
327
|
+
if (!l1ConsensusHostUrls || l1ConsensusHostUrls.length === 0) {
|
|
328
|
+
return undefined;
|
|
329
|
+
}
|
|
330
|
+
// If no supernodes, no point resolving the slot
|
|
331
|
+
if (this.superNodeHostIndexes && this.superNodeHostIndexes.size === 0) {
|
|
332
|
+
return undefined;
|
|
333
|
+
}
|
|
334
|
+
return this.getSlotNumber(blockHash, opts?.parentBeaconBlockRoot, opts?.l1BlockTimestamp);
|
|
335
|
+
}
|
|
336
|
+
/**
|
|
337
|
+
* Try all supernode consensus hosts for blob sidecars.
|
|
338
|
+
* Skips hosts that were detected as non-supernodes during testSources().
|
|
339
|
+
*/ async tryConsensusHosts(getSlotNumber, getMissingBlobHashes, fillResults, ctx) {
|
|
340
|
+
const { l1ConsensusHostUrls } = this.config;
|
|
341
|
+
if (!l1ConsensusHostUrls || l1ConsensusHostUrls.length === 0) {
|
|
342
|
+
return;
|
|
343
|
+
}
|
|
344
|
+
const slotNumber = await getSlotNumber();
|
|
345
|
+
if (!slotNumber) {
|
|
346
|
+
return;
|
|
347
|
+
}
|
|
348
|
+
for(let l1ConsensusHostIndex = 0; l1ConsensusHostIndex < l1ConsensusHostUrls.length; l1ConsensusHostIndex++){
|
|
349
|
+
const missingHashes = getMissingBlobHashes();
|
|
350
|
+
if (missingHashes.length === 0) {
|
|
351
|
+
break;
|
|
352
|
+
}
|
|
353
|
+
// Skip non-supernode hosts if we've already detected supernodes
|
|
354
|
+
if (this.superNodeHostIndexes && !this.superNodeHostIndexes.has(l1ConsensusHostIndex)) {
|
|
355
|
+
this.log.trace(`Skipping non-supernode consensus host`, {
|
|
356
|
+
l1ConsensusHostUrl: l1ConsensusHostUrls[l1ConsensusHostIndex]
|
|
357
|
+
});
|
|
358
|
+
continue;
|
|
359
|
+
}
|
|
360
|
+
const l1ConsensusHostUrl = l1ConsensusHostUrls[l1ConsensusHostIndex];
|
|
361
|
+
this.log.trace(`Attempting to get ${missingHashes.length} blobs from consensus host`, {
|
|
362
|
+
slotNumber,
|
|
363
|
+
l1ConsensusHostUrl,
|
|
364
|
+
...ctx
|
|
365
|
+
});
|
|
366
|
+
const blobs = await this.getBlobsFromHost(l1ConsensusHostUrl, slotNumber, l1ConsensusHostIndex, missingHashes);
|
|
367
|
+
const result = await fillResults(blobs);
|
|
368
|
+
this.log.debug(`Got ${blobs.length} blobs from consensus host (total: ${result.length}/${ctx.blobHashes.length})`, {
|
|
369
|
+
slotNumber,
|
|
370
|
+
l1ConsensusHostUrl,
|
|
371
|
+
...ctx
|
|
372
|
+
});
|
|
373
|
+
}
|
|
374
|
+
}
|
|
302
375
|
/**
|
|
303
376
|
* Try all filestores once (shuffled for load distribution).
|
|
304
377
|
* @param getMissingBlobHashes - Function to get remaining blob hashes to fetch
|
|
@@ -390,7 +463,8 @@ export class HttpBlobClient {
|
|
|
390
463
|
url,
|
|
391
464
|
...options
|
|
392
465
|
});
|
|
393
|
-
|
|
466
|
+
// No retry here — this is called inside the main retry loop in getBlobSidecar
|
|
467
|
+
return fetch(url, options);
|
|
394
468
|
}
|
|
395
469
|
async getLatestSlotNumber(hostUrl, l1ConsensusHostIndex) {
|
|
396
470
|
try {
|
|
@@ -5,9 +5,7 @@ import type { Blob } from '@aztec/blob-lib';
|
|
|
5
5
|
export interface GetBlobSidecarOptions {
|
|
6
6
|
/**
|
|
7
7
|
* True if the archiver is catching up (historical sync), false if near tip.
|
|
8
|
-
*
|
|
9
|
-
* - Historical: FileStore first (data should exist), then L1 consensus, then archive (eg. blobscan)
|
|
10
|
-
* - Near tip: FileStore first with no retries (data should exist), L1 consensus second (freshest data), then FileStore with retries, then archive (eg. blobscan)
|
|
8
|
+
* Historical sync uses a shorter retry backoff since blobs should already exist.
|
|
11
9
|
*/
|
|
12
10
|
isHistoricalSync?: boolean;
|
|
13
11
|
/**
|
|
@@ -36,4 +34,4 @@ export interface BlobClientInterface {
|
|
|
36
34
|
/** Returns true if this client can upload blobs to filestore. */
|
|
37
35
|
canUpload(): boolean;
|
|
38
36
|
}
|
|
39
|
-
//# sourceMappingURL=data:application/json;base64,
|
|
37
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiaW50ZXJmYWNlLmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi9zcmMvY2xpZW50L2ludGVyZmFjZS50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSxPQUFPLEtBQUssRUFBRSxJQUFJLEVBQUUsTUFBTSxpQkFBaUIsQ0FBQztBQUU1Qzs7R0FFRztBQUNILE1BQU0sV0FBVyxxQkFBcUI7SUFDcEM7OztPQUdHO0lBQ0gsZ0JBQWdCLENBQUMsRUFBRSxPQUFPLENBQUM7SUFDM0I7OztPQUdHO0lBQ0gscUJBQXFCLENBQUMsRUFBRSxNQUFNLENBQUM7SUFDL0I7Ozs7T0FJRztJQUNILGdCQUFnQixDQUFDLEVBQUUsTUFBTSxDQUFDO0NBQzNCO0FBRUQsTUFBTSxXQUFXLG1CQUFtQjtJQUNsQywwRUFBMEU7SUFDMUUsb0JBQW9CLENBQUMsS0FBSyxFQUFFLElBQUksRUFBRSxHQUFHLE9BQU8sQ0FBQyxPQUFPLENBQUMsQ0FBQztJQUN0RCxxRUFBcUU7SUFDckUsY0FBYyxDQUFDLE9BQU8sRUFBRSxNQUFNLEVBQUUsVUFBVSxDQUFDLEVBQUUsTUFBTSxFQUFFLEVBQUUsSUFBSSxDQUFDLEVBQUUscUJBQXFCLEdBQUcsT0FBTyxDQUFDLElBQUksRUFBRSxDQUFDLENBQUM7SUFDdEcsNkVBQTZFO0lBQzdFLEtBQUssQ0FBQyxJQUFJLE9BQU8sQ0FBQyxJQUFJLENBQUMsQ0FBQztJQUN4QixvRkFBb0Y7SUFDcEYsV0FBVyxJQUFJLE9BQU8sQ0FBQyxJQUFJLENBQUMsQ0FBQztJQUM3QiwwREFBMEQ7SUFDMUQsSUFBSSxDQUFDLElBQUksSUFBSSxDQUFDO0lBQ2QsaUVBQWlFO0lBQ2pFLFNBQVMsSUFBSSxPQUFPLENBQUM7Q0FDdEIifQ==
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"interface.d.ts","sourceRoot":"","sources":["../../src/client/interface.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAC;AAE5C;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC
|
|
1
|
+
{"version":3,"file":"interface.d.ts","sourceRoot":"","sources":["../../src/client/interface.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAC;AAE5C;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC;;;OAGG;IACH,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B;;;OAGG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B;;;;OAIG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAED,MAAM,WAAW,mBAAmB;IAClC,0EAA0E;IAC1E,oBAAoB,CAAC,KAAK,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IACtD,qEAAqE;IACrE,cAAc,CAAC,OAAO,EAAE,MAAM,EAAE,UAAU,CAAC,EAAE,MAAM,EAAE,EAAE,IAAI,CAAC,EAAE,qBAAqB,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;IACtG,6EAA6E;IAC7E,KAAK,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACxB,oFAAoF;IACpF,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAC7B,0DAA0D;IAC1D,IAAI,CAAC,IAAI,IAAI,CAAC;IACd,iEAAiE;IACjE,SAAS,IAAI,OAAO,CAAC;CACtB"}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { type Logger } from '@aztec/foundation/log';
|
|
2
|
+
import { type HttpFileStoreOptions } from '@aztec/stdlib/file-store';
|
|
2
3
|
import { FileStoreBlobClient } from './filestore_blob_client.js';
|
|
3
4
|
/**
|
|
4
5
|
* Metadata required to construct the base path for blob storage.
|
|
@@ -25,8 +26,8 @@ export declare function makeBlobBasePath(metadata: BlobFileStoreMetadata): strin
|
|
|
25
26
|
* @param logger - Optional logger
|
|
26
27
|
* @returns A FileStoreBlobClient for reading blobs, or undefined if storeUrl is undefined
|
|
27
28
|
*/
|
|
28
|
-
export declare function createReadOnlyFileStoreBlobClient(storeUrl: string, metadata: BlobFileStoreMetadata, logger?: Logger): Promise<FileStoreBlobClient>;
|
|
29
|
-
export declare function createReadOnlyFileStoreBlobClient(storeUrl: string | undefined, metadata: BlobFileStoreMetadata, logger?: Logger): Promise<FileStoreBlobClient | undefined>;
|
|
29
|
+
export declare function createReadOnlyFileStoreBlobClient(storeUrl: string, metadata: BlobFileStoreMetadata, logger?: Logger, httpOptions?: HttpFileStoreOptions): Promise<FileStoreBlobClient>;
|
|
30
|
+
export declare function createReadOnlyFileStoreBlobClient(storeUrl: string | undefined, metadata: BlobFileStoreMetadata, logger?: Logger, httpOptions?: HttpFileStoreOptions): Promise<FileStoreBlobClient | undefined>;
|
|
30
31
|
/**
|
|
31
32
|
* Creates multiple read-only FileStoreBlobClients from an array of URLs.
|
|
32
33
|
*
|
|
@@ -35,7 +36,7 @@ export declare function createReadOnlyFileStoreBlobClient(storeUrl: string | und
|
|
|
35
36
|
* @param logger - Optional logger
|
|
36
37
|
* @returns Array of FileStoreBlobClients (excludes any that failed to create)
|
|
37
38
|
*/
|
|
38
|
-
export declare function createReadOnlyFileStoreBlobClients(storeUrls: string[] | undefined, metadata: BlobFileStoreMetadata, logger?: Logger): Promise<FileStoreBlobClient[]>;
|
|
39
|
+
export declare function createReadOnlyFileStoreBlobClients(storeUrls: string[] | undefined, metadata: BlobFileStoreMetadata, logger?: Logger, httpOptions?: HttpFileStoreOptions): Promise<FileStoreBlobClient[]>;
|
|
39
40
|
/**
|
|
40
41
|
* Creates a writable FileStoreBlobClient for uploading blobs.
|
|
41
42
|
* Note: https:// URLs are not supported for upload, only s3://, gs://, or file://.
|
|
@@ -47,4 +48,4 @@ export declare function createReadOnlyFileStoreBlobClients(storeUrls: string[] |
|
|
|
47
48
|
*/
|
|
48
49
|
export declare function createWritableFileStoreBlobClient(storeUrl: string, metadata: BlobFileStoreMetadata, logger?: Logger): Promise<FileStoreBlobClient>;
|
|
49
50
|
export declare function createWritableFileStoreBlobClient(storeUrl: string | undefined, metadata: BlobFileStoreMetadata, logger?: Logger): Promise<FileStoreBlobClient | undefined>;
|
|
50
|
-
//# sourceMappingURL=data:application/json;base64,
|
|
51
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiZmFjdG9yeS5kLnRzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vLi4vc3JjL2ZpbGVzdG9yZS9mYWN0b3J5LnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLE9BQU8sRUFBRSxLQUFLLE1BQU0sRUFBZ0IsTUFBTSx1QkFBdUIsQ0FBQztBQUNsRSxPQUFPLEVBRUwsS0FBSyxvQkFBb0IsRUFJMUIsTUFBTSwwQkFBMEIsQ0FBQztBQUVsQyxPQUFPLEVBQUUsbUJBQW1CLEVBQUUsTUFBTSw0QkFBNEIsQ0FBQztBQUVqRTs7O0dBR0c7QUFDSCxNQUFNLFdBQVcscUJBQXFCO0lBQ3BDLHNCQUFzQjtJQUN0QixTQUFTLEVBQUUsTUFBTSxDQUFDO0lBQ2xCLHlCQUF5QjtJQUN6QixhQUFhLEVBQUUsTUFBTSxDQUFDO0lBQ3RCLDhEQUE4RDtJQUM5RCxhQUFhLEVBQUUsTUFBTSxDQUFDO0NBQ3ZCO0FBRUQ7OztHQUdHO0FBQ0gsd0JBQWdCLGdCQUFnQixDQUFDLFFBQVEsRUFBRSxxQkFBcUIsR0FBRyxNQUFNLENBS3hFO0FBRUQ7Ozs7Ozs7R0FPRztBQUNILHdCQUFzQixpQ0FBaUMsQ0FDckQsUUFBUSxFQUFFLE1BQU0sRUFDaEIsUUFBUSxFQUFFLHFCQUFxQixFQUMvQixNQUFNLENBQUMsRUFBRSxNQUFNLEVBQ2YsV0FBVyxDQUFDLEVBQUUsb0JBQW9CLEdBQ2pDLE9BQU8sQ0FBQyxtQkFBbUIsQ0FBQyxDQUFDO0FBQ2hDLHdCQUFzQixpQ0FBaUMsQ0FDckQsUUFBUSxFQUFFLE1BQU0sR0FBRyxTQUFTLEVBQzVCLFFBQVEsRUFBRSxxQkFBcUIsRUFDL0IsTUFBTSxDQUFDLEVBQUUsTUFBTSxFQUNmLFdBQVcsQ0FBQyxFQUFFLG9CQUFvQixHQUNqQyxPQUFPLENBQUMsbUJBQW1CLEdBQUcsU0FBUyxDQUFDLENBQUM7QUFvQjVDOzs7Ozs7O0dBT0c7QUFDSCx3QkFBc0Isa0NBQWtDLENBQ3RELFNBQVMsRUFBRSxNQUFNLEVBQUUsR0FBRyxTQUFTLEVBQy9CLFFBQVEsRUFBRSxxQkFBcUIsRUFDL0IsTUFBTSxDQUFDLEVBQUUsTUFBTSxFQUNmLFdBQVcsQ0FBQyxFQUFFLG9CQUFvQixHQUNqQyxPQUFPLENBQUMsbUJBQW1CLEVBQUUsQ0FBQyxDQW9CaEM7QUFFRDs7Ozs7Ozs7R0FRRztBQUNILHdCQUFzQixpQ0FBaUMsQ0FDckQsUUFBUSxFQUFFLE1BQU0sRUFDaEIsUUFBUSxFQUFFLHFCQUFxQixFQUMvQixNQUFNLENBQUMsRUFBRSxNQUFNLEdBQ2QsT0FBTyxDQUFDLG1CQUFtQixDQUFDLENBQUM7QUFDaEMsd0JBQXNCLGlDQUFpQyxDQUNyRCxRQUFRLEVBQUUsTUFBTSxHQUFHLFNBQVMsRUFDNUIsUUFBUSxFQUFFLHFCQUFxQixFQUMvQixNQUFNLENBQUMsRUFBRSxNQUFNLEdBQ2QsT0FBTyxDQUFDLG1CQUFtQixHQUFHLFNBQVMsQ0FBQyxDQUFDIn0=
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"factory.d.ts","sourceRoot":"","sources":["../../src/filestore/factory.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,MAAM,EAAgB,MAAM,uBAAuB,CAAC;
|
|
1
|
+
{"version":3,"file":"factory.d.ts","sourceRoot":"","sources":["../../src/filestore/factory.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,MAAM,EAAgB,MAAM,uBAAuB,CAAC;AAClE,OAAO,EAEL,KAAK,oBAAoB,EAI1B,MAAM,0BAA0B,CAAC;AAElC,OAAO,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAC;AAEjE;;;GAGG;AACH,MAAM,WAAW,qBAAqB;IACpC,sBAAsB;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,yBAAyB;IACzB,aAAa,EAAE,MAAM,CAAC;IACtB,8DAA8D;IAC9D,aAAa,EAAE,MAAM,CAAC;CACvB;AAED;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,qBAAqB,GAAG,MAAM,CAKxE;AAED;;;;;;;GAOG;AACH,wBAAsB,iCAAiC,CACrD,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,qBAAqB,EAC/B,MAAM,CAAC,EAAE,MAAM,EACf,WAAW,CAAC,EAAE,oBAAoB,GACjC,OAAO,CAAC,mBAAmB,CAAC,CAAC;AAChC,wBAAsB,iCAAiC,CACrD,QAAQ,EAAE,MAAM,GAAG,SAAS,EAC5B,QAAQ,EAAE,qBAAqB,EAC/B,MAAM,CAAC,EAAE,MAAM,EACf,WAAW,CAAC,EAAE,oBAAoB,GACjC,OAAO,CAAC,mBAAmB,GAAG,SAAS,CAAC,CAAC;AAoB5C;;;;;;;GAOG;AACH,wBAAsB,kCAAkC,CACtD,SAAS,EAAE,MAAM,EAAE,GAAG,SAAS,EAC/B,QAAQ,EAAE,qBAAqB,EAC/B,MAAM,CAAC,EAAE,MAAM,EACf,WAAW,CAAC,EAAE,oBAAoB,GACjC,OAAO,CAAC,mBAAmB,EAAE,CAAC,CAoBhC;AAED;;;;;;;;GAQG;AACH,wBAAsB,iCAAiC,CACrD,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,qBAAqB,EAC/B,MAAM,CAAC,EAAE,MAAM,GACd,OAAO,CAAC,mBAAmB,CAAC,CAAC;AAChC,wBAAsB,iCAAiC,CACrD,QAAQ,EAAE,MAAM,GAAG,SAAS,EAC5B,QAAQ,EAAE,qBAAqB,EAC/B,MAAM,CAAC,EAAE,MAAM,GACd,OAAO,CAAC,mBAAmB,GAAG,SAAS,CAAC,CAAC"}
|
|
@@ -10,7 +10,7 @@ import { FileStoreBlobClient } from './filestore_blob_client.js';
|
|
|
10
10
|
const normalizedAddress = rollupAddress.toLowerCase().replace(/^0x/, '');
|
|
11
11
|
return `aztec-${l1ChainId}-${rollupVersion}-0x${normalizedAddress}`;
|
|
12
12
|
}
|
|
13
|
-
export async function createReadOnlyFileStoreBlobClient(storeUrl, metadata, logger) {
|
|
13
|
+
export async function createReadOnlyFileStoreBlobClient(storeUrl, metadata, logger, httpOptions) {
|
|
14
14
|
if (!storeUrl) {
|
|
15
15
|
return undefined;
|
|
16
16
|
}
|
|
@@ -20,7 +20,7 @@ export async function createReadOnlyFileStoreBlobClient(storeUrl, metadata, logg
|
|
|
20
20
|
storeUrl,
|
|
21
21
|
basePath
|
|
22
22
|
});
|
|
23
|
-
const store = await createReadOnlyFileStore(storeUrl, log);
|
|
23
|
+
const store = await createReadOnlyFileStore(storeUrl, log, httpOptions);
|
|
24
24
|
return new FileStoreBlobClient(store, basePath, log);
|
|
25
25
|
}
|
|
26
26
|
/**
|
|
@@ -30,7 +30,7 @@ export async function createReadOnlyFileStoreBlobClient(storeUrl, metadata, logg
|
|
|
30
30
|
* @param metadata - Chain metadata for constructing the base path
|
|
31
31
|
* @param logger - Optional logger
|
|
32
32
|
* @returns Array of FileStoreBlobClients (excludes any that failed to create)
|
|
33
|
-
*/ export async function createReadOnlyFileStoreBlobClients(storeUrls, metadata, logger) {
|
|
33
|
+
*/ export async function createReadOnlyFileStoreBlobClients(storeUrls, metadata, logger, httpOptions) {
|
|
34
34
|
if (!storeUrls || storeUrls.length === 0) {
|
|
35
35
|
return [];
|
|
36
36
|
}
|
|
@@ -38,7 +38,7 @@ export async function createReadOnlyFileStoreBlobClient(storeUrl, metadata, logg
|
|
|
38
38
|
const clients = [];
|
|
39
39
|
for (const storeUrl of storeUrls){
|
|
40
40
|
try {
|
|
41
|
-
const client = await createReadOnlyFileStoreBlobClient(storeUrl, metadata, log);
|
|
41
|
+
const client = await createReadOnlyFileStoreBlobClient(storeUrl, metadata, log, httpOptions);
|
|
42
42
|
if (client) {
|
|
43
43
|
clients.push(client);
|
|
44
44
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@aztec/blob-client",
|
|
3
|
-
"version": "0.0.1-commit.
|
|
3
|
+
"version": "0.0.1-commit.9ef841308",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"bin": "./dest/client/bin/index.js",
|
|
6
6
|
"exports": {
|
|
@@ -56,12 +56,12 @@
|
|
|
56
56
|
]
|
|
57
57
|
},
|
|
58
58
|
"dependencies": {
|
|
59
|
-
"@aztec/blob-lib": "0.0.1-commit.
|
|
60
|
-
"@aztec/ethereum": "0.0.1-commit.
|
|
61
|
-
"@aztec/foundation": "0.0.1-commit.
|
|
62
|
-
"@aztec/kv-store": "0.0.1-commit.
|
|
63
|
-
"@aztec/stdlib": "0.0.1-commit.
|
|
64
|
-
"@aztec/telemetry-client": "0.0.1-commit.
|
|
59
|
+
"@aztec/blob-lib": "0.0.1-commit.9ef841308",
|
|
60
|
+
"@aztec/ethereum": "0.0.1-commit.9ef841308",
|
|
61
|
+
"@aztec/foundation": "0.0.1-commit.9ef841308",
|
|
62
|
+
"@aztec/kv-store": "0.0.1-commit.9ef841308",
|
|
63
|
+
"@aztec/stdlib": "0.0.1-commit.9ef841308",
|
|
64
|
+
"@aztec/telemetry-client": "0.0.1-commit.9ef841308",
|
|
65
65
|
"express": "^4.21.2",
|
|
66
66
|
"snappy": "^7.2.2",
|
|
67
67
|
"source-map-support": "^0.5.21",
|
package/src/client/config.ts
CHANGED
|
@@ -59,6 +59,12 @@ export interface BlobClientConfig extends BlobArchiveApiConfig {
|
|
|
59
59
|
|
|
60
60
|
/** Timeout for HTTP requests to the L1 RPC node in ms. */
|
|
61
61
|
l1HttpTimeoutMS?: number;
|
|
62
|
+
|
|
63
|
+
/** Whether to prefer filestores over consensus clients when fetching blobs. Default: false (consensus first). */
|
|
64
|
+
blobPreferFilestores?: boolean;
|
|
65
|
+
|
|
66
|
+
/** Timeout in ms for HTTP requests to the blob file store. Default: 10000 (10s). */
|
|
67
|
+
blobFileStoreTimeoutMs?: number;
|
|
62
68
|
}
|
|
63
69
|
|
|
64
70
|
export const blobClientConfigMapping: ConfigMappingsType<BlobClientConfig> = {
|
|
@@ -117,6 +123,16 @@ export const blobClientConfigMapping: ConfigMappingsType<BlobClientConfig> = {
|
|
|
117
123
|
description: 'Timeout for HTTP requests to the L1 RPC node in ms.',
|
|
118
124
|
...optionalNumberConfigHelper(),
|
|
119
125
|
},
|
|
126
|
+
blobPreferFilestores: {
|
|
127
|
+
env: 'BLOB_PREFER_FILESTORES',
|
|
128
|
+
description: 'Whether to prefer filestores over consensus clients when fetching blobs. Default: false.',
|
|
129
|
+
...booleanConfigHelper(false),
|
|
130
|
+
},
|
|
131
|
+
blobFileStoreTimeoutMs: {
|
|
132
|
+
env: 'BLOB_FILE_STORE_TIMEOUT_MS',
|
|
133
|
+
description: 'Timeout in ms for HTTP requests to the blob file store. Default: 10000 (10s).',
|
|
134
|
+
...optionalNumberConfigHelper(),
|
|
135
|
+
},
|
|
120
136
|
...blobArchiveApiConfigMappings,
|
|
121
137
|
};
|
|
122
138
|
|
package/src/client/factory.ts
CHANGED
|
@@ -76,8 +76,15 @@ export async function createBlobClientWithFileStores(
|
|
|
76
76
|
rollupAddress: config.l1Contracts.rollupAddress.toString(),
|
|
77
77
|
};
|
|
78
78
|
|
|
79
|
+
// Disable internal retries for blob file stores — retry logic is handled by HttpBlobClient.
|
|
80
|
+
// Set a configurable timeout (default 10s) to avoid hanging on slow stores.
|
|
81
|
+
const httpOptions = {
|
|
82
|
+
retryBackoff: [] as number[],
|
|
83
|
+
timeoutMs: config.blobFileStoreTimeoutMs ?? 10_000,
|
|
84
|
+
};
|
|
85
|
+
|
|
79
86
|
const [fileStoreClients, fileStoreUploadClient] = await Promise.all([
|
|
80
|
-
createReadOnlyFileStoreBlobClients(config.blobFileStoreUrls, fileStoreMetadata, log),
|
|
87
|
+
createReadOnlyFileStoreBlobClients(config.blobFileStoreUrls, fileStoreMetadata, log, httpOptions),
|
|
81
88
|
createWritableFileStoreBlobClient(config.blobFileStoreUploadUrl, fileStoreMetadata, log),
|
|
82
89
|
]);
|
|
83
90
|
|
package/src/client/http.ts
CHANGED
|
@@ -30,6 +30,9 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
30
30
|
/** Cached beacon slot duration in seconds. Fetched once at startup. */
|
|
31
31
|
private beaconSecondsPerSlot?: number;
|
|
32
32
|
|
|
33
|
+
/** Indexes of consensus hosts that serve blob sidecars (supernodes). Populated by testSources(). */
|
|
34
|
+
private superNodeHostIndexes?: Set<number>;
|
|
35
|
+
|
|
33
36
|
constructor(
|
|
34
37
|
config?: BlobClientConfig,
|
|
35
38
|
private readonly opts: {
|
|
@@ -95,44 +98,75 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
95
98
|
const archiveUrl = this.archiveClient?.getBaseUrl();
|
|
96
99
|
this.log.info(`Testing configured blob sources`, { l1ConsensusHostUrls, archiveUrl });
|
|
97
100
|
|
|
98
|
-
let
|
|
101
|
+
let consensusSuperNodes = 0;
|
|
102
|
+
let consensusNonSuperNodes = 0;
|
|
103
|
+
let archiveSources = 0;
|
|
104
|
+
let blobSinks = 0;
|
|
105
|
+
|
|
106
|
+
const detectedSuperNodes = new Set<number>();
|
|
99
107
|
|
|
100
108
|
if (l1ConsensusHostUrls && l1ConsensusHostUrls.length > 0) {
|
|
101
109
|
for (let l1ConsensusHostIndex = 0; l1ConsensusHostIndex < l1ConsensusHostUrls.length; l1ConsensusHostIndex++) {
|
|
102
110
|
const l1ConsensusHostUrl = l1ConsensusHostUrls[l1ConsensusHostIndex];
|
|
103
111
|
try {
|
|
104
112
|
const { url, ...options } = getBeaconNodeFetchOptions(
|
|
105
|
-
`${l1ConsensusHostUrl}/eth/v1/beacon/headers`,
|
|
113
|
+
`${l1ConsensusHostUrl}/eth/v1/beacon/headers/head`,
|
|
106
114
|
this.config,
|
|
107
115
|
l1ConsensusHostIndex,
|
|
108
116
|
);
|
|
109
117
|
const res = await this.fetch(url, options);
|
|
110
|
-
if (res.ok) {
|
|
111
|
-
this.log.info(`L1 consensus host is reachable`, { l1ConsensusHostUrl });
|
|
112
|
-
successfulSourceCount++;
|
|
113
|
-
} else {
|
|
118
|
+
if (!res.ok) {
|
|
114
119
|
this.log.error(`Failure reaching L1 consensus host: ${res.statusText} (${res.status})`, {
|
|
115
120
|
l1ConsensusHostUrl,
|
|
116
121
|
});
|
|
122
|
+
continue;
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
this.log.info(`L1 consensus host is reachable`, { l1ConsensusHostUrl });
|
|
126
|
+
|
|
127
|
+
// Check if the host serves blob sidecars (supernode/semi-supernode).
|
|
128
|
+
// Post-Fusaka (PeerDAS), non-supernode beacon nodes no longer serve the
|
|
129
|
+
// blob sidecar endpoint. A 200 response (even with an empty data array
|
|
130
|
+
// for a slot with no blobs) means the node supports serving blob sidecars.
|
|
131
|
+
const body = await res.json();
|
|
132
|
+
const headSlot = body?.data?.header?.message?.slot;
|
|
133
|
+
if (headSlot) {
|
|
134
|
+
const { url: blobUrl, ...blobOptions } = getBeaconNodeFetchOptions(
|
|
135
|
+
`${l1ConsensusHostUrl}/eth/v1/beacon/blobs/${headSlot}`,
|
|
136
|
+
this.config,
|
|
137
|
+
l1ConsensusHostIndex,
|
|
138
|
+
);
|
|
139
|
+
const blobRes = await this.fetch(blobUrl, blobOptions);
|
|
140
|
+
if (blobRes.ok) {
|
|
141
|
+
this.log.info(`L1 consensus host serves blob sidecars (supernode)`, { l1ConsensusHostUrl });
|
|
142
|
+
detectedSuperNodes.add(l1ConsensusHostIndex);
|
|
143
|
+
consensusSuperNodes++;
|
|
144
|
+
} else {
|
|
145
|
+
this.log.info(`L1 consensus host does not serve blob sidecars, skipping for blob fetching`, {
|
|
146
|
+
l1ConsensusHostUrl,
|
|
147
|
+
});
|
|
148
|
+
consensusNonSuperNodes++;
|
|
149
|
+
}
|
|
150
|
+
} else {
|
|
151
|
+
this.log.info(`L1 consensus host is reachable but could not determine head slot`, { l1ConsensusHostUrl });
|
|
152
|
+
consensusNonSuperNodes++;
|
|
117
153
|
}
|
|
118
154
|
} catch (err) {
|
|
119
155
|
this.log.error(`Error reaching L1 consensus host`, err, { l1ConsensusHostUrl });
|
|
120
156
|
}
|
|
121
157
|
}
|
|
122
|
-
} else {
|
|
123
|
-
this.log.warn('No L1 consensus host urls configured');
|
|
124
158
|
}
|
|
125
159
|
|
|
160
|
+
this.superNodeHostIndexes = detectedSuperNodes;
|
|
161
|
+
|
|
126
162
|
if (this.archiveClient) {
|
|
127
163
|
try {
|
|
128
164
|
const latest = await this.archiveClient.getLatestBlock();
|
|
129
165
|
this.log.info(`Archive client is reachable and synced to L1 block ${latest.number}`, { latest, archiveUrl });
|
|
130
|
-
|
|
166
|
+
archiveSources++;
|
|
131
167
|
} catch (err) {
|
|
132
168
|
this.log.error(`Error reaching archive client`, err, { archiveUrl });
|
|
133
169
|
}
|
|
134
|
-
} else {
|
|
135
|
-
this.log.warn('No archive client configured');
|
|
136
170
|
}
|
|
137
171
|
|
|
138
172
|
if (this.fileStoreClients.length > 0) {
|
|
@@ -141,7 +175,7 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
141
175
|
const accessible = await fileStoreClient.testConnection();
|
|
142
176
|
if (accessible) {
|
|
143
177
|
this.log.info(`FileStore is reachable`, { url: fileStoreClient.getBaseUrl() });
|
|
144
|
-
|
|
178
|
+
blobSinks++;
|
|
145
179
|
} else {
|
|
146
180
|
this.log.warn(`FileStore is not accessible`, { url: fileStoreClient.getBaseUrl() });
|
|
147
181
|
}
|
|
@@ -151,12 +185,24 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
151
185
|
}
|
|
152
186
|
}
|
|
153
187
|
|
|
188
|
+
// Emit a single summary after validating all sources
|
|
189
|
+
const successfulSourceCount = consensusSuperNodes + archiveSources + blobSinks;
|
|
190
|
+
|
|
191
|
+
let summary = `Blob client running with consensusSuperNodes=${consensusSuperNodes} archiveSources=${archiveSources} blobSinks=${blobSinks}`;
|
|
192
|
+
if (consensusNonSuperNodes > 0) {
|
|
193
|
+
summary += `. ${consensusNonSuperNodes} consensus client(s) ignored because they are not running in supernode or semi-supernode mode`;
|
|
194
|
+
}
|
|
195
|
+
|
|
154
196
|
if (successfulSourceCount === 0) {
|
|
155
197
|
if (this.config.blobAllowEmptySources) {
|
|
156
|
-
this.log.warn(
|
|
198
|
+
this.log.warn(summary);
|
|
157
199
|
} else {
|
|
158
|
-
throw new Error(
|
|
200
|
+
throw new Error(summary);
|
|
159
201
|
}
|
|
202
|
+
} else if (consensusSuperNodes === 0) {
|
|
203
|
+
this.log.warn(summary);
|
|
204
|
+
} else {
|
|
205
|
+
this.log.info(summary);
|
|
160
206
|
}
|
|
161
207
|
}
|
|
162
208
|
|
|
@@ -182,18 +228,15 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
182
228
|
}
|
|
183
229
|
|
|
184
230
|
/**
|
|
185
|
-
* Get the blob sidecar
|
|
186
|
-
*
|
|
187
|
-
* If requesting from the blob client, we send the blobkHash
|
|
188
|
-
* If requesting from the beacon node, we send the slot number
|
|
231
|
+
* Get the blob sidecar.
|
|
189
232
|
*
|
|
190
|
-
*
|
|
191
|
-
*
|
|
192
|
-
*
|
|
233
|
+
* Alternates between two primary sources (consensus and filestore) in a retry loop,
|
|
234
|
+
* then falls back to archive if blobs are still missing. The order of the primary
|
|
235
|
+
* sources is configurable via `blobPreferFilestores`.
|
|
193
236
|
*
|
|
194
237
|
* @param blockHash - The block hash
|
|
195
238
|
* @param blobHashes - The blob hashes to fetch
|
|
196
|
-
* @param opts - Options
|
|
239
|
+
* @param opts - Options for slot resolution
|
|
197
240
|
* @returns The blobs
|
|
198
241
|
*/
|
|
199
242
|
public async getBlobSidecar(
|
|
@@ -206,12 +249,11 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
206
249
|
return [];
|
|
207
250
|
}
|
|
208
251
|
|
|
209
|
-
const isHistoricalSync = opts?.isHistoricalSync ?? false;
|
|
210
252
|
// Accumulate blobs across sources, preserving order and handling duplicates
|
|
211
253
|
// resultBlobs[i] will contain the blob for blobHashes[i], or undefined if not yet found
|
|
212
254
|
const resultBlobs: (Blob | undefined)[] = new Array(blobHashes.length).fill(undefined);
|
|
213
255
|
|
|
214
|
-
// Helper to get
|
|
256
|
+
// Helper to get missing blob hashes that we still need to fetch
|
|
215
257
|
const getMissingBlobHashes = (): Buffer[] =>
|
|
216
258
|
blobHashes
|
|
217
259
|
.map((bh, i) => (resultBlobs[i] === undefined ? bh : undefined))
|
|
@@ -240,84 +282,60 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
240
282
|
return blobs;
|
|
241
283
|
};
|
|
242
284
|
|
|
243
|
-
const { l1ConsensusHostUrls } = this.config;
|
|
244
|
-
|
|
245
285
|
const ctx = { blockHash, blobHashes: blobHashes.map(bufferToHex) };
|
|
246
286
|
|
|
247
|
-
//
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
287
|
+
// Lazily resolve the slot number — only resolved when consensus hosts are actually tried.
|
|
288
|
+
let slotNumber: number | undefined;
|
|
289
|
+
let slotResolved = false;
|
|
290
|
+
const getSlotNumber = async (): Promise<number | undefined> => {
|
|
291
|
+
if (!slotResolved) {
|
|
292
|
+
slotNumber = await this.resolveSlotNumber(blockHash, opts);
|
|
293
|
+
slotResolved = true;
|
|
252
294
|
}
|
|
253
|
-
|
|
295
|
+
return slotNumber;
|
|
296
|
+
};
|
|
254
297
|
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
const consensusCtx = { l1ConsensusHostUrls, ...ctx };
|
|
259
|
-
this.log.trace(`Attempting to get slot number for block hash`, consensusCtx);
|
|
260
|
-
const slotNumber = await this.getSlotNumber(blockHash, opts?.parentBeaconBlockRoot, opts?.l1BlockTimestamp);
|
|
261
|
-
this.log.debug(`Got slot number ${slotNumber} from consensus host for querying blobs`, consensusCtx);
|
|
262
|
-
|
|
263
|
-
if (slotNumber) {
|
|
264
|
-
let l1ConsensusHostUrl: string;
|
|
265
|
-
for (let l1ConsensusHostIndex = 0; l1ConsensusHostIndex < l1ConsensusHostUrls.length; l1ConsensusHostIndex++) {
|
|
266
|
-
const missingHashes = getMissingBlobHashes();
|
|
267
|
-
if (missingHashes.length === 0) {
|
|
268
|
-
break;
|
|
269
|
-
}
|
|
298
|
+
// Build the two source-try functions. The order depends on the config.
|
|
299
|
+
const tryConsensus = () => this.tryConsensusHosts(getSlotNumber, getMissingBlobHashes, fillResults, ctx);
|
|
300
|
+
const tryFilestores = () => this.tryFileStores(getMissingBlobHashes, fillResults, ctx);
|
|
270
301
|
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
slotNumber,
|
|
274
|
-
l1ConsensusHostUrl,
|
|
275
|
-
...ctx,
|
|
276
|
-
});
|
|
277
|
-
const blobs = await this.getBlobsFromHost(
|
|
278
|
-
l1ConsensusHostUrl,
|
|
279
|
-
slotNumber,
|
|
280
|
-
l1ConsensusHostIndex,
|
|
281
|
-
getMissingBlobHashes(),
|
|
282
|
-
);
|
|
283
|
-
const result = await fillResults(blobs);
|
|
284
|
-
this.log.debug(
|
|
285
|
-
`Got ${blobs.length} blobs from consensus host (total: ${result.length}/${blobHashes.length})`,
|
|
286
|
-
{ slotNumber, l1ConsensusHostUrl, ...ctx },
|
|
287
|
-
);
|
|
288
|
-
if (result.length === blobHashes.length) {
|
|
289
|
-
return returnWithCallback(result);
|
|
290
|
-
}
|
|
291
|
-
}
|
|
292
|
-
}
|
|
293
|
-
}
|
|
302
|
+
const preferFilestores = this.config.blobPreferFilestores ?? false;
|
|
303
|
+
const [trySourceA, trySourceB] = preferFilestores ? [tryFilestores, tryConsensus] : [tryConsensus, tryFilestores];
|
|
294
304
|
|
|
295
|
-
//
|
|
296
|
-
//
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
305
|
+
// Historical sync: blobs should already exist, use shorter backoff for transient errors.
|
|
306
|
+
// Near-tip sync: blobs may still be uploading, use longer backoff for eventual consistency.
|
|
307
|
+
const isHistoricalSync = opts?.isHistoricalSync ?? false;
|
|
308
|
+
const backoff = isHistoricalSync ? [1, 1] : [1, 1, 1, 2, 2];
|
|
309
|
+
|
|
310
|
+
// Retry loop: alternate between the two primary sources with backoff.
|
|
311
|
+
try {
|
|
312
|
+
await retry(
|
|
313
|
+
async () => {
|
|
314
|
+
if (getMissingBlobHashes().length > 0) {
|
|
315
|
+
await trySourceA();
|
|
316
|
+
}
|
|
317
|
+
if (getMissingBlobHashes().length > 0) {
|
|
318
|
+
await trySourceB();
|
|
319
|
+
}
|
|
320
|
+
if (getMissingBlobHashes().length > 0) {
|
|
321
|
+
throw new Error('Still missing blobs after trying all primary sources');
|
|
322
|
+
}
|
|
323
|
+
},
|
|
324
|
+
'blob retrieval',
|
|
325
|
+
makeBackoff(backoff),
|
|
326
|
+
this.log,
|
|
327
|
+
true, // failSilently — expected during eventual consistency
|
|
328
|
+
);
|
|
329
|
+
return returnWithCallback(getFilledBlobs());
|
|
330
|
+
} catch {
|
|
331
|
+
// Exhausted retries, continue to archive fallback
|
|
315
332
|
}
|
|
316
333
|
|
|
317
|
-
|
|
318
|
-
|
|
334
|
+
// Archive fallback
|
|
335
|
+
const missingAfterPrimary = getMissingBlobHashes();
|
|
336
|
+
if (missingAfterPrimary.length > 0 && this.archiveClient) {
|
|
319
337
|
const archiveCtx = { archiveUrl: this.archiveClient.getBaseUrl(), ...ctx };
|
|
320
|
-
this.log.trace(`Attempting to get ${
|
|
338
|
+
this.log.trace(`Attempting to get ${missingAfterPrimary.length} blobs from archive`, archiveCtx);
|
|
321
339
|
const allBlobs = await this.archiveClient.getBlobsFromBlock(blockHash);
|
|
322
340
|
if (!allBlobs) {
|
|
323
341
|
this.log.debug('No blobs found from archive client', archiveCtx);
|
|
@@ -339,7 +357,7 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
339
357
|
this.log.warn(
|
|
340
358
|
`Failed to fetch all blobs for ${blockHash} from all blob sources (got ${result.length}/${blobHashes.length})`,
|
|
341
359
|
{
|
|
342
|
-
l1ConsensusHostUrls,
|
|
360
|
+
l1ConsensusHostUrls: this.config.l1ConsensusHostUrls,
|
|
343
361
|
archiveUrl: this.archiveClient?.getBaseUrl(),
|
|
344
362
|
fileStoreUrls: this.fileStoreClients.map(c => c.getBaseUrl()),
|
|
345
363
|
},
|
|
@@ -348,6 +366,71 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
348
366
|
return returnWithCallback(result);
|
|
349
367
|
}
|
|
350
368
|
|
|
369
|
+
/** Resolves the beacon slot number for the given block hash. Returns undefined if no consensus hosts. */
|
|
370
|
+
private resolveSlotNumber(
|
|
371
|
+
blockHash: `0x${string}`,
|
|
372
|
+
opts?: GetBlobSidecarOptions,
|
|
373
|
+
): Promise<number | undefined> | undefined {
|
|
374
|
+
const { l1ConsensusHostUrls } = this.config;
|
|
375
|
+
if (!l1ConsensusHostUrls || l1ConsensusHostUrls.length === 0) {
|
|
376
|
+
return undefined;
|
|
377
|
+
}
|
|
378
|
+
// If no supernodes, no point resolving the slot
|
|
379
|
+
if (this.superNodeHostIndexes && this.superNodeHostIndexes.size === 0) {
|
|
380
|
+
return undefined;
|
|
381
|
+
}
|
|
382
|
+
return this.getSlotNumber(blockHash, opts?.parentBeaconBlockRoot, opts?.l1BlockTimestamp);
|
|
383
|
+
}
|
|
384
|
+
|
|
385
|
+
/**
|
|
386
|
+
* Try all supernode consensus hosts for blob sidecars.
|
|
387
|
+
* Skips hosts that were detected as non-supernodes during testSources().
|
|
388
|
+
*/
|
|
389
|
+
private async tryConsensusHosts(
|
|
390
|
+
getSlotNumber: () => Promise<number | undefined>,
|
|
391
|
+
getMissingBlobHashes: () => Buffer[],
|
|
392
|
+
fillResults: (blobs: BlobJson[]) => Promise<Blob[]>,
|
|
393
|
+
ctx: { blockHash: string; blobHashes: string[] },
|
|
394
|
+
): Promise<void> {
|
|
395
|
+
const { l1ConsensusHostUrls } = this.config;
|
|
396
|
+
if (!l1ConsensusHostUrls || l1ConsensusHostUrls.length === 0) {
|
|
397
|
+
return;
|
|
398
|
+
}
|
|
399
|
+
|
|
400
|
+
const slotNumber = await getSlotNumber();
|
|
401
|
+
if (!slotNumber) {
|
|
402
|
+
return;
|
|
403
|
+
}
|
|
404
|
+
|
|
405
|
+
for (let l1ConsensusHostIndex = 0; l1ConsensusHostIndex < l1ConsensusHostUrls.length; l1ConsensusHostIndex++) {
|
|
406
|
+
const missingHashes = getMissingBlobHashes();
|
|
407
|
+
if (missingHashes.length === 0) {
|
|
408
|
+
break;
|
|
409
|
+
}
|
|
410
|
+
|
|
411
|
+
// Skip non-supernode hosts if we've already detected supernodes
|
|
412
|
+
if (this.superNodeHostIndexes && !this.superNodeHostIndexes.has(l1ConsensusHostIndex)) {
|
|
413
|
+
this.log.trace(`Skipping non-supernode consensus host`, {
|
|
414
|
+
l1ConsensusHostUrl: l1ConsensusHostUrls[l1ConsensusHostIndex],
|
|
415
|
+
});
|
|
416
|
+
continue;
|
|
417
|
+
}
|
|
418
|
+
|
|
419
|
+
const l1ConsensusHostUrl = l1ConsensusHostUrls[l1ConsensusHostIndex];
|
|
420
|
+
this.log.trace(`Attempting to get ${missingHashes.length} blobs from consensus host`, {
|
|
421
|
+
slotNumber,
|
|
422
|
+
l1ConsensusHostUrl,
|
|
423
|
+
...ctx,
|
|
424
|
+
});
|
|
425
|
+
const blobs = await this.getBlobsFromHost(l1ConsensusHostUrl, slotNumber, l1ConsensusHostIndex, missingHashes);
|
|
426
|
+
const result = await fillResults(blobs);
|
|
427
|
+
this.log.debug(
|
|
428
|
+
`Got ${blobs.length} blobs from consensus host (total: ${result.length}/${ctx.blobHashes.length})`,
|
|
429
|
+
{ slotNumber, l1ConsensusHostUrl, ...ctx },
|
|
430
|
+
);
|
|
431
|
+
}
|
|
432
|
+
}
|
|
433
|
+
|
|
351
434
|
/**
|
|
352
435
|
* Try all filestores once (shuffled for load distribution).
|
|
353
436
|
* @param getMissingBlobHashes - Function to get remaining blob hashes to fetch
|
|
@@ -465,7 +548,8 @@ export class HttpBlobClient implements BlobClientInterface {
|
|
|
465
548
|
|
|
466
549
|
const { url, ...options } = getBeaconNodeFetchOptions(baseUrl, this.config, l1ConsensusHostIndex);
|
|
467
550
|
this.log.debug(`Fetching blob sidecar for ${blockHashOrSlot}`, { url, ...options });
|
|
468
|
-
|
|
551
|
+
// No retry here — this is called inside the main retry loop in getBlobSidecar
|
|
552
|
+
return fetch(url, options);
|
|
469
553
|
}
|
|
470
554
|
|
|
471
555
|
private async getLatestSlotNumber(hostUrl: string, l1ConsensusHostIndex?: number): Promise<number | undefined> {
|
package/src/client/interface.ts
CHANGED
|
@@ -6,9 +6,7 @@ import type { Blob } from '@aztec/blob-lib';
|
|
|
6
6
|
export interface GetBlobSidecarOptions {
|
|
7
7
|
/**
|
|
8
8
|
* True if the archiver is catching up (historical sync), false if near tip.
|
|
9
|
-
*
|
|
10
|
-
* - Historical: FileStore first (data should exist), then L1 consensus, then archive (eg. blobscan)
|
|
11
|
-
* - Near tip: FileStore first with no retries (data should exist), L1 consensus second (freshest data), then FileStore with retries, then archive (eg. blobscan)
|
|
9
|
+
* Historical sync uses a shorter retry backoff since blobs should already exist.
|
|
12
10
|
*/
|
|
13
11
|
isHistoricalSync?: boolean;
|
|
14
12
|
/**
|
package/src/filestore/factory.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { type Logger, createLogger } from '@aztec/foundation/log';
|
|
2
2
|
import {
|
|
3
3
|
type FileStore,
|
|
4
|
+
type HttpFileStoreOptions,
|
|
4
5
|
type ReadOnlyFileStore,
|
|
5
6
|
createFileStore,
|
|
6
7
|
createReadOnlyFileStore,
|
|
@@ -44,16 +45,19 @@ export async function createReadOnlyFileStoreBlobClient(
|
|
|
44
45
|
storeUrl: string,
|
|
45
46
|
metadata: BlobFileStoreMetadata,
|
|
46
47
|
logger?: Logger,
|
|
48
|
+
httpOptions?: HttpFileStoreOptions,
|
|
47
49
|
): Promise<FileStoreBlobClient>;
|
|
48
50
|
export async function createReadOnlyFileStoreBlobClient(
|
|
49
51
|
storeUrl: string | undefined,
|
|
50
52
|
metadata: BlobFileStoreMetadata,
|
|
51
53
|
logger?: Logger,
|
|
54
|
+
httpOptions?: HttpFileStoreOptions,
|
|
52
55
|
): Promise<FileStoreBlobClient | undefined>;
|
|
53
56
|
export async function createReadOnlyFileStoreBlobClient(
|
|
54
57
|
storeUrl: string | undefined,
|
|
55
58
|
metadata: BlobFileStoreMetadata,
|
|
56
59
|
logger?: Logger,
|
|
60
|
+
httpOptions?: HttpFileStoreOptions,
|
|
57
61
|
): Promise<FileStoreBlobClient | undefined> {
|
|
58
62
|
if (!storeUrl) {
|
|
59
63
|
return undefined;
|
|
@@ -64,7 +68,7 @@ export async function createReadOnlyFileStoreBlobClient(
|
|
|
64
68
|
|
|
65
69
|
log.debug(`Creating read-only filestore blob client`, { storeUrl, basePath });
|
|
66
70
|
|
|
67
|
-
const store: ReadOnlyFileStore = await createReadOnlyFileStore(storeUrl, log);
|
|
71
|
+
const store: ReadOnlyFileStore = await createReadOnlyFileStore(storeUrl, log, httpOptions);
|
|
68
72
|
return new FileStoreBlobClient(store, basePath, log);
|
|
69
73
|
}
|
|
70
74
|
|
|
@@ -80,6 +84,7 @@ export async function createReadOnlyFileStoreBlobClients(
|
|
|
80
84
|
storeUrls: string[] | undefined,
|
|
81
85
|
metadata: BlobFileStoreMetadata,
|
|
82
86
|
logger?: Logger,
|
|
87
|
+
httpOptions?: HttpFileStoreOptions,
|
|
83
88
|
): Promise<FileStoreBlobClient[]> {
|
|
84
89
|
if (!storeUrls || storeUrls.length === 0) {
|
|
85
90
|
return [];
|
|
@@ -90,7 +95,7 @@ export async function createReadOnlyFileStoreBlobClients(
|
|
|
90
95
|
|
|
91
96
|
for (const storeUrl of storeUrls) {
|
|
92
97
|
try {
|
|
93
|
-
const client = await createReadOnlyFileStoreBlobClient(storeUrl, metadata, log);
|
|
98
|
+
const client = await createReadOnlyFileStoreBlobClient(storeUrl, metadata, log, httpOptions);
|
|
94
99
|
if (client) {
|
|
95
100
|
clients.push(client);
|
|
96
101
|
}
|