@aztec/blob-client 0.0.1-commit.03f7ef2 → 0.0.1-commit.04852196a

package/src/client/http.ts

@@ -9,6 +9,7 @@ import { type RpcBlock, createPublicClient, fallback, http } from 'viem';
 import { createBlobArchiveClient } from '../archive/factory.js';
 import type { BlobArchiveClient } from '../archive/interface.js';
 import type { FileStoreBlobClient } from '../filestore/filestore_blob_client.js';
+import { DEFAULT_HEALTHCHECK_UPLOAD_INTERVAL_MINUTES } from '../filestore/healthcheck.js';
 import { type BlobClientConfig, getBlobClientConfigFromEnv } from './config.js';
 import type { BlobClientInterface, GetBlobSidecarOptions } from './interface.js';
 
@@ -21,6 +22,12 @@ export class HttpBlobClient implements BlobClientInterface {
   protected readonly fileStoreUploadClient: FileStoreBlobClient | undefined;
 
   private disabled = false;
+  private healthcheckUploadIntervalId?: NodeJS.Timeout;
+
+  /** Cached beacon genesis time (seconds since Unix epoch). Fetched once at startup. */
+  private beaconGenesisTime?: bigint;
+  /** Cached beacon slot duration in seconds. Fetched once at startup. */
+  private beaconSecondsPerSlot?: number;
 
   constructor(
     config?: BlobClientConfig,
@@ -213,8 +220,8 @@ export class HttpBlobClient implements BlobClientInterface {
     const getFilledBlobs = (): Blob[] => resultBlobs.filter((b): b is Blob => b !== undefined);
 
     // Helper to fill in results from fetched blobs
-    const fillResults = (fetchedBlobs: BlobJson[]): Blob[] => {
-      const blobs = processFetchedBlobs(fetchedBlobs, blobHashes, this.log);
+    const fillResults = async (fetchedBlobs: BlobJson[]): Promise<Blob[]> => {
+      const blobs = await processFetchedBlobs(fetchedBlobs, blobHashes, this.log);
       // Fill in any missing positions with matching blobs
       for (let i = 0; i < blobHashes.length; i++) {
         if (resultBlobs[i] === undefined) {
@@ -249,7 +256,7 @@ export class HttpBlobClient implements BlobClientInterface {
       // The beacon api can query by slot number, so we get that first
       const consensusCtx = { l1ConsensusHostUrls, ...ctx };
       this.log.trace(`Attempting to get slot number for block hash`, consensusCtx);
-      const slotNumber = await this.getSlotNumber(blockHash);
+      const slotNumber = await this.getSlotNumber(blockHash, opts?.parentBeaconBlockRoot, opts?.l1BlockTimestamp);
       this.log.debug(`Got slot number ${slotNumber} from consensus host for querying blobs`, consensusCtx);
 
       if (slotNumber) {
@@ -266,8 +273,13 @@ export class HttpBlobClient implements BlobClientInterface {
             l1ConsensusHostUrl,
             ...ctx,
           });
-          const blobs = await this.getBlobsFromHost(l1ConsensusHostUrl, slotNumber, l1ConsensusHostIndex);
-          const result = fillResults(blobs);
+          const blobs = await this.getBlobsFromHost(
+            l1ConsensusHostUrl,
+            slotNumber,
+            l1ConsensusHostIndex,
+            getMissingBlobHashes(),
+          );
+          const result = await fillResults(blobs);
           this.log.debug(
             `Got ${blobs.length} blobs from consensus host (total: ${result.length}/${blobHashes.length})`,
             { slotNumber, l1ConsensusHostUrl, ...ctx },
@@ -310,7 +322,7 @@ export class HttpBlobClient implements BlobClientInterface {
         this.log.debug('No blobs found from archive client', archiveCtx);
       } else {
         this.log.trace(`Got ${allBlobs.length} blobs from archive client before filtering`, archiveCtx);
-        const result = fillResults(allBlobs);
+        const result = await fillResults(allBlobs);
         this.log.debug(
           `Got ${allBlobs.length} blobs from archive client (total: ${result.length}/${blobHashes.length})`,
           archiveCtx,
@@ -343,7 +355,7 @@ export class HttpBlobClient implements BlobClientInterface {
    */
   private async tryFileStores(
     getMissingBlobHashes: () => Buffer[],
-    fillResults: (blobs: BlobJson[]) => Blob[],
+    fillResults: (blobs: BlobJson[]) => Promise<Blob[]>,
     ctx: { blockHash: string; blobHashes: string[] },
   ): Promise<void> {
     // Shuffle clients for load distribution
@@ -364,7 +376,7 @@ export class HttpBlobClient implements BlobClientInterface {
         });
         const blobs = await client.getBlobsByHashes(blobHashStrings);
         if (blobs.length > 0) {
-          const result = fillResults(blobs);
+          const result = await fillResults(blobs);
           this.log.debug(
             `Got ${blobs.length} blobs from filestore (total: ${result.length}/${ctx.blobHashes.length})`,
             {
@@ -385,19 +397,20 @@ export class HttpBlobClient implements BlobClientInterface {
     blobHashes: Buffer[] = [],
     l1ConsensusHostIndex?: number,
   ): Promise<Blob[]> {
-    const blobs = await this.getBlobsFromHost(hostUrl, blockHashOrSlot, l1ConsensusHostIndex);
-    return processFetchedBlobs(blobs, blobHashes, this.log).filter((b): b is Blob => b !== undefined);
+    const blobs = await this.getBlobsFromHost(hostUrl, blockHashOrSlot, l1ConsensusHostIndex, blobHashes);
+    return (await processFetchedBlobs(blobs, blobHashes, this.log)).filter((b): b is Blob => b !== undefined);
   }
 
   public async getBlobsFromHost(
     hostUrl: string,
     blockHashOrSlot: string | number,
     l1ConsensusHostIndex?: number,
+    blobHashes?: Buffer[],
   ): Promise<BlobJson[]> {
     try {
-      let res = await this.fetchBlobSidecars(hostUrl, blockHashOrSlot, l1ConsensusHostIndex);
+      let res = await this.fetchBlobSidecars(hostUrl, blockHashOrSlot, l1ConsensusHostIndex, blobHashes);
       if (res.ok) {
-        return parseBlobJsonsFromResponse(await res.json(), this.log);
+        return await parseBlobJsonsFromResponse(await res.json(), this.log);
       }
 
       if (res.status === 404 && typeof blockHashOrSlot === 'number') {
@@ -412,9 +425,9 @@ export class HttpBlobClient implements BlobClientInterface {
         let currentSlot = blockHashOrSlot + 1;
         while (res.status === 404 && maxRetries > 0 && latestSlot !== undefined && currentSlot <= latestSlot) {
           this.log.debug(`Trying slot ${currentSlot}`);
-          res = await this.fetchBlobSidecars(hostUrl, currentSlot, l1ConsensusHostIndex);
+          res = await this.fetchBlobSidecars(hostUrl, currentSlot, l1ConsensusHostIndex, blobHashes);
           if (res.ok) {
-            return parseBlobJsonsFromResponse(await res.json(), this.log);
+            return await parseBlobJsonsFromResponse(await res.json(), this.log);
           }
           currentSlot++;
           maxRetries--;
@@ -437,8 +450,17 @@ export class HttpBlobClient implements BlobClientInterface {
     hostUrl: string,
     blockHashOrSlot: string | number,
     l1ConsensusHostIndex?: number,
+    blobHashes?: Buffer[],
   ): Promise<Response> {
-    const baseUrl = `${hostUrl}/eth/v1/beacon/blob_sidecars/${blockHashOrSlot}`;
+    let baseUrl = `${hostUrl}/eth/v1/beacon/blobs/${blockHashOrSlot}`;
+
+    if (blobHashes && blobHashes.length > 0) {
+      const params = new URLSearchParams();
+      for (const hash of blobHashes) {
+        params.append('versioned_hashes', `0x${hash.toString('hex')}`);
+      }
+      baseUrl += `?${params.toString()}`;
+    }
 
     const { url, ...options } = getBeaconNodeFetchOptions(baseUrl, this.config, l1ConsensusHostIndex);
     this.log.debug(`Fetching blob sidecar for ${blockHashOrSlot}`, { url, ...options });
@@ -480,34 +502,50 @@ export class HttpBlobClient implements BlobClientInterface {
    * @param blockHash - The block hash
    * @returns The slot number
    */
-  private async getSlotNumber(blockHash: `0x${string}`): Promise<number | undefined> {
+  private async getSlotNumber(
+    blockHash: `0x${string}`,
+    parentBeaconBlockRoot?: string,
+    l1BlockTimestamp?: bigint,
+  ): Promise<number | undefined> {
     const { l1ConsensusHostUrls, l1RpcUrls } = this.config;
     if (!l1ConsensusHostUrls || l1ConsensusHostUrls.length === 0) {
       this.log.debug('No consensus host url configured');
       return undefined;
     }
 
-    if (!l1RpcUrls || l1RpcUrls.length === 0) {
-      this.log.debug('No execution host url configured');
-      return undefined;
+    // Primary path: compute slot from timestamp if genesis config is cached (no network call needed)
+    if (
+      l1BlockTimestamp !== undefined &&
+      this.beaconGenesisTime !== undefined &&
+      this.beaconSecondsPerSlot !== undefined
+    ) {
+      const slot = Number((l1BlockTimestamp - this.beaconGenesisTime) / BigInt(this.beaconSecondsPerSlot));
+      this.log.debug(`Computed slot ${slot} from L1 block timestamp`, { l1BlockTimestamp });
+      return slot;
     }
 
-    // Ping execution node to get the parentBeaconBlockRoot for this block
-    let parentBeaconBlockRoot: string | undefined;
-    const client = createPublicClient({
-      transport: fallback(l1RpcUrls.map(url => http(url, { batch: false }))),
-    });
-    try {
-      const res: RpcBlock = await client.request({
-        method: 'eth_getBlockByHash',
-        params: [blockHash, /*tx flag*/ false],
+    if (!parentBeaconBlockRoot) {
+      // parentBeaconBlockRoot not provided by caller — fetch it from the execution RPC
+      if (!l1RpcUrls || l1RpcUrls.length === 0) {
+        this.log.debug('No execution host url configured');
+        return undefined;
+      }
+
+      const client = createPublicClient({
+        transport: fallback(l1RpcUrls.map(url => http(url, { batch: false }))),
       });
+      try {
+        const res: RpcBlock = await client.request({
+          method: 'eth_getBlockByHash',
+          params: [blockHash, /*tx flag*/ false],
+        });
 
-      if (res.parentBeaconBlockRoot) {
-        parentBeaconBlockRoot = res.parentBeaconBlockRoot;
+        if (res.parentBeaconBlockRoot) {
+          parentBeaconBlockRoot = res.parentBeaconBlockRoot;
+        }
+      } catch (err) {
+        this.log.error(`Error getting parent beacon block root`, err);
       }
-    } catch (err) {
-      this.log.error(`Error getting parent beacon block root`, err);
     }
 
     if (!parentBeaconBlockRoot) {
@@ -545,12 +583,105 @@ export class HttpBlobClient implements BlobClientInterface {
   public getArchiveClient(): BlobArchiveClient | undefined {
     return this.archiveClient;
   }
+
+  /** Returns true if this client can upload blobs to filestore. */
+  public canUpload(): boolean {
+    return this.fileStoreUploadClient !== undefined;
+  }
+
+  /**
+   * Start the blob client.
+   * Fetches and caches beacon genesis config for timestamp-based slot resolution,
+   * then uploads the initial healthcheck file (awaited) and starts periodic uploads.
+   */
+  public async start(): Promise<void> {
+    await this.fetchBeaconConfig();
+
+    if (!this.fileStoreUploadClient) {
+      return;
+    }
+
+    await this.fileStoreUploadClient.uploadHealthcheck();
+    this.log.debug('Initial healthcheck file uploaded');
+
+    this.startPeriodicHealthcheckUpload();
+  }
+
+  /**
+   * Start periodic healthcheck upload to the file store to ensure it remains available even if accidentally deleted.
+   */
+  private startPeriodicHealthcheckUpload(): void {
+    const intervalMs =
+      (this.config.blobHealthcheckUploadIntervalMinutes ?? DEFAULT_HEALTHCHECK_UPLOAD_INTERVAL_MINUTES) * 60 * 1000;
+
+    this.healthcheckUploadIntervalId = setInterval(() => {
+      void this.fileStoreUploadClient!.uploadHealthcheck().catch(err => {
+        this.log.warn('Failed to upload periodic healthcheck file', err);
+      });
+    }, intervalMs);
+  }
+
+  /**
+   * Fetches and caches beacon genesis time and slot duration from the first available consensus host.
+   * These static values enable timestamp-based slot resolution, eliminating the per-fetch headers call.
+   * Logs a warning and leaves fields undefined if all hosts fail, callers fall back gracefully.
+   */
+  private async fetchBeaconConfig(): Promise<void> {
+    const { l1ConsensusHostUrls } = this.config;
+    if (!l1ConsensusHostUrls || l1ConsensusHostUrls.length === 0) {
+      return;
+    }
+
+    for (let i = 0; i < l1ConsensusHostUrls.length; i++) {
+      try {
+        const { url: genesisUrl, ...genesisOptions } = getBeaconNodeFetchOptions(
+          `${l1ConsensusHostUrls[i]}/eth/v1/config/genesis`,
+          this.config,
+          i,
+        );
+        const { url: specUrl, ...specOptions } = getBeaconNodeFetchOptions(
+          `${l1ConsensusHostUrls[i]}/eth/v1/config/spec`,
+          this.config,
+          i,
+        );
+
+        const [genesisRes, specRes] = await Promise.all([
+          this.fetch(genesisUrl, genesisOptions),
+          this.fetch(specUrl, specOptions),
+        ]);
+
+        if (genesisRes.ok && specRes.ok) {
+          const genesis = await genesisRes.json();
+          const spec = await specRes.json();
+          this.beaconGenesisTime = BigInt(genesis.data.genesisTime);
+          this.beaconSecondsPerSlot = parseInt(spec.data.secondsPerSlot);
+          this.log.debug(`Fetched beacon genesis config`, {
+            genesisTime: this.beaconGenesisTime,
+            secondsPerSlot: this.beaconSecondsPerSlot,
+          });
+          return;
+        }
+      } catch (err) {
+        this.log.warn(`Failed to fetch beacon config from host ${l1ConsensusHostUrls[i]}`, err);
+      }
+    }
+    this.log.warn('Could not fetch beacon genesis config from any consensus host — will use headers call fallback');
+  }
+
+  /**
+   * Stop the blob client, clearing any periodic tasks.
+   */
+  public stop(): void {
+    if (this.healthcheckUploadIntervalId) {
+      clearInterval(this.healthcheckUploadIntervalId);
+      this.healthcheckUploadIntervalId = undefined;
+    }
+  }
 }
 
-function parseBlobJsonsFromResponse(response: any, logger: Logger): BlobJson[] {
+async function parseBlobJsonsFromResponse(response: any, logger: Logger): Promise<BlobJson[]> {
   try {
-    const blobs = response.data.map(parseBlobJson);
-    return blobs;
+    return await Promise.all((response.data as string[]).map(parseBlobJson));
   } catch (err) {
     logger.error(`Error parsing blob json from response`, err);
     return [];
@@ -561,16 +692,19 @@ function parseBlobJsonsFromResponse(response: any, logger: Logger): BlobJson[] {
 // https://ethereum.github.io/beacon-APIs/?urls.primaryName=dev#/Beacon/getBlobSidecars
 // Here we attempt to parse the response data to Buffer, and check the lengths (via Blob's constructor), to avoid
 // throwing an error down the line when calling Blob.fromJson().
-function parseBlobJson(data: any): BlobJson {
-  const blobBuffer = Buffer.from(data.blob.slice(2), 'hex');
-  const commitmentBuffer = Buffer.from(data.kzg_commitment.slice(2), 'hex');
-  const blob = new Blob(blobBuffer, commitmentBuffer);
+async function parseBlobJson(rawHex: string): Promise<BlobJson> {
+  const blobBuffer = Buffer.from(rawHex.slice(2), 'hex');
+  const blob = await Blob.fromBlobBuffer(blobBuffer);
   return blob.toJSON();
 }
 
 // Returns an array that maps each blob hash to the corresponding blob, or undefined if the blob is not found
 // or the data does not match the commitment.
-function processFetchedBlobs(blobs: BlobJson[], blobHashes: Buffer[], logger: Logger): (Blob | undefined)[] {
+async function processFetchedBlobs(
+  blobs: BlobJson[],
+  blobHashes: Buffer[],
+  logger: Logger,
+): Promise<(Blob | undefined)[]> {
   const requestedBlobHashes = new Set<string>(blobHashes.map(bufferToHex));
   const hashToBlob = new Map<string, Blob>();
   for (const blobJson of blobs) {
@@ -580,7 +714,7 @@ function processFetchedBlobs(blobs: BlobJson[], blobHashes: Buffer[], logger: Lo
     }
 
     try {
-      const blob = Blob.fromJson(blobJson);
+      const blob = await Blob.fromJson(blobJson);
       hashToBlob.set(hashHex, blob);
     } catch (err) {
       // If the above throws, it's likely that the blob commitment does not match the hash of the blob data.

package/src/client/interface.ts

@@ -11,6 +11,17 @@ export interface GetBlobSidecarOptions {
    * - Near tip: FileStore first with no retries (data should exist), L1 consensus second (freshest data), then FileStore with retries, then archive (eg. blobscan)
    */
   isHistoricalSync?: boolean;
+  /**
+   * The parent beacon block root for the L1 block containing the blobs.
+   * If provided, skips the eth_getBlockByHash execution RPC call inside getSlotNumber.
+   */
+  parentBeaconBlockRoot?: string;
+  /**
+   * The timestamp of the L1 execution block containing the blobs.
+   * When provided alongside a cached beacon genesis config (fetched at startup), allows computing
+   * the beacon slot directly via timestamp math, skipping the beacon headers network call entirely.
+   */
+  l1BlockTimestamp?: bigint;
 }
 
 export interface BlobClientInterface {
@@ -18,6 +29,12 @@ export interface BlobClientInterface {
   sendBlobsToFilestore(blobs: Blob[]): Promise<boolean>;
   /** Fetches the given blob sidecars by block hash and blob hashes. */
   getBlobSidecar(blockId: string, blobHashes?: Buffer[], opts?: GetBlobSidecarOptions): Promise<Blob[]>;
+  /** Starts the blob client (e.g., uploads healthcheck file if not exists). */
+  start?(): Promise<void>;
   /** Tests all configured blob sources and logs whether they are reachable or not. */
   testSources(): Promise<void>;
+  /** Stops the blob client, clearing any periodic tasks. */
+  stop?(): void;
+  /** Returns true if this client can upload blobs to filestore. */
+  canUpload(): boolean;
 }

package/src/client/local.ts

@@ -22,4 +22,9 @@ export class LocalBlobClient implements BlobClientInterface {
   public getBlobSidecar(_blockId: string, blobHashes: Buffer[], _opts?: GetBlobSidecarOptions): Promise<Blob[]> {
     return this.blobStore.getBlobsByHashes(blobHashes);
   }
+
+  /** Returns true if this client can upload blobs. Always true for local client. */
+  public canUpload(): boolean {
+    return true;
+  }
 }

package/src/client/tests.ts

@@ -28,7 +28,7 @@ export function runBlobClientTests(
   });
 
   it('should send and retrieve blobs by hash', async () => {
-    const blob = makeRandomBlob(5);
+    const blob = await makeRandomBlob(5);
     const blobHash = blob.getEthVersionedBlobHash();
 
     await client.sendBlobsToFilestore([blob]);
@@ -39,7 +39,7 @@ export function runBlobClientTests(
   });
 
   it('should handle multiple blobs', async () => {
-    const blobs = Array.from({ length: 3 }, () => makeRandomBlob(7));
+    const blobs = await Promise.all(Array.from({ length: 3 }, () => makeRandomBlob(7)));
     const blobHashes = blobs.map(blob => blob.getEthVersionedBlobHash());
 
     await client.sendBlobsToFilestore(blobs);

package/src/filestore/filestore_blob_client.ts

@@ -3,6 +3,7 @@ import { type Logger, createLogger } from '@aztec/foundation/log';
 import type { FileStore, ReadOnlyFileStore } from '@aztec/stdlib/file-store';
 
 import { inboundTransform, outboundTransform } from '../encoding/index.js';
+import { HEALTHCHECK_CONTENT, HEALTHCHECK_FILENAME } from './healthcheck.js';
 
 /**
  * A blob client that uses a FileStore (S3/GCS/local) as the data source.
@@ -27,6 +28,14 @@ export class FileStoreBlobClient {
     return `${this.basePath}/blobs/${versionedBlobHash}.data`;
   }
 
+  /**
+   * Get the path for the healthcheck file.
+   * Format: basePath/.healthcheck
+   */
+  private healthcheckPath(): string {
+    return `${this.basePath}/${HEALTHCHECK_FILENAME}`;
+  }
+
   /**
    * Fetch blobs by their versioned hashes.
    * @param blobHashes - Array of versioned blob hashes (0x-prefixed hex strings)
@@ -104,12 +113,31 @@ export class FileStoreBlobClient {
   }
 
   /**
-   * Test if the filestore connection is working.
+   * Test if the filestore connection is working by checking for healthcheck file.
+   * The healthcheck file is uploaded periodically by writable clients via HttpBlobClient.start().
+   * This provides a uniform connection test across all store types (S3/GCS/Local/HTTP).
+   */
+  async testConnection(): Promise<boolean> {
+    try {
+      return await this.store.exists(this.healthcheckPath());
+    } catch (err: any) {
+      this.log.warn(`Connection test failed: ${err?.message ?? String(err)}`);
+      return false;
+    }
+  }
+
+  /**
+   * Upload the healthcheck file if it doesn't already exist.
+   * This enables read-only clients (HTTP) to verify connectivity.
    */
-  testConnection(): Promise<boolean> {
-    // This implementation will be improved in a separate PR
-    // Currently underlying filestore implementations do not expose an easy way to test connectivitiy
-    return Promise.resolve(true);
+  async uploadHealthcheck(): Promise<void> {
+    if (!this.isWritable()) {
+      this.log.trace('Cannot upload healthcheck: store is read-only');
+      return;
+    }
+    const path = this.healthcheckPath();
+    await (this.store as FileStore).save(path, Buffer.from(HEALTHCHECK_CONTENT));
+    this.log.debug(`Uploaded healthcheck file to ${path}`);
   }
 
   /**

package/src/filestore/healthcheck.ts

@@ -0,0 +1,5 @@
+/** Constants for healthcheck file used to test file store connectivity. */
+
+export const HEALTHCHECK_FILENAME = '.healthcheck';
+export const HEALTHCHECK_CONTENT = 'ok';
+export const DEFAULT_HEALTHCHECK_UPLOAD_INTERVAL_MINUTES = 60; // 1 hour