@aztec/blob-client 3.0.0-nightly.20251223

package/src/archive/index.ts

@@ -0,0 +1,2 @@
+export * from './interface.js';
+export * from './factory.js';

package/src/archive/instrumentation.ts

@@ -0,0 +1,41 @@
+import { Attributes, Metrics, type TelemetryClient, type UpDownCounter, ValueType } from '@aztec/telemetry-client';
+
+export class BlobArchiveClientInstrumentation {
+  private blockRequestCounter: UpDownCounter;
+  private blobRequestCounter: UpDownCounter;
+  private retrievedBlobs: UpDownCounter;
+
+  constructor(
+    client: TelemetryClient,
+    private httpHost: string,
+    name: string,
+  ) {
+    const meter = client.getMeter(name);
+    this.blockRequestCounter = meter.createUpDownCounter(Metrics.BLOB_SINK_ARCHIVE_BLOCK_REQUEST_COUNT, {
+      description: 'Number of requests made to retrieve blocks from the blob archive',
+      valueType: ValueType.INT,
+    });
+
+    this.blobRequestCounter = meter.createUpDownCounter(Metrics.BLOB_SINK_ARCHIVE_BLOB_REQUEST_COUNT, {
+      description: 'Number of requests made to retrieve blobs from the blob archive',
+      valueType: ValueType.INT,
+    });
+
+    this.retrievedBlobs = meter.createUpDownCounter(Metrics.BLOB_SINK_ARCHIVE_BLOB_COUNT, {
+      description: 'Number of blobs retrieved from the blob archive',
+      valueType: ValueType.INT,
+    });
+  }
+
+  incRequest(type: 'blocks' | 'blobs', status: number) {
+    const counter = type === 'blocks' ? this.blockRequestCounter : this.blobRequestCounter;
+    counter.add(1, {
+      [Attributes.HTTP_RESPONSE_STATUS_CODE]: status,
+      [Attributes.HTTP_REQUEST_HOST]: this.httpHost,
+    });
+  }
+
+  incRetrievedBlobs(count: number) {
+    this.retrievedBlobs.add(count);
+  }
+}

package/src/archive/interface.ts

@@ -0,0 +1,9 @@
+import type { BlobJson } from '@aztec/blob-lib/types';
+
+/** Interface to an blob archiving service. */
+export interface BlobArchiveClient {
+  getBlobData(id: string): Promise<Buffer | undefined>;
+  getBlobsFromBlock(blockId: string): Promise<BlobJson[] | undefined>;
+  getLatestBlock(): Promise<{ hash: string; number: number; slot: number }>;
+  getBaseUrl(): string;
+}

package/src/blobstore/blob_store_test_suite.ts

@@ -0,0 +1,137 @@
+import { Blob } from '@aztec/blob-lib';
+import { Fr } from '@aztec/foundation/curves/bn254';
+
+import { BlobWithIndex } from '../types/index.js';
+import type { BlobStore } from './interface.js';
+
+export function describeBlobStore(getBlobStore: () => Promise<BlobStore>) {
+  let blobStore: BlobStore;
+
+  beforeEach(async () => {
+    blobStore = await getBlobStore();
+  });
+
+  it('should store and retrieve a blob by hash', async () => {
+    // Create a test blob with random fields
+    const testFields = [Fr.random(), Fr.random(), Fr.random()];
+    const blob = Blob.fromFields(testFields);
+    const blobWithIndex = new BlobWithIndex(blob, 0);
+    const blobHash = blob.getEthVersionedBlobHash();
+
+    // Store the blob
+    await blobStore.addBlobs([blobWithIndex]);
+
+    // Retrieve the blob by hash
+    const retrievedBlobs = await blobStore.getBlobsByHashes([blobHash]);
+
+    // Verify the blob was retrieved and matches
+    expect(retrievedBlobs.length).toBe(1);
+    expect(retrievedBlobs[0].blob).toEqual(blob);
+  });
+
+  it('should handle multiple blobs stored and retrieved by their hashes', async () => {
+    // Create two different blobs
+    const blob1 = Blob.fromFields([Fr.random(), Fr.random()]);
+    const blob2 = Blob.fromFields([Fr.random(), Fr.random(), Fr.random()]);
+    const blobWithIndex1 = new BlobWithIndex(blob1, 0);
+    const blobWithIndex2 = new BlobWithIndex(blob2, 1);
+
+    const blobHash1 = blob1.getEthVersionedBlobHash();
+    const blobHash2 = blob2.getEthVersionedBlobHash();
+
+    // Store both blobs
+    await blobStore.addBlobs([blobWithIndex1, blobWithIndex2]);
+
+    // Retrieve and verify both blobs
+    const retrievedBlobs = await blobStore.getBlobsByHashes([blobHash1, blobHash2]);
+
+    expect(retrievedBlobs.length).toBe(2);
+    expect(retrievedBlobs[0].blob).toEqual(blob1);
+    expect(retrievedBlobs[1].blob).toEqual(blob2);
+  });
+
+  it('should return empty array for non-existent blob hash', async () => {
+    // Create a random hash that doesn't exist
+    const nonExistentHash = Buffer.alloc(32);
+    nonExistentHash.fill(0xff);
+
+    const retrievedBlobs = await blobStore.getBlobsByHashes([nonExistentHash]);
+    expect(retrievedBlobs).toEqual([]);
+  });
+
+  it('should handle storing blobs with different indices', async () => {
+    // Create blobs with different indices
+    const blob1 = Blob.fromFields([Fr.random()]);
+    const blob2 = Blob.fromFields([Fr.random()]);
+    const blobWithIndex1 = new BlobWithIndex(blob1, 0);
+    const blobWithIndex2 = new BlobWithIndex(blob2, 1);
+
+    await blobStore.addBlobs([blobWithIndex1, blobWithIndex2]);
+
+    const blobHash1 = blob1.getEthVersionedBlobHash();
+    const blobHash2 = blob2.getEthVersionedBlobHash();
+
+    const retrievedBlobs = await blobStore.getBlobsByHashes([blobHash1, blobHash2]);
+
+    expect(retrievedBlobs[0].index).toBe(0);
+    expect(retrievedBlobs[1].index).toBe(1);
+  });
+
+  it('should handle retrieving subset of stored blobs', async () => {
+    // Store multiple blobs
+    const blob1 = Blob.fromFields([Fr.random()]);
+    const blob2 = Blob.fromFields([Fr.random()]);
+    const blob3 = Blob.fromFields([Fr.random()]);
+
+    await blobStore.addBlobs([new BlobWithIndex(blob1, 0), new BlobWithIndex(blob2, 1), new BlobWithIndex(blob3, 2)]);
+
+    // Retrieve only some of them
+    const blobHash1 = blob1.getEthVersionedBlobHash();
+    const blobHash3 = blob3.getEthVersionedBlobHash();
+
+    const retrievedBlobs = await blobStore.getBlobsByHashes([blobHash1, blobHash3]);
+
+    expect(retrievedBlobs.length).toBe(2);
+    expect(retrievedBlobs[0].blob).toEqual(blob1);
+    expect(retrievedBlobs[1].blob).toEqual(blob3);
+  });
+
+  it('should handle duplicate blob hashes in request', async () => {
+    const blob = Blob.fromFields([Fr.random()]);
+    const blobWithIndex = new BlobWithIndex(blob, 0);
+    const blobHash = blob.getEthVersionedBlobHash();
+
+    await blobStore.addBlobs([blobWithIndex]);
+
+    // Request the same blob hash multiple times
+    const retrievedBlobs = await blobStore.getBlobsByHashes([blobHash, blobHash]);
+
+    // Implementation may return duplicates or deduplicate - both are valid
+    expect(retrievedBlobs.length).toBeGreaterThanOrEqual(1);
+    expect(retrievedBlobs[0].blob).toEqual(blob);
+  });
+
+  it('should overwrite blob when storing with same hash', async () => {
+    // Create two blobs that will have the same hash (same content)
+    const fields = [Fr.random(), Fr.random()];
+    const blob1 = Blob.fromFields(fields);
+    const blob2 = Blob.fromFields(fields);
+
+    // Store with different indices
+    const blobWithIndex1 = new BlobWithIndex(blob1, 0);
+    const blobWithIndex2 = new BlobWithIndex(blob2, 5);
+
+    const blobHash = blob1.getEthVersionedBlobHash();
+
+    // Store first blob
+    await blobStore.addBlobs([blobWithIndex1]);
+
+    // Overwrite with second blob (same hash, different index)
+    await blobStore.addBlobs([blobWithIndex2]);
+
+    // Retrieve and verify it's the second blob (with index 5)
+    const retrievedBlobs = await blobStore.getBlobsByHashes([blobHash]);
+    expect(retrievedBlobs.length).toBe(1);
+    expect(retrievedBlobs[0].index).toBe(5);
+  });
+}

package/src/blobstore/index.ts

@@ -0,0 +1,2 @@
+export * from './memory_blob_store.js';
+export * from './interface.js';

package/src/blobstore/interface.ts

@@ -0,0 +1,12 @@
+import type { BlobWithIndex } from '../types/index.js';
+
+export interface BlobStore {
+  /**
+   * Get blobs by their hashes
+   */
+  getBlobsByHashes: (blobHashes: Buffer[]) => Promise<BlobWithIndex[]>;
+  /**
+   * Add blobs to the store, indexed by their hashes
+   */
+  addBlobs: (blobs: BlobWithIndex[]) => Promise<void>;
+}

package/src/blobstore/memory_blob_store.ts

@@ -0,0 +1,31 @@
+import { bufferToHex } from '@aztec/foundation/string';
+
+import { BlobWithIndex } from '../types/index.js';
+import type { BlobStore } from './interface.js';
+
+export class MemoryBlobStore implements BlobStore {
+  private blobs: Map<string, Buffer> = new Map();
+
+  public getBlobsByHashes(blobHashes: Buffer[]): Promise<BlobWithIndex[]> {
+    const results: BlobWithIndex[] = [];
+
+    for (const blobHash of blobHashes) {
+      const key = bufferToHex(blobHash);
+      const blobBuffer = this.blobs.get(key);
+      if (blobBuffer) {
+        results.push(BlobWithIndex.fromBuffer(blobBuffer));
+      }
+    }
+
+    return Promise.resolve(results);
+  }
+
+  public addBlobs(blobs: BlobWithIndex[]): Promise<void> {
+    for (const blob of blobs) {
+      const blobHash = blob.blob.getEthVersionedBlobHash();
+      const key = bufferToHex(blobHash);
+      this.blobs.set(key, blob.toBuffer());
+    }
+    return Promise.resolve();
+  }
+}

package/src/client/bin/index.ts

@@ -0,0 +1,35 @@
+#!/usr/bin/env node
+/* eslint-disable no-console */
+import { createLogger } from '@aztec/foundation/log';
+import { bufferToHex, hexToBuffer } from '@aztec/foundation/string';
+
+import { getBlobClientConfigFromEnv } from '../config.js';
+import { createBlobClient } from '../factory.js';
+
+async function main() {
+  const logger = createLogger('blob-client');
+  const blockHash = process.argv[2];
+  if (!blockHash) {
+    logger.error('Please provide a block hash as an argument.');
+    process.exit(1);
+  }
+  const blobHashes = process.argv.slice(3).map(hexToBuffer);
+  logger.info(`Fetching blobs for block hash ${blockHash}`);
+  if (blobHashes.length > 0) {
+    logger.info(`Filtering by blob hashes ${blobHashes.map(bufferToHex).join(', ')}`);
+  }
+
+  const blobClient = createBlobClient(getBlobClientConfigFromEnv());
+  const blobs = await blobClient.getBlobSidecar(blockHash, blobHashes);
+  logger.info(`Got ${blobs.length} blobs`);
+  for (const blob of blobs) {
+    console.log(blob.toJSON());
+  }
+}
+
+// Example usage:
+// $ L1_CHAIN_ID=11155111 LOG_LEVEL=trace yarn blob-client 0x7d81980a40426c40544f0f729ada953be406730b877b5865d6cdc35cc8f9c84e 0x010657f37554c781402a22917dee2f75def7ab966d7b770905398eba3c444014
+main().catch(err => {
+  console.error(err);
+  process.exit(1);
+});

package/src/client/config.ts

@@ -0,0 +1,117 @@
+import {
+  type ConfigMappingsType,
+  SecretValue,
+  booleanConfigHelper,
+  getConfigFromMappings,
+} from '@aztec/foundation/config';
+
+import { type BlobArchiveApiConfig, blobArchiveApiConfigMappings } from '../archive/config.js';
+
+/**
+ * The configuration for the blob client
+ */
+export interface BlobClientConfig extends BlobArchiveApiConfig {
+  /**
+   * List of URLs for L1 RPC Execution clients
+   */
+  l1RpcUrls?: string[];
+
+  /**
+   * List of URLs of the Ethereum consensus nodes that services will connect to (comma separated)
+   */
+  l1ConsensusHostUrls?: string[];
+
+  /**
+   * List of API keys for the corresponding L1 consensus client URLs. Added at the end of the URL as "?key=<api-key>" unless a header is defined
+   */
+  l1ConsensusHostApiKeys?: SecretValue<string>[];
+
+  /**
+   * List of header names for the corresponding L1 consensus client API keys, if needed. Added as "<api-key-header>: <api-key>"
+   */
+  l1ConsensusHostApiKeyHeaders?: string[];
+
+  /**
+   * The map size to be provided to LMDB for each blob sink DB, optional, will inherit from the general dataStoreMapSizeKb if not specified
+   */
+  blobSinkMapSizeKb?: number;
+
+  /**
+   * Whether to allow having no blob sources configured during startup
+   */
+  blobAllowEmptySources?: boolean;
+
+  /**
+   * URLs for reading blobs from filestore (s3://, gs://, file://, https://). Tried in order until blobs are found.
+   */
+  blobFileStoreUrls?: string[];
+
+  /**
+   * URL for uploading blobs to filestore (s3://, gs://, file://)
+   */
+  blobFileStoreUploadUrl?: string;
+}
+
+export const blobClientConfigMapping: ConfigMappingsType<BlobClientConfig> = {
+  l1RpcUrls: {
+    env: 'ETHEREUM_HOSTS',
+    description: 'List of URLs for L1 RPC Execution clients',
+    parseEnv: (val: string) => val.split(',').map(url => url.trim()),
+  },
+  l1ConsensusHostUrls: {
+    env: 'L1_CONSENSUS_HOST_URLS',
+    description: 'List of URLs of the Ethereum consensus nodes that services will connect to (comma separated)',
+    parseEnv: (val: string) => val.split(',').map(url => url.trim().replace(/\/$/, '')),
+  },
+  l1ConsensusHostApiKeys: {
+    env: 'L1_CONSENSUS_HOST_API_KEYS',
+    description:
+      'List of API keys for the corresponding L1 consensus clients, if needed. Added to the end of the corresponding URL as "?key=<api-key>" unless a header is defined',
+    parseEnv: (val: string) => val.split(',').map(key => new SecretValue(key.trim())),
+  },
+  l1ConsensusHostApiKeyHeaders: {
+    env: 'L1_CONSENSUS_HOST_API_KEY_HEADERS',
+    description:
+      'List of header names for the corresponding L1 consensus client API keys, if needed. Added to the corresponding request as "<api-key-header>: <api-key>"',
+    parseEnv: (val: string) => val.split(',').map(url => url.trim()),
+  },
+  blobSinkMapSizeKb: {
+    env: 'BLOB_SINK_MAP_SIZE_KB',
+    description: 'The maximum possible size of the blob sink DB in KB. Overwrites the general dataStoreMapSizeKb.',
+    parseEnv: (val: string | undefined) => (val ? +val : undefined),
+  },
+  blobAllowEmptySources: {
+    env: 'BLOB_ALLOW_EMPTY_SOURCES',
+    description: 'Whether to allow having no blob sources configured during startup',
+    ...booleanConfigHelper(false),
+  },
+  blobFileStoreUrls: {
+    env: 'BLOB_FILE_STORE_URLS',
+    description: 'URLs for filestore blob archive, comma-separated. Tried in order until blobs are found.',
+    parseEnv: (val: string) =>
+      val
+        .split(',')
+        .map(url => url.trim())
+        .filter(url => url.length > 0),
+  },
+  blobFileStoreUploadUrl: {
+    env: 'BLOB_FILE_STORE_UPLOAD_URL',
+    description: 'URL for uploading blobs to filestore (s3://, gs://, file://)',
+  },
+  ...blobArchiveApiConfigMappings,
+};
+
+/**
+ * Returns the blob client configuration from the environment variables.
+ * @returns The blob client configuration.
+ */
+export function getBlobClientConfigFromEnv(): BlobClientConfig {
+  return getConfigFromMappings<BlobClientConfig>(blobClientConfigMapping);
+}
+
+/**
+ * Returns whether the given blob client config has any remote sources defined.
+ */
+export function hasRemoteBlobSources(config: BlobClientConfig = {}): boolean {
+  return !!(config.l1ConsensusHostUrls?.length || config.archiveApiUrl || config.blobFileStoreUrls?.length);
+}

package/src/client/factory.ts

@@ -0,0 +1,88 @@
+import { type Logger, createLogger } from '@aztec/foundation/log';
+
+import { MemoryBlobStore } from '../blobstore/memory_blob_store.js';
+import {
+  type BlobFileStoreMetadata,
+  createReadOnlyFileStoreBlobClients,
+  createWritableFileStoreBlobClient,
+} from '../filestore/factory.js';
+import type { FileStoreBlobClient } from '../filestore/filestore_blob_client.js';
+import { type BlobClientConfig, hasRemoteBlobSources } from './config.js';
+import { HttpBlobClient } from './http.js';
+import type { BlobClientInterface } from './interface.js';
+import { LocalBlobClient } from './local.js';
+
+export interface CreateBlobClientDeps {
+  logger?: Logger;
+  /** FileStore clients for reading blobs */
+  fileStoreClients?: FileStoreBlobClient[];
+  /** FileStore client for uploading blobs */
+  fileStoreUploadClient?: FileStoreBlobClient;
+}
+
+export function createBlobClient(config?: BlobClientConfig, deps?: CreateBlobClientDeps): BlobClientInterface {
+  const log = deps?.logger ?? createLogger('blob-client');
+  if (!hasRemoteBlobSources(config)) {
+    log.info(`Creating local blob client.`);
+    const blobStore = new MemoryBlobStore();
+    return new LocalBlobClient(blobStore);
+  }
+
+  log.info(`Creating blob client.`, {
+    l1ConsensusHostUrls: config?.l1ConsensusHostUrls,
+    archiveApiUrl: config?.archiveApiUrl,
+    fileStoreCount: deps?.fileStoreClients?.length ?? 0,
+    hasFileStoreUpload: !!deps?.fileStoreUploadClient,
+  });
+  return new HttpBlobClient(config, {
+    logger: log,
+    fileStoreClients: deps?.fileStoreClients,
+    fileStoreUploadClient: deps?.fileStoreUploadClient,
+  });
+}
+
+/**
+ * Configuration required for creating a blob client with file stores.
+ * Extends BlobClientConfig with the metadata needed to construct file store paths.
+ */
+export interface BlobClientWithFileStoresConfig extends BlobClientConfig {
+  l1ChainId: number;
+  rollupVersion: number;
+  l1Contracts: { rollupAddress: { toString(): string } };
+}
+
+/**
+ * Creates a BlobClient with FileStore clients for reading and uploading blobs.
+ * This is a convenience function that handles the common pattern of:
+ * 1. Building BlobFileStoreMetadata from config
+ * 2. Creating read-only FileStore clients
+ * 3. Creating a writable FileStore client for uploads
+ * 4. Creating the BlobClient with these dependencies
+ *
+ * @param config - Configuration containing blob client settings and chain metadata
+ * @param logger - Optional logger for the blob client
+ * @returns A BlobClientInterface configured with file store support
+ */
+export async function createBlobClientWithFileStores(
+  config: BlobClientWithFileStoresConfig,
+  logger?: Logger,
+): Promise<BlobClientInterface> {
+  const log = logger ?? createLogger('blob-client');
+
+  const fileStoreMetadata: BlobFileStoreMetadata = {
+    l1ChainId: config.l1ChainId,
+    rollupVersion: config.rollupVersion,
+    rollupAddress: config.l1Contracts.rollupAddress.toString(),
+  };
+
+  const [fileStoreClients, fileStoreUploadClient] = await Promise.all([
+    createReadOnlyFileStoreBlobClients(config.blobFileStoreUrls, fileStoreMetadata, log),
+    createWritableFileStoreBlobClient(config.blobFileStoreUploadUrl, fileStoreMetadata, log),
+  ]);
+
+  return createBlobClient(config, {
+    logger: log,
+    fileStoreClients,
+    fileStoreUploadClient,
+  });
+}