@aztec/blob-client 3.0.0-nightly.20251223

package/src/client/tests.ts

@@ -0,0 +1,78 @@
+import { makeRandomBlob } from '@aztec/blob-lib/testing';
+
+import type { Hex } from 'viem';
+
+import type { BlobClientInterface } from './interface.js';
+
+/**
+ * Shared test suite for blob clients
+ * @param createClient - Function that creates a client instance for testing
+ * @param cleanup - Optional cleanup function to run after each test
+ */
+export function runBlobClientTests(
+  createClient: () => Promise<{ client: BlobClientInterface; cleanup: () => Promise<void> }>,
+) {
+  let blockId: Hex;
+  let client: BlobClientInterface;
+  let cleanup: () => Promise<void>;
+
+  beforeEach(async () => {
+    blockId = '0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef';
+    const setup = await createClient();
+    client = setup.client;
+    cleanup = setup.cleanup;
+  });
+
+  afterEach(async () => {
+    await cleanup();
+  });
+
+  it('should send and retrieve blobs by hash', async () => {
+    const blob = makeRandomBlob(5);
+    const blobHash = blob.getEthVersionedBlobHash();
+
+    await client.sendBlobsToFilestore([blob]);
+
+    const retrievedBlobs = await client.getBlobSidecar(blockId, [blobHash]);
+    expect(retrievedBlobs).toHaveLength(1);
+    expect(retrievedBlobs[0].blob).toEqual(blob);
+  });
+
+  it('should handle multiple blobs', async () => {
+    const blobs = Array.from({ length: 3 }, () => makeRandomBlob(7));
+    const blobHashes = blobs.map(blob => blob.getEthVersionedBlobHash());
+
+    await client.sendBlobsToFilestore(blobs);
+
+    const retrievedBlobs = await client.getBlobSidecar(blockId, blobHashes);
+    expect(retrievedBlobs.length).toBe(3);
+
+    for (let i = 0; i < blobs.length; i++) {
+      expect(retrievedBlobs[i].blob).toEqual(blobs[i]);
+    }
+  });
+
+  it('should return empty array for non-existent blob hash', async () => {
+    const nonExistentHash = Buffer.alloc(32);
+    nonExistentHash.fill(0xff);
+
+    const retrievedBlobs = await client.getBlobSidecar(blockId, [nonExistentHash]);
+    expect(retrievedBlobs).toEqual([]);
+  });
+
+  it('should preserve blob indices', async () => {
+    const blobs = Array.from({ length: 3 }, () => makeRandomBlob(7));
+    const blobHashes = blobs.map(blob => blob.getEthVersionedBlobHash());
+
+    await client.sendBlobsToFilestore(blobs);
+
+    const retrievedBlobs = await client.getBlobSidecar(blockId, blobHashes);
+    expect(retrievedBlobs.length).toBe(blobs.length);
+
+    // Indices should be assigned sequentially based on the order they were sent
+    for (let i = 0; i < blobs.length; i++) {
+      expect(retrievedBlobs[i].blob).toEqual(blobs[i]);
+      expect(retrievedBlobs[i].index).toBe(i);
+    }
+  });
+}

package/src/encoding/index.ts

@@ -0,0 +1,21 @@
+import { compressSync, uncompressSync } from 'snappy';
+
+/**
+ * Snappy decompress the blob buffer
+ *
+ * @param data - The blob buffer
+ * @returns The decompressed blob buffer
+ */
+export function inboundTransform(data: Buffer): Buffer {
+  return Buffer.from(uncompressSync(data, { asBuffer: true }));
+}
+
+/**
+ * Snappy compress the blob buffer
+ *
+ * @param data - The blob buffer
+ * @returns The compressed blob buffer
+ */
+export function outboundTransform(data: Buffer): Buffer {
+  return Buffer.from(compressSync(data));
+}

package/src/filestore/factory.ts

@@ -0,0 +1,145 @@
+import { type Logger, createLogger } from '@aztec/foundation/log';
+import {
+  type FileStore,
+  type ReadOnlyFileStore,
+  createFileStore,
+  createReadOnlyFileStore,
+} from '@aztec/stdlib/file-store';
+
+import { FileStoreBlobClient } from './filestore_blob_client.js';
+
+/**
+ * Metadata required to construct the base path for blob storage.
+ * Path format: aztec-{l1ChainId}-{rollupVersion}-{rollupAddress}/
+ */
+export interface BlobFileStoreMetadata {
+  /** The L1 chain ID */
+  l1ChainId: number;
+  /** The rollup version */
+  rollupVersion: number;
+  /** The rollup contract address (with or without 0x prefix) */
+  rollupAddress: string;
+}
+
+/**
+ * Constructs the base path for blob storage.
+ * Format: aztec-{l1ChainId}-{rollupVersion}-{rollupAddress}
+ */
+export function makeBlobBasePath(metadata: BlobFileStoreMetadata): string {
+  const { l1ChainId, rollupVersion, rollupAddress } = metadata;
+  // Normalize rollup address to lowercase without 0x prefix for consistency
+  const normalizedAddress = rollupAddress.toLowerCase().replace(/^0x/, '');
+  return `aztec-${l1ChainId}-${rollupVersion}-0x${normalizedAddress}`;
+}
+
+/**
+ * Creates a read-only FileStoreBlobClient for fetching blobs.
+ *
+ * @param storeUrl - The URL of the filestore (s3://, gs://, file://, https://)
+ * @param metadata - Chain metadata for constructing the base path
+ * @param logger - Optional logger
+ * @returns A FileStoreBlobClient for reading blobs, or undefined if storeUrl is undefined
+ */
+export async function createReadOnlyFileStoreBlobClient(
+  storeUrl: string,
+  metadata: BlobFileStoreMetadata,
+  logger?: Logger,
+): Promise<FileStoreBlobClient>;
+export async function createReadOnlyFileStoreBlobClient(
+  storeUrl: string | undefined,
+  metadata: BlobFileStoreMetadata,
+  logger?: Logger,
+): Promise<FileStoreBlobClient | undefined>;
+export async function createReadOnlyFileStoreBlobClient(
+  storeUrl: string | undefined,
+  metadata: BlobFileStoreMetadata,
+  logger?: Logger,
+): Promise<FileStoreBlobClient | undefined> {
+  if (!storeUrl) {
+    return undefined;
+  }
+
+  const log = logger ?? createLogger('blob-client:filestore-factory');
+  const basePath = makeBlobBasePath(metadata);
+
+  log.debug(`Creating read-only filestore blob client`, { storeUrl, basePath });
+
+  const store: ReadOnlyFileStore = await createReadOnlyFileStore(storeUrl, log);
+  return new FileStoreBlobClient(store, basePath, log);
+}
+
+/**
+ * Creates multiple read-only FileStoreBlobClients from an array of URLs.
+ *
+ * @param storeUrls - Array of filestore URLs
+ * @param metadata - Chain metadata for constructing the base path
+ * @param logger - Optional logger
+ * @returns Array of FileStoreBlobClients (excludes any that failed to create)
+ */
+export async function createReadOnlyFileStoreBlobClients(
+  storeUrls: string[] | undefined,
+  metadata: BlobFileStoreMetadata,
+  logger?: Logger,
+): Promise<FileStoreBlobClient[]> {
+  if (!storeUrls || storeUrls.length === 0) {
+    return [];
+  }
+
+  const log = logger ?? createLogger('blob-client:filestore-factory');
+  const clients: FileStoreBlobClient[] = [];
+
+  for (const storeUrl of storeUrls) {
+    try {
+      const client = await createReadOnlyFileStoreBlobClient(storeUrl, metadata, log);
+      if (client) {
+        clients.push(client);
+      }
+    } catch (err) {
+      log.error(`Failed to create read-only filestore blob client for ${storeUrl}`, err);
+    }
+  }
+
+  return clients;
+}
+
+/**
+ * Creates a writable FileStoreBlobClient for uploading blobs.
+ * Note: https:// URLs are not supported for upload, only s3://, gs://, or file://.
+ *
+ * @param storeUrl - The URL of the filestore (s3://, gs://, file://)
+ * @param metadata - Chain metadata for constructing the base path
+ * @param logger - Optional logger
+ * @returns A writable FileStoreBlobClient, or undefined if storeUrl is undefined
+ */
+export async function createWritableFileStoreBlobClient(
+  storeUrl: string,
+  metadata: BlobFileStoreMetadata,
+  logger?: Logger,
+): Promise<FileStoreBlobClient>;
+export async function createWritableFileStoreBlobClient(
+  storeUrl: string | undefined,
+  metadata: BlobFileStoreMetadata,
+  logger?: Logger,
+): Promise<FileStoreBlobClient | undefined>;
+export async function createWritableFileStoreBlobClient(
+  storeUrl: string | undefined,
+  metadata: BlobFileStoreMetadata,
+  logger?: Logger,
+): Promise<FileStoreBlobClient | undefined> {
+  if (!storeUrl) {
+    return undefined;
+  }
+
+  const log = logger ?? createLogger('blob-client:filestore-factory');
+  const basePath = makeBlobBasePath(metadata);
+
+  log.debug(`Creating writable filestore blob client`, { storeUrl, basePath });
+
+  const store: FileStore | undefined = await createFileStore(storeUrl, log);
+  if (!store) {
+    log.warn(`Failed to create writable filestore for ${storeUrl}`);
+    return undefined;
+  }
+
+  return new FileStoreBlobClient(store, basePath, log);
+}

package/src/filestore/filestore_blob_client.ts

@@ -0,0 +1,129 @@
+import { Blob, type BlobJson, computeEthVersionedBlobHash } from '@aztec/blob-lib';
+import { type Logger, createLogger } from '@aztec/foundation/log';
+import type { FileStore, ReadOnlyFileStore } from '@aztec/stdlib/file-store';
+
+import { inboundTransform, outboundTransform } from '../encoding/index.js';
+import { BlobWithIndex } from '../types/blob_with_index.js';
+
+/**
+ * A blob client that uses a FileStore (S3/GCS/local) as the data source.
+ * Blobs are stored as JSON files keyed by their versioned blob hash.
+ */
+export class FileStoreBlobClient {
+  private readonly log: Logger;
+
+  constructor(
+    private readonly store: ReadOnlyFileStore | FileStore,
+    private readonly basePath: string,
+    logger?: Logger,
+  ) {
+    this.log = logger ?? createLogger('blob-client:filestore-client');
+  }
+
+  /**
+   * Get the path for a blob file.
+   * Format: basePath/blobs/{versionedBlobHash}.data
+   */
+  private blobPath(versionedBlobHash: string): string {
+    return `${this.basePath}/blobs/${versionedBlobHash}.data`;
+  }
+
+  /**
+   * Fetch blobs by their versioned hashes.
+   * @param blobHashes - Array of versioned blob hashes (0x-prefixed hex strings)
+   * @returns Array of BlobJson objects for found blobs
+   */
+  async getBlobsByHashes(blobHashes: string[]): Promise<BlobJson[]> {
+    const blobs: BlobJson[] = [];
+
+    for (let i = 0; i < blobHashes.length; i++) {
+      try {
+        const path = this.blobPath(blobHashes[i]);
+        if (!(await this.store.exists(path))) {
+          continue;
+        }
+
+        const data = await this.store.read(path);
+        const json = JSON.parse(inboundTransform(data).toString()) as BlobJson;
+        // We don't know the actual index when fetching from filestore - use -1 to indicate this deliberately
+        blobs.push({ ...json, index: '-1' });
+      } catch (err) {
+        this.log.warn(`Failed to read blob ${blobHashes[i]} from filestore`, err);
+      }
+    }
+
+    return blobs;
+  }
+
+  /**
+   * Check if a blob exists in the store.
+   * @param versionedBlobHash - The versioned blob hash (0x-prefixed hex string)
+   */
+  exists(versionedBlobHash: string): Promise<boolean> {
+    return this.store.exists(this.blobPath(versionedBlobHash));
+  }
+
+  /**
+   * Save a single blob to the store.
+   * @param blob - The blob to save
+   * @param skipIfExists - Skip saving if blob already exists (default: true)
+   * @throws Error if the store is read-only
+   */
+  async saveBlob(blob: Blob, skipIfExists = true): Promise<void> {
+    if (!this.isWritable()) {
+      throw new Error('FileStore is read-only');
+    }
+
+    const versionedHash = `0x${computeEthVersionedBlobHash(blob.commitment).toString('hex')}`;
+
+    if (skipIfExists && (await this.store.exists(this.blobPath(versionedHash)))) {
+      this.log.trace(`Blob ${versionedHash} already exists, skipping`);
+      return;
+    }
+
+    // index=-1 is deliberate as we don't know the actual blob index in most cases when the filestores are used (blobs are saved in the filestores before we ever fetch them from L1)
+    const json = blob.toJson(-1);
+    await (this.store as FileStore).save(
+      this.blobPath(versionedHash),
+      outboundTransform(Buffer.from(JSON.stringify(json))),
+    );
+    this.log.debug(`Saved blob ${versionedHash} to filestore`);
+  }
+
+  /**
+   * Save multiple blobs to the store in parallel.
+   * @param blobs - The blobs to save (either Blob[] or BlobWithIndex[])
+   * @param skipIfExists - Skip saving if blob already exists (default: true)
+   */
+  async saveBlobs(blobs: Blob[] | BlobWithIndex[], skipIfExists = true): Promise<void> {
+    await Promise.all(
+      blobs.map(b => {
+        const blob = 'blob' in b ? b.blob : b;
+        return this.saveBlob(blob, skipIfExists);
+      }),
+    );
+  }
+
+  /**
+   * Get the base URL/path of the filestore.
+   */
+  getBaseUrl(): string {
+    return this.basePath;
+  }
+
+  /**
+   * Test if the filestore connection is working.
+   */
+  testConnection(): Promise<boolean> {
+    // This implementation will be improved in a separate PR
+    // Currently underlying filestore implementations do not expose an easy way to test connectivitiy
+    return Promise.resolve(true);
+  }
+
+  /**
+   * Check if the store supports write operations.
+   */
+  private isWritable(): boolean {
+    return 'save' in this.store;
+  }
+}

package/src/filestore/index.ts

@@ -0,0 +1,2 @@
+export * from './filestore_blob_client.js';
+export * from './factory.js';

package/src/types/api.ts

@@ -0,0 +1,50 @@
+import type { Hex } from 'viem';
+import { z } from 'zod';
+
+export interface PostBlobSidecarRequest {
+  blobs: Array<{
+    index: number;
+    blob: {
+      type: string;
+      data: string;
+    };
+  }>;
+}
+
+export const blockRootSchema = z
+  .string()
+  .regex(/^0x[0-9a-fA-F]{0,64}$/)
+  .max(66)
+  .transform(str => str as Hex);
+
+export const slotSchema = z.number().int().positive();
+
+// Define the Zod schema for an array of numbers
+export const indicesSchema = z.optional(
+  z
+    .string()
+    .refine(str => str.split(',').every(item => !isNaN(Number(item))), {
+      message: 'All items in the query must be valid numbers.',
+    })
+    .transform(str => str.split(',').map(Number)),
+); // Convert to an array of numbers
+
+/**
+ * Block identifier. The spec allows for <hex encoded blockRoot with 0x prefix>, <slot>, "head", "genesis", "finalized", but we only support the block root.
+ * See https://ethereum.github.io/beacon-APIs/?urls.primaryName=dev#/Beacon/getBlobSidecars.
+ */
+export const blockIdSchema = blockRootSchema;
+
+export const postBlobSidecarSchema = z.object({
+  // eslint-disable-next-line camelcase
+  block_id: blockIdSchema,
+  blobs: z.array(
+    z.object({
+      index: z.number().int().nonnegative(),
+      blob: z.object({
+        type: z.string(),
+        data: z.string(),
+      }),
+    }),
+  ),
+});

package/src/types/blob_with_index.ts

@@ -0,0 +1,48 @@
+import { Blob, type BlobJson } from '@aztec/blob-lib';
+import { BufferReader, serializeToBuffer } from '@aztec/foundation/serialize';
+
+/** Serialized an array of blobs with their indexes to be stored at a given block id */
+export class BlobsWithIndexes {
+  constructor(public blobs: BlobWithIndex[]) {}
+
+  public toBuffer(): Buffer {
+    return serializeToBuffer(this.blobs.length, this.blobs);
+  }
+
+  public static fromBuffer(buffer: Buffer | BufferReader): BlobsWithIndexes {
+    const reader = BufferReader.asReader(buffer);
+    return new BlobsWithIndexes(reader.readArray(reader.readNumber(), BlobWithIndex));
+  }
+
+  public getBlobsFromIndices(indices: number[]): BlobWithIndex[] {
+    return this.blobs.filter((_, index) => indices.includes(index));
+  }
+}
+
+/** We store blobs alongside their index in the block */
+export class BlobWithIndex {
+  constructor(
+    /** The blob */
+    public blob: Blob,
+    /** The index of the blob in the block */
+    public index: number,
+  ) {}
+
+  public toBuffer(): Buffer {
+    return serializeToBuffer([this.blob, this.index]);
+  }
+
+  public static fromBuffer(buffer: Buffer | BufferReader): BlobWithIndex {
+    const reader = BufferReader.asReader(buffer);
+    return new BlobWithIndex(reader.readObject(Blob), reader.readNumber());
+  }
+
+  public static fromJson(json: BlobJson): BlobWithIndex {
+    return new BlobWithIndex(Blob.fromJson(json), parseInt(json.index));
+  }
+
+  // Follows the structure the beacon node api expects
+  public toJSON(): BlobJson {
+    return this.blob.toJson(this.index);
+  }
+}

package/src/types/index.ts

@@ -0,0 +1 @@
+export * from './blob_with_index.js';