@aztec/blob-client 0.0.1-commit.03f7ef2

package/src/client/tests.ts

@@ -0,0 +1,62 @@
+import { makeRandomBlob } from '@aztec/blob-lib/testing';
+
+import type { Hex } from 'viem';
+
+import type { BlobClientInterface } from './interface.js';
+
+/**
+ * Shared test suite for blob clients
+ * @param createClient - Function that creates a client instance for testing
+ * @param cleanup - Optional cleanup function to run after each test
+ */
+export function runBlobClientTests(
+  createClient: () => Promise<{ client: BlobClientInterface; cleanup: () => Promise<void> }>,
+) {
+  let blockId: Hex;
+  let client: BlobClientInterface;
+  let cleanup: () => Promise<void>;
+
+  beforeEach(async () => {
+    blockId = '0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef';
+    const setup = await createClient();
+    client = setup.client;
+    cleanup = setup.cleanup;
+  });
+
+  afterEach(async () => {
+    await cleanup();
+  });
+
+  it('should send and retrieve blobs by hash', async () => {
+    const blob = makeRandomBlob(5);
+    const blobHash = blob.getEthVersionedBlobHash();
+
+    await client.sendBlobsToFilestore([blob]);
+
+    const retrievedBlobs = await client.getBlobSidecar(blockId, [blobHash]);
+    expect(retrievedBlobs).toHaveLength(1);
+    expect(retrievedBlobs[0]).toEqual(blob);
+  });
+
+  it('should handle multiple blobs', async () => {
+    const blobs = Array.from({ length: 3 }, () => makeRandomBlob(7));
+    const blobHashes = blobs.map(blob => blob.getEthVersionedBlobHash());
+
+    await client.sendBlobsToFilestore(blobs);
+
+    const retrievedBlobs = await client.getBlobSidecar(blockId, blobHashes);
+    expect(retrievedBlobs.length).toBe(3);
+
+    for (let i = 0; i < blobs.length; i++) {
+      expect(retrievedBlobs[i]).toEqual(blobs[i]);
+    }
+  });
+
+  it('should return empty array for non-existent blob hash', async () => {
+    const nonExistentHash = Buffer.alloc(32);
+    nonExistentHash.fill(0xff);
+
+    const retrievedBlobs = await client.getBlobSidecar(blockId, [nonExistentHash]);
+    expect(retrievedBlobs).toEqual([]);
+  });
+}

package/src/encoding/index.ts

@@ -0,0 +1,21 @@
+import { compressSync, uncompressSync } from 'snappy';
+
+/**
+ * Snappy decompress the blob buffer
+ *
+ * @param data - The blob buffer
+ * @returns The decompressed blob buffer
+ */
+export function inboundTransform(data: Buffer): Buffer {
+  return Buffer.from(uncompressSync(data, { asBuffer: true }));
+}
+
+/**
+ * Snappy compress the blob buffer
+ *
+ * @param data - The blob buffer
+ * @returns The compressed blob buffer
+ */
+export function outboundTransform(data: Buffer): Buffer {
+  return Buffer.from(compressSync(data));
+}

package/src/filestore/factory.ts

@@ -0,0 +1,145 @@
+import { type Logger, createLogger } from '@aztec/foundation/log';
+import {
+  type FileStore,
+  type ReadOnlyFileStore,
+  createFileStore,
+  createReadOnlyFileStore,
+} from '@aztec/stdlib/file-store';
+
+import { FileStoreBlobClient } from './filestore_blob_client.js';
+
+/**
+ * Metadata required to construct the base path for blob storage.
+ * Path format: aztec-{l1ChainId}-{rollupVersion}-{rollupAddress}/
+ */
+export interface BlobFileStoreMetadata {
+  /** The L1 chain ID */
+  l1ChainId: number;
+  /** The rollup version */
+  rollupVersion: number;
+  /** The rollup contract address (with or without 0x prefix) */
+  rollupAddress: string;
+}
+
+/**
+ * Constructs the base path for blob storage.
+ * Format: aztec-{l1ChainId}-{rollupVersion}-{rollupAddress}
+ */
+export function makeBlobBasePath(metadata: BlobFileStoreMetadata): string {
+  const { l1ChainId, rollupVersion, rollupAddress } = metadata;
+  // Normalize rollup address to lowercase without 0x prefix for consistency
+  const normalizedAddress = rollupAddress.toLowerCase().replace(/^0x/, '');
+  return `aztec-${l1ChainId}-${rollupVersion}-0x${normalizedAddress}`;
+}
+
+/**
+ * Creates a read-only FileStoreBlobClient for fetching blobs.
+ *
+ * @param storeUrl - The URL of the filestore (s3://, gs://, file://, https://)
+ * @param metadata - Chain metadata for constructing the base path
+ * @param logger - Optional logger
+ * @returns A FileStoreBlobClient for reading blobs, or undefined if storeUrl is undefined
+ */
+export async function createReadOnlyFileStoreBlobClient(
+  storeUrl: string,
+  metadata: BlobFileStoreMetadata,
+  logger?: Logger,
+): Promise<FileStoreBlobClient>;
+export async function createReadOnlyFileStoreBlobClient(
+  storeUrl: string | undefined,
+  metadata: BlobFileStoreMetadata,
+  logger?: Logger,
+): Promise<FileStoreBlobClient | undefined>;
+export async function createReadOnlyFileStoreBlobClient(
+  storeUrl: string | undefined,
+  metadata: BlobFileStoreMetadata,
+  logger?: Logger,
+): Promise<FileStoreBlobClient | undefined> {
+  if (!storeUrl) {
+    return undefined;
+  }
+
+  const log = logger ?? createLogger('blob-client:filestore-factory');
+  const basePath = makeBlobBasePath(metadata);
+
+  log.debug(`Creating read-only filestore blob client`, { storeUrl, basePath });
+
+  const store: ReadOnlyFileStore = await createReadOnlyFileStore(storeUrl, log);
+  return new FileStoreBlobClient(store, basePath, log);
+}
+
+/**
+ * Creates multiple read-only FileStoreBlobClients from an array of URLs.
+ *
+ * @param storeUrls - Array of filestore URLs
+ * @param metadata - Chain metadata for constructing the base path
+ * @param logger - Optional logger
+ * @returns Array of FileStoreBlobClients (excludes any that failed to create)
+ */
+export async function createReadOnlyFileStoreBlobClients(
+  storeUrls: string[] | undefined,
+  metadata: BlobFileStoreMetadata,
+  logger?: Logger,
+): Promise<FileStoreBlobClient[]> {
+  if (!storeUrls || storeUrls.length === 0) {
+    return [];
+  }
+
+  const log = logger ?? createLogger('blob-client:filestore-factory');
+  const clients: FileStoreBlobClient[] = [];
+
+  for (const storeUrl of storeUrls) {
+    try {
+      const client = await createReadOnlyFileStoreBlobClient(storeUrl, metadata, log);
+      if (client) {
+        clients.push(client);
+      }
+    } catch (err) {
+      log.error(`Failed to create read-only filestore blob client for ${storeUrl}`, err);
+    }
+  }
+
+  return clients;
+}
+
+/**
+ * Creates a writable FileStoreBlobClient for uploading blobs.
+ * Note: https:// URLs are not supported for upload, only s3://, gs://, or file://.
+ *
+ * @param storeUrl - The URL of the filestore (s3://, gs://, file://)
+ * @param metadata - Chain metadata for constructing the base path
+ * @param logger - Optional logger
+ * @returns A writable FileStoreBlobClient, or undefined if storeUrl is undefined
+ */
+export async function createWritableFileStoreBlobClient(
+  storeUrl: string,
+  metadata: BlobFileStoreMetadata,
+  logger?: Logger,
+): Promise<FileStoreBlobClient>;
+export async function createWritableFileStoreBlobClient(
+  storeUrl: string | undefined,
+  metadata: BlobFileStoreMetadata,
+  logger?: Logger,
+): Promise<FileStoreBlobClient | undefined>;
+export async function createWritableFileStoreBlobClient(
+  storeUrl: string | undefined,
+  metadata: BlobFileStoreMetadata,
+  logger?: Logger,
+): Promise<FileStoreBlobClient | undefined> {
+  if (!storeUrl) {
+    return undefined;
+  }
+
+  const log = logger ?? createLogger('blob-client:filestore-factory');
+  const basePath = makeBlobBasePath(metadata);
+
+  log.debug(`Creating writable filestore blob client`, { storeUrl, basePath });
+
+  const store: FileStore | undefined = await createFileStore(storeUrl, log);
+  if (!store) {
+    log.warn(`Failed to create writable filestore for ${storeUrl}`);
+    return undefined;
+  }
+
+  return new FileStoreBlobClient(store, basePath, log);
+}

package/src/filestore/filestore_blob_client.ts

@@ -0,0 +1,121 @@
+import { Blob, type BlobJson, computeEthVersionedBlobHash } from '@aztec/blob-lib';
+import { type Logger, createLogger } from '@aztec/foundation/log';
+import type { FileStore, ReadOnlyFileStore } from '@aztec/stdlib/file-store';
+
+import { inboundTransform, outboundTransform } from '../encoding/index.js';
+
+/**
+ * A blob client that uses a FileStore (S3/GCS/local) as the data source.
+ * Blobs are stored as JSON files keyed by their versioned blob hash.
+ */
+export class FileStoreBlobClient {
+  private readonly log: Logger;
+
+  constructor(
+    private readonly store: ReadOnlyFileStore | FileStore,
+    private readonly basePath: string,
+    logger?: Logger,
+  ) {
+    this.log = logger ?? createLogger('blob-client:filestore-client');
+  }
+
+  /**
+   * Get the path for a blob file.
+   * Format: basePath/blobs/{versionedBlobHash}.data
+   */
+  private blobPath(versionedBlobHash: string): string {
+    return `${this.basePath}/blobs/${versionedBlobHash}.data`;
+  }
+
+  /**
+   * Fetch blobs by their versioned hashes.
+   * @param blobHashes - Array of versioned blob hashes (0x-prefixed hex strings)
+   * @returns Array of BlobJson objects for found blobs
+   */
+  async getBlobsByHashes(blobHashes: string[]): Promise<BlobJson[]> {
+    const blobs: BlobJson[] = [];
+
+    for (let i = 0; i < blobHashes.length; i++) {
+      try {
+        const path = this.blobPath(blobHashes[i]);
+        if (!(await this.store.exists(path))) {
+          continue;
+        }
+
+        const data = await this.store.read(path);
+        const json = JSON.parse(inboundTransform(data).toString()) as BlobJson;
+        blobs.push(json);
+      } catch (err) {
+        this.log.warn(`Failed to read blob ${blobHashes[i]} from filestore`, err);
+      }
+    }
+
+    return blobs;
+  }
+
+  /**
+   * Check if a blob exists in the store.
+   * @param versionedBlobHash - The versioned blob hash (0x-prefixed hex string)
+   */
+  exists(versionedBlobHash: string): Promise<boolean> {
+    return this.store.exists(this.blobPath(versionedBlobHash));
+  }
+
+  /**
+   * Save a single blob to the store.
+   * @param blob - The blob to save
+   * @param skipIfExists - Skip saving if blob already exists (default: true)
+   * @throws Error if the store is read-only
+   */
+  async saveBlob(blob: Blob, skipIfExists = true): Promise<void> {
+    if (!this.isWritable()) {
+      throw new Error('FileStore is read-only');
+    }
+
+    const versionedHash = `0x${computeEthVersionedBlobHash(blob.commitment).toString('hex')}`;
+
+    if (skipIfExists && (await this.store.exists(this.blobPath(versionedHash)))) {
+      this.log.trace(`Blob ${versionedHash} already exists, skipping`);
+      return;
+    }
+
+    const json = blob.toJSON();
+    await (this.store as FileStore).save(
+      this.blobPath(versionedHash),
+      outboundTransform(Buffer.from(JSON.stringify(json))),
+    );
+    this.log.debug(`Saved blob ${versionedHash} to filestore`);
+  }
+
+  /**
+   * Save multiple blobs to the store in parallel.
+   * @param blobs - The blobs to save
+   * @param skipIfExists - Skip saving if blob already exists (default: true)
+   */
+  async saveBlobs(blobs: Blob[], skipIfExists = true): Promise<void> {
+    await Promise.all(blobs.map(blob => this.saveBlob(blob, skipIfExists)));
+  }
+
+  /**
+   * Get the base URL/path of the filestore.
+   */
+  getBaseUrl(): string {
+    return this.basePath;
+  }
+
+  /**
+   * Test if the filestore connection is working.
+   */
+  testConnection(): Promise<boolean> {
+    // This implementation will be improved in a separate PR
+    // Currently underlying filestore implementations do not expose an easy way to test connectivitiy
+    return Promise.resolve(true);
+  }
+
+  /**
+   * Check if the store supports write operations.
+   */
+  private isWritable(): boolean {
+    return 'save' in this.store;
+  }
+}

package/src/filestore/index.ts

@@ -0,0 +1,2 @@
+export * from './filestore_blob_client.js';
+export * from './factory.js';