@aztec/stdlib 0.0.1-commit.d431d1c → 0.0.1-commit.d58ff9d0

package/src/database-version/version_manager.ts

@@ -1,90 +1,24 @@
 import { EthAddress } from '@aztec/foundation/eth-address';
-import { jsonParseWithSchema, jsonStringify } from '@aztec/foundation/json-rpc';
 import { type Logger, createLogger } from '@aztec/foundation/log';
 
 import fs from 'fs/promises';
-import { inspect } from 'node:util';
 import { join } from 'path';
-import { z } from 'zod';
 
-/**
- * Represents a version record for storing in a version file.
- */
-export class DatabaseVersion {
-  constructor(
-    /** The version of the data on disk. Used to perform upgrades */
-    public readonly schemaVersion: number,
-    /** The rollup the data pertains to */
-    public readonly rollupAddress: EthAddress,
-  ) {}
-
-  public toBuffer(): Buffer {
-    return Buffer.from(jsonStringify(this));
-  }
-
-  public static fromBuffer(buf: Buffer): DatabaseVersion {
-    try {
-      return jsonParseWithSchema(buf.toString('utf-8'), DatabaseVersion.schema);
-    } catch (err) {
-      throw new Error(`Failed to deserialize version information: ${err}`, { cause: err });
-    }
-  }
-
-  /**
-   * Compares two versions. If the rollups addresses are different then it returns undefined
-   */
-  public cmp(other: DatabaseVersion): undefined | -1 | 0 | 1 {
-    if (this.rollupAddress.equals(other.rollupAddress)) {
-      if (this.schemaVersion < other.schemaVersion) {
-        return -1;
-      } else if (this.schemaVersion > other.schemaVersion) {
-        return 1;
-      } else {
-        return 0;
-      }
-    }
-    return undefined;
-  }
-
-  /**
-   * Checks if two versions exactly match
-   */
-  public equals(other: DatabaseVersion): boolean {
-    return this.cmp(other) === 0;
-  }
-
-  /**
-   * Returns the schema for this class
-   */
-  static get schema() {
-    return z
-      .object({
-        schemaVersion: z.number(),
-        rollupAddress: EthAddress.schema,
-      })
-      .transform(({ schemaVersion, rollupAddress }) => new DatabaseVersion(schemaVersion, rollupAddress));
-  }
-
-  /** Allows for better introspection. */
-  public [inspect.custom](): string {
-    return this.toString();
-  }
-
-  public toString(): string {
-    return `DatabaseVersion{schemaVersion=${this.schemaVersion},rollupAddress=${this.rollupAddress}"}`;
-  }
+import { DatabaseVersion } from './database_version.js';
 
-  /**
-   * Returns an empty instance
-   */
-  static empty() {
-    return new DatabaseVersion(0, EthAddress.ZERO);
-  }
-}
-
-export type DatabaseVersionManagerFs = Pick<typeof fs, 'readFile' | 'writeFile' | 'rm' | 'mkdir'>;
+export type DatabaseVersionManagerFs = Pick<typeof fs, 'readFile' | 'rm' | 'mkdir' | 'rename' | 'open'>;
 
 export const DATABASE_VERSION_FILE_NAME = 'db_version';
+export type SchemaVersionMismatchPolicy = 'reset' | 'throw';
+
+/**
+ * How to react when the version file exists but cannot be read (permissions, IO error, truncation).
+ * `'reset'` (default) treats the store as unversioned and lets the reset path run — safe for stores
+ * where an empty state is legitimate. `'throw'` refuses to open, leaving data untouched — required
+ * for protection stores (e.g. signing protection) that must never be silently wiped by a transient
+ * filesystem error. A genuinely missing file (ENOENT) is a first boot and is never affected by this.
+ */
+export type VersionFileReadFailurePolicy = 'reset' | 'throw';
 
 export type DatabaseVersionManagerOptions<T> = {
   schemaVersion: number;
@@ -92,6 +26,8 @@ export type DatabaseVersionManagerOptions<T> = {
   dataDirectory: string;
   onOpen: (dataDir: string) => Promise<T>;
   onUpgrade?: (dataDir: string, currentVersion: number, latestVersion: number) => Promise<void>;
+  schemaVersionMismatchPolicy?: SchemaVersionMismatchPolicy;
+  versionFileReadFailurePolicy?: VersionFileReadFailurePolicy;
   fileSystem?: DatabaseVersionManagerFs;
   log?: Logger;
 };
@@ -110,6 +46,8 @@ export class DatabaseVersionManager<T> {
   private dataDirectory: string;
   private onOpen: (dataDir: string) => Promise<T>;
   private onUpgrade?: (dataDir: string, currentVersion: number, latestVersion: number) => Promise<void>;
+  private schemaVersionMismatchPolicy: SchemaVersionMismatchPolicy;
+  private versionFileReadFailurePolicy: VersionFileReadFailurePolicy;
   private fileSystem: DatabaseVersionManagerFs;
   private log: Logger;
 
@@ -120,7 +58,11 @@ export class DatabaseVersionManager<T> {
    * @param rollupAddress - The rollup contract address
    * @param dataDirectory - The directory where version information will be stored
    * @param onOpen - A callback to the open the database at the given location
-   * @param onUpgrade - An optional callback to upgrade the database before opening. If not provided it will reset the database
+   * @param onUpgrade - An optional callback to upgrade the database before opening. If not provided it will reset the
+   *   database. Must be idempotent: since the version marker is written only after a successful open, a crash after
+   *   onUpgrade but before the marker is written re-runs onUpgrade on the next start.
+   * @param schemaVersionMismatchPolicy - Whether schema mismatches should reset data or throw
+   * @param versionFileReadFailurePolicy - Whether an unreadable (non-missing) version file should reset data or throw
    * @param fileSystem - An interface to access the filesystem
    * @param log - Optional custom logger
    * @param options - Configuration options
@@ -131,6 +73,8 @@ export class DatabaseVersionManager<T> {
     dataDirectory,
     onOpen,
     onUpgrade,
+    schemaVersionMismatchPolicy = 'reset',
+    versionFileReadFailurePolicy = 'reset',
     fileSystem = fs,
     log = createLogger(`foundation:version-manager`),
   }: DatabaseVersionManagerOptions<T>) {
@@ -144,13 +88,41 @@ export class DatabaseVersionManager<T> {
     this.dataDirectory = dataDirectory;
     this.onOpen = onOpen;
     this.onUpgrade = onUpgrade;
+    this.schemaVersionMismatchPolicy = schemaVersionMismatchPolicy;
+    this.versionFileReadFailurePolicy = versionFileReadFailurePolicy;
     this.fileSystem = fileSystem;
     this.log = log;
   }
 
   static async writeVersion(version: DatabaseVersion, dataDir: string, fileSystem: DatabaseVersionManagerFs = fs) {
     await fileSystem.mkdir(dataDir, { recursive: true });
-    return fileSystem.writeFile(join(dataDir, DatabaseVersionManager.VERSION_FILE), version.toBuffer());
+    const finalPath = join(dataDir, DatabaseVersionManager.VERSION_FILE);
+    const tmpPath = `${finalPath}.tmp`;
+
+    // Atomic durable write: fill a temp file, fsync it, then rename it into place. The marker only
+    // becomes visible under its final name once its bytes are durably on disk, so a crash mid-write
+    // can never leave a "valid" version file sitting over an empty or partially-populated data dir.
+    const handle = await fileSystem.open(tmpPath, 'w');
+    try {
+      await handle.writeFile(version.toBuffer());
+      await handle.sync();
+    } finally {
+      await handle.close();
+    }
+    await fileSystem.rename(tmpPath, finalPath);
+
+    // Best-effort fsync of the containing directory so the rename itself survives a crash. Not all
+    // filesystems support directory fsync, so a failure here is non-fatal.
+    try {
+      const dirHandle = await fileSystem.open(dataDir, 'r');
+      try {
+        await dirHandle.sync();
+      } finally {
+        await dirHandle.close();
+      }
+    } catch {
+      // directory fsync is best-effort
+    }
   }
 
   /**
@@ -175,6 +147,17 @@ export class DatabaseVersionManager<T> {
         storedVersion = DatabaseVersion.empty();
         // only turn off these logs if the data dir didn't exist before
         shouldLogDataReset = false;
+      } else if (this.versionFileReadFailurePolicy === 'throw') {
+        // The version file exists but could not be read/parsed (permissions, IO error, truncation).
+        // Treating this as "unversioned" would reset the data directory, silently wiping a store that
+        // must fail closed. Refuse to open instead, leaving the data untouched for the operator.
+        const code = (err as Error & { code?: string })?.code;
+        const message = err instanceof Error ? err.message : String(err);
+        throw new Error(
+          `Failed to read database version file at ${this.versionFile} (${code ?? 'unknown error'}: ${message}). ` +
+            `Refusing to open the database; data was NOT reset. Resolve the underlying filesystem error and retry.`,
+          { cause: err },
+        );
       } else {
         this.log.warn(`Failed to read stored version information: ${err}. Defaulting to empty version`);
         storedVersion = DatabaseVersion.empty();
@@ -191,10 +174,21 @@ export class DatabaseVersionManager<T> {
         try {
           await this.onUpgrade(this.dataDirectory, storedVersion.schemaVersion, this.currentVersion.schemaVersion);
         } catch (error) {
+          if (this.schemaVersionMismatchPolicy === 'throw') {
+            throw new Error(
+              `Failed to upgrade database at ${this.dataDirectory} from schema version ${storedVersion.schemaVersion} to ${this.currentVersion.schemaVersion}`,
+              { cause: error },
+            );
+          }
           this.log.error(`Failed to upgrade: ${error}. Falling back to reset.`);
           needsReset = true;
         }
       } else if (cmp !== 0) {
+        if (this.schemaVersionMismatchPolicy === 'throw') {
+          throw new Error(
+            `Cannot open database at ${this.dataDirectory}: stored schema version ${storedVersion.schemaVersion} is incompatible with expected schema version ${this.currentVersion.schemaVersion}`,
+          );
+        }
         if (shouldLogDataReset) {
           this.log.info(
             `Can't upgrade from version ${storedVersion} to ${this.currentVersion}. Resetting database at ${this.dataDirectory}`,
@@ -218,10 +212,47 @@ export class DatabaseVersionManager<T> {
       await this.resetDataDirectory();
     }
 
-    // Write the current version to disk
-    await this.writeVersion();
+    // Open the database first, then record the version marker. Writing the marker only after a
+    // successful open makes it a post-commit record: if the process crashes between the reset and a
+    // durable open, no marker is left behind, so the next startup re-runs the reset instead of
+    // trusting a marker that sits over empty or partially-initialized data.
+    const instance = await this.onOpen(this.dataDirectory);
+
+    // Only (re)write the marker when it would actually change — first boot, reset, or upgrade. On a
+    // normal boot it already matches, so skipping avoids an unnecessary fsync (and the directory
+    // write permission the temp-file+rename needs) and, more importantly, any window where a
+    // marker-write failure would orphan the database we just opened.
+    if (!storedVersion.equals(this.currentVersion)) {
+      try {
+        await this.writeVersion();
+      } catch (err) {
+        // The database opened but recording the marker failed; close the freshly opened instance so
+        // we do not leak its file handles / locks before propagating the failure.
+        await this.closeQuietly(instance);
+        throw err;
+      }
+    }
+
+    return [instance, needsReset];
+  }
 
-    return [await this.onOpen(this.dataDirectory), needsReset];
+  /**
+   * Best-effort close of a just-opened database instance, used to avoid leaking handles/locks when a
+   * post-open step fails. Swallows close errors so the original failure is the one that propagates.
+   */
+  private async closeQuietly(instance: T): Promise<void> {
+    const closable = instance as
+      | { close?: () => Promise<void> | void; [Symbol.asyncDispose]?: () => Promise<void> }
+      | undefined;
+    const dispose = closable?.close ?? closable?.[Symbol.asyncDispose];
+    if (typeof dispose !== 'function') {
+      return;
+    }
+    try {
+      await dispose.call(closable);
+    } catch (err) {
+      this.log.warn(`Failed to close database after version-write failure: ${err}`);
+    }
   }
 
   /**

package/src/delayed_public_mutable/delayed_public_mutable_values.ts

@@ -68,23 +68,22 @@ export class DelayedPublicMutableValues {
   }
 
   static async readFromTree(delayedPublicMutableSlot: Fr, readStorage: (storageSlot: Fr) => Promise<Fr>) {
-    const fields = [];
-    for (let i = 0; i < DELAYED_PUBLIC_MUTABLE_VALUES_LEN; i++) {
-      fields.push(await readStorage(delayedPublicMutableSlot.add(new Fr(i))));
-    }
-    return DelayedPublicMutableValues.fromFields(fields);
+    const fieldPromises = Array.from({ length: DELAYED_PUBLIC_MUTABLE_VALUES_LEN }).map((_, i) =>
+      readStorage(delayedPublicMutableSlot.add(new Fr(i))),
+    );
+    return DelayedPublicMutableValues.fromFields(await Promise.all(fieldPromises));
   }
 
   isEmpty(): boolean {
     return this.svc.isEmpty() && this.sdc.isEmpty();
   }
 
-  async writeToTree(delayedPublicMutableSlot: Fr, storageWrite: (storageSlot: Fr, value: Fr) => Promise<void>) {
-    const fields = this.toFields();
+  writeToTree(delayedPublicMutableSlot: Fr, storageWrite: (storageSlot: Fr, value: Fr) => Promise<void>) {
+    const fieldPromises = this.toFields().map((field, i) =>
+      storageWrite(delayedPublicMutableSlot.add(new Fr(i)), field),
+    );
 
-    for (let i = 0; i < fields.length; i++) {
-      await storageWrite(delayedPublicMutableSlot.add(new Fr(i)), fields[i]);
-    }
+    return Promise.all(fieldPromises);
   }
 
   async hash(): Promise<Fr> {

package/src/deserialization/index.ts

@@ -11,8 +11,41 @@ export const MAX_TXS_PER_BLOCK = 2 ** 16;
 /** Max committee size - theoretical max from bitmap design (256 bytes × 8 bits) */
 export const MAX_COMMITTEE_SIZE = 2048;
 
-/** Max blocks per checkpoint */
-export const MAX_BLOCKS_PER_CHECKPOINT = 72;
+/**
+ * Maximum number of L2 blocks in a provable checkpoint. Used for deserialization and when ingesting
+ * checkpoints already accepted by L1.
+ *
+ * This MUST be >= the number of blocks the proving system can carry, or a structurally-valid, provable
+ * checkpoint that L1 accepts would be rejected on ingest (`validateCheckpoint`) and wedge the archiver.
+ * The circuits and L1 impose no explicit per-checkpoint block count; the ceiling comes from the
+ * blob-field budget:
+ *
+ *   budget = BLOBS_PER_CHECKPOINT * FIELDS_PER_BLOB = 6 * 4096 = 24,576 fields
+ *
+ *   per-block minimal blob footprint:
+ *     first block  (may be empty)   = 7 fields   (NUM_FIRST_BLOCK_END_BLOB_FIELDS)
+ *     every other  (needs >= 1 tx)  = 6 + 4 = 10 fields
+ *                                     (NUM_BLOCK_END_BLOB_FIELDS + a 4-field minimal tx:
+ *                                      tx start marker + tx hash + fee + 1 mandatory nullifier)
+ *     + 1 checkpoint-end marker      (NUM_CHECKPOINT_END_MARKER_FIELDS)
+ *
+ *   max N:  7 + 10*(N - 1) + 1 <= 24,576  =>  10*(N - 1) <= 24,568  =>  N <= 2,457
+ *
+ * Only the first block may be empty; every other block needs >= 1 tx (the circuits have no empty-tx
+ * variant for non-first blocks), so 2457 is the largest provable checkpoint. The blob format can encode
+ * more blocks than this (up to ~4095 with all-empty blocks), but such a checkpoint is unprovable and
+ * can only reach L1 with a malicious committee supermajority — a terminal network compromise where a
+ * wedged archiver is an acceptable outcome. We therefore bound ingest to the provable maximum.
+ * Invariant checked by the unit test in deserialization.test.ts.
+ */
+export const MAX_CAPACITY_BLOCKS_PER_CHECKPOINT = 2457;
+
+/**
+ * Max blocks per checkpoint we are willing to build or attest to (conservative client policy, and the
+ * default block-count limit for `validateCheckpoint`). Distinct from {@link MAX_CAPACITY_BLOCKS_PER_CHECKPOINT},
+ * which is the higher ceiling we tolerate when deserializing checkpoints already accepted by L1.
+ */
+export const MAX_ATTESTABLE_BLOCKS_PER_CHECKPOINT = 72;
 
 /** Max tx effects per body (based on blob capacity per checkpoint) */
 export const MAX_TX_EFFECTS_PER_BODY = BLOBS_PER_CHECKPOINT * FIELDS_PER_BLOB;

package/src/epoch-helpers/index.ts

@@ -11,6 +11,8 @@ export type L1RollupConstants = {
   epochDuration: number;
   ethereumSlotDuration: number;
   proofSubmissionEpochs: number;
+  targetCommitteeSize: number;
+  rollupManaLimit: number;
 };
 
 export const EmptyL1RollupConstants: L1RollupConstants = {
@@ -20,6 +22,8 @@ export const EmptyL1RollupConstants: L1RollupConstants = {
   slotDuration: 1,
   ethereumSlotDuration: 1,
   proofSubmissionEpochs: 1,
+  targetCommitteeSize: 48,
+  rollupManaLimit: Number.MAX_SAFE_INTEGER,
 };
 
 export const L1RollupConstantsSchema = zodFor<L1RollupConstants>()(
@@ -30,6 +34,8 @@ export const L1RollupConstantsSchema = zodFor<L1RollupConstants>()(
     epochDuration: z.number(),
     ethereumSlotDuration: z.number(),
     proofSubmissionEpochs: z.number(),
+    targetCommitteeSize: z.number(),
+    rollupManaLimit: z.number(),
   }),
 );
 
@@ -51,6 +57,34 @@ export function getSlotAtTimestamp(
     : SlotNumber.fromBigInt((ts - constants.l1GenesisTime) / BigInt(constants.slotDuration));
 }
 
+/** Returns the timestamp of the next L1 slot boundary after the given wall-clock time. */
+export function getNextL1SlotTimestamp(
+  nowInSeconds: number,
+  constants: Pick<L1RollupConstants, 'l1GenesisTime' | 'ethereumSlotDuration'>,
+): bigint {
+  const now = BigInt(nowInSeconds);
+  const elapsed = now - constants.l1GenesisTime;
+  const currentL1Slot = elapsed / BigInt(constants.ethereumSlotDuration);
+  return constants.l1GenesisTime + (currentL1Slot + 1n) * BigInt(constants.ethereumSlotDuration);
+}
+
+/** Returns the timestamp of the last L1 slot within a given L2 slot. */
+export function getLastL1SlotTimestampForL2Slot(
+  slot: SlotNumber,
+  constants: Pick<L1RollupConstants, 'l1GenesisTime' | 'slotDuration' | 'ethereumSlotDuration'>,
+): bigint {
+  return getTimestampForSlot(slot, constants) + BigInt(constants.slotDuration - constants.ethereumSlotDuration);
+}
+
+/** Returns the L2 slot number at the next L1 block based on the current timestamp. */
+export function getSlotAtNextL1Block(
+  currentL1Timestamp: bigint,
+  constants: Pick<L1RollupConstants, 'l1GenesisTime' | 'slotDuration' | 'ethereumSlotDuration'>,
+): SlotNumber {
+  const nextL1BlockTimestamp = currentL1Timestamp + BigInt(constants.ethereumSlotDuration);
+  return getSlotAtTimestamp(nextL1BlockTimestamp, constants);
+}
+
 /** Returns the epoch number for a given timestamp. */
 export function getEpochNumberAtTimestamp(
   ts: bigint,
@@ -69,7 +103,7 @@ export function getSlotRangeForEpoch(
   epochNumber: EpochNumber,
   constants: Pick<L1RollupConstants, 'epochDuration'>,
 ): [SlotNumber, SlotNumber] {
-  const startSlot = SlotNumber(epochNumber * constants.epochDuration);
+  const startSlot = SlotNumber(Number(epochNumber) * constants.epochDuration);
   return [startSlot, SlotNumber(startSlot + constants.epochDuration - 1)];
 }
 
@@ -128,6 +162,11 @@ export function getProofSubmissionDeadlineTimestamp(
   return getTimestampForSlot(deadlineSlot, constants);
 }
 
+/** Computes the quorum size required for a committee (⌊2n/3⌋ + 1). */
+export function computeQuorum(committeeSize: number): number {
+  return Math.floor((committeeSize * 2) / 3) + 1;
+}
+
 /** Returns the timestamp to start building a block for a given L2 slot. Computed as the start timestamp of the slot minus one L1 slot duration. */
 export function getSlotStartBuildTimestamp(
   slotNumber: SlotNumber,

package/src/file-store/factory.ts

@@ -1,9 +1,10 @@
 import { type Logger, createLogger } from '@aztec/foundation/log';
 
 import { GoogleCloudFileStore } from './gcs.js';
-import { HttpFileStore } from './http.js';
+import { HttpFileStore, type HttpFileStoreOptions } from './http.js';
 import type { FileStore, ReadOnlyFileStore } from './interface.js';
 import { LocalFileStore } from './local.js';
+import { InMemoryFileStore } from './memory.js';
 import { S3FileStore } from './s3.js';
 
 const supportedExamples = [
@@ -21,6 +22,12 @@ export async function createFileStore(
 ): Promise<FileStore | undefined> {
   if (config === undefined) {
     return undefined;
+  } else if (config.startsWith('mem://')) {
+    // In-memory store for tests (see InMemoryFileStore). The namespace is everything after the
+    // scheme, so `mem://X` and a later `mem://X` share data, like two file:// stores over one dir.
+    const namespace = config.slice('mem://'.length).replace(/\/+$/, '');
+    logger.info(`Creating in-memory file store at ${namespace}`);
+    return new InMemoryFileStore(namespace);
   } else if (config.startsWith('file://')) {
     const url = new URL(config);
     if (url.host) {
@@ -59,17 +66,26 @@ export async function createFileStore(
   }
 }
 
-export async function createReadOnlyFileStore(config: string, logger?: Logger): Promise<ReadOnlyFileStore>;
-export async function createReadOnlyFileStore(config: undefined, logger?: Logger): Promise<undefined>;
+export async function createReadOnlyFileStore(
+  config: string,
+  logger?: Logger,
+  options?: HttpFileStoreOptions,
+): Promise<ReadOnlyFileStore>;
+export async function createReadOnlyFileStore(
+  config: undefined,
+  logger?: Logger,
+  options?: HttpFileStoreOptions,
+): Promise<undefined>;
 export async function createReadOnlyFileStore(
   config: string | undefined,
   logger = createLogger('stdlib:file-store'),
+  options?: HttpFileStoreOptions,
 ): Promise<ReadOnlyFileStore | undefined> {
   if (config === undefined) {
     return undefined;
   } else if (config.startsWith('http://') || config.startsWith('https://')) {
     logger.info(`Creating read-only HTTP file store at ${config}`);
-    return new HttpFileStore(config, logger);
+    return new HttpFileStore(config, logger, options);
   } else {
     return await createFileStore(config, logger);
   }

package/src/file-store/http.ts

@@ -10,6 +10,14 @@ import { pipeline } from 'stream/promises';
 
 import type { ReadOnlyFileStore } from './interface.js';
 
+/** Options for configuring HttpFileStore behavior. */
+export interface HttpFileStoreOptions {
+  /** Retry backoff intervals in seconds. Empty array disables retries. Default: [1, 1, 3]. */
+  retryBackoff?: number[];
+  /** Request timeout in milliseconds. Default: no timeout (axios default). */
+  timeoutMs?: number;
+}
+
 export class HttpFileStore implements ReadOnlyFileStore {
   private readonly axiosInstance: AxiosInstance;
   private readonly fetch: <T>(config: AxiosRequestConfig) => Promise<AxiosResponse<T>>;
@@ -17,17 +25,28 @@ export class HttpFileStore implements ReadOnlyFileStore {
   constructor(
     private readonly baseUrl: string,
     private readonly log: Logger = createLogger('stdlib:http-file-store'),
+    options?: HttpFileStoreOptions,
   ) {
-    this.axiosInstance = axios.create();
-    this.fetch = async <T>(config: AxiosRequestConfig) => {
-      return await retry(
-        () => this.axiosInstance.request<T>(config),
-        `Fetching ${config.url}`,
-        makeBackoff([1, 1, 3]),
-        this.log,
-        /*failSilently=*/ true,
-      );
-    };
+    this.axiosInstance = axios.create({
+      ...(options?.timeoutMs !== undefined && { timeout: options.timeoutMs }),
+    });
+
+    const retryBackoff = options?.retryBackoff ?? [1, 1, 3];
+    if (retryBackoff.length > 0) {
+      this.fetch = async <T>(config: AxiosRequestConfig) => {
+        return await retry(
+          () => this.axiosInstance.request<T>(config),
+          `Fetching ${config.url}`,
+          makeBackoff(retryBackoff),
+          this.log,
+          /*failSilently=*/ true,
+        );
+      };
+    } else {
+      this.fetch = async <T>(config: AxiosRequestConfig) => {
+        return await this.axiosInstance.request<T>(config);
+      };
+    }
   }
 
   public async read(pathOrUrl: string): Promise<Buffer> {

package/src/file-store/index.ts

@@ -1,2 +1,4 @@
 export * from './interface.js';
 export * from './factory.js';
+export { type HttpFileStoreOptions } from './http.js';
+export { InMemoryFileStore } from './memory.js';

package/src/file-store/local.ts

@@ -1,15 +1,21 @@
 import { access, mkdir, readFile, writeFile } from 'fs/promises';
 import { dirname, resolve } from 'path';
+import { promisify } from 'util';
+import { gunzip as gunzipCb, gzip as gzipCb } from 'zlib';
 
-import type { FileStore } from './interface.js';
+import type { FileStore, FileStoreSaveOptions } from './interface.js';
+
+const gzip = promisify(gzipCb);
+const gunzip = promisify(gunzipCb);
 
 export class LocalFileStore implements FileStore {
   constructor(private readonly basePath: string) {}
 
-  public async save(path: string, data: Buffer): Promise<string> {
+  public async save(path: string, data: Buffer, opts?: FileStoreSaveOptions): Promise<string> {
     const fullPath = this.getFullPath(path);
     await mkdir(dirname(fullPath), { recursive: true });
-    await writeFile(fullPath, data);
+    const toWrite = opts?.compress ? await gzip(data) : data;
+    await writeFile(fullPath, toWrite);
     return `file://${fullPath}`;
   }
 
@@ -18,9 +24,13 @@ export class LocalFileStore implements FileStore {
     return this.save(destPath, data);
   }
 
-  public read(pathOrUrlStr: string): Promise<Buffer> {
+  public async read(pathOrUrlStr: string): Promise<Buffer> {
     const fullPath = this.getFullPath(pathOrUrlStr);
-    return readFile(fullPath);
+    const data = await readFile(fullPath);
+    if (data.length >= 2 && data[0] === 0x1f && data[1] === 0x8b) {
+      return await gunzip(data);
+    }
+    return data;
   }
 
   public async download(pathOrUrlStr: string, destPath: string): Promise<void> {