@aztec/stdlib 0.0.1-commit.f504929 → 0.0.1-commit.f5a9928

package/src/database-version/version_manager.ts

@@ -6,9 +6,19 @@ import { join } from 'path';
 
 import { DatabaseVersion } from './database_version.js';
 
-export type DatabaseVersionManagerFs = Pick<typeof fs, 'readFile' | 'writeFile' | 'rm' | 'mkdir'>;
+export type DatabaseVersionManagerFs = Pick<typeof fs, 'readFile' | 'rm' | 'mkdir' | 'rename' | 'open'>;
 
 export const DATABASE_VERSION_FILE_NAME = 'db_version';
+export type SchemaVersionMismatchPolicy = 'reset' | 'throw';
+
+/**
+ * How to react when the version file exists but cannot be read (permissions, IO error, truncation).
+ * `'reset'` (default) treats the store as unversioned and lets the reset path run — safe for stores
+ * where an empty state is legitimate. `'throw'` refuses to open, leaving data untouched — required
+ * for protection stores (e.g. signing protection) that must never be silently wiped by a transient
+ * filesystem error. A genuinely missing file (ENOENT) is a first boot and is never affected by this.
+ */
+export type VersionFileReadFailurePolicy = 'reset' | 'throw';
 
 export type DatabaseVersionManagerOptions<T> = {
   schemaVersion: number;
@@ -16,6 +26,8 @@ export type DatabaseVersionManagerOptions<T> = {
   dataDirectory: string;
   onOpen: (dataDir: string) => Promise<T>;
   onUpgrade?: (dataDir: string, currentVersion: number, latestVersion: number) => Promise<void>;
+  schemaVersionMismatchPolicy?: SchemaVersionMismatchPolicy;
+  versionFileReadFailurePolicy?: VersionFileReadFailurePolicy;
   fileSystem?: DatabaseVersionManagerFs;
   log?: Logger;
 };
@@ -34,6 +46,8 @@ export class DatabaseVersionManager<T> {
   private dataDirectory: string;
   private onOpen: (dataDir: string) => Promise<T>;
   private onUpgrade?: (dataDir: string, currentVersion: number, latestVersion: number) => Promise<void>;
+  private schemaVersionMismatchPolicy: SchemaVersionMismatchPolicy;
+  private versionFileReadFailurePolicy: VersionFileReadFailurePolicy;
   private fileSystem: DatabaseVersionManagerFs;
   private log: Logger;
 
@@ -44,7 +58,11 @@ export class DatabaseVersionManager<T> {
    * @param rollupAddress - The rollup contract address
    * @param dataDirectory - The directory where version information will be stored
    * @param onOpen - A callback to the open the database at the given location
-   * @param onUpgrade - An optional callback to upgrade the database before opening. If not provided it will reset the database
+   * @param onUpgrade - An optional callback to upgrade the database before opening. If not provided it will reset the
+   *   database. Must be idempotent: since the version marker is written only after a successful open, a crash after
+   *   onUpgrade but before the marker is written re-runs onUpgrade on the next start.
+   * @param schemaVersionMismatchPolicy - Whether schema mismatches should reset data or throw
+   * @param versionFileReadFailurePolicy - Whether an unreadable (non-missing) version file should reset data or throw
    * @param fileSystem - An interface to access the filesystem
    * @param log - Optional custom logger
    * @param options - Configuration options
@@ -55,6 +73,8 @@ export class DatabaseVersionManager<T> {
     dataDirectory,
     onOpen,
     onUpgrade,
+    schemaVersionMismatchPolicy = 'reset',
+    versionFileReadFailurePolicy = 'reset',
     fileSystem = fs,
     log = createLogger(`foundation:version-manager`),
   }: DatabaseVersionManagerOptions<T>) {
@@ -68,13 +88,41 @@ export class DatabaseVersionManager<T> {
     this.dataDirectory = dataDirectory;
     this.onOpen = onOpen;
     this.onUpgrade = onUpgrade;
+    this.schemaVersionMismatchPolicy = schemaVersionMismatchPolicy;
+    this.versionFileReadFailurePolicy = versionFileReadFailurePolicy;
     this.fileSystem = fileSystem;
     this.log = log;
   }
 
   static async writeVersion(version: DatabaseVersion, dataDir: string, fileSystem: DatabaseVersionManagerFs = fs) {
     await fileSystem.mkdir(dataDir, { recursive: true });
-    return fileSystem.writeFile(join(dataDir, DatabaseVersionManager.VERSION_FILE), version.toBuffer());
+    const finalPath = join(dataDir, DatabaseVersionManager.VERSION_FILE);
+    const tmpPath = `${finalPath}.tmp`;
+
+    // Atomic durable write: fill a temp file, fsync it, then rename it into place. The marker only
+    // becomes visible under its final name once its bytes are durably on disk, so a crash mid-write
+    // can never leave a "valid" version file sitting over an empty or partially-populated data dir.
+    const handle = await fileSystem.open(tmpPath, 'w');
+    try {
+      await handle.writeFile(version.toBuffer());
+      await handle.sync();
+    } finally {
+      await handle.close();
+    }
+    await fileSystem.rename(tmpPath, finalPath);
+
+    // Best-effort fsync of the containing directory so the rename itself survives a crash. Not all
+    // filesystems support directory fsync, so a failure here is non-fatal.
+    try {
+      const dirHandle = await fileSystem.open(dataDir, 'r');
+      try {
+        await dirHandle.sync();
+      } finally {
+        await dirHandle.close();
+      }
+    } catch {
+      // directory fsync is best-effort
+    }
   }
 
   /**
@@ -99,6 +147,17 @@ export class DatabaseVersionManager<T> {
         storedVersion = DatabaseVersion.empty();
         // only turn off these logs if the data dir didn't exist before
         shouldLogDataReset = false;
+      } else if (this.versionFileReadFailurePolicy === 'throw') {
+        // The version file exists but could not be read/parsed (permissions, IO error, truncation).
+        // Treating this as "unversioned" would reset the data directory, silently wiping a store that
+        // must fail closed. Refuse to open instead, leaving the data untouched for the operator.
+        const code = (err as Error & { code?: string })?.code;
+        const message = err instanceof Error ? err.message : String(err);
+        throw new Error(
+          `Failed to read database version file at ${this.versionFile} (${code ?? 'unknown error'}: ${message}). ` +
+            `Refusing to open the database; data was NOT reset. Resolve the underlying filesystem error and retry.`,
+          { cause: err },
+        );
       } else {
         this.log.warn(`Failed to read stored version information: ${err}. Defaulting to empty version`);
         storedVersion = DatabaseVersion.empty();
@@ -115,10 +174,21 @@ export class DatabaseVersionManager<T> {
         try {
           await this.onUpgrade(this.dataDirectory, storedVersion.schemaVersion, this.currentVersion.schemaVersion);
         } catch (error) {
+          if (this.schemaVersionMismatchPolicy === 'throw') {
+            throw new Error(
+              `Failed to upgrade database at ${this.dataDirectory} from schema version ${storedVersion.schemaVersion} to ${this.currentVersion.schemaVersion}`,
+              { cause: error },
+            );
+          }
           this.log.error(`Failed to upgrade: ${error}. Falling back to reset.`);
           needsReset = true;
         }
       } else if (cmp !== 0) {
+        if (this.schemaVersionMismatchPolicy === 'throw') {
+          throw new Error(
+            `Cannot open database at ${this.dataDirectory}: stored schema version ${storedVersion.schemaVersion} is incompatible with expected schema version ${this.currentVersion.schemaVersion}`,
+          );
+        }
         if (shouldLogDataReset) {
           this.log.info(
             `Can't upgrade from version ${storedVersion} to ${this.currentVersion}. Resetting database at ${this.dataDirectory}`,
@@ -142,10 +212,47 @@ export class DatabaseVersionManager<T> {
       await this.resetDataDirectory();
     }
 
-    // Write the current version to disk
-    await this.writeVersion();
+    // Open the database first, then record the version marker. Writing the marker only after a
+    // successful open makes it a post-commit record: if the process crashes between the reset and a
+    // durable open, no marker is left behind, so the next startup re-runs the reset instead of
+    // trusting a marker that sits over empty or partially-initialized data.
+    const instance = await this.onOpen(this.dataDirectory);
 
-    return [await this.onOpen(this.dataDirectory), needsReset];
+    // Only (re)write the marker when it would actually change — first boot, reset, or upgrade. On a
+    // normal boot it already matches, so skipping avoids an unnecessary fsync (and the directory
+    // write permission the temp-file+rename needs) and, more importantly, any window where a
+    // marker-write failure would orphan the database we just opened.
+    if (!storedVersion.equals(this.currentVersion)) {
+      try {
+        await this.writeVersion();
+      } catch (err) {
+        // The database opened but recording the marker failed; close the freshly opened instance so
+        // we do not leak its file handles / locks before propagating the failure.
+        await this.closeQuietly(instance);
+        throw err;
+      }
+    }
+
+    return [instance, needsReset];
+  }
+
+  /**
+   * Best-effort close of a just-opened database instance, used to avoid leaking handles/locks when a
+   * post-open step fails. Swallows close errors so the original failure is the one that propagates.
+   */
+  private async closeQuietly(instance: T): Promise<void> {
+    const closable = instance as
+      | { close?: () => Promise<void> | void; [Symbol.asyncDispose]?: () => Promise<void> }
+      | undefined;
+    const dispose = closable?.close ?? closable?.[Symbol.asyncDispose];
+    if (typeof dispose !== 'function') {
+      return;
+    }
+    try {
+      await dispose.call(closable);
+    } catch (err) {
+      this.log.warn(`Failed to close database after version-write failure: ${err}`);
+    }
   }
 
   /**

package/src/deserialization/index.ts

@@ -11,8 +11,41 @@ export const MAX_TXS_PER_BLOCK = 2 ** 16;
 /** Max committee size - theoretical max from bitmap design (256 bytes × 8 bits) */
 export const MAX_COMMITTEE_SIZE = 2048;
 
-/** Max blocks per checkpoint */
-export const MAX_BLOCKS_PER_CHECKPOINT = 72;
+/**
+ * Maximum number of L2 blocks in a provable checkpoint. Used for deserialization and when ingesting
+ * checkpoints already accepted by L1.
+ *
+ * This MUST be >= the number of blocks the proving system can carry, or a structurally-valid, provable
+ * checkpoint that L1 accepts would be rejected on ingest (`validateCheckpoint`) and wedge the archiver.
+ * The circuits and L1 impose no explicit per-checkpoint block count; the ceiling comes from the
+ * blob-field budget:
+ *
+ *   budget = BLOBS_PER_CHECKPOINT * FIELDS_PER_BLOB = 6 * 4096 = 24,576 fields
+ *
+ *   per-block minimal blob footprint:
+ *     first block  (may be empty)   = 7 fields   (NUM_FIRST_BLOCK_END_BLOB_FIELDS)
+ *     every other  (needs >= 1 tx)  = 6 + 4 = 10 fields
+ *                                     (NUM_BLOCK_END_BLOB_FIELDS + a 4-field minimal tx:
+ *                                      tx start marker + tx hash + fee + 1 mandatory nullifier)
+ *     + 1 checkpoint-end marker      (NUM_CHECKPOINT_END_MARKER_FIELDS)
+ *
+ *   max N:  7 + 10*(N - 1) + 1 <= 24,576  =>  10*(N - 1) <= 24,568  =>  N <= 2,457
+ *
+ * Only the first block may be empty; every other block needs >= 1 tx (the circuits have no empty-tx
+ * variant for non-first blocks), so 2457 is the largest provable checkpoint. The blob format can encode
+ * more blocks than this (up to ~4095 with all-empty blocks), but such a checkpoint is unprovable and
+ * can only reach L1 with a malicious committee supermajority — a terminal network compromise where a
+ * wedged archiver is an acceptable outcome. We therefore bound ingest to the provable maximum.
+ * Invariant checked by the unit test in deserialization.test.ts.
+ */
+export const MAX_CAPACITY_BLOCKS_PER_CHECKPOINT = 2457;
+
+/**
+ * Max blocks per checkpoint we are willing to build or attest to (conservative client policy, and the
+ * default block-count limit for `validateCheckpoint`). Distinct from {@link MAX_CAPACITY_BLOCKS_PER_CHECKPOINT},
+ * which is the higher ceiling we tolerate when deserializing checkpoints already accepted by L1.
+ */
+export const MAX_ATTESTABLE_BLOCKS_PER_CHECKPOINT = 72;
 
 /** Max tx effects per body (based on blob capacity per checkpoint) */
 export const MAX_TX_EFFECTS_PER_BODY = BLOBS_PER_CHECKPOINT * FIELDS_PER_BLOB;

package/src/epoch-helpers/index.ts

@@ -12,6 +12,7 @@ export type L1RollupConstants = {
   ethereumSlotDuration: number;
   proofSubmissionEpochs: number;
   targetCommitteeSize: number;
+  rollupManaLimit: number;
 };
 
 export const EmptyL1RollupConstants: L1RollupConstants = {
@@ -22,6 +23,7 @@ export const EmptyL1RollupConstants: L1RollupConstants = {
   ethereumSlotDuration: 1,
   proofSubmissionEpochs: 1,
   targetCommitteeSize: 48,
+  rollupManaLimit: Number.MAX_SAFE_INTEGER,
 };
 
 export const L1RollupConstantsSchema = zodFor<L1RollupConstants>()(
@@ -33,6 +35,7 @@ export const L1RollupConstantsSchema = zodFor<L1RollupConstants>()(
     ethereumSlotDuration: z.number(),
     proofSubmissionEpochs: z.number(),
     targetCommitteeSize: z.number(),
+    rollupManaLimit: z.number(),
   }),
 );
 
@@ -54,6 +57,25 @@ export function getSlotAtTimestamp(
     : SlotNumber.fromBigInt((ts - constants.l1GenesisTime) / BigInt(constants.slotDuration));
 }
 
+/** Returns the timestamp of the next L1 slot boundary after the given wall-clock time. */
+export function getNextL1SlotTimestamp(
+  nowInSeconds: number,
+  constants: Pick<L1RollupConstants, 'l1GenesisTime' | 'ethereumSlotDuration'>,
+): bigint {
+  const now = BigInt(nowInSeconds);
+  const elapsed = now - constants.l1GenesisTime;
+  const currentL1Slot = elapsed / BigInt(constants.ethereumSlotDuration);
+  return constants.l1GenesisTime + (currentL1Slot + 1n) * BigInt(constants.ethereumSlotDuration);
+}
+
+/** Returns the timestamp of the last L1 slot within a given L2 slot. */
+export function getLastL1SlotTimestampForL2Slot(
+  slot: SlotNumber,
+  constants: Pick<L1RollupConstants, 'l1GenesisTime' | 'slotDuration' | 'ethereumSlotDuration'>,
+): bigint {
+  return getTimestampForSlot(slot, constants) + BigInt(constants.slotDuration - constants.ethereumSlotDuration);
+}
+
 /** Returns the L2 slot number at the next L1 block based on the current timestamp. */
 export function getSlotAtNextL1Block(
   currentL1Timestamp: bigint,
@@ -140,6 +162,11 @@ export function getProofSubmissionDeadlineTimestamp(
   return getTimestampForSlot(deadlineSlot, constants);
 }
 
+/** Computes the quorum size required for a committee (⌊2n/3⌋ + 1). */
+export function computeQuorum(committeeSize: number): number {
+  return Math.floor((committeeSize * 2) / 3) + 1;
+}
+
 /** Returns the timestamp to start building a block for a given L2 slot. Computed as the start timestamp of the slot minus one L1 slot duration. */
 export function getSlotStartBuildTimestamp(
   slotNumber: SlotNumber,

package/src/file-store/factory.ts

@@ -1,9 +1,10 @@
 import { type Logger, createLogger } from '@aztec/foundation/log';
 
 import { GoogleCloudFileStore } from './gcs.js';
-import { HttpFileStore } from './http.js';
+import { HttpFileStore, type HttpFileStoreOptions } from './http.js';
 import type { FileStore, ReadOnlyFileStore } from './interface.js';
 import { LocalFileStore } from './local.js';
+import { InMemoryFileStore } from './memory.js';
 import { S3FileStore } from './s3.js';
 
 const supportedExamples = [
@@ -21,6 +22,12 @@ export async function createFileStore(
 ): Promise<FileStore | undefined> {
   if (config === undefined) {
     return undefined;
+  } else if (config.startsWith('mem://')) {
+    // In-memory store for tests (see InMemoryFileStore). The namespace is everything after the
+    // scheme, so `mem://X` and a later `mem://X` share data, like two file:// stores over one dir.
+    const namespace = config.slice('mem://'.length).replace(/\/+$/, '');
+    logger.info(`Creating in-memory file store at ${namespace}`);
+    return new InMemoryFileStore(namespace);
   } else if (config.startsWith('file://')) {
     const url = new URL(config);
     if (url.host) {
@@ -59,17 +66,26 @@ export async function createFileStore(
   }
 }
 
-export async function createReadOnlyFileStore(config: string, logger?: Logger): Promise<ReadOnlyFileStore>;
-export async function createReadOnlyFileStore(config: undefined, logger?: Logger): Promise<undefined>;
+export async function createReadOnlyFileStore(
+  config: string,
+  logger?: Logger,
+  options?: HttpFileStoreOptions,
+): Promise<ReadOnlyFileStore>;
+export async function createReadOnlyFileStore(
+  config: undefined,
+  logger?: Logger,
+  options?: HttpFileStoreOptions,
+): Promise<undefined>;
 export async function createReadOnlyFileStore(
   config: string | undefined,
   logger = createLogger('stdlib:file-store'),
+  options?: HttpFileStoreOptions,
 ): Promise<ReadOnlyFileStore | undefined> {
   if (config === undefined) {
     return undefined;
   } else if (config.startsWith('http://') || config.startsWith('https://')) {
     logger.info(`Creating read-only HTTP file store at ${config}`);
-    return new HttpFileStore(config, logger);
+    return new HttpFileStore(config, logger, options);
   } else {
     return await createFileStore(config, logger);
   }

package/src/file-store/http.ts

@@ -10,6 +10,14 @@ import { pipeline } from 'stream/promises';
 
 import type { ReadOnlyFileStore } from './interface.js';
 
+/** Options for configuring HttpFileStore behavior. */
+export interface HttpFileStoreOptions {
+  /** Retry backoff intervals in seconds. Empty array disables retries. Default: [1, 1, 3]. */
+  retryBackoff?: number[];
+  /** Request timeout in milliseconds. Default: no timeout (axios default). */
+  timeoutMs?: number;
+}
+
 export class HttpFileStore implements ReadOnlyFileStore {
   private readonly axiosInstance: AxiosInstance;
   private readonly fetch: <T>(config: AxiosRequestConfig) => Promise<AxiosResponse<T>>;
@@ -17,17 +25,28 @@ export class HttpFileStore implements ReadOnlyFileStore {
   constructor(
     private readonly baseUrl: string,
     private readonly log: Logger = createLogger('stdlib:http-file-store'),
+    options?: HttpFileStoreOptions,
   ) {
-    this.axiosInstance = axios.create();
-    this.fetch = async <T>(config: AxiosRequestConfig) => {
-      return await retry(
-        () => this.axiosInstance.request<T>(config),
-        `Fetching ${config.url}`,
-        makeBackoff([1, 1, 3]),
-        this.log,
-        /*failSilently=*/ true,
-      );
-    };
+    this.axiosInstance = axios.create({
+      ...(options?.timeoutMs !== undefined && { timeout: options.timeoutMs }),
+    });
+
+    const retryBackoff = options?.retryBackoff ?? [1, 1, 3];
+    if (retryBackoff.length > 0) {
+      this.fetch = async <T>(config: AxiosRequestConfig) => {
+        return await retry(
+          () => this.axiosInstance.request<T>(config),
+          `Fetching ${config.url}`,
+          makeBackoff(retryBackoff),
+          this.log,
+          /*failSilently=*/ true,
+        );
+      };
+    } else {
+      this.fetch = async <T>(config: AxiosRequestConfig) => {
+        return await this.axiosInstance.request<T>(config);
+      };
+    }
   }
 
   public async read(pathOrUrl: string): Promise<Buffer> {

package/src/file-store/index.ts

@@ -1,2 +1,4 @@
 export * from './interface.js';
 export * from './factory.js';
+export { type HttpFileStoreOptions } from './http.js';
+export { InMemoryFileStore } from './memory.js';

package/src/file-store/memory.ts

@@ -0,0 +1,94 @@
+import { writeFile } from 'fs/promises';
+import { promisify } from 'util';
+import { gunzip as gunzipCb, gzip as gzipCb } from 'zlib';
+
+import type { FileStore, FileStoreSaveOptions } from './interface.js';
+
+const gzip = promisify(gzipCb);
+const gunzip = promisify(gunzipCb);
+
+// Backing data is shared per namespace so that a store created for writing and a (read-only) store
+// created later from the same `mem://` URL observe the same files — mirroring how two `file://`
+// stores over the same directory share state on disk. Without this, `FileStoreTxSource`, which
+// builds its own store from the URL, would never see what a test wrote through a different instance.
+const namespaces = new Map<string, Map<string, Buffer>>();
+
+/**
+ * In-memory {@link FileStore}, addressed via `mem://<namespace>/...`. Reads and writes are synchronous
+ * map operations with no disk I/O, so it is deterministic — useful for tests that exercise consumers
+ * of a file store (e.g. the tx file store) rather than the on-disk store itself, where a real
+ * filesystem can introduce timing flakes.
+ */
+export class InMemoryFileStore implements FileStore {
+  private readonly files: Map<string, Buffer>;
+
+  constructor(private readonly namespace: string) {
+    let files = namespaces.get(namespace);
+    if (!files) {
+      files = new Map();
+      namespaces.set(namespace, files);
+    }
+    this.files = files;
+  }
+
+  /** Clears all in-memory file store data, or a single namespace. Intended for test isolation. */
+  static clear(namespace?: string): void {
+    if (namespace === undefined) {
+      namespaces.clear();
+    } else {
+      namespaces.delete(namespace);
+    }
+  }
+
+  public async save(path: string, data: Buffer, opts?: FileStoreSaveOptions): Promise<string> {
+    const toStore = opts?.compress ? await gzip(data) : data;
+    const key = this.key(path);
+    this.files.set(key, Buffer.from(toStore));
+    return `mem://${this.namespace}/${key}`;
+  }
+
+  public async upload(destPath: string, srcPath: string, opts?: FileStoreSaveOptions): Promise<string> {
+    const { readFile } = await import('fs/promises');
+    return this.save(destPath, await readFile(srcPath), opts);
+  }
+
+  public async read(pathOrUrl: string): Promise<Buffer> {
+    const data = this.files.get(this.key(pathOrUrl));
+    if (data === undefined) {
+      throw new Error(`File not found in memory store: ${pathOrUrl}`);
+    }
+    // Match LocalFileStore: transparently gunzip content that was stored compressed.
+    if (data.length >= 2 && data[0] === 0x1f && data[1] === 0x8b) {
+      return await gunzip(data);
+    }
+    return data;
+  }
+
+  public async download(pathOrUrl: string, destPath: string): Promise<void> {
+    await writeFile(destPath, await this.read(pathOrUrl));
+  }
+
+  public exists(pathOrUrl: string): Promise<boolean> {
+    return Promise.resolve(this.files.has(this.key(pathOrUrl)));
+  }
+
+  /** Lists stored file keys, optionally restricted to those under `prefix`. Not part of {@link FileStore}. */
+  public listFiles(prefix = ''): string[] {
+    const keys = [...this.files.keys()];
+    const normalizedPrefix = prefix.replace(/^\/+/, '');
+    return normalizedPrefix ? keys.filter(k => k.startsWith(normalizedPrefix)) : keys;
+  }
+
+  /** Resolves a relative path or a `mem://` URI (as returned by `save`) to the registry key. */
+  private key(pathOrUrl: string): string {
+    const ownPrefix = `mem://${this.namespace}/`;
+    if (pathOrUrl.startsWith(ownPrefix)) {
+      return pathOrUrl.slice(ownPrefix.length);
+    }
+    if (pathOrUrl.startsWith('mem://')) {
+      // A mem:// URI for some namespace: drop the scheme and the leading namespace segment.
+      return pathOrUrl.slice('mem://'.length).replace(/^[^/]*\//, '');
+    }
+    return pathOrUrl.replace(/^\/+/, '');
+  }
+}