@aztec-labs/foundation 6.0.0-nightly.20260829

package/src/queue/fifo_memory_queue.ts

@@ -0,0 +1,39 @@
+import type { Logger } from '../log/index.js';
+import { BaseMemoryQueue } from './base_memory_queue.js';
+
+/**
+ * A simple fifo queue. It can grow unbounded. It can have multiple producers and consumers.
+ * Putting an item onto the queue always succeeds, unless either end() or cancel() has been called in which case
+ * the item being pushed is simply discarded.
+ */
+export class FifoMemoryQueue<T> extends BaseMemoryQueue<T> {
+  private container = new FifoQueue<T>();
+
+  constructor(log?: Logger) {
+    super(log);
+  }
+
+  protected override get items() {
+    return this.container;
+  }
+}
+
+class FifoQueue<T> {
+  private items: T[] = [];
+
+  public put(item: T): void {
+    this.items.push(item);
+  }
+
+  public get(): T | undefined {
+    return this.items.shift();
+  }
+
+  public get length(): number {
+    return this.items.length;
+  }
+
+  public clear() {
+    this.items = [];
+  }
+}

package/src/queue/index.ts

@@ -0,0 +1,6 @@
+export * from './fifo_memory_queue.js';
+export * from './priority_memory_queue.js';
+export * from './serial_queue.js';
+export * from './bounded_serial_queue.js';
+export * from './semaphore.js';
+export * from './batch_queue.js';

package/src/queue/priority_memory_queue.ts

@@ -0,0 +1,20 @@
+import { BaseMemoryQueue } from './base_memory_queue.js';
+import { PriorityQueue } from './priority_queue.js';
+
+/**
+ * A priority queue. It can grow unbounded. It can have multiple producers and consumers.
+ * Putting an item onto the queue always succeeds, unless either end() or cancel() has been called in which case
+ * the item being pushed is simply discarded.
+ */
+export class PriorityMemoryQueue<T> extends BaseMemoryQueue<T> {
+  private container: PriorityQueue<T>;
+
+  constructor(comparator: (a: T, b: T) => number) {
+    super();
+    this.container = new PriorityQueue(comparator);
+  }
+
+  protected override get items() {
+    return this.container;
+  }
+}

package/src/queue/priority_queue.ts

@@ -0,0 +1,34 @@
+/**
+ * Priority queue implementation based on a custom comparator.
+ */
+export class PriorityQueue<T> {
+  private items: T[];
+
+  constructor(private comparator: (a: T, b: T) => number) {
+    this.items = [];
+  }
+
+  public put(item: T): void {
+    let i = 0;
+    while (i < this.items.length && this.comparator(item, this.items[i]) >= 0) {
+      i++;
+    }
+    this.items.splice(i, 0, item);
+  }
+
+  public get(): T | undefined {
+    return this.items.shift();
+  }
+
+  public peek(): T | undefined {
+    return this.items[0];
+  }
+
+  public clear() {
+    this.items = [];
+  }
+
+  public get length(): number {
+    return this.items.length;
+  }
+}

package/src/queue/semaphore.ts

@@ -0,0 +1,38 @@
+import { FifoMemoryQueue } from './fifo_memory_queue.js';
+
+export interface ISemaphore {
+  acquire(): Promise<void>;
+  release(): void;
+}
+
+/**
+ * Allows the acquiring of up to `size` tokens before calls to acquire block, waiting for a call to release().
+ */
+export class Semaphore {
+  private readonly queue = new FifoMemoryQueue<boolean>();
+
+  constructor(size: number) {
+    new Array(size).fill(true).map(() => this.queue.put(true));
+  }
+
+  /**
+   * Acquires a token from the Semaphore, allowing access to a limited resource.
+   * If no tokens are available, the call will block and wait until a token is released.
+   * Use in conjunction with the release() method to manage access to resources with limited capacity.
+   *
+   * @returns A Promise that resolves when a token is acquired.
+   */
+  public async acquire() {
+    await this.queue.get();
+  }
+
+  /**
+   * Releases a token back into the semaphore, allowing another acquire call to proceed.
+   * If there are any pending calls to acquire(), one of them will be unblocked and allowed to proceed.
+   * This method should only be called by the holder of the acquired token to ensure proper functionality
+   * and avoid unexpected behavior.
+   */
+  public release() {
+    this.queue.put(true);
+  }
+}

package/src/queue/serial_queue.ts

@@ -0,0 +1,86 @@
+import { FifoMemoryQueue } from './fifo_memory_queue.js';
+
+/**
+ * A more specialized fifo queue that enqueues functions to execute. Enqueued functions are executed in serial.
+ */
+export class SerialQueue {
+  private readonly queue = new FifoMemoryQueue<() => Promise<void>>();
+  private runningPromises: Promise<void>[] = [];
+  private started = false;
+
+  /**
+   * Initializes the execution of enqueued functions in the serial queue.
+   * Functions are executed in the order they were added to the queue, with each function
+   * waiting for the completion of the previous one before starting its execution.
+   * This method should be called once to start processing the queue.
+   */
+  public start(numWorkers = 1) {
+    if (this.started) {
+      return;
+    }
+    this.runningPromises = Array.from({ length: numWorkers }, () => this.queue.process(fn => fn()));
+    this.started = true;
+  }
+
+  /**
+   * Returns the current number of enqueued functions in the serial queue.
+   * This provides a way to check the size of the queue and monitor its progress.
+   *
+   * @returns The length of the serial queue as a number.
+   */
+  public length() {
+    return this.queue.length();
+  }
+
+  /**
+   * Cancels the processing of the remaining functions in the serial queue and resolves the running promise.
+   * Any enqueued functions that have not yet been executed will be discarded. The queue can still accept new
+   * functions after cancellation, but the previously enqueued functions will not be re-processed.
+   *
+   * @returns The running promise which resolves when the current executing function (if any) completes.
+   */
+  public cancel() {
+    this.queue.cancel();
+    return Promise.all(this.runningPromises).then(() => {});
+  }
+
+  /**
+   * Signals the SerialQueue that it should finish processing its current task and stop accepting new tasks.
+   * The returned Promise resolves when all enqueued tasks have completed execution.
+   *
+   * @returns A Promise that resolves when the queue is completely emptied and no new tasks are allowed.
+   */
+  public end() {
+    this.queue.end();
+    return Promise.all(this.runningPromises).then(() => {});
+  }
+
+  /**
+   * Enqueues fn for execution on the serial queue.
+   * Returns the result of the function after execution.
+   * @param fn - The function to enqueue.
+   * @returns A resolution promise. Rejects if the function does, or if the function could not be enqueued.
+   */
+  public put<T>(fn: () => T | Promise<T>): Promise<Awaited<T>> {
+    return new Promise((resolve, reject) => {
+      const accepted = this.queue.put(async () => {
+        try {
+          const res = await fn();
+          resolve(res);
+        } catch (e) {
+          reject(e);
+        }
+      });
+      if (!accepted) {
+        reject(new Error('Could not enqueue function'));
+      }
+    });
+  }
+
+  /**
+   * Awaiting this ensures the queue is empty before resuming.
+   */
+  public async syncPoint() {
+    await this.put(async () => {});
+  }
+}

package/src/retry/index.ts

@@ -0,0 +1,185 @@
+import { TimeoutError } from '../error/index.js';
+import { type Logger, createLogger } from '../log/index.js';
+import { sleep } from '../sleep/index.js';
+import { type DateProvider, Timer } from '../timer/index.js';
+
+/** An error that indicates that the operation should not be retried. */
+export class NoRetryError extends Error {}
+
+/**
+ * Generates a backoff sequence for retrying operations with an increasing delay.
+ * The backoff sequence follows this pattern: 1, 1, 1, 2, 4, 8, 16, 32, 64, ...
+ * This generator can be used in combination with the `retry` function to perform
+ * retries with exponential backoff and capped at 64 seconds between attempts.
+ *
+ * @returns A generator that yields the next backoff value in seconds as an integer.
+ */
+export function* backoffGenerator() {
+  const v = [1, 1, 1, 2, 4, 8, 16, 32, 64];
+  let i = 0;
+  while (true) {
+    yield v[Math.min(i++, v.length - 1)];
+  }
+}
+
+/**
+ * Generates a backoff sequence based on the array of retry intervals to use with the `retry` function.
+ * @param retries - Intervals to retry (in seconds).
+ * @returns A generator sequence.
+ */
+export function* makeBackoff(retries: number[]) {
+  for (const retry of retries) {
+    yield retry;
+  }
+}
+
+/**
+ * Retry a given asynchronous function with a specific backoff strategy, until it succeeds or backoff generator ends.
+ * It logs the error and retry interval in case an error is caught. The function can be named for better log output.
+ *
+ * @param fn - The asynchronous function to be retried.
+ * @param name - The optional name of the operation, used for logging purposes.
+ * @param backoff - The optional backoff generator providing the intervals in seconds between retries. Defaults to a predefined series.
+ * @param log - Logger to use for logging.
+ * @param failSilently - Do not log errors while retrying.
+ * @returns A Promise that resolves with the successful result of the provided function, or rejects if backoff generator ends.
+ * @throws If `NoRetryError` is thrown by the `fn`, it is rethrown.
+ */
+export async function retry<Result>(
+  fn: () => Promise<Result>,
+  name = 'Operation',
+  backoff = backoffGenerator(),
+  log: Logger = createLogger('foundation:retry'),
+  failSilently = false,
+) {
+  while (true) {
+    try {
+      return await fn();
+    } catch (err: any) {
+      if (err instanceof NoRetryError) {
+        // A special error that indicates that the operation should not be retried. Rethrow it.
+        throw err;
+      }
+      const s = backoff.next().value;
+      if (s === undefined) {
+        throw err;
+      }
+      log?.debug(`${name} failed. Will retry in ${s}s...`);
+      !failSilently && log?.error(`Error while retrying ${name}`, err);
+      await sleep(s * 1000);
+      continue;
+    }
+  }
+}
+
+/**
+ * Timeout specification accepted by {@link retryUntil}. Either a plain number of seconds, an explicit
+ * `{ timeout }` in seconds, or an absolute `{ deadline }` with an optional {@link DateProvider} used to
+ * read the current time. A deadline is converted to the remaining seconds at call time; a deadline that
+ * has already passed yields a zero remaining budget, matching the immediate-timeout semantics of a
+ * non-positive numeric timeout.
+ */
+export type RetryUntilTimeout = number | { timeout: number } | { deadline: Date; dateProvider?: DateProvider };
+
+/**
+ * Resolves a {@link RetryUntilTimeout} to a numeric timeout in seconds for the legacy timer-based loop.
+ * A numeric/`{ timeout }` value passes through unchanged (0 means never time out, negative times out on the
+ * first interval). A `{ deadline }` is the remaining seconds until the deadline; when the deadline is already
+ * at or past `now` it resolves to a negative value so the loop times out immediately instead of being read as
+ * the never-timeout sentinel 0.
+ */
+function resolveRetryUntilTimeoutSeconds(timeout: RetryUntilTimeout): number {
+  if (typeof timeout === 'number') {
+    return timeout;
+  }
+  if ('deadline' in timeout) {
+    const now = timeout.dateProvider?.now() ?? Date.now();
+    const remainingSeconds = (timeout.deadline.getTime() - now) / 1000;
+    return remainingSeconds > 0 ? remainingSeconds : -1;
+  }
+  return timeout.timeout;
+}
+
+/**
+ * Retry an asynchronous function until it returns a truthy value or the specified timeout is exceeded.
+ * The function is retried periodically with a fixed interval between attempts. The operation can be named for better error messages.
+ * Will never timeout if the value is 0.
+ *
+ * @param fn - The asynchronous function to be retried, which should return a truthy value upon success or undefined otherwise.
+ * @param name - The optional name of the operation, used for generating timeout error message.
+ * @param timeout - The maximum time to keep retrying before throwing a timeout error. Accepts a number of
+ * seconds (0 = never time out), an explicit `{ timeout }` in seconds, or an absolute `{ deadline }` with an
+ * optional `dateProvider`. A deadline already in the past times out on the first interval, the same as a
+ * zero/negative numeric timeout. Defaults to 0 (never timeout).
+ * @param interval - The optional interval, in seconds, between retry attempts. Defaults to 1 second.
+ * @returns A Promise that resolves with the successful (truthy) result of the provided function, or rejects if timeout is exceeded.
+ */
+export async function retryUntil<T>(
+  fn: () => (T | undefined) | Promise<T | undefined>,
+  name = '',
+  timeout: RetryUntilTimeout = 0,
+  interval = 1,
+) {
+  const timeoutSeconds = resolveRetryUntilTimeoutSeconds(timeout);
+  const timer = new Timer();
+  while (true) {
+    const result = await fn();
+    if (result) {
+      return result;
+    }
+
+    await sleep(interval * 1000);
+
+    if (timeoutSeconds && timer.s() > timeoutSeconds) {
+      throw new TimeoutError(name ? `Timeout awaiting ${name}` : 'Timeout');
+    }
+  }
+}
+
+/**
+ * Retry an asynchronous function until it returns a truthy value or the maximum number of retries is exceeded.
+ * The function is retried periodically with a fixed interval between attempts.
+ *
+ * @param fn - The asynchronous function to be retried, which should return a truthy value upon success or undefined otherwise.
+ * @param name - The optional name of the operation, used for generating error messages.
+ * @param maxRetries - The maximum number of retry attempts before throwing an error.
+ * @param retryInterval - The optional interval, in seconds, between retry attempts. Defaults to 1 second.
+ * @returns A Promise that resolves with the successful (truthy) result of the provided function, or rejects if retries are exhausted.
+ */
+export async function retryTimes<T>(
+  fn: () => (T | undefined) | Promise<T | undefined>,
+  name = '',
+  maxRetries: number,
+  retryInterval = 1,
+) {
+  for (let attempt = 0; attempt <= maxRetries; attempt++) {
+    const result = await fn();
+    if (result) {
+      return result;
+    }
+
+    if (attempt < maxRetries) {
+      await sleep(retryInterval * 1000);
+    }
+  }
+
+  throw new Error(name ? `Retries exhausted awaiting ${name}` : 'Retries exhausted');
+}
+
+/**
+ * Convenience wrapper around retryUntil with fast polling for tests.
+ * Uses 10s timeout and 100ms polling interval by default.
+ *
+ * @param fn - The function to retry until it returns a truthy value.
+ * @param name - Description of what we're waiting for (for error messages).
+ * @param timeout - Optional timeout in seconds. Defaults to 10s.
+ * @param interval - Optional interval in seconds. Defaults to 0.1s (100ms).
+ */
+export function retryFastUntil<T>(
+  fn: () => (T | undefined) | Promise<T | undefined>,
+  name = '',
+  timeout = 10,
+  interval = 0.1,
+) {
+  return retryUntil(fn, name, timeout, interval);
+}

package/src/running-promise/index.ts

@@ -0,0 +1 @@
+export * from '../promise/running-promise.js';

package/src/schemas/api.ts

@@ -0,0 +1,47 @@
+import type { z } from 'zod';
+
+import type { ZodNullableOptional } from './utils.js';
+
+// Forces usage of schemas that accept nulls for optional parameters.
+type ZodParameterTypeFor<T> = undefined extends T ? ZodNullableOptional<z.ZodType<T, any>> : z.ZodType<T, any>;
+
+type ZodReturnTypeFor<T> = z.ZodType<T, any>;
+
+type ZodMapParameterTypes<T extends readonly unknown[]> = {
+  [K in keyof T]-?: ZodParameterTypeFor<T[K]>;
+};
+
+/** Maps all functions in an interface to their schema representation. */
+export type ApiSchemaFor<T> = {
+  [K in keyof T]: T[K] extends (...args: infer Args) => Promise<infer Ret>
+    ? z.ZodFunction<z.ZodTuple<ZodMapParameterTypes<Args>, null>, ZodReturnTypeFor<Ret>>
+    : never;
+};
+
+/** Generic Api schema not bounded to a specific implementation. */
+export type ApiSchema = {
+  [key: string]: z.ZodFunction<z.ZodTuple<any, any>, z.ZodTypeAny>;
+};
+
+/** Return whether an API schema defines a valid function schema for a given method name. */
+export function schemaHasMethod<T extends ApiSchema>(
+  schema: T,
+  methodName: string,
+): methodName is Extract<keyof T, string> {
+  return (
+    typeof methodName === 'string' &&
+    Object.hasOwn(schema, methodName) &&
+    schema[methodName].def.input !== undefined &&
+    schema[methodName].def.output !== undefined
+  );
+}
+
+/** Returns the parameter tuple schema for an API method schema. */
+export function getSchemaParameters<T extends z.ZodFunction<z.ZodTuple<any, any>, z.ZodTypeAny>>(schema: T) {
+  return schema.def.input;
+}
+
+/** Returns the return value schema for an API method schema. */
+export function getSchemaReturnType<T extends z.ZodFunction<z.ZodTuple<any, any>, z.ZodTypeAny>>(schema: T) {
+  return schema.def.output;
+}

package/src/schemas/index.ts

@@ -0,0 +1,5 @@
+export * from './api.js';
+export * from './parse.js';
+export * from './utils.js';
+export * from './types.js';
+export * from './schemas.js';

package/src/schemas/parse.ts

@@ -0,0 +1,32 @@
+import { z } from 'zod';
+
+import { times } from '../collection/array.js';
+
+/** Parses the given arguments using a tuple from the provided schemas. */
+export function parse<T extends [] | [z.ZodTypeAny, ...z.ZodTypeAny[]]>(args: IArguments, ...schemas: T) {
+  return schemas.length === 0
+    ? z.tuple([]).parse(args)
+    : z.tuple(schemas as [z.ZodTypeAny, ...z.ZodTypeAny[]]).parse(args);
+}
+
+/**
+ * Parses the given arguments against a tuple, allowing empty for optional items.
+ * @dev Zod doesn't like tuples with optional items. See https://github.com/colinhacks/zod/discussions/949.
+ */
+export function parseWithOptionals<T extends z.ZodTuple<any, any>>(args: any[], schema: T): Promise<z.output<T>> {
+  const items = schema.def.items;
+  const missingCount = items.length - args.length;
+  const optionalCount = items.filter(isOptional).length;
+  const toParse =
+    missingCount > 0 && missingCount <= optionalCount ? args.concat(times(missingCount, () => undefined)) : args;
+  return schema.parseAsync(toParse);
+}
+
+function isOptional(schema: z.ZodTypeAny) {
+  try {
+    return schema.isOptional();
+  } catch {
+    // See https://github.com/colinhacks/zod/issues/1911
+    return schema.def.type === 'optional';
+  }
+}

package/src/schemas/schemas.ts

@@ -0,0 +1,93 @@
+import { z } from 'zod';
+
+import { EpochNumber, EpochNumberSchema } from '../branded-types/epoch.js';
+import { SlotNumber, SlotNumberSchema } from '../branded-types/slot.js';
+import { Buffer32 } from '../buffer/buffer32.js';
+import { SecretValue } from '../config/secret_value.js';
+import { Fq, Fr } from '../curves/bn254/field.js';
+import { Point } from '../curves/grumpkin/point.js';
+import { EthAddress } from '../eth-address/index.js';
+import { isHex, withHexPrefix, withoutHexPrefix } from '../string/index.js';
+import { bufferSchema, hexSchema } from './utils.js';
+
+export const schemas = {
+  /** Accepts a number, bigint, or string as an EpochNumber. */
+  EpochNumber: EpochNumberSchema,
+
+  /** Accepts a number, bigint, or string as a SlotNumber. */
+  SlotNumber: SlotNumberSchema,
+
+  /** Accepts a hex string. */
+  EthAddress: EthAddress.schema,
+
+  /** Accepts a hex string. */
+  Fr: Fr.schema,
+
+  /** Accepts a hex string. */
+  Fq: Fq.schema,
+
+  /** Point. Serialized as a hex string. */
+  Point: Point.schema,
+
+  /** Coerces truthy-like string values to boolean. */
+  Boolean: z.union([
+    z.boolean(),
+    z
+      .number()
+      .refine(arg => arg === 0 || arg === 1, { message: `Numeric value for a boolean variable must be 0 or 1` })
+      .transform(arg => arg === 1),
+    z
+      .string()
+      .transform(arg => arg.trim().toLowerCase())
+      .refine(arg => ['true', 'false', '1', '0'].includes(arg))
+      .transform(arg => arg === '1' || arg === 'true'),
+  ]),
+
+  /** Coerces any input to bigint. */
+  BigInt: z.union([z.bigint(), z.number(), z.string()]).pipe(z.coerce.bigint<string | number | bigint>()),
+
+  /** Coerces any input to integer number. */
+  Integer: z.union([z.bigint(), z.number(), z.string()]).pipe(z.coerce.number<string | number | bigint>().int()),
+
+  /** Coerces input to UInt32. */
+  UInt32: z.union([z.bigint(), z.number(), z.string()]).pipe(
+    z.coerce
+      .number<string | number | bigint>()
+      .int()
+      .min(0)
+      .max(2 ** 32 - 1),
+  ),
+
+  /** Coerces input to UInt64. */
+  UInt64: z.union([z.bigint(), z.number(), z.string()]).pipe(
+    z.coerce
+      .bigint<string | number | bigint>()
+      .min(0n)
+      .max(2n ** 64n - 1n),
+  ),
+
+  /** Accepts a hex string as a Buffer32 type. */
+  Buffer32: z.string().refine(isHex, 'Not a valid hex string').transform(Buffer32.fromString),
+
+  /** Accepts a base64 string or an object `{ type: 'Buffer', data: [byte, byte...] }` as a buffer. */
+  Buffer: bufferSchema,
+
+  /** Accepts a hex string as a buffer. */
+  BufferHex: z
+    .string()
+    .refine(isHex, 'Not a valid hex string')
+    .transform(withoutHexPrefix)
+    .transform(data => Buffer.from(data, 'hex')),
+
+  /** Hex string with an optional 0x prefix which gets removed as part of the parsing. */
+  HexString: hexSchema,
+
+  /** Hex string with an optional 0x prefix which gets enforced as part of the parsing. */
+  HexStringWith0x: z.string().refine(isHex, 'Not a valid hex string').transform(withHexPrefix),
+
+  /** A secret config value */
+  SecretValue: SecretValue.schema,
+};
+
+// These are needed to avoid errors such as: "The inferred type of 'YourClassSchema' cannot be named without a reference to..."
+export type { EpochNumber, EthAddress, Fq, Fr, Point, SlotNumber };

package/src/schemas/types.ts

@@ -0,0 +1,36 @@
+import type { ZodType } from 'zod';
+
+export type ZodFor<T> = ZodType<T, any>;
+
+/**
+ * Creates a schema validator that enforces all properties of type T are present in the schema.
+ * This provides compile-time safety to ensure schemas don't miss optional properties.
+ *
+ * @example
+ * ```ts
+ * interface MyConfig {
+ *   foo: string;
+ *   bar?: number;
+ * }
+ *
+ * // ✅ This will work - all keys present
+ * const schema1 = zodFor<MyConfig>()(z.object({
+ *   foo: z.string(),
+ *   bar: z.number().optional(),
+ * }));
+ *
+ * // ❌ This will error - 'bar' is missing
+ * const schema2 = zodFor<MyConfig>()(z.object({
+ *   foo: z.string(),
+ * }));
+ * ```
+ */
+export function zodFor<T>() {
+  return (schema => schema) as <S extends ZodType<any, any>>(
+    schema: keyof T extends keyof S['_output']
+      ? keyof S['_output'] extends keyof T
+        ? S
+        : S & { __error__: 'Schema has extra keys not in type'; __extra__: Exclude<keyof S['_output'], keyof T> }
+      : S & { __error__: 'Schema is missing keys from type'; __missing__: Exclude<keyof T, keyof S['_output']> },
+  ) => S;
+}