@aztec-labs/foundation 6.0.0-nightly.20260829

package/src/profiler/profiler.ts

@@ -0,0 +1,125 @@
+import { AsyncLocalStorage } from 'node:async_hooks';
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import { performance } from 'node:perf_hooks';
+
+interface Span {
+  label: string;
+  start: number;
+  dur: number;
+  count: number;
+  children: Span[];
+  parent: Span | undefined;
+}
+
+interface ProfileData {
+  spans: SerializedSpan[];
+  timestamp: string;
+  totalTime: number;
+}
+
+interface SerializedSpan {
+  label: string;
+  dur: number;
+  count: number;
+  children: SerializedSpan[];
+}
+
+const als = new AsyncLocalStorage<Span>();
+const roots: Span[] = [];
+
+function reset(): void {
+  roots.length = 0;
+}
+
+// Strip out circular references (parent) and unused fields (start) for JSON serialization
+function serializeSpans(spans: Span[]): SerializedSpan[] {
+  return spans.map(span => ({
+    label: span.label,
+    dur: span.dur,
+    count: span.count,
+    children: serializeSpans(span.children),
+  }));
+}
+
+let i = 0;
+function save(): void {
+  if (roots.length === 0) {
+    return;
+  }
+
+  // Find max single execution time across all spans (dur/count since dur is accumulated)
+  const findMaxSingleDuration = (spans: Span[]): number => {
+    let max = 0;
+    for (const span of spans) {
+      const singleDur = span.dur / span.count;
+      max = Math.max(max, singleDur);
+      if (span.children.length > 0) {
+        max = Math.max(max, findMaxSingleDuration(span.children));
+      }
+    }
+    return max;
+  };
+
+  const profileData: ProfileData = {
+    spans: serializeSpans(roots),
+    timestamp: new Date().toISOString(),
+    totalTime: findMaxSingleDuration(roots),
+  };
+
+  const profilePath = path.join(process.cwd(), `profile-${i++}.json`);
+  process.stdout.write(`Writing profile data to ${profilePath}\n`);
+  fs.writeFileSync(profilePath, JSON.stringify(profileData, null, 2));
+}
+
+// Hook into Jest to save after each test
+if (typeof afterEach === 'function') {
+  afterEach(() => {
+    save();
+    reset();
+  });
+}
+
+// Also save on process exit for non-Jest environments
+process.on('exit', () => {
+  save();
+});
+
+// Wrapper for async functions to maintain context properly
+async function runAsync<ReturnType>(label: string, fn: () => Promise<ReturnType>): Promise<ReturnType> {
+  const parent = als.getStore();
+
+  // Check if we already have a span with this label in the current context
+  let existingSpan: Span | undefined;
+  if (parent) {
+    existingSpan = parent.children.find(c => c.label === label);
+  } else {
+    existingSpan = roots.find(r => r.label === label);
+  }
+
+  let span: Span;
+  if (existingSpan) {
+    // Reuse existing span and increment count
+    span = existingSpan;
+    span.count++;
+  } else {
+    // Create new span
+    span = { label, start: performance.now(), dur: 0, count: 1, children: [], parent };
+    if (parent) {
+      parent.children.push(span);
+    } else {
+      roots.push(span);
+    }
+  }
+
+  const startTime = performance.now();
+  const result: ReturnType = await als.run(span, fn);
+  const elapsed = performance.now() - startTime;
+
+  // Add to total duration (for averaging)
+  span.dur += elapsed;
+
+  return result;
+}
+
+export const profiler = { reset, runAsync };

package/src/promise/index.ts

@@ -0,0 +1,2 @@
+export * from './running-promise.js';
+export * from './utils.js';

package/src/promise/running-promise.ts

@@ -0,0 +1,138 @@
+import { InterruptError } from '../error/index.js';
+import { type Logger, createLogger } from '../log/pino-logger.js';
+import { InterruptibleSleep } from '../sleep/index.js';
+import { type PromiseWithResolvers, promiseWithResolvers } from './utils.js';
+
+const EXIT = Symbol.for('RunningPromise.EXIT');
+
+export type ErrorHandler = (err: unknown) => typeof EXIT | void | Promise<typeof EXIT | void>;
+
+export function makeLoggingErrorHandler(
+  logger: Logger,
+  ...ignoredErrors: (new (...args: any[]) => Error)[]
+): ErrorHandler {
+  return err => {
+    if (err instanceof Error && !ignoredErrors.some(ErrorType => err instanceof ErrorType)) {
+      logger.error('Error in running promise', err);
+    }
+  };
+}
+
+/**
+ * RunningPromise is a utility class that helps manage the execution of an asynchronous function
+ * at a specified polling interval. It allows starting, stopping, and checking the status of the
+ * internally managed promise. The class also supports interrupting the polling process when stopped.
+ */
+export class RunningPromise {
+  private running = false;
+  private runningPromise = Promise.resolve();
+  private interruptibleSleep = new InterruptibleSleep();
+  private requested: PromiseWithResolvers<void> | undefined = undefined;
+
+  public static readonly EXIT: typeof EXIT = EXIT;
+
+  constructor(
+    private fn: () => void | Promise<void>,
+    private logger = createLogger('running-promise'),
+    private pollingIntervalMS = 10000,
+    private handleError: ErrorHandler = makeLoggingErrorHandler(logger),
+  ) {}
+
+  /**
+   * Starts the running promise.
+   */
+  public start() {
+    if (this.running) {
+      this.logger.warn(`Attempted to start running promise that was already started`);
+      return this;
+    }
+    this.running = true;
+
+    const poll = async () => {
+      while (this.running) {
+        const requested = this.requested;
+        this.requested = undefined;
+        try {
+          await this.fn();
+        } catch (err) {
+          const code = await this.handleError(err);
+          if (code === RunningPromise.EXIT) {
+            this.logger.warn('Error handler has requested to exit', { err });
+            this.running = false;
+          }
+        }
+
+        // If an immediate run had been requested *before* the function started running, resolve the request.
+        if (requested) {
+          requested.resolve();
+        }
+
+        // If no immediate run was requested, sleep for the polling interval.
+        if (this.requested === undefined && this.running) {
+          await this.interruptibleSleep.sleep(this.pollingIntervalMS);
+        }
+      }
+
+      // A trigger that arrived after the final pass started will never be served by any pass, so settle it as
+      // failed rather than leaving its caller waiting forever.
+      const unserved = this.requested;
+      this.requested = undefined;
+      unserved?.reject(new InterruptError('RunningPromise stopped before serving trigger'));
+    };
+    this.runningPromise = poll();
+    return this;
+  }
+
+  /**
+   * Stops the running promise, resolves any pending interruptible sleep,
+   * and waits for the currently executing function to complete.
+   */
+  async stop(): Promise<void> {
+    if (!this.running) {
+      return;
+    }
+    this.running = false;
+    this.interruptibleSleep.interrupt();
+    await this.runningPromise;
+  }
+
+  /**
+   * Checks if the running promise is currently active.
+   * @returns True if the promise is running.
+   */
+  public isRunning() {
+    return this.running;
+  }
+
+  /**
+   * Triggers an immediate run of the function, bypassing the polling interval.
+   *
+   * Resolves only after a complete run of the function that *started after* this call: a run already in flight is
+   * allowed to finish first, and the function is then called again. Concurrent callers coalesce onto a single such
+   * run. Rejects if the loop stops (via `stop()` or an error handler requesting exit) before that run happens.
+   *
+   * Calling this from inside the function itself and awaiting the result deadlocks, since the awaited run cannot
+   * start until the current one returns. That usage is not supported.
+   */
+  public async trigger(): Promise<void> {
+    if (!this.running) {
+      return this.fn();
+    }
+
+    let requested = this.requested;
+    if (!requested) {
+      requested = promiseWithResolvers<void>();
+      this.requested = requested;
+      this.interruptibleSleep.interrupt();
+    }
+    await requested.promise;
+  }
+
+  /**
+   * Updates the polling interval. The new interval will take effect after the next poll.
+   * @param pollingIntervalMS The polling interval in milliseconds.
+   */
+  setPollingIntervalMS(pollingIntervalMS: number) {
+    this.pollingIntervalMS = pollingIntervalMS;
+  }
+}

package/src/promise/utils.ts

@@ -0,0 +1,60 @@
+/**
+ * Like `Promise.all`, but runs every promise to completion before resolving or rejecting.
+ *
+ * `Promise.all` rejects as soon as one input rejects, abandoning the still-running siblings. When those siblings have
+ * side effects, the caller observes the rejection and reacts to the failure while the abandoned work is still
+ * producing effects. This helper instead guarantees that by the time it rejects, no input is still running.
+ *
+ * The signature mirrors `Promise.all`'s declaration in TypeScript's standard library, so call sites type exactly as
+ * they did with `Promise.all` (in particular, heterogeneous tuples keep their per-position types).
+ */
+export async function allToCompletion<T extends readonly unknown[] | []>(
+  promises: T,
+): Promise<{ -readonly [P in keyof T]: Awaited<T[P]> }> {
+  const results = await Promise.allSettled<unknown>(promises);
+  const failures = results.filter((result): result is PromiseRejectedResult => result.status === 'rejected');
+  if (failures.length === 1) {
+    // Rethrow a lone failure as-is so error identity is preserved for callers that inspect error types.
+    throw failures[0].reason;
+  } else if (failures.length > 1) {
+    throw new AggregateError(
+      failures.map(failure => failure.reason),
+      `${failures.length} of ${results.length} concurrent operations failed: ${failures
+        .map(failure => (failure.reason instanceof Error ? failure.reason.message : String(failure.reason)))
+        .join(' | ')}`,
+    );
+  }
+  return results.map(result => (result as PromiseFulfilledResult<unknown>).value) as {
+    -readonly [P in keyof T]: Awaited<T[P]>;
+  };
+}
+
+export type PromiseWithResolvers<T> = {
+  promise: Promise<T>;
+  resolve: (value: T) => void;
+  reject: (reason?: any) => void;
+};
+
+/**
+ * A polyfill for the Promise.withResolvers proposed API.
+ * @see https://github.com/tc39/proposal-promise-with-resolvers
+ * @returns A promise with resolvers.
+ */
+export function promiseWithResolvers<T>(): PromiseWithResolvers<T> {
+  // use ! operator to avoid TS error
+  let resolve!: (value: T) => void;
+  let reject!: (reason?: any) => void;
+
+  // the ES spec guarantees that the promise executor is called synchronously
+  // so the resolve and reject functions will be defined
+  const promise = new Promise<T>((res, rej) => {
+    resolve = res;
+    reject = rej;
+  });
+
+  return {
+    promise,
+    resolve,
+    reject,
+  };
+}

package/src/queue/base_memory_queue.ts

@@ -0,0 +1,138 @@
+import { TimeoutError } from '../error/index.js';
+import { createLogger } from '../log/index.js';
+
+export abstract class BaseMemoryQueue<T> {
+  private waiting: ((item: T | null) => void)[] = [];
+  private flushing = false;
+
+  constructor(private log = createLogger('foundation:memory_fifo')) {}
+
+  protected abstract get items(): {
+    length: number;
+    get(): T | undefined;
+    put(item: T): void;
+    clear: () => void;
+  };
+
+  /**
+   * Returns the current number of items in the queue.
+   * The length represents the size of the queue at the time of invocation and may change as new items are added or consumed.
+   *
+   * @returns The number of items in the queue.
+   */
+  public length() {
+    return this.items.length;
+  }
+
+  /**
+   * Returns next item within the queue, or undefined if the queue is empty. Does not block.
+   * @returns The next item in the queue.
+   */
+  public getImmediate(): T | undefined {
+    return this.items.get();
+  }
+
+  /**
+   * Returns next item within the queue, or blocks until an item has been put into the queue.
+   *
+   * If given a timeout, the promise will reject if no item is received after `timeoutSec` seconds.
+   * If the timeout is undefined (default), this call will block until an item is available or the queue is closed.
+   * If the timeout is 0 and there are no items available then the queue will immediately reject with a TimeoutError.
+   *
+   * If the queue is flushing, `null` is returned.
+   * @param timeoutSec - The timeout in seconds.
+   * @returns A result promise.
+   */
+  public get(timeoutSec?: number): Promise<T | null> {
+    if (this.items.length) {
+      return Promise.resolve(this.items.get()!);
+    }
+
+    if (this.items.length === 0 && this.flushing) {
+      return Promise.resolve(null);
+    }
+
+    // if the caller doesn't want to wait for an item to be available
+    // immediately reject with a Timeout error
+    if (timeoutSec === 0) {
+      return Promise.reject(new TimeoutError('Timeout getting item from queue.'));
+    }
+
+    return new Promise<T | null>((resolve, reject) => {
+      this.waiting.push(resolve);
+
+      if (timeoutSec) {
+        setTimeout(() => {
+          const index = this.waiting.findIndex(r => r === resolve);
+          if (index > -1) {
+            this.waiting.splice(index, 1);
+            const err = new TimeoutError('Timeout getting item from queue.');
+            reject(err);
+          }
+        }, timeoutSec * 1000);
+      }
+    });
+  }
+
+  /**
+   * Put an item onto back of the queue.
+   * @param item - The item to enqueue.
+   * @returns A boolean indicating whether the item was successfully added to the queue.
+   */
+  public put(item: T): boolean {
+    if (this.flushing) {
+      this.log.warn('Discarding item because queue is flushing');
+      return false;
+    } else if (this.waiting.length) {
+      this.waiting.shift()!(item);
+      return true;
+    } else {
+      this.items.put(item);
+      return true;
+    }
+  }
+
+  /**
+   * Once ended, no further items are added to queue. Consumers will consume remaining items within the queue.
+   * The queue is not reusable after calling `end()`.
+   * Any consumers waiting for an item receive null.
+   */
+  public end() {
+    this.flushing = true;
+    this.waiting.forEach(resolve => resolve(null));
+  }
+
+  /**
+   * Once cancelled, all items are discarded from the queue, and no further items are added to the queue.
+   * The queue is not reusable after calling `cancel()`.
+   * Any consumers waiting for an item receive null.
+   */
+  public cancel() {
+    this.flushing = true;
+    this.items.clear();
+    this.waiting.forEach(resolve => resolve(null));
+  }
+
+  /**
+   * Process items from the queue using a provided handler function.
+   * The function iterates over items in the queue, invoking the handler for each item until the queue is empty and flushing.
+   * If the handler throws an error, it will be caught and logged as 'Queue handler exception:', but the iteration will continue.
+   * The process function returns a promise that resolves when there are no more items in the queue and the queue is flushing.
+   *
+   * @param handler - A function that takes an item of type T and returns a Promise<void> after processing the item.
+   * @returns A Promise<void> that resolves when the queue is finished processing.
+   */
+  public async process(handler: (item: T) => Promise<void> | void) {
+    try {
+      while (true) {
+        const item = await this.get();
+        if (item === null) {
+          break;
+        }
+        await handler(item);
+      }
+    } catch (err) {
+      this.log.error('Queue handler exception', err);
+    }
+  }
+}

package/src/queue/batch_queue.ts

@@ -0,0 +1,121 @@
+import { createLogger } from '../log/pino-logger.js';
+import { type PromiseWithResolvers, promiseWithResolvers } from '../promise/utils.js';
+import { FifoMemoryQueue } from './fifo_memory_queue.js';
+
+type Batch<T, K> = {
+  items: Array<T>;
+  key: K;
+  deferred: PromiseWithResolvers<void>;
+  enqueueTimeout: ReturnType<typeof setTimeout>;
+};
+
+/**
+ * A queue that groups items into batches based on a group key.
+ *
+ * The batching algorithm is greedy, meaning that as long as consecutive items have the same group key then they will
+ * be batched together. As soon as an item with a different group key is encountered, the old batch is flushed to the
+ * queue and a new batch is started.
+ *
+ * A batch can also be flushed to the queue if:
+ * - it reaches the selected batch size limit
+ * - or the batch duration limit is hit (in milliseconds)
+ *
+ * This ensures that batches don't grow too big and that they are flushed at a minimum rate of 1 batch every interval.
+ *
+ * The consumer side of this queue will process batches as quickly as possible.
+ */
+export class BatchQueue<T, K extends string | number> {
+  private container = new FifoMemoryQueue<Batch<T, K>>();
+  private currentBatch?: Batch<T, K>;
+  private runningPromise?: Promise<void>;
+
+  constructor(
+    private processBatch: (items: Array<T>, key: K) => void | Promise<void>,
+    private maxBatchSize: number,
+    private maxBatchDuration: number,
+    private log = createLogger('foundation:batch_queue'),
+  ) {}
+
+  /**
+   * Put an item in the queue. It will be routed based on the given key
+   * @param item - The item to add
+   * @param key - The group key for this item
+   * @returns A promise that resolves or rejects when the batch this item is part of is processed
+   */
+  public put(item: T, key: K): Promise<void> {
+    if (!this.runningPromise) {
+      return Promise.reject(new Error('BatchQueue is not started'));
+    }
+
+    let currentBatch = this.currentBatch;
+    if (!currentBatch || currentBatch.key !== key || currentBatch.items.length >= this.maxBatchSize) {
+      this.flushCurrentBatch();
+
+      this.log.trace('Creating new batch', { key });
+      currentBatch = {
+        items: [],
+        key,
+        deferred: promiseWithResolvers(),
+        enqueueTimeout: setTimeout(this.flushCurrentBatch, this.maxBatchDuration),
+      };
+
+      this.currentBatch = currentBatch;
+    }
+
+    currentBatch.items.push(item);
+    if (currentBatch.items.length >= this.maxBatchSize) {
+      this.flushCurrentBatch();
+    }
+
+    return currentBatch.deferred.promise;
+  }
+
+  /**
+   * Immediately flushes the current batch, starting a new one
+   */
+  public flushCurrentBatch = (): void => {
+    if (this.currentBatch) {
+      this.log.trace('Flushing batch', { size: this.currentBatch.items.length, key: this.currentBatch.key });
+      clearTimeout(this.currentBatch.enqueueTimeout);
+      this.container.put(this.currentBatch);
+      this.currentBatch = undefined;
+    }
+  };
+
+  /**
+   * Starts the queue.
+   */
+  public start() {
+    if (this.runningPromise) {
+      return;
+    }
+
+    this.runningPromise = this.container.process(this.execProcessor);
+  }
+
+  /**
+   * Stops the queue. Any items in the queue will continue to be processed but new items won't be accepted anymore
+   * @returns A promise that resolves when the queue is drained completely
+   */
+  public stop(): Promise<void> {
+    const runningPromise = this.runningPromise;
+    this.runningPromise = undefined;
+
+    if (!runningPromise) {
+      return Promise.resolve();
+    }
+
+    this.flushCurrentBatch();
+    this.container.end();
+    return runningPromise;
+  }
+
+  private execProcessor = async (batch: Batch<T, K>): Promise<void> => {
+    try {
+      await this.processBatch(batch.items, batch.key);
+      batch.deferred.resolve();
+    } catch (err) {
+      batch.deferred.reject(err);
+    }
+  };
+}

package/src/queue/bounded_serial_queue.ts

@@ -0,0 +1,103 @@
+import { createLogger } from '../log/index.js';
+import { Semaphore } from './semaphore.js';
+import { SerialQueue } from './serial_queue.js';
+
+/**
+ * Leverages the unbounded SerialQueue and Semaphore to create a SerialQueue that will block when putting an item
+ * if the queue size = maxQueueSize.
+ */
+export class BoundedSerialQueue {
+  private readonly queue = new SerialQueue();
+  private semaphore: Semaphore;
+
+  constructor(
+    maxQueueSize: number,
+    private log = createLogger('foundation:bounded_serial_queue'),
+  ) {
+    this.semaphore = new Semaphore(maxQueueSize);
+  }
+
+  /**
+   * Initializes the underlying SerialQueue instance, allowing items to be processed from the queue.
+   * The start method should be called before using the BoundedSerialQueue to ensure proper functionality.
+   */
+  public start() {
+    this.queue.start();
+  }
+
+  /**
+   * Returns the current number of items in the queue.
+   * This is useful for monitoring the size of BoundedSerialQueue and understanding its utilization.
+   *
+   * @returns The length of the queue as an integer value.
+   */
+  public length() {
+    return this.queue.length();
+  }
+
+  /**
+   * Cancels the current operation in the SerialQueue, if any, and clears the queue.
+   * Any pending tasks in the queue will not be executed, and the queue will be emptied.
+   * This method is useful for cleaning up resources and stopping ongoing processes when they are no longer needed.
+   * @returns A promise, resolved once cancelled.
+   */
+  public cancel() {
+    return this.queue.cancel();
+  }
+
+  /**
+   * Ends the queue processing gracefully, preventing new items from being added.
+   * The currently executing item, if any, will complete and remaining queued items
+   * will be processed in order. Once all items have been processed, the queue becomes
+   * permanently unusable.
+   *
+   * @returns A promise that resolves when all items in the queue have been processed.
+   */
+  public end() {
+    return this.queue.end();
+  }
+
+  /**
+   * The caller will block until fn is successfully enqueued.
+   * The fn itself is execute asynchronously and its result discarded.
+   * TODO(AD) do we need this if we have exec()?
+   * @param fn - The function to call once unblocked.
+   */
+  public async put(fn: () => Promise<void>): Promise<void> {
+    await this.semaphore.acquire();
+    this.queue
+      .put(async () => {
+        try {
+          await fn();
+        } finally {
+          this.semaphore.release();
+        }
+      })
+      .catch(err => {
+        this.log.error('BoundedSerialQueue handler exception:', err);
+      });
+  }
+
+  /**
+   * The caller will block until fn is successfully executed, and it's result returned.
+   * @param fn - The function.
+   * @returns A promise that resolves with the result once executed.
+   */
+  public async exec<T>(fn: () => Promise<T>): Promise<T> {
+    await this.semaphore.acquire();
+    return this.queue.put(async () => {
+      try {
+        return await fn();
+      } finally {
+        this.semaphore.release();
+      }
+    });
+  }
+
+  /**
+   * Awaiting this ensures the queue is empty before resuming.
+   */
+  public async syncPoint() {
+    await this.queue.syncPoint();
+  }
+}