@aztec/foundation 0.0.0-test.0

package/src/promise/running-promise.ts

@@ -0,0 +1,125 @@
+import { type Logger, createLogger } from '../log/pino-logger.js';
+import { InterruptibleSleep } from '../sleep/index.js';
+import { type PromiseWithResolvers, promiseWithResolvers } from './utils.js';
+
+const EXIT = Symbol.for('RunningPromise.EXIT');
+
+export type ErrorHandler = (err: unknown) => typeof EXIT | void | Promise<typeof EXIT | void>;
+
+export function makeLoggingErrorHandler(
+  logger: Logger,
+  ...ignoredErrors: (new (...args: any[]) => Error)[]
+): ErrorHandler {
+  return err => {
+    if (err instanceof Error && !ignoredErrors.some(ErrorType => err instanceof ErrorType)) {
+      logger.error('Error in running promise', err);
+    }
+  };
+}
+
+/**
+ * RunningPromise is a utility class that helps manage the execution of an asynchronous function
+ * at a specified polling interval. It allows starting, stopping, and checking the status of the
+ * internally managed promise. The class also supports interrupting the polling process when stopped.
+ */
+export class RunningPromise {
+  private running = false;
+  private runningPromise = Promise.resolve();
+  private interruptibleSleep = new InterruptibleSleep();
+  private requested: PromiseWithResolvers<void> | undefined = undefined;
+
+  public static readonly EXIT: typeof EXIT = EXIT;
+
+  constructor(
+    private fn: () => void | Promise<void>,
+    private logger = createLogger('running-promise'),
+    private pollingIntervalMS = 10000,
+    private handleError: ErrorHandler = makeLoggingErrorHandler(logger),
+  ) {}
+
+  /**
+   * Starts the running promise.
+   */
+  public start() {
+    if (this.running) {
+      this.logger.warn(`Attempted to start running promise that was already started`);
+      return;
+    }
+    this.running = true;
+
+    const poll = async () => {
+      while (this.running) {
+        const hasRequested = this.requested !== undefined;
+        try {
+          await this.fn();
+        } catch (err) {
+          const code = await this.handleError(err);
+          if (code === RunningPromise.EXIT) {
+            this.logger.warn('Error handler has requested to exit', { err });
+            this.running = false;
+          }
+        }
+
+        // If an immediate run had been requested *before* the function started running, resolve the request.
+        if (hasRequested) {
+          this.requested!.resolve();
+          this.requested = undefined;
+        }
+
+        // If no immediate run was requested, sleep for the polling interval.
+        if (this.requested === undefined && this.running) {
+          await this.interruptibleSleep.sleep(this.pollingIntervalMS);
+        }
+      }
+    };
+    this.runningPromise = poll();
+  }
+
+  /**
+   * Stops the running promise, resolves any pending interruptible sleep,
+   * and waits for the currently executing function to complete.
+   */
+  async stop(): Promise<void> {
+    if (!this.running) {
+      this.logger.warn(`Running promise was not started`);
+      return;
+    }
+    this.running = false;
+    this.interruptibleSleep.interrupt();
+    await this.runningPromise;
+  }
+
+  /**
+   * Checks if the running promise is currently active.
+   * @returns True if the promise is running.
+   */
+  public isRunning() {
+    return this.running;
+  }
+
+  /**
+   * Triggers an immediate run of the function, bypassing the polling interval.
+   * If the function is currently running, it will be allowed to continue and then called again immediately.
+   */
+  public async trigger() {
+    if (!this.running) {
+      return this.fn();
+    }
+
+    let requested = this.requested;
+    if (!requested) {
+      requested = promiseWithResolvers<void>();
+      this.requested = requested;
+      this.interruptibleSleep.interrupt();
+    }
+    await requested!.promise;
+  }
+
+  /**
+   * Updates the polling interval. The new interval will take effect after the next poll.
+   * @param pollingIntervalMS The polling interval in milliseconds.
+   */
+  setPollingIntervalMS(pollingIntervalMS: number) {
+    this.pollingIntervalMS = pollingIntervalMS;
+  }
+}

package/src/promise/utils.ts

@@ -0,0 +1,29 @@
+export type PromiseWithResolvers<T> = {
+  promise: Promise<T>;
+  resolve: (value: T) => void;
+  reject: (reason?: any) => void;
+};
+
+/**
+ * A polyfill for the Promise.withResolvers proposed API.
+ * @see https://github.com/tc39/proposal-promise-with-resolvers
+ * @returns A promise with resolvers.
+ */
+export function promiseWithResolvers<T>(): PromiseWithResolvers<T> {
+  // use ! operator to avoid TS error
+  let resolve!: (value: T) => void;
+  let reject!: (reason?: any) => void;
+
+  // the ES spec guarantees that the promise executor is called synchronously
+  // so the resolve and reject functions will be defined
+  const promise = new Promise<T>((res, rej) => {
+    resolve = res;
+    reject = rej;
+  });
+
+  return {
+    promise,
+    resolve,
+    reject,
+  };
+}

package/src/queue/base_memory_queue.ts

@@ -0,0 +1,138 @@
+import { TimeoutError } from '../error/index.js';
+import { createLogger } from '../log/index.js';
+
+export abstract class BaseMemoryQueue<T> {
+  private waiting: ((item: T | null) => void)[] = [];
+  private flushing = false;
+
+  constructor(private log = createLogger('foundation:memory_fifo')) {}
+
+  protected abstract get items(): {
+    length: number;
+    get(): T | undefined;
+    put(item: T): void;
+    clear: () => void;
+  };
+
+  /**
+   * Returns the current number of items in the queue.
+   * The length represents the size of the queue at the time of invocation and may change as new items are added or consumed.
+   *
+   * @returns The number of items in the queue.
+   */
+  public length() {
+    return this.items.length;
+  }
+
+  /**
+   * Returns next item within the queue, or undefined if the queue is empty. Does not block.
+   * @returns The next item in the queue.
+   */
+  public getImmediate(): T | undefined {
+    return this.items.get();
+  }
+
+  /**
+   * Returns next item within the queue, or blocks until an item has been put into the queue.
+   *
+   * If given a timeout, the promise will reject if no item is received after `timeoutSec` seconds.
+   * If the timeout is undefined (default), this call will block until an item is available or the queue is closed.
+   * If the timeout is 0 and there are no items available then the queue will immediately reject with a TimeoutError.
+   *
+   * If the queue is flushing, `null` is returned.
+   * @param timeoutSec - The timeout in seconds.
+   * @returns A result promise.
+   */
+  public get(timeoutSec?: number): Promise<T | null> {
+    if (this.items.length) {
+      return Promise.resolve(this.items.get()!);
+    }
+
+    if (this.items.length === 0 && this.flushing) {
+      return Promise.resolve(null);
+    }
+
+    // if the caller doesn't want to wait for an item to be available
+    // immediately reject with a Timeout error
+    if (timeoutSec === 0) {
+      return Promise.reject(new TimeoutError('Timeout getting item from queue.'));
+    }
+
+    return new Promise<T | null>((resolve, reject) => {
+      this.waiting.push(resolve);
+
+      if (timeoutSec) {
+        setTimeout(() => {
+          const index = this.waiting.findIndex(r => r === resolve);
+          if (index > -1) {
+            this.waiting.splice(index, 1);
+            const err = new TimeoutError('Timeout getting item from queue.');
+            reject(err);
+          }
+        }, timeoutSec * 1000);
+      }
+    });
+  }
+
+  /**
+   * Put an item onto back of the queue.
+   * @param item - The item to enqueue.
+   * @returns A boolean indicating whether the item was successfully added to the queue.
+   */
+  public put(item: T): boolean {
+    if (this.flushing) {
+      this.log.warn('Discarding item because queue is flushing');
+      return false;
+    } else if (this.waiting.length) {
+      this.waiting.shift()!(item);
+      return true;
+    } else {
+      this.items.put(item);
+      return true;
+    }
+  }
+
+  /**
+   * Once ended, no further items are added to queue. Consumers will consume remaining items within the queue.
+   * The queue is not reusable after calling `end()`.
+   * Any consumers waiting for an item receive null.
+   */
+  public end() {
+    this.flushing = true;
+    this.waiting.forEach(resolve => resolve(null));
+  }
+
+  /**
+   * Once cancelled, all items are discarded from the queue, and no further items are added to the queue.
+   * The queue is not reusable after calling `cancel()`.
+   * Any consumers waiting for an item receive null.
+   */
+  public cancel() {
+    this.flushing = true;
+    this.items.clear();
+    this.waiting.forEach(resolve => resolve(null));
+  }
+
+  /**
+   * Process items from the queue using a provided handler function.
+   * The function iterates over items in the queue, invoking the handler for each item until the queue is empty and flushing.
+   * If the handler throws an error, it will be caught and logged as 'Queue handler exception:', but the iteration will continue.
+   * The process function returns a promise that resolves when there are no more items in the queue and the queue is flushing.
+   *
+   * @param handler - A function that takes an item of type T and returns a Promise<void> after processing the item.
+   * @returns A Promise<void> that resolves when the queue is finished processing.
+   */
+  public async process(handler: (item: T) => Promise<void>) {
+    try {
+      while (true) {
+        const item = await this.get();
+        if (item === null) {
+          break;
+        }
+        await handler(item);
+      }
+    } catch (err) {
+      this.log.error('Queue handler exception', err);
+    }
+  }
+}

package/src/queue/batch_queue.ts

@@ -0,0 +1,120 @@
+import { createLogger } from '../log/pino-logger.js';
+import { type PromiseWithResolvers, promiseWithResolvers } from '../promise/utils.js';
+import { FifoMemoryQueue } from './fifo_memory_queue.js';
+
+type Batch<T, K> = {
+  items: Array<T>;
+  key: K;
+  deferred: PromiseWithResolvers<void>;
+  enqueueTimeout: ReturnType<typeof setTimeout>;
+};
+
+/**
+ * A queue that groups items into batches based on a group key.
+ *
+ * The batching algorithm is greedy, meaning that as long as consecutive items have the same group key then they will
+ * be batvched together. As soon as an item with a different group key is encountered, the old batch is flushed to the
+ * queue and a new batch is started.
+ *
+ * A batch can also be flushed to the queue if:
+ * - it reaches the selected batch size limit
+ * - or the batch duration limit is hit (in milliseconds)
+ *
+ * This ensures that batches don't grow too big and that they are flushed at a minimum rate of 1 batch every interval.
+ *
+ * The consumer side of this queue will process batches as quickly as possible.
+ */
+export class BatchQueue<T, K extends string | number> {
+  private container = new FifoMemoryQueue<Batch<T, K>>();
+  private currentBatch?: Batch<T, K>;
+  private runningPromise?: Promise<void>;
+
+  constructor(
+    private processBatch: (items: Array<T>, key: K) => void | Promise<void>,
+    private maxBatchSize: number,
+    private maxBatchDuration: number,
+    private log = createLogger('foundation:batch_queue'),
+  ) {}
+
+  /**
+   * Put an item in the queue. It will be routed based on the given key
+   * @param item - The item to add
+   * @param key - The group key for this item
+   * @returns A promise that resolves or rejects when the batch this item is part of is processed
+   */
+  public put(item: T, key: K): Promise<void> {
+    if (!this.runningPromise) {
+      return Promise.reject(new Error('BatchQueue is not started'));
+    }
+
+    let currentBatch = this.currentBatch;
+    if (!currentBatch || currentBatch.key !== key || currentBatch.items.length >= this.maxBatchSize) {
+      this.flushCurrentBatch();
+
+      this.log.trace('Creating new batch', { key });
+      currentBatch = {
+        items: [],
+        key,
+        deferred: promiseWithResolvers(),
+        enqueueTimeout: setTimeout(this.flushCurrentBatch, this.maxBatchDuration),
+      };
+
+      this.currentBatch = currentBatch;
+    }
+
+    currentBatch.items.push(item);
+    if (currentBatch.items.length >= this.maxBatchSize) {
+      this.flushCurrentBatch();
+    }
+
+    return currentBatch.deferred.promise;
+  }
+
+  /**
+   * Immediately flushes the current batch, starting a new one
+   */
+  public flushCurrentBatch = (): void => {
+    if (this.currentBatch) {
+      this.log.trace('Flushing batch', { size: this.currentBatch.items.length, key: this.currentBatch.key });
+      clearTimeout(this.currentBatch.enqueueTimeout);
+      this.container.put(this.currentBatch);
+      this.currentBatch = undefined;
+    }
+  };
+
+  /**
+   * Starts the queue.
+   */
+  public start() {
+    if (this.runningPromise) {
+      return;
+    }
+
+    this.runningPromise = this.container.process(this.execProcessor);
+  }
+
+  /**
+   * Stops the queue. Any items in the queue will continue to be processed but new items won't be accepted anymore
+   * @returns A promise that resolves when the queue is drained completely
+   */
+  public stop(): Promise<void> {
+    const runningPromise = this.runningPromise;
+    this.runningPromise = undefined;
+
+    if (!runningPromise) {
+      return Promise.resolve();
+    }
+
+    this.container.end();
+    return runningPromise;
+  }
+
+  private execProcessor = async (batch: Batch<T, K>): Promise<void> => {
+    try {
+      await this.processBatch(batch.items, batch.key);
+      batch.deferred.resolve();
+    } catch (err) {
+      batch.deferred.reject(err);
+    }
+  };
+}

package/src/queue/bounded_serial_queue.ts

@@ -0,0 +1,100 @@
+import { createLogger } from '../log/index.js';
+import { Semaphore } from './semaphore.js';
+import { SerialQueue } from './serial_queue.js';
+
+/**
+ * Leverages the unbounded SerialQueue and Semaphore to create a SerialQueue that will block when putting an item
+ * if the queue size = maxQueueSize.
+ */
+export class BoundedSerialQueue {
+  private readonly queue = new SerialQueue();
+  private semaphore: Semaphore;
+
+  constructor(maxQueueSize: number, private log = createLogger('foundation:bounded_serial_queue')) {
+    this.semaphore = new Semaphore(maxQueueSize);
+  }
+
+  /**
+   * Initializes the underlying SerialQueue instance, allowing items to be processed from the queue.
+   * The start method should be called before using the BoundedSerialQueue to ensure proper functionality.
+   */
+  public start() {
+    this.queue.start();
+  }
+
+  /**
+   * Returns the current number of items in the queue.
+   * This is useful for monitoring the size of BoundedSerialQueue and understanding its utilization.
+   *
+   * @returns The length of the queue as an integer value.
+   */
+  public length() {
+    return this.queue.length();
+  }
+
+  /**
+   * Cancels the current operation in the SerialQueue, if any, and clears the queue.
+   * Any pending tasks in the queue will not be executed, and the queue will be emptied.
+   * This method is useful for cleaning up resources and stopping ongoing processes when they are no longer needed.
+   * @returns A promise, resolved once cancelled.
+   */
+  public cancel() {
+    return this.queue.cancel();
+  }
+
+  /**
+   * Ends the queue processing gracefully, preventing new items from being added.
+   * The currently executing item, if any, will complete and remaining queued items
+   * will be processed in order. Once all items have been processed, the queue becomes
+   * permanently unusable.
+   *
+   * @returns A promise that resolves when all items in the queue have been processed.
+   */
+  public end() {
+    return this.queue.end();
+  }
+
+  /**
+   * The caller will block until fn is successfully enqueued.
+   * The fn itself is execute asynchronously and its result discarded.
+   * TODO(AD) do we need this if we have exec()?
+   * @param fn - The function to call once unblocked.
+   */
+  public async put(fn: () => Promise<void>): Promise<void> {
+    await this.semaphore.acquire();
+    this.queue
+      .put(async () => {
+        try {
+          await fn();
+        } finally {
+          this.semaphore.release();
+        }
+      })
+      .catch(err => {
+        this.log.error('BoundedSerialQueue handler exception:', err);
+      });
+  }
+
+  /**
+   * The caller will block until fn is successfully executed, and it's result returned.
+   * @param fn - The function.
+   * @returns A promise that resolves with the result once executed.
+   */
+  public async exec<T>(fn: () => Promise<T>): Promise<T> {
+    await this.semaphore.acquire();
+    return this.queue.put(async () => {
+      try {
+        return await fn();
+      } finally {
+        this.semaphore.release();
+      }
+    });
+  }
+
+  /**
+   * Awaiting this ensures the queue is empty before resuming.
+   */
+  public async syncPoint() {
+    await this.queue.syncPoint();
+  }
+}

package/src/queue/fifo_memory_queue.ts

@@ -0,0 +1,39 @@
+import type { Logger } from '../log/index.js';
+import { BaseMemoryQueue } from './base_memory_queue.js';
+
+/**
+ * A simple fifo queue. It can grow unbounded. It can have multiple producers and consumers.
+ * Putting an item onto the queue always succeeds, unless either end() or cancel() has been called in which case
+ * the item being pushed is simply discarded.
+ */
+export class FifoMemoryQueue<T> extends BaseMemoryQueue<T> {
+  private container = new FifoQueue<T>();
+
+  constructor(log?: Logger) {
+    super(log);
+  }
+
+  protected override get items() {
+    return this.container;
+  }
+}
+
+class FifoQueue<T> {
+  private items: T[] = [];
+
+  public put(item: T): void {
+    this.items.push(item);
+  }
+
+  public get(): T | undefined {
+    return this.items.shift();
+  }
+
+  public get length(): number {
+    return this.items.length;
+  }
+
+  public clear() {
+    this.items = [];
+  }
+}

package/src/queue/index.ts

@@ -0,0 +1,6 @@
+export * from './fifo_memory_queue.js';
+export * from './priority_memory_queue.js';
+export * from './serial_queue.js';
+export * from './bounded_serial_queue.js';
+export * from './semaphore.js';
+export * from './batch_queue.js';

package/src/queue/priority_memory_queue.ts

@@ -0,0 +1,20 @@
+import { BaseMemoryQueue } from './base_memory_queue.js';
+import { PriorityQueue } from './priority_queue.js';
+
+/**
+ * A priority queue. It can grow unbounded. It can have multiple producers and consumers.
+ * Putting an item onto the queue always succeeds, unless either end() or cancel() has been called in which case
+ * the item being pushed is simply discarded.
+ */
+export class PriorityMemoryQueue<T> extends BaseMemoryQueue<T> {
+  private container: PriorityQueue<T>;
+
+  constructor(comparator: (a: T, b: T) => number) {
+    super();
+    this.container = new PriorityQueue(comparator);
+  }
+
+  protected override get items() {
+    return this.container;
+  }
+}

package/src/queue/priority_queue.ts

@@ -0,0 +1,34 @@
+/**
+ * Priority queue implementation based on a custom comparator.
+ */
+export class PriorityQueue<T> {
+  private items: T[];
+
+  constructor(private comparator: (a: T, b: T) => number) {
+    this.items = [];
+  }
+
+  public put(item: T): void {
+    let i = 0;
+    while (i < this.items.length && this.comparator(item, this.items[i]) >= 0) {
+      i++;
+    }
+    this.items.splice(i, 0, item);
+  }
+
+  public get(): T | undefined {
+    return this.items.shift();
+  }
+
+  public peek(): T | undefined {
+    return this.items[0];
+  }
+
+  public clear() {
+    this.items = [];
+  }
+
+  public get length(): number {
+    return this.items.length;
+  }
+}

package/src/queue/semaphore.ts

@@ -0,0 +1,33 @@
+import { FifoMemoryQueue } from './fifo_memory_queue.js';
+
+/**
+ * Allows the acquiring of up to `size` tokens before calls to acquire block, waiting for a call to release().
+ */
+export class Semaphore {
+  private readonly queue = new FifoMemoryQueue<boolean>();
+
+  constructor(size: number) {
+    new Array(size).fill(true).map(() => this.queue.put(true));
+  }
+
+  /**
+   * Acquires a token from the Semaphore, allowing access to a limited resource.
+   * If no tokens are available, the call will block and wait until a token is released.
+   * Use in conjunction with the release() method to manage access to resources with limited capacity.
+   *
+   * @returns A Promise that resolves when a token is acquired.
+   */
+  public async acquire() {
+    await this.queue.get();
+  }
+
+  /**
+   * Releases a token back into the semaphore, allowing another acquire call to proceed.
+   * If there are any pending calls to acquire(), one of them will be unblocked and allowed to proceed.
+   * This method should only be called by the holder of the acquired token to ensure proper functionality
+   * and avoid unexpected behavior.
+   */
+  public release() {
+    this.queue.put(true);
+  }
+}