@aztec/foundation 0.1.0-alpha10

package/src/transport/interface/transferable.ts

@@ -0,0 +1,121 @@
+const $transferable = Symbol('thread.transferable');
+
+/**
+ * Represents a descriptor for transferable objects in multi-threaded environments.
+ * Provides a structure for marking certain objects as transferable and managing the ownership transfer
+ * between threads, particularly useful when working with Web Workers.
+ */
+export interface TransferDescriptor<T = any> {
+  /**
+   * A unique symbol indicating that an object is a TransferDescriptor.
+   */
+  [$transferable]: true;
+  /**
+   * The transferable data to be sent between threads.
+   */
+  send: T;
+  /**
+   * An array of objects that can be transferred between threads without serialization and deserialization.
+   */
+  transferables: Transferable[];
+}
+
+/**
+ * Determines if the provided object is transferable.
+ * Transferable objects are instances of a certain set of classes,
+ * such as ArrayBuffer or MessagePort, which can be transferred between
+ * different execution contexts (e.g., workers) without incurring the
+ * overhead of serialization and deserialization.
+ *
+ * This function checks for the basic transferable criteria, but does not
+ * perform an exhaustive check for all possible transferable types. As new
+ * transferable types are added to JavaScript, they may be supported without
+ * needing to modify this function.
+ *
+ * @param thing - The object to check for transferability.
+ * @returns A boolean indicating whether the object is transferable.
+ */
+function isTransferable(thing: any): thing is Transferable {
+  if (!thing || typeof thing !== 'object') return false;
+  // Don't check too thoroughly, since the list of transferable things in JS might grow over time
+  return true;
+}
+
+/**
+ * Determines whether a given object is a TransferDescriptor.
+ * A TransferDescriptor is an object with a [$transferable] property set to true and used for
+ * transferring ownership of transferable objects between threads.
+ * This function checks if the input object has the required properties to be considered
+ * a valid TransferDescriptor.
+ *
+ * @param thing - The object to be checked for being a TransferDescriptor.
+ * @returns True if the object is a TransferDescriptor, false otherwise.
+ */
+export function isTransferDescriptor(thing: any): thing is TransferDescriptor {
+  return thing && typeof thing === 'object' && thing[$transferable];
+}
+
+/**
+ * Mark a transferable object as such, so it will no be serialized and
+ * deserialized on messaging with the main thread, but to transfer
+ * ownership of it to the receiving thread.
+ *
+ * Only works with array buffers, message ports and few more special
+ * types of objects, but it's much faster than serializing and
+ * deserializing them.
+ *
+ * Note:
+ * The transferable object cannot be accessed by this thread again
+ * unless the receiving thread transfers it back again!
+ *
+ * @param transferable - Array buffer, message port or similar.
+ * @see https://developers.google.com/web/updates/2011/12/Transferable-Objects-Lightning-Fast
+ */
+export function Transfer<T>(transferable: Transferable): TransferDescriptor<T>;
+
+/**
+ * Mark transferable objects within an arbitrary object or array as
+ * being a transferable object. They will then not be serialized
+ * and deserialized on messaging with the main thread, but ownership
+ * of them will be tranferred to the receiving thread.
+ *
+ * Only array buffers, message ports and few more special types of
+ * objects can be transferred, but it's much faster than serializing and
+ * deserializing them.
+ *
+ * Note:
+ * The transferable object cannot be accessed by this thread again
+ * unless the receiving thread transfers it back again!
+ *
+ * @param transferable - Array buffer, message port or similar.
+ * @see https://developers.google.com/web/updates/2011/12/Transferable-Objects-Lightning-Fast
+ */
+export function Transfer<T>(payload: T, transferables: Transferable[]): TransferDescriptor<T>;
+
+/**
+ * Create a TransferDescriptor for transferable objects within an arbitrary object or array, allowing
+ * them to be transferred between threads instead of being serialized and deserialized.
+ * This method is particularly useful when working with Web Workers and other multi-threaded environments.
+ * Transferable objects include ArrayBuffers, MessagePorts, and a few other special types.
+ * Note that after transferring, the original thread will lose access to the transferred object unless
+ * it's transferred back again.
+ *
+ * @param payload - The transferable object or an object containing transferable properties.
+ * @param transferables - Optional array of Transferable objects found in the payload. If not provided,
+ *                        the payload itself should be a Transferable object.
+ * @returns A TransferDescriptor<T> containing the payload and transferables, marked as transferable.
+ * @throws Error if payload is not transferable and transferables array is not provided.
+ * @see https://developers.google.com/web/updates/2011/12/Transferable-Objects-Lightning-Fast
+ */
+export function Transfer<T>(payload: T, transferables?: Transferable[]): TransferDescriptor<T> {
+  if (!transferables) {
+    if (!isTransferable(payload)) throw Error();
+    transferables = [payload];
+  }
+
+  return {
+    [$transferable]: true,
+    send: payload,
+    transferables,
+  };
+}

package/src/transport/node/index.ts

@@ -0,0 +1,2 @@
+export * from './node_connector.js';
+export * from './node_listener.js';

package/src/transport/node/node_connector.ts

@@ -0,0 +1,29 @@
+import { Worker } from 'worker_threads';
+import { Connector } from '../interface/connector.js';
+import { NodeConnectorSocket } from './node_connector_socket.js';
+
+/**
+ * The NodeConnector class is a concrete implementation of the Connector interface, utilizing worker_threads for
+ * efficient parallel execution. This class provides an easy way to establish a connection with a Worker instance,
+ * allowing seamless communication via sockets.
+ *
+ * @example
+ * const worker = new Worker('./path/to/worker.js');
+ * const nodeConnector = new NodeConnector(worker);
+ * const socket = await nodeConnector.createSocket();
+ * socket.send('Hello from main thread!');
+ */
+export class NodeConnector implements Connector {
+  constructor(private worker: Worker) {}
+
+  /**
+   * Creates a new instance of NodeConnectorSocket using the worker provided in the constructor.
+   * The createSocket method is used to establish connections using the worker_threads module,
+   * allowing for efficient and fast communication between different parts of the application.
+   *
+   * @returns A Promise that resolves to a newly created NodeConnectorSocket instance.
+   */
+  createSocket() {
+    return Promise.resolve(new NodeConnectorSocket(this.worker));
+  }
+}

package/src/transport/node/node_connector_socket.ts

@@ -0,0 +1,51 @@
+import { TransferListItem, Worker } from 'worker_threads';
+import { Socket } from '../interface/socket.js';
+
+/**
+ * NodeConnectorSocket is a wrapper class that implements the Socket interface for messaging between
+ * the main thread and worker threads in a Node.js environment. It uses the Worker API for
+ * communication by sending and receiving messages through postMessage and handling messages using
+ * event listeners.
+ *
+ * The send method sends messages to the worker thread, and the registerHandler method registers a
+ * callback function to handle incoming messages from the worker. The close method cleans up
+ * resources when the socket is no longer needed.
+ */
+export class NodeConnectorSocket implements Socket {
+  constructor(private worker: Worker) {}
+
+  /**
+   * Sends a message from the NodeConnectorSocket instance to the associated worker thread.
+   * The 'msg' can be any data type and 'transfer' is an optional array of transferable objects
+   * that can be transferred with zero-copy semantics. The function returns a resolved Promise
+   * once the message has been posted.
+   *
+   * @param msg - The message to send to the worker thread.
+   * @param transfer - Optional array of Transferable objects to transfer ownership alongside the message.
+   * @returns A Promise that resolves when the message has been posted.
+   */
+  send(msg: any, transfer: Transferable[] = []): Promise<void> {
+    this.worker.postMessage(msg, transfer as TransferListItem[]);
+    return Promise.resolve();
+  }
+
+  /**
+   * Registers a callback function to handle incoming messages from the worker.
+   * The provided callback will be executed whenever a message is received from
+   * the worker, passing the message as its single argument.
+   *
+   * @param cb - The callback function to be called when a message is received.
+   */
+  registerHandler(cb: (msg: any) => any): void {
+    this.worker.on('message', cb);
+  }
+
+  /**
+   * Closes the worker connection and removes all event listeners.
+   * Sends an undefined message to the worker for graceful termination.
+   */
+  close() {
+    void this.send(undefined);
+    this.worker.removeAllListeners();
+  }
+}

package/src/transport/node/node_listener.ts

@@ -0,0 +1,33 @@
+import { parentPort } from 'worker_threads';
+import EventEmitter from 'events';
+import { Listener } from '../interface/listener.js';
+import { NodeListenerSocket } from './node_listener_socket.js';
+
+/**
+ * NodeListener is an event-driven class that extends EventEmitter and implements the Listener interface.
+ * It provides methods to open and close communication with a worker thread using the NodeListenerSocket.
+ * The 'new_socket' event is emitted when a new NodeListenerSocket instance is created, allowing for
+ * efficient processing of incoming messages from the parent thread.
+ */
+export class NodeListener extends EventEmitter implements Listener {
+  constructor() {
+    super();
+  }
+
+  /**
+   * Opens a new connection to a parent worker thread and emits an event with the created NodeListenerSocket instance.
+   * The 'new_socket' event can be listened for, providing access to the newly created NodeListenerSocket.
+   *
+   * Fires NodeListener#new_socket.
+   */
+  open() {
+    this.emit('new_socket', new NodeListenerSocket(parentPort as any));
+  }
+
+  /**
+   * Closes the NodeListener instance.
+   * This method currently has no implementation, as there is no need to perform any actions
+   * when closing a NodeListener. It exists for compatibility with the Listener interface.
+   */
+  close() {}
+}

package/src/transport/node/node_listener_socket.ts

@@ -0,0 +1,47 @@
+import { MessagePort, TransferListItem } from 'worker_threads';
+import { Socket } from '../interface/socket.js';
+
+/**
+ * An implementation of a TransportSocket using MessagePorts.
+ */
+export class NodeListenerSocket implements Socket {
+  constructor(private port: MessagePort) {}
+
+  /**
+   * Sends a message through the MessagePort along with any provided Transferables.
+   * The transfer list allows for efficient sending of certain types of data,
+   * such as ArrayBuffer, ImageBitmap, and MessagePort.
+   * The Promise resolves once the message has been successfully sent.
+   *
+   * @param msg - The message to be sent through the MessagePort.
+   * @param transfer - An optional array of Transferable objects to be transferred.
+   * @returns A Promise that resolves once the message has been sent.
+   */
+  send(msg: any, transfer: Transferable[] = []): Promise<void> {
+    this.port.postMessage(msg, transfer as TransferListItem[]);
+    return Promise.resolve();
+  }
+
+  /**
+   * Registers a callback function to handle incoming messages from the MessagePort.
+   * When a message is received, the provided callback function will be invoked with
+   * the received message as its argument. This method allows for efficient and
+   * dynamic handling of incoming data in a NodeListenerSocket instance.
+   *
+   * @param cb - The callback function to process incoming messages.
+   */
+  registerHandler(cb: (msg: any) => any): void {
+    this.port.on('message', cb);
+  }
+
+  /**
+   * Closes the NodeListenerSocket instance, removing all listeners and closing the underlying MessagePort.
+   * Sends an undefined message to notify any connected ports about the closure before removing event listeners
+   * and cleaning up resources. This method should be called when the socket is no longer needed to avoid memory leaks.
+   */
+  close() {
+    void this.send(undefined);
+    this.port.removeAllListeners();
+    this.port.close();
+  }
+}

package/src/transport/transport_client.ts

@@ -0,0 +1,125 @@
+import EventEmitter from 'events';
+import { EventMessage, isEventMessage, ResponseMessage } from './dispatch/messages.js';
+import { Connector } from './interface/connector.js';
+import { Socket } from './interface/socket.js';
+import { createDebugLogger } from '../log/index.js';
+
+const debug = createDebugLogger('aztec:transport_client');
+
+/**
+ * Represents a pending request in the TransportClient.
+ * Contains information about the message ID, and resolve/reject functions for handling responses.
+ * Used to track and manage asynchronous request/response communication with the TransportServer.
+ */
+interface PendingRequest {
+  /**
+   * The unique message identifier used for tracking and matching request/response pairs.
+   */
+  msgId: number;
+  resolve(data: any): void;
+  reject(error: Error): void;
+}
+
+/**
+ * Represents a transport client for communication between TransportServer and clients.
+ * Provides request/response functionality, event handling, and multiplexing support
+ * for efficient and concurrent communication with a corresponding TransportServer.
+ */
+export interface TransportClient<Payload> extends EventEmitter {
+  on(name: 'event_msg', handler: (payload: Payload) => void): this;
+  emit(name: 'event_msg', payload: Payload): boolean;
+}
+
+/**
+ * A TransportClient provides a request/response and event api to a corresponding TransportServer.
+ * If `broadcast` is called on TransportServer, TransportClients will emit an `event_msg`.
+ * The `request` method will block until a response is returned from the TransportServer's dispatch function.
+ * Request multiplexing is supported.
+ */
+export class TransportClient<Payload> extends EventEmitter {
+  private msgId = 0;
+  private pendingRequests: PendingRequest[] = [];
+  private socket?: Socket;
+
+  constructor(private transportConnect: Connector) {
+    super();
+  }
+
+  /**
+   * Initializes and opens the socket connection for the TransportClient.
+   * This method creates a new Socket instance using the provided Connector,
+   * registers a handler for incoming messages, and establishes the connection.
+   * It should be called before making any requests or handling events.
+   *
+   * @throws An error if the socket is already open or there's an issue opening the connection.
+   * @returns A Promise that resolves when the socket connection is successfully opened.
+   */
+  async open() {
+    this.socket = await this.transportConnect.createSocket();
+    this.socket.registerHandler(msg => this.handleSocketMessage(msg));
+  }
+
+  /**
+   * Close the transport client's socket connection and remove all event listeners.
+   * This method should be called when the client is no longer needed to ensure proper cleanup
+   * and prevent potential memory leaks. Once closed, the client cannot be reused and a new
+   * instance must be created if another connection is needed.
+   */
+  close() {
+    this.socket?.close();
+    this.socket = undefined;
+    this.removeAllListeners();
+  }
+
+  /**
+   * Sends a request to the TransportServer with the given payload and transferable objects.
+   * The method will block until a response from the TransportServer's dispatch function is returned.
+   * Request multiplexing is supported, allowing multiple requests to be sent concurrently.
+   *
+   * @param payload - The message payload to send to the server.
+   * @param transfer - An optional array of ArrayBuffer, MessagePort, or ImageBitmap objects to transfer ownership.
+   * @returns A Promise that resolves with the server's response data or rejects with an error message.
+   */
+  request(payload: Payload, transfer?: Transferable[]) {
+    if (!this.socket) {
+      throw new Error('Socket not open.');
+    }
+    const msgId = this.msgId++;
+    const msg = { msgId, payload };
+    debug(`->`, msg);
+    return new Promise<any>((resolve, reject) => {
+      this.pendingRequests.push({ resolve, reject, msgId });
+      this.socket!.send(msg, transfer).catch(reject);
+    });
+  }
+
+  /**
+   * Handles incoming socket messages from the TransportServer, such as ResponseMessage and EventMessage.
+   * If it's an EventMessage, emits an 'event_msg' event with the payload.
+   * If it's a ResponseMessage, resolves or rejects the corresponding pending request based on the message content.
+   *
+   * @param msg - The ResponseMessage or EventMessage received from the TransportServer, or undefined if the remote socket closed.
+   */
+  private handleSocketMessage(msg: ResponseMessage<Payload> | EventMessage<Payload> | undefined) {
+    if (msg === undefined) {
+      // The remote socket closed.
+      this.close();
+      return;
+    }
+    debug(`<-`, msg);
+    if (isEventMessage(msg)) {
+      this.emit('event_msg', msg.payload);
+      return;
+    }
+    const reqIndex = this.pendingRequests.findIndex(r => r.msgId === msg.msgId);
+    if (reqIndex === -1) {
+      return;
+    }
+    const [pending] = this.pendingRequests.splice(reqIndex, 1);
+    if (msg.error) {
+      pending.reject(new Error(msg.error));
+    } else {
+      pending.resolve(msg.payload);
+    }
+  }
+}

package/src/transport/transport_server.ts

@@ -0,0 +1,108 @@
+import { RequestMessage, ResponseMessage } from './dispatch/messages.js';
+import { Listener } from './interface/listener.js';
+import { Socket } from './interface/socket.js';
+import { isTransferDescriptor } from './interface/transferable.js';
+
+/**
+ * Keeps track of clients, providing a broadcast, and request/response api with multiplexing.
+ */
+export class TransportServer<Payload> {
+  private sockets: Socket[] = [];
+
+  constructor(private listener: Listener, private msgHandlerFn: (msg: Payload) => Promise<any>) {}
+
+  /**
+   * Starts the TransportServer, allowing it to accept new connections and handle incoming messages.
+   * The server will listen for 'new_socket' events from the underlying listener and invoke the provided message handler function
+   * for each received message. The server remains active until the 'stop' method is called.
+   */
+  start() {
+    this.listener.on('new_socket', client => this.handleNewSocket(client));
+    this.listener.open();
+  }
+
+  /**
+   * Stops accepting new connections. It doesn't close existing sockets.
+   * It's expected the clients will gracefully complete by closing their end, sending an `undefined` message.
+   */
+  stop() {
+    this.listener.close();
+  }
+
+  /**
+   * Sends a broadcast message to all connected clients.
+   * The given payload will be sent to all the clients currently connected to the TransportServer.
+   * It waits for all the messages to be sent and resolves when they are all sent successfully.
+   *
+   * @param msg - The payload to broadcast to all connected clients.
+   * @returns A Promise that resolves when all messages have been sent successfully.
+   */
+  async broadcast(msg: Payload) {
+    await Promise.all(this.sockets.map(s => s.send({ payload: msg })));
+  }
+
+  /**
+   * Handles the addition of a new socket to the server by registering a message handler for the client
+   * and adding the socket to the list of active sockets. The message handler processes incoming messages
+   * from the client, including detecting client disconnection and removing the closed socket.
+   *
+   * @param socket - The new Socket instance that has connected to the server.
+   */
+  private handleNewSocket(socket: Socket) {
+    socket.registerHandler(async msg => {
+      if (msg === undefined) {
+        // Client socket has closed. Remove it from the list of sockets. Call close on it for any cleanup.
+        const socketIndex = this.sockets.findIndex(s => s === socket);
+        const [closingSocket] = this.sockets.splice(socketIndex, 1);
+        closingSocket.close();
+        return;
+      }
+      return await this.handleSocketMessage(socket, msg);
+    });
+    this.sockets.push(socket);
+  }
+
+  /**
+   * Detect the 'transferables' argument to our socket from our message
+   * handler return type.
+   * @param data - The compound payload data.
+   * @returns The split data and transferables.
+   */
+  private getPayloadAndTransfers(data: any): [any, Transferable[]] {
+    if (isTransferDescriptor(data)) {
+      // We treat PayloadWithTransfers specially so that we're able to
+      // attach transferables while keeping a simple return-type based usage
+      return [data.send, data.transferables];
+    }
+    if (data instanceof Uint8Array) {
+      // We may want to devise a better solution to this. We maybe given a view over a non cloneable/transferrable
+      // ArrayBuffer (such as a view over wasm memory). In this case we want to take a copy, and then transfer it.
+      const respPayload = data instanceof Uint8Array && ArrayBuffer.isView(data) ? new Uint8Array(data) : data;
+      const transferables = data instanceof Uint8Array ? [respPayload.buffer] : [];
+      return [respPayload, transferables];
+    }
+    return [data, []];
+  }
+  /**
+   * Handles incoming socket messages, processing the request and sending back a response.
+   * This function is responsible for invoking the registered message handler function with the received
+   * payload, extracting the result and transferables, and sending a response message back to the client.
+   * In case of an error during message handling, it sends an error response with the stack trace.
+   *
+   * @param socket - The Socket instance from which the message was received.
+   * @param msg - The RequestMessage object containing the message ID and payload.
+   */
+  private async handleSocketMessage(socket: Socket, { msgId, payload }: RequestMessage<Payload>) {
+    try {
+      const data = await this.msgHandlerFn(payload);
+
+      const [respPayload, transferables] = this.getPayloadAndTransfers(data);
+      const rep: ResponseMessage<Payload> = { msgId, payload: respPayload };
+
+      await socket.send(rep, transferables);
+    } catch (err: any) {
+      const rep: ResponseMessage<Payload> = { msgId, error: err.stack };
+      await socket.send(rep);
+    }
+  }
+}

package/src/types/index.ts

@@ -0,0 +1,7 @@
+/**
+ * Strips methods of a type.
+ */
+export type FieldsOf<T> = {
+  // eslint-disable-next-line @typescript-eslint/ban-types
+  [P in keyof T as T[P] extends Function ? never : P]: T[P];
+};

package/src/wasm/README.md

@@ -0,0 +1,6 @@
+# wasm
+
+Functionality to:
+
+1.  Call WebAssembly functions
+2.  Create asynchronous workers that host webassembly.

package/src/wasm/fixtures/gcd.wat

@@ -0,0 +1,27 @@
+(module
+  (func $gcd (param i32 i32) (result i32)
+    (local i32)
+    block  ;; label = @1
+      block  ;; label = @2
+        local.get 0
+        br_if 0 (;@2;)
+        local.get 1
+        local.set 2
+        br 1 (;@1;)
+      end
+      loop  ;; label = @2
+        local.get 1
+        local.get 0
+        local.tee 2
+        i32.rem_u
+        local.set 0
+        local.get 2
+        local.set 1
+        local.get 0
+        br_if 0 (;@2;)
+      end
+    end
+    local.get 2
+  )
+  (export "gcd" (func $gcd))
+)

package/src/wasm/index.ts

@@ -0,0 +1,4 @@
+export { WebDataStore } from './worker/browser/index.js';
+export { NodeDataStore } from './worker/node/index.js';
+export { WasmWorker, WorkerPool } from './worker/index.js';
+export { WasmModule, IWasmModule } from './wasm/index.js';