@aztec/bb.js 3.0.0-canary.a9708bd → 3.0.0-manual.20251030

package/src/bb_backends/node/native_socket.ts

@@ -0,0 +1,312 @@
+import { spawn, ChildProcess } from 'child_process';
+import * as net from 'net';
+import * as fs from 'fs';
+import * as os from 'os';
+import * as path from 'path';
+import { IMsgpackBackendAsync } from '../interface.js';
+import { findPackageRoot } from './platform.js';
+
+/**
+ * Asynchronous native backend that communicates with bb binary via Unix Domain Socket.
+ * Uses event-based I/O with a state machine to handle partial reads.
+ *
+ * Architecture: bb acts as the SERVER, TypeScript is the CLIENT
+ * - bb creates the socket and listens for connections
+ * - TypeScript waits for socket file to exist, then connects
+ *
+ * Protocol:
+ * - Request: 4-byte little-endian length + msgpack buffer
+ * - Response: 4-byte little-endian length + msgpack buffer
+ */
+export class BarretenbergNativeSocketAsyncBackend implements IMsgpackBackendAsync {
+  private process: ChildProcess;
+  private socket: net.Socket | null = null;
+  private socketPath: string;
+  private connectionPromise: Promise<void>;
+  private connectionTimeout: NodeJS.Timeout | null = null;
+
+  // Queue of pending callbacks for pipelined requests
+  // Responses come back in FIFO order, so we match them with queued callbacks
+  private pendingCallbacks: Array<{
+    resolve: (data: Uint8Array) => void;
+    reject: (error: Error) => void;
+  }> = [];
+
+  // State machine for reading responses
+  private readingLength: boolean = true;
+  private lengthBuffer: Buffer = Buffer.alloc(4);
+  private lengthBytesRead: number = 0;
+  private responseLength: number = 0;
+  private responseBuffer: Buffer | null = null;
+  private responseBytesRead: number = 0;
+
+  constructor(bbBinaryPath: string, threads?: number) {
+    // Create a unique socket path in temp directory
+    this.socketPath = path.join(os.tmpdir(), `bb-${process.pid}-${Date.now()}.sock`);
+
+    // Ensure socket path doesn't already exist (cleanup from previous crashes)
+    if (fs.existsSync(this.socketPath)) {
+      fs.unlinkSync(this.socketPath);
+    }
+
+    let connectionResolve: (() => void) | null = null;
+    let connectionReject: ((error: Error) => void) | null = null;
+
+    this.connectionPromise = new Promise<void>((resolve, reject) => {
+      connectionResolve = resolve;
+      connectionReject = reject;
+    });
+
+    // Set HARDWARE_CONCURRENCY if threads specified
+    const env = threads !== undefined ? { ...process.env, HARDWARE_CONCURRENCY: threads.toString() } : process.env;
+
+    // Spawn bb process - it will create the socket server
+    const args = [bbBinaryPath, 'msgpack', 'run', '--input', this.socketPath];
+    this.process = spawn(findPackageRoot() + '/scripts/kill_wrapper.sh', args, {
+      stdio: ['ignore', 'ignore', 'ignore'],
+      env,
+    });
+    // Disconnect from event loop so process can exit. The kill wrapper will reap bb once parent (node) dies.
+    this.process.unref();
+
+    this.process.on('error', err => {
+      if (connectionReject) {
+        connectionReject(new Error(`Native backend process error: ${err.message}`));
+        connectionReject = null;
+        connectionResolve = null;
+      }
+      // Reject all pending callbacks
+      const error = new Error(`Native backend process error: ${err.message}`);
+      for (const callback of this.pendingCallbacks) {
+        callback.reject(error);
+      }
+      this.pendingCallbacks = [];
+    });
+
+    this.process.on('exit', (code, signal) => {
+      const errorMsg =
+        code !== null && code !== 0
+          ? `Native backend process exited with code ${code}`
+          : signal && signal !== 'SIGTERM'
+            ? `Native backend process killed with signal ${signal}`
+            : 'Native backend process exited unexpectedly';
+
+      if (connectionReject) {
+        connectionReject(new Error(errorMsg));
+        connectionReject = null;
+        connectionResolve = null;
+      }
+      // Reject all pending callbacks
+      const error = new Error(errorMsg);
+      for (const callback of this.pendingCallbacks) {
+        callback.reject(error);
+      }
+      this.pendingCallbacks = [];
+    });
+
+    // Wait for bb to create socket file, then connect
+    this.waitForSocketAndConnect()
+      .then(() => {
+        if (connectionResolve) {
+          connectionResolve();
+          connectionResolve = null;
+          connectionReject = null;
+        }
+      })
+      .catch(err => {
+        if (connectionReject) {
+          connectionReject(err);
+          connectionReject = null;
+          connectionResolve = null;
+        }
+      });
+
+    // Set a timeout for connection
+    this.connectionTimeout = setTimeout(() => {
+      if (connectionReject) {
+        connectionReject(new Error('Timeout waiting for bb socket connection'));
+        connectionReject = null;
+        connectionResolve = null;
+        this.cleanup();
+      }
+    }, 5000);
+  }
+
+  private async waitForSocketAndConnect(): Promise<void> {
+    // Poll for socket file to exist (bb is creating it)
+    const startTime = Date.now();
+    while (!fs.existsSync(this.socketPath)) {
+      if (Date.now() - startTime > 5000) {
+        throw new Error('Timeout waiting for bb to create socket file');
+      }
+      await new Promise(resolve => setTimeout(resolve, 50));
+    }
+
+    // Additional check: ensure it's actually a socket
+    const stats = fs.statSync(this.socketPath);
+    if (!stats.isSocket()) {
+      throw new Error(`Path exists but is not a socket: ${this.socketPath}`);
+    }
+
+    // Connect to bb's socket server as a client
+    return new Promise<void>((resolve, reject) => {
+      this.socket = net.connect(this.socketPath);
+
+      // Disable Nagle's algorithm for lower latency
+      this.socket.setNoDelay(true);
+
+      // Set up event handlers
+      this.socket.once('connect', () => {
+        this.socket!.unref();
+
+        // Clear connection timeout on successful connection
+        if (this.connectionTimeout) {
+          clearTimeout(this.connectionTimeout);
+          this.connectionTimeout = null;
+        }
+        resolve();
+      });
+
+      this.socket.once('error', err => {
+        reject(new Error(`Failed to connect to bb socket: ${err.message}`));
+      });
+
+      // Set up data handler after connection is established
+      this.socket.on('data', (chunk: Buffer) => {
+        this.handleData(chunk);
+      });
+
+      // Handle ongoing errors after initial connection
+      this.socket.on('error', err => {
+        // Reject all pending callbacks
+        const error = new Error(`Socket error: ${err.message}`);
+        for (const callback of this.pendingCallbacks) {
+          callback.reject(error);
+        }
+        this.pendingCallbacks = [];
+      });
+
+      this.socket.on('end', () => {
+        // Reject all pending callbacks
+        const error = new Error('Socket connection ended unexpectedly');
+        for (const callback of this.pendingCallbacks) {
+          callback.reject(error);
+        }
+        this.pendingCallbacks = [];
+      });
+    });
+  }
+
+  private handleData(chunk: Buffer): void {
+    let offset = 0;
+
+    while (offset < chunk.length) {
+      if (this.readingLength) {
+        // Reading 4-byte length prefix
+        const bytesToCopy = Math.min(4 - this.lengthBytesRead, chunk.length - offset);
+        chunk.copy(this.lengthBuffer, this.lengthBytesRead, offset, offset + bytesToCopy);
+        this.lengthBytesRead += bytesToCopy;
+        offset += bytesToCopy;
+
+        if (this.lengthBytesRead === 4) {
+          // Length is complete, switch to reading data
+          this.responseLength = this.lengthBuffer.readUInt32LE(0);
+          this.responseBuffer = Buffer.alloc(this.responseLength);
+          this.responseBytesRead = 0;
+          this.readingLength = false;
+        }
+      } else {
+        // Reading response data
+        const bytesToCopy = Math.min(this.responseLength - this.responseBytesRead, chunk.length - offset);
+        chunk.copy(this.responseBuffer!, this.responseBytesRead, offset, offset + bytesToCopy);
+        this.responseBytesRead += bytesToCopy;
+        offset += bytesToCopy;
+
+        if (this.responseBytesRead === this.responseLength) {
+          // Response is complete - dequeue the next pending callback (FIFO)
+          const callback = this.pendingCallbacks.shift();
+          if (callback) {
+            callback.resolve(new Uint8Array(this.responseBuffer!));
+          } else {
+            // This shouldn't happen - response without a pending request
+            console.warn('Received response but no pending callback');
+          }
+
+          // Reset state for next message
+          this.readingLength = true;
+          this.lengthBytesRead = 0;
+          this.responseLength = 0;
+          this.responseBuffer = null;
+          this.responseBytesRead = 0;
+        }
+      }
+    }
+  }
+
+  async call(inputBuffer: Uint8Array): Promise<Uint8Array> {
+    // Wait for connection to be established
+    await this.connectionPromise;
+
+    if (!this.socket) {
+      throw new Error('Socket not connected');
+    }
+
+    return new Promise((resolve, reject) => {
+      // Enqueue this promise's callbacks (FIFO order)
+      this.pendingCallbacks.push({ resolve, reject });
+
+      // Write request: 4-byte little-endian length + msgpack data
+      // Socket will buffer these if needed, maintaining order
+      const lengthBuf = Buffer.alloc(4);
+      lengthBuf.writeUInt32LE(inputBuffer.length, 0);
+      this.socket!.write(lengthBuf);
+      this.socket!.write(inputBuffer);
+    });
+  }
+
+  private cleanup(): void {
+    // Reject any remaining pending callbacks
+    const error = new Error('Backend connection closed');
+    for (const callback of this.pendingCallbacks) {
+      callback.reject(error);
+    }
+    this.pendingCallbacks = [];
+
+    try {
+      // Remove all event listeners to prevent hanging
+      if (this.socket) {
+        this.socket.removeAllListeners();
+        // Unref so socket doesn't keep event loop alive
+        // this.socket.unref();
+        this.socket.destroy();
+      }
+    } catch (e) {
+      // Ignore errors during cleanup
+    }
+
+    // Clear connection timeout if still pending
+    if (this.connectionTimeout) {
+      clearTimeout(this.connectionTimeout);
+      this.connectionTimeout = null;
+    }
+
+    // Remove process event listeners and unref to not block event loop
+    this.process.removeAllListeners();
+    // this.process.unref();
+
+    // Don't try to unlink socket - bb owns it and will clean it up
+  }
+
+  async destroy(): Promise<void> {
+    // Cleanup first (closes socket, unrefs everything)
+    this.cleanup();
+
+    // Send SIGTERM for graceful shutdown
+    // Process is unref'd so won't block event loop - just kill and return
+    try {
+      this.process.kill('SIGTERM');
+    } catch (e) {
+      // Already dead
+    }
+  }
+}

package/src/bb_backends/node/platform.ts

@@ -0,0 +1,151 @@
+import * as path from 'path';
+import * as fs from 'fs';
+import { fileURLToPath } from 'url';
+
+function getCurrentDir() {
+  if (typeof __dirname !== 'undefined') {
+    return __dirname;
+  } else {
+    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+    // @ts-ignore
+    return path.dirname(fileURLToPath(import.meta.url));
+  }
+}
+
+/**
+ * Find package root by climbing directory tree until package.json is found.
+ * @param startDir Starting directory to search from
+ * @returns Absolute path to package root, or null if not found
+ */
+export function findPackageRoot(): string | null {
+  let currentDir = getCurrentDir();
+  const root = path.parse(currentDir).root;
+
+  while (currentDir !== root) {
+    const packageJsonPath = path.join(currentDir, 'package.json');
+    if (fs.existsSync(packageJsonPath)) {
+      return currentDir;
+    }
+    currentDir = path.dirname(currentDir);
+  }
+
+  return null;
+}
+
+/**
+ * Supported platform/architecture combinations.
+ */
+export type Platform = 'x86_64-linux' | 'x86_64-darwin' | 'aarch64-linux' | 'aarch64-darwin';
+
+/**
+ * Map from Platform to build directory name.
+ */
+const PLATFORM_TO_BUILD_DIR: Record<Platform, string> = {
+  'x86_64-linux': 'amd64-linux',
+  'x86_64-darwin': 'amd64-macos',
+  'aarch64-linux': 'arm64-linux',
+  'aarch64-darwin': 'arm64-macos',
+};
+
+/**
+ * Detect the current platform and architecture.
+ * @returns Platform identifier or null if unsupported
+ */
+export function detectPlatform(): Platform | null {
+  const arch = process.arch; // 'x64' | 'arm64' | ...
+  const platform = process.platform; // 'linux' | 'darwin' | 'win32' | ...
+
+  if (arch === 'x64' && platform === 'linux') {
+    return 'x86_64-linux';
+  }
+  if (arch === 'x64' && platform === 'darwin') {
+    return 'x86_64-darwin';
+  }
+  if (arch === 'arm64' && platform === 'linux') {
+    return 'aarch64-linux';
+  }
+  if (arch === 'arm64' && platform === 'darwin') {
+    return 'aarch64-darwin';
+  }
+
+  return null;
+}
+
+/**
+ * Find the bb binary for the native backend.
+ * @param customPath Optional custom path to bb binary (overrides automatic detection)
+ * @returns Absolute path to bb binary, or null if not found
+ *
+ * Search order:
+ * 1. If customPath is provided and exists, return it
+ * 2. Otherwise search in <package-root>/build/<platform>/bb
+ */
+export function findBbBinary(customPath?: string): string | null {
+  // Check custom path first if provided
+  if (customPath) {
+    if (fs.existsSync(customPath)) {
+      return path.resolve(customPath);
+    }
+    // Custom path provided but doesn't exist - return null
+    return null;
+  }
+
+  // Automatic detection
+  const platform = detectPlatform();
+  if (!platform) {
+    return null;
+  }
+
+  const buildDir = PLATFORM_TO_BUILD_DIR[platform];
+
+  // Get package root by climbing directory tree to find package.json
+  const packageRoot = findPackageRoot();
+
+  if (!packageRoot) {
+    return null;
+  }
+
+  // Check in build/<platform>/bb
+  const bbPath = path.join(packageRoot, 'build', buildDir, 'bb');
+
+  if (fs.existsSync(bbPath)) {
+    return bbPath;
+  }
+
+  return null;
+}
+
+export function findNapiBinary(customPath?: string): string | null {
+  // Check custom path first if provided
+  if (customPath) {
+    if (fs.existsSync(customPath)) {
+      return path.resolve(customPath);
+    }
+    // Custom path provided but doesn't exist - return null
+    return null;
+  }
+
+  // Automatic detection
+  const platform = detectPlatform();
+  if (!platform) {
+    return null;
+  }
+
+  const buildDir = PLATFORM_TO_BUILD_DIR[platform];
+
+  // Get package root by climbing directory tree to find package.json
+  const packageRoot = findPackageRoot();
+
+  if (!packageRoot) {
+    return null;
+  }
+
+  // Check in build/<platform>/nodejs_module.node
+  const bbPath = path.join(packageRoot, 'build', buildDir, 'nodejs_module.node');
+
+  if (fs.existsSync(bbPath)) {
+    return bbPath;
+  }
+
+  return null;
+}

package/src/bb_backends/sync_to_async_adapter.ts

@@ -0,0 +1,21 @@
+import { IMsgpackBackendSync, IMsgpackBackendAsync } from './interface.js';
+
+/**
+ * Adapter that wraps a synchronous backend to provide an async interface.
+ * The sync backend's blocking calls are simply wrapped in Promise.resolve().
+ *
+ * This is useful for backends like shared memory where the call is actually
+ * synchronous but we want to use it with the async API.
+ */
+export class SyncToAsyncAdapter implements IMsgpackBackendAsync {
+  constructor(private syncBackend: IMsgpackBackendSync) {}
+
+  async call(inputBuffer: Uint8Array): Promise<Uint8Array> {
+    // The sync backend blocks until complete, so just wrap in a resolved promise
+    return Promise.resolve(this.syncBackend.call(inputBuffer));
+  }
+
+  async destroy(): Promise<void> {
+    this.syncBackend.destroy();
+  }
+}

package/src/bb_backends/wasm.ts

@@ -0,0 +1,103 @@
+import { BarretenbergWasmMain, BarretenbergWasmMainWorker } from '../barretenberg_wasm/barretenberg_wasm_main/index.js';
+import { fetchModuleAndThreads } from '../barretenberg_wasm/index.js';
+import { IMsgpackBackendSync, IMsgpackBackendAsync } from './interface.js';
+import { createMainWorker } from '../barretenberg_wasm/barretenberg_wasm_main/factory/node/index.js';
+import { getRemoteBarretenbergWasm } from '../barretenberg_wasm/helpers/index.js';
+import { createDebugLogger } from '../log/index.js';
+import { proxy } from 'comlink';
+
+/**
+ * Synchronous WASM backend that wraps BarretenbergWasmMain.
+ * Encapsulates all WASM initialization and memory management.
+ */
+export class BarretenbergWasmSyncBackend implements IMsgpackBackendSync {
+  private constructor(private wasm: BarretenbergWasmMain) {}
+
+  /**
+   * Create and initialize a synchronous WASM backend.
+   * @param wasmPath Optional path to WASM files
+   * @param logger Optional logging function
+   */
+  static async new(wasmPath?: string, logger?: (msg: string) => void): Promise<BarretenbergWasmSyncBackend> {
+    const wasm = new BarretenbergWasmMain();
+    const { module, threads } = await fetchModuleAndThreads(1, wasmPath, logger);
+    await wasm.init(module, threads, logger);
+    return new BarretenbergWasmSyncBackend(wasm);
+  }
+
+  call(inputBuffer: Uint8Array): Uint8Array {
+    return this.wasm.cbindCall('bbapi', inputBuffer);
+  }
+
+  destroy(): void {
+    // BarretenbergWasmMain has async destroy, but for sync API we call it without awaiting
+    // This is consistent with the synchronous semantics expected by the caller
+    void this.wasm.destroy();
+  }
+}
+
+/**
+ * Asynchronous WASM backend that supports both direct WASM and worker-based modes.
+ *
+ * Worker mode (default): Runs WASM on a worker thread to avoid blocking the main thread. Used in browsers.
+ * Direct mode: Runs WASM directly on the calling thread. Used by node.js for better performance.
+ */
+export class BarretenbergWasmAsyncBackend implements IMsgpackBackendAsync {
+  private constructor(
+    private wasm: BarretenbergWasmMain | BarretenbergWasmMainWorker,
+    private worker?: any,
+  ) {}
+
+  /**
+   * Create and initialize an asynchronous WASM backend.
+   * @param options.threads Number of threads (defaults to hardware max, up to 32 for parallel proving)
+   * @param options.wasmPath Optional path to WASM files
+   * @param options.logger Optional logging function
+   * @param options.memory Optional initial and maximum memory configuration
+   * @param options.useWorker Run on worker thread (default: true for browser safety)
+   */
+  static async new(
+    options: {
+      threads?: number;
+      wasmPath?: string;
+      logger?: (msg: string) => void;
+      memory?: { initial?: number; maximum?: number };
+      useWorker?: boolean;
+    } = {},
+  ): Promise<BarretenbergWasmAsyncBackend> {
+    // Default to worker mode for browser safety
+    const useWorker = options.useWorker ?? true;
+
+    if (useWorker) {
+      // Worker-based mode: runs on worker thread (browser-safe)
+      const worker = await createMainWorker();
+      const wasm = getRemoteBarretenbergWasm<BarretenbergWasmMainWorker>(worker);
+      const { module, threads } = await fetchModuleAndThreads(options.threads, options.wasmPath, options.logger);
+      await wasm.init(
+        module,
+        threads,
+        proxy(options.logger ?? createDebugLogger('bb_wasm_async')),
+        options.memory?.initial,
+        options.memory?.maximum,
+      );
+      return new BarretenbergWasmAsyncBackend(wasm, worker);
+    } else {
+      // Direct mode: runs on calling thread (faster but blocks thread)
+      const wasm = new BarretenbergWasmMain();
+      const { module, threads } = await fetchModuleAndThreads(options.threads, options.wasmPath, options.logger);
+      await wasm.init(module, threads, options.logger, options.memory?.initial, options.memory?.maximum);
+      return new BarretenbergWasmAsyncBackend(wasm);
+    }
+  }
+
+  async call(inputBuffer: Uint8Array): Promise<Uint8Array> {
+    return this.wasm.cbindCall('bbapi', inputBuffer);
+  }
+
+  async destroy(): Promise<void> {
+    await this.wasm.destroy();
+    if (this.worker) {
+      await this.worker.terminate();
+    }
+  }
+}

package/src/bbapi_exception.ts

@@ -0,0 +1,13 @@
+/**
+ * Exception thrown when barretenberg API operations fail
+ */
+export class BBApiException extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = 'BBApiException';
+    // Maintains proper stack trace for where our error was thrown (only available on V8)
+    if (Error.captureStackTrace) {
+      Error.captureStackTrace(this, BBApiException);
+    }
+  }
+}

package/src/cbind/README.md

@@ -1 +1 @@
-Derive bindings from the reported scheme of msgpack from bb. Currently redundant with bindgen, until that is supplanted.
+Derives bindings from the reported msgpack schema from bb.