@aztec/bb.js 3.0.0-devnet.2 → 3.0.0-devnet.2-patch.1

package/src/bb_backends/node/native_shm_async.ts

@@ -0,0 +1,261 @@
+import { createRequire } from 'module';
+import { spawn, ChildProcess } from 'child_process';
+import { openSync, closeSync } from 'fs';
+import { IMsgpackBackendAsync } from '../interface.js';
+import { findNapiBinary, findPackageRoot } from './platform.js';
+
+let instanceCounter = 0;
+
+/**
+ * Asynchronous shared memory backend that communicates with bb binary via shared memory.
+ * Uses NAPI module with background thread polling for async operations.
+ * Supports request pipelining - multiple requests can be in flight simultaneously.
+ *
+ * Architecture (matches socket backend pattern):
+ * - bb acts as the SERVER, TypeScript is the CLIENT
+ * - bb creates the shared memory region
+ * - TypeScript connects via NAPI wrapper (MsgpackClientAsync)
+ * - TypeScript manages promise queue (single-threaded, no mutex needed)
+ * - C++ background thread polls for responses, calls JavaScript callback
+ * - JavaScript callback pops queue and resolves promises in FIFO order
+ */
+export class BarretenbergNativeShmAsyncBackend implements IMsgpackBackendAsync {
+  private process: ChildProcess;
+  private client: any; // NAPI MsgpackClientAsync instance
+  private logFd?: number; // File descriptor for logs
+
+  // Queue of pending callbacks for pipelined requests
+  // Responses come back in FIFO order, so we match them with queued callbacks
+  private pendingCallbacks: Array<{
+    resolve: (data: Uint8Array) => void;
+    reject: (error: Error) => void;
+  }> = [];
+
+  private constructor(process: ChildProcess, client: any, logFd?: number) {
+    this.process = process;
+    this.client = client;
+    this.logFd = logFd;
+
+    // Register our response handler with the C++ client
+    // This callback will be invoked from the background thread via ThreadSafeFunction
+    this.client.setResponseCallback((responseBuffer: Buffer) => {
+      this.handleResponse(responseBuffer);
+    });
+  }
+
+  /**
+   * Handle response from C++ background thread
+   * Dequeues the next pending callback and resolves it (FIFO order)
+   */
+  private handleResponse(responseBuffer: Buffer): void {
+    // Response is complete - dequeue the next pending callback (FIFO)
+    const callback = this.pendingCallbacks.shift();
+    if (callback) {
+      callback.resolve(new Uint8Array(responseBuffer));
+    } else {
+      // This shouldn't happen - response without a pending request
+      console.warn('Received response but no pending callback');
+    }
+
+    // If no more pending callbacks, release ref to allow process to exit
+    if (this.pendingCallbacks.length === 0) {
+      this.client.release();
+    }
+  }
+
+  /**
+   * Create and initialize an async shared memory backend.
+   * @param bbBinaryPath Path to bb binary
+   * @param threads Optional number of threads (defaults to min(32, num_cpus))
+   * @param logger Optional logger function for bb output
+   */
+  static async new(
+    bbBinaryPath: string,
+    napiPath: string,
+    threads?: number,
+    logger?: (msg: string) => void,
+  ): Promise<BarretenbergNativeShmAsyncBackend> {
+    // Import the NAPI module
+    // The addon is built to the nodejs_module directory
+    const addonPath = findNapiBinary(napiPath);
+    // Try loading
+    let addon: any = null;
+    try {
+      const require = createRequire(findPackageRoot()!);
+      addon = require(addonPath!);
+    } catch (err) {
+      // Addon not built yet or not available
+      throw new Error('Shared memory async NAPI not available.');
+    }
+
+    // Create a unique shared memory name
+    const shmName = `bb-async-${process.pid}-${instanceCounter++}`;
+
+    // If threads not set use num cpu cores, max 16 (same as socket backend)
+    const hwc = threads ? threads.toString() : '16';
+    const env = { ...process.env, HARDWARE_CONCURRENCY: hwc };
+
+    // Set up file logging if logger is provided
+    // Direct file redirection bypasses Node event loop - logs are written even if process hangs
+    let logFd: number | undefined;
+    let logPath: string | undefined;
+    if (logger) {
+      logPath = `/tmp/${shmName}.log`;
+      logFd = openSync(logPath, 'w');
+      logger(`BB process logs redirected to: ${logPath}`);
+    }
+
+    // Spawn bb process with shared memory mode
+    // Use larger ring buffers for async mode to support pipelining
+    const args = [
+      'msgpack',
+      'run',
+      '--input',
+      `${shmName}.shm`,
+      '--request-ring-size',
+      `${1024 * 1024 * 4}`,
+      '--response-ring-size',
+      `${1024 * 1024 * 4}`,
+    ];
+    const bbProcess = spawn(bbBinaryPath, args, {
+      stdio: ['ignore', logFd ?? 'ignore', logFd ?? 'ignore'],
+      env,
+    });
+
+    // Disconnect from event loop so process can exit without waiting for bb
+    // The bb process has parent death monitoring (prctl on Linux, kqueue on macOS)
+    // so it will automatically exit when Node.js exits
+    bbProcess.unref();
+
+    // Track if process has exited
+    let processExited = false;
+    let exitError: Error | null = null;
+
+    bbProcess.on('error', err => {
+      processExited = true;
+      exitError = new Error(`Native backend process error: ${err.message}`);
+    });
+
+    bbProcess.on('exit', (code, signal) => {
+      processExited = true;
+      if (code !== null && code !== 0) {
+        exitError = new Error(`Native backend process exited with code ${code}`);
+      } else if (signal && signal !== 'SIGTERM') {
+        exitError = new Error(`Native backend process killed with signal ${signal}`);
+      }
+    });
+
+    // Wait for bb to create shared memory
+    // Retry connection every 100ms for up to 5 seconds (longer than sync for thread startup)
+    const retryInterval = 100; // ms
+    const timeout = 5000; // ms
+    const maxAttempts = Math.floor(timeout / retryInterval);
+    let client: any = null;
+
+    try {
+      for (let attempt = 0; attempt < maxAttempts; attempt++) {
+        // Check if bb process has exited before attempting connection
+        if (processExited) {
+          throw exitError || new Error('Native backend process exited unexpectedly during startup');
+        }
+
+        // Wait before attempting connection (except first attempt)
+        if (attempt > 0) {
+          await new Promise(resolve => setTimeout(resolve, retryInterval));
+        }
+
+        try {
+          // Create NAPI async client
+          client = new addon.MsgpackClientAsync(shmName);
+          break; // Success!
+        } catch (err: any) {
+          // Connection failed, will retry
+          if (attempt === maxAttempts - 1) {
+            // Last attempt failed - check one more time if process exited
+            if (processExited && exitError) {
+              throw exitError;
+            }
+            throw new Error(`Failed to connect to shared memory after ${timeout}ms: ${err.message}`);
+          }
+        }
+      }
+
+      if (!client) {
+        throw new Error('Failed to create client connection');
+      }
+
+      return new BarretenbergNativeShmAsyncBackend(bbProcess, client, logFd);
+    } finally {
+      // If we failed to connect, ensure the process is killed and log file closed
+      if (!client) {
+        bbProcess.kill('SIGKILL');
+        if (logFd !== undefined) {
+          try {
+            closeSync(logFd);
+          } catch (e) {
+            // Ignore errors during cleanup
+          }
+        }
+      }
+    }
+  }
+
+  /**
+   * Send a msgpack request asynchronously.
+   * Supports pipelining - can be called multiple times before awaiting responses.
+   * Use Promise.all() to send multiple requests concurrently.
+   *
+   * Example:
+   *   const results = await Promise.all([
+   *     backend.call(buf1),
+   *     backend.call(buf2),
+   *     backend.call(buf3)
+   *   ]);
+   *
+   * @param inputBuffer The msgpack-encoded request
+   * @returns Promise resolving to msgpack-encoded response
+   */
+  async call(inputBuffer: Uint8Array): Promise<Uint8Array> {
+    return new Promise((resolve, reject) => {
+      // If this is the first pending callback, acquire ref to keep event loop alive
+      if (this.pendingCallbacks.length === 0) {
+        this.client.acquire();
+      }
+
+      // Enqueue this promise's callbacks (FIFO order)
+      this.pendingCallbacks.push({ resolve, reject });
+
+      try {
+        // Send request to shared memory (synchronous write)
+        // C++ call() no longer returns a promise - we manage them here
+        this.client.call(Buffer.from(inputBuffer));
+      } catch (err: any) {
+        // Send failed - dequeue the callback we just added and reject
+        this.pendingCallbacks.pop();
+
+        // If queue is now empty, release ref to allow exit
+        if (this.pendingCallbacks.length === 0) {
+          this.client.release();
+        }
+
+        reject(new Error(`Shared memory async call failed: ${err.message}`));
+      }
+    });
+  }
+
+  async destroy(): Promise<void> {
+    // Kill the bb process
+    // Background thread and callbacks will be cleaned up by OS on process exit
+    this.process.kill('SIGTERM');
+    this.process.removeAllListeners();
+
+    // Close log file if open
+    if (this.logFd !== undefined) {
+      try {
+        closeSync(this.logFd);
+      } catch (e) {
+        // Ignore errors during cleanup
+      }
+    }
+  }
+}

package/src/bb_backends/node/native_socket.ts

@@ -4,7 +4,7 @@ import * as fs from 'fs';
 import * as os from 'os';
 import * as path from 'path';
 import { IMsgpackBackendAsync } from '../interface.js';
-import { findPackageRoot } from './platform.js';
+import readline from 'readline';
 
 /**
  * Asynchronous native backend that communicates with bb binary via Unix Domain Socket.
@@ -40,7 +40,7 @@ export class BarretenbergNativeSocketAsyncBackend implements IMsgpackBackendAsyn
   private responseBuffer: Buffer | null = null;
   private responseBytesRead: number = 0;
 
-  constructor(bbBinaryPath: string, threads?: number) {
+  constructor(bbBinaryPath: string, threads?: number, logger?: (msg: string) => void) {
     // Create a unique socket path in temp directory
     this.socketPath = path.join(os.tmpdir(), `bb-${process.pid}-${Date.now()}.sock`);
 
@@ -57,18 +57,28 @@ export class BarretenbergNativeSocketAsyncBackend implements IMsgpackBackendAsyn
       connectionReject = reject;
     });
 
-    // Set HARDWARE_CONCURRENCY if threads specified
-    const env = threads !== undefined ? { ...process.env, HARDWARE_CONCURRENCY: threads.toString() } : process.env;
+    // If threads not set use num cpu cores, max 16.
+    const hwc = threads ? threads.toString() : Math.min(16, os.cpus().length).toString();
+    const env = { ...process.env, HARDWARE_CONCURRENCY: hwc };
 
     // Spawn bb process - it will create the socket server
-    const args = [bbBinaryPath, 'msgpack', 'run', '--input', this.socketPath];
-    this.process = spawn(findPackageRoot() + '/scripts/kill_wrapper.sh', args, {
-      stdio: ['ignore', 'ignore', 'ignore'],
+    const args = ['msgpack', 'run', '--input', this.socketPath];
+    this.process = spawn(bbBinaryPath, args, {
+      stdio: ['ignore', logger ? 'pipe' : 'ignore', logger ? 'pipe' : 'ignore'],
       env,
     });
-    // Disconnect from event loop so process can exit. The kill wrapper will reap bb once parent (node) dies.
+
+    // Disconnect from event loop so process can exit without waiting for bb
+    // The bb process has parent death monitoring (prctl on Linux, kqueue on macOS)
+    // so it will automatically exit when Node.js exits
     this.process.unref();
 
+    if (logger) {
+      logger("Logger attached to bb process. DON'T FORGET TO DESTROY THE BACKEND to allow Node.js to exit.");
+      readline.createInterface({ input: this.process.stdout! }).on('line', logger);
+      readline.createInterface({ input: this.process.stderr! }).on('line', logger);
+    }
+
     this.process.on('error', err => {
       if (connectionReject) {
         connectionReject(new Error(`Native backend process error: ${err.message}`));
@@ -157,7 +167,7 @@ export class BarretenbergNativeSocketAsyncBackend implements IMsgpackBackendAsyn
 
       // Set up event handlers
       this.socket.once('connect', () => {
-        this.socket!.unref();
+        // Socket starts referenced - will be unreferenced when no callbacks pending
 
         // Clear connection timeout on successful connection
         if (this.connectionTimeout) {
@@ -232,6 +242,11 @@ export class BarretenbergNativeSocketAsyncBackend implements IMsgpackBackendAsyn
             console.warn('Received response but no pending callback');
           }
 
+          // If no more pending callbacks, unref socket to allow process to exit
+          if (this.pendingCallbacks.length === 0 && this.socket) {
+            this.socket.unref();
+          }
+
           // Reset state for next message
           this.readingLength = true;
           this.lengthBytesRead = 0;
@@ -252,6 +267,11 @@ export class BarretenbergNativeSocketAsyncBackend implements IMsgpackBackendAsyn
     }
 
     return new Promise((resolve, reject) => {
+      // If this is the first pending callback, ref the socket to keep event loop alive
+      if (this.pendingCallbacks.length === 0) {
+        this.socket!.ref();
+      }
+
       // Enqueue this promise's callbacks (FIFO order)
       this.pendingCallbacks.push({ resolve, reject });
 
@@ -298,15 +318,8 @@ export class BarretenbergNativeSocketAsyncBackend implements IMsgpackBackendAsyn
   }
 
   async destroy(): Promise<void> {
-    // Cleanup first (closes socket, unrefs everything)
     this.cleanup();
-
-    // Send SIGTERM for graceful shutdown
-    // Process is unref'd so won't block event loop - just kill and return
-    try {
-      this.process.kill('SIGTERM');
-    } catch (e) {
-      // Already dead
-    }
+    this.process.kill('SIGTERM');
+    this.process.removeAllListeners();
   }
 }

package/src/bb_backends/node/platform.ts

@@ -24,7 +24,12 @@ export function findPackageRoot(): string | null {
   while (currentDir !== root) {
     const packageJsonPath = path.join(currentDir, 'package.json');
     if (fs.existsSync(packageJsonPath)) {
-      return currentDir;
+      // Check if this is the actual package root by verifying it has a 'build' directory
+      // This ensures we skip intermediate package.json files (e.g., in dest/node-cjs/)
+      const buildDir = path.join(currentDir, 'build');
+      if (fs.existsSync(buildDir)) {
+        return currentDir;
+      }
     }
     currentDir = path.dirname(currentDir);
   }

package/src/bb_backends/wasm.ts

@@ -3,7 +3,6 @@ import { fetchModuleAndThreads } from '../barretenberg_wasm/index.js';
 import { IMsgpackBackendSync, IMsgpackBackendAsync } from './interface.js';
 import { createMainWorker } from '../barretenberg_wasm/barretenberg_wasm_main/factory/node/index.js';
 import { getRemoteBarretenbergWasm } from '../barretenberg_wasm/helpers/index.js';
-import { createDebugLogger } from '../log/index.js';
 import { proxy } from 'comlink';
 
 /**
@@ -76,7 +75,7 @@ export class BarretenbergWasmAsyncBackend implements IMsgpackBackendAsync {
       await wasm.init(
         module,
         threads,
-        proxy(options.logger ?? createDebugLogger('bb_wasm_async')),
+        proxy(options.logger ?? (() => {})),
         options.memory?.initial,
         options.memory?.maximum,
       );

package/src/bbapi/exception_handling.test.ts

@@ -0,0 +1,54 @@
+import { BarretenbergWasmSyncBackend } from '../bb_backends/wasm.js';
+import { SyncApi } from '../cbind/generated/sync.js';
+import { BBApiException } from '../bbapi_exception.js';
+
+describe('BBApi Exception Handling from bb.js', () => {
+  let backend: BarretenbergWasmSyncBackend;
+  let api: SyncApi;
+
+  beforeAll(async () => {
+    backend = await BarretenbergWasmSyncBackend.new();
+    api = new SyncApi(backend);
+  }, 60000);
+
+  afterAll(() => {
+    backend.destroy();
+  });
+
+  it('should catch CRS initialization exceptions from WASM', () => {
+    // Create an SrsInitSrs command with invalid data that will cause an exception in C++
+    // We pass buffers that are too small, which will cause validation to fail
+    const invalidCommand = {
+      numPoints: 100, // Request 100 points (requires 6400 bytes)
+      pointsBuf: new Uint8Array(10), // Only 10 bytes - will cause exception
+      g2Point: new Uint8Array(10), // Only 10 bytes (needs 128) - will cause exception
+    };
+
+    // In WASM builds, throw_or_abort calls abort directly which throws a generic Error
+    // In native builds with exceptions, our try-catch in bbapi converts it to ErrorResponse
+    // This test verifies that errors are catchable from bb.js (even if as generic Error in WASM)
+    expect(() => {
+      api.srsInitSrs(invalidCommand);
+    }).toThrow();
+  });
+
+  it('should return error message from caught exception', () => {
+    const invalidCommand = {
+      numPoints: 100,
+      pointsBuf: new Uint8Array(10),
+      g2Point: new Uint8Array(10),
+    };
+
+    try {
+      api.srsInitSrs(invalidCommand);
+      fail('Expected exception to be thrown');
+    } catch (error) {
+      // Error is catchable and contains a useful message
+      expect(error).toBeInstanceOf(Error);
+      expect((error as Error).message).toBeTruthy();
+      expect((error as Error).message.length).toBeGreaterThan(0);
+      expect((error as Error).message).toContain('g1_identity');
+      console.log('Successfully caught exception from bb.js with message:', (error as Error).message);
+    }
+  });
+});

package/src/bbapi_exception.ts

@@ -0,0 +1,13 @@
+/**
+ * Exception thrown when barretenberg API operations fail
+ */
+export class BBApiException extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = 'BBApiException';
+    // Maintains proper stack trace for where our error was thrown (only available on V8)
+    if (Error.captureStackTrace) {
+      Error.captureStackTrace(this, BBApiException);
+    }
+  }
+}

package/src/bin/index.ts

@@ -0,0 +1,14 @@
+#!/usr/bin/env node
+import { findBbBinary } from '../bb_backends/node/platform.js';
+import { spawnSync } from 'node:child_process';
+
+const bin = findBbBinary();
+
+if (!bin) {
+  console.error('Could not find bb binary. Please ensure it is built and accessible.');
+  process.exit(1);
+}
+
+const result = spawnSync(bin, process.argv.slice(2), { stdio: 'inherit' });
+
+process.exit(result.status ?? 1);

package/src/cbind/generate.ts

@@ -7,6 +7,7 @@ import { dirname, join } from 'path';
 import { exec } from 'child_process';
 import { promisify } from 'util';
 import { fileURLToPath } from 'url';
+import { unpack } from 'msgpackr';
 import {
   createSharedTypesCompiler,
   createSyncApiCompiler,
@@ -73,9 +74,107 @@ async function generate() {
     console.log(`✓ ${config.name}: ${outputPath}`);
   }
 
+  // Generate curve constants
+  console.log('\nGenerating curve constants...');
+  await generateCurveConstants(bbBuildPath, outputDir);
+
   console.log('\nGeneration complete!');
 }
 
+async function generateCurveConstants(bbBuildPath: string, outputDir: string) {
+  // Get curve constants from bb as msgpack binary
+  const { stdout: constantsBuffer } = await execAsync(`${bbBuildPath} msgpack curve_constants`, {
+    encoding: 'buffer',
+    maxBuffer: 10 * 1024 * 1024, // 10MB buffer
+  });
+
+  // Decode msgpack
+  const constants = unpack(constantsBuffer as Buffer);
+
+  // Helper to convert Uint8Array to hex string
+  const toHex = (bytes: Uint8Array) => '0x' + Buffer.from(bytes).toString('hex');
+
+  // Helper to convert Uint8Array to bigint (big-endian)
+  const toBigInt = (bytes: Uint8Array) => {
+    let result = 0n;
+    for (const byte of bytes) {
+      result = (result << 8n) | BigInt(byte);
+    }
+    return result;
+  };
+
+  // Helper to serialize point coordinate (handles both Uint8Array and array of Uint8Array for field2)
+  const serializeCoordinate = (coord: Uint8Array | Uint8Array[]) => {
+    if (Array.isArray(coord)) {
+      // For field2 (like BN254 G2), we have array of two Uint8Arrays
+      return `[${coord.map(c => `new Uint8Array([${Array.from(c).join(', ')}])`).join(', ')}]`;
+    } else {
+      // For regular fields, single Uint8Array
+      return `new Uint8Array([${Array.from(coord).join(', ')}])`;
+    }
+  };
+
+  // Generate TypeScript file
+  const content = `/**
+ * Curve constants generated from barretenberg native binary.
+ * DO NOT EDIT - This file is auto-generated by generate.ts
+ */
+
+/**
+ * BN254 curve constants
+ */
+export const BN254_FR_MODULUS = ${toBigInt(constants.bn254_fr_modulus)}n;
+export const BN254_FQ_MODULUS = ${toBigInt(constants.bn254_fq_modulus)}n;
+
+export const BN254_G1_GENERATOR = {
+  x: ${serializeCoordinate(constants.bn254_g1_generator.x)},
+  y: ${serializeCoordinate(constants.bn254_g1_generator.y)},
+} as const;
+
+export const BN254_G2_GENERATOR = {
+  x: ${serializeCoordinate(constants.bn254_g2_generator.x)},
+  y: ${serializeCoordinate(constants.bn254_g2_generator.y)},
+} as const;
+
+/**
+ * Grumpkin curve constants
+ */
+export const GRUMPKIN_FR_MODULUS = ${toBigInt(constants.grumpkin_fr_modulus)}n;
+export const GRUMPKIN_FQ_MODULUS = ${toBigInt(constants.grumpkin_fq_modulus)}n;
+
+export const GRUMPKIN_G1_GENERATOR = {
+  x: ${serializeCoordinate(constants.grumpkin_g1_generator.x)},
+  y: ${serializeCoordinate(constants.grumpkin_g1_generator.y)},
+} as const;
+
+/**
+ * Secp256k1 curve constants
+ */
+export const SECP256K1_FR_MODULUS = ${toBigInt(constants.secp256k1_fr_modulus)}n;
+export const SECP256K1_FQ_MODULUS = ${toBigInt(constants.secp256k1_fq_modulus)}n;
+
+export const SECP256K1_G1_GENERATOR = {
+  x: ${serializeCoordinate(constants.secp256k1_g1_generator.x)},
+  y: ${serializeCoordinate(constants.secp256k1_g1_generator.y)},
+} as const;
+
+/**
+ * Secp256r1 curve constants
+ */
+export const SECP256R1_FR_MODULUS = ${toBigInt(constants.secp256r1_fr_modulus)}n;
+export const SECP256R1_FQ_MODULUS = ${toBigInt(constants.secp256r1_fq_modulus)}n;
+
+export const SECP256R1_G1_GENERATOR = {
+  x: ${serializeCoordinate(constants.secp256r1_g1_generator.x)},
+  y: ${serializeCoordinate(constants.secp256r1_g1_generator.y)},
+} as const;
+`;
+
+  const outputPath = join(outputDir, 'curve_constants.ts');
+  writeFileSync(outputPath, content);
+  console.log(`✓ Curve constants: ${outputPath}`);
+}
+
 // Run the generator
 generate().catch(error => {
   console.error('Generation failed:', error);