@aztec/bb.js 0.0.1-commit.24de95ac → 0.0.1-commit.2e2504e2

package/src/bb_backends/node/native_shm_async.ts

@@ -0,0 +1,262 @@
+import { createRequire } from 'module';
+import { spawn, ChildProcess } from 'child_process';
+import { openSync, closeSync } from 'fs';
+import { IMsgpackBackendAsync } from '../interface.js';
+import { findNapiBinary, findPackageRoot } from './platform.js';
+import { threadId } from 'worker_threads';
+
+let instanceCounter = 0;
+
+/**
+ * Asynchronous shared memory backend that communicates with bb binary via shared memory.
+ * Uses NAPI module with background thread polling for async operations.
+ * Supports request pipelining - multiple requests can be in flight simultaneously.
+ *
+ * Architecture (matches socket backend pattern):
+ * - bb acts as the SERVER, TypeScript is the CLIENT
+ * - bb creates the shared memory region
+ * - TypeScript connects via NAPI wrapper (MsgpackClientAsync)
+ * - TypeScript manages promise queue (single-threaded, no mutex needed)
+ * - C++ background thread polls for responses, calls JavaScript callback
+ * - JavaScript callback pops queue and resolves promises in FIFO order
+ */
+export class BarretenbergNativeShmAsyncBackend implements IMsgpackBackendAsync {
+  private process: ChildProcess;
+  private client: any; // NAPI MsgpackClientAsync instance
+  private logFd?: number; // File descriptor for logs
+
+  // Queue of pending callbacks for pipelined requests
+  // Responses come back in FIFO order, so we match them with queued callbacks
+  private pendingCallbacks: Array<{
+    resolve: (data: Uint8Array) => void;
+    reject: (error: Error) => void;
+  }> = [];
+
+  private constructor(process: ChildProcess, client: any, logFd?: number) {
+    this.process = process;
+    this.client = client;
+    this.logFd = logFd;
+
+    // Register our response handler with the C++ client
+    // This callback will be invoked from the background thread via ThreadSafeFunction
+    this.client.setResponseCallback((responseBuffer: Buffer) => {
+      this.handleResponse(responseBuffer);
+    });
+  }
+
+  /**
+   * Handle response from C++ background thread
+   * Dequeues the next pending callback and resolves it (FIFO order)
+   */
+  private handleResponse(responseBuffer: Buffer): void {
+    // Response is complete - dequeue the next pending callback (FIFO)
+    const callback = this.pendingCallbacks.shift();
+    if (callback) {
+      callback.resolve(new Uint8Array(responseBuffer));
+    } else {
+      // This shouldn't happen - response without a pending request
+      console.warn('Received response but no pending callback');
+    }
+
+    // If no more pending callbacks, release ref to allow process to exit
+    if (this.pendingCallbacks.length === 0) {
+      this.client.release();
+    }
+  }
+
+  /**
+   * Create and initialize an async shared memory backend.
+   * @param bbBinaryPath Path to bb binary
+   * @param threads Optional number of threads (defaults to min(32, num_cpus))
+   * @param logger Optional logger function for bb output
+   */
+  static async new(
+    bbBinaryPath: string,
+    napiPath: string,
+    threads?: number,
+    logger?: (msg: string) => void,
+  ): Promise<BarretenbergNativeShmAsyncBackend> {
+    // Import the NAPI module
+    // The addon is built to the nodejs_module directory
+    const addonPath = findNapiBinary(napiPath);
+    // Try loading
+    let addon: any = null;
+    try {
+      const require = createRequire(findPackageRoot()!);
+      addon = require(addonPath!);
+    } catch (err) {
+      // Addon not built yet or not available
+      throw new Error('Shared memory async NAPI not available.');
+    }
+
+    // Create a unique shared memory name
+    const shmName = `bb-async-${process.pid}-${threadId}-${instanceCounter++}`;
+
+    // If threads not set use num cpu cores, max 16 (same as socket backend)
+    const hwc = threads ? threads.toString() : '16';
+    const env = { ...process.env, HARDWARE_CONCURRENCY: hwc };
+
+    // Set up file logging if logger is provided
+    // Direct file redirection bypasses Node event loop - logs are written even if process hangs
+    let logFd: number | undefined;
+    let logPath: string | undefined;
+    if (logger) {
+      logPath = `/tmp/${shmName}.log`;
+      logFd = openSync(logPath, 'w');
+      logger(`BB process logs redirected to: ${logPath}`);
+    }
+
+    // Spawn bb process with shared memory mode
+    // Use larger ring buffers for async mode to support pipelining
+    const args = [
+      'msgpack',
+      'run',
+      '--input',
+      `${shmName}.shm`,
+      '--request-ring-size',
+      `${1024 * 1024 * 4}`,
+      '--response-ring-size',
+      `${1024 * 1024 * 4}`,
+    ];
+    const bbProcess = spawn(bbBinaryPath, args, {
+      stdio: ['ignore', logFd ?? 'ignore', logFd ?? 'ignore'],
+      env,
+    });
+
+    // Disconnect from event loop so process can exit without waiting for bb
+    // The bb process has parent death monitoring (prctl on Linux, kqueue on macOS)
+    // so it will automatically exit when Node.js exits
+    bbProcess.unref();
+
+    // Track if process has exited
+    let processExited = false;
+    let exitError: Error | null = null;
+
+    bbProcess.on('error', err => {
+      processExited = true;
+      exitError = new Error(`Native backend process error: ${err.message}`);
+    });
+
+    bbProcess.on('exit', (code, signal) => {
+      processExited = true;
+      if (code !== null && code !== 0) {
+        exitError = new Error(`Native backend process exited with code ${code}`);
+      } else if (signal && signal !== 'SIGTERM') {
+        exitError = new Error(`Native backend process killed with signal ${signal}`);
+      }
+    });
+
+    // Wait for bb to create shared memory
+    // Retry connection every 100ms for up to 5 seconds (longer than sync for thread startup)
+    const retryInterval = 100; // ms
+    const timeout = 5000; // ms
+    const maxAttempts = Math.floor(timeout / retryInterval);
+    let client: any = null;
+
+    try {
+      for (let attempt = 0; attempt < maxAttempts; attempt++) {
+        // Check if bb process has exited before attempting connection
+        if (processExited) {
+          throw exitError || new Error('Native backend process exited unexpectedly during startup');
+        }
+
+        // Wait before attempting connection (except first attempt)
+        if (attempt > 0) {
+          await new Promise(resolve => setTimeout(resolve, retryInterval));
+        }
+
+        try {
+          // Create NAPI async client
+          client = new addon.MsgpackClientAsync(shmName);
+          break; // Success!
+        } catch (err: any) {
+          // Connection failed, will retry
+          if (attempt === maxAttempts - 1) {
+            // Last attempt failed - check one more time if process exited
+            if (processExited && exitError) {
+              throw exitError;
+            }
+            throw new Error(`Failed to connect to shared memory after ${timeout}ms: ${err.message}`);
+          }
+        }
+      }
+
+      if (!client) {
+        throw new Error('Failed to create client connection');
+      }
+
+      return new BarretenbergNativeShmAsyncBackend(bbProcess, client, logFd);
+    } finally {
+      // If we failed to connect, ensure the process is killed and log file closed
+      if (!client) {
+        bbProcess.kill('SIGKILL');
+        if (logFd !== undefined) {
+          try {
+            closeSync(logFd);
+          } catch (e) {
+            // Ignore errors during cleanup
+          }
+        }
+      }
+    }
+  }
+
+  /**
+   * Send a msgpack request asynchronously.
+   * Supports pipelining - can be called multiple times before awaiting responses.
+   * Use Promise.all() to send multiple requests concurrently.
+   *
+   * Example:
+   *   const results = await Promise.all([
+   *     backend.call(buf1),
+   *     backend.call(buf2),
+   *     backend.call(buf3)
+   *   ]);
+   *
+   * @param inputBuffer The msgpack-encoded request
+   * @returns Promise resolving to msgpack-encoded response
+   */
+  async call(inputBuffer: Uint8Array): Promise<Uint8Array> {
+    return new Promise((resolve, reject) => {
+      // If this is the first pending callback, acquire ref to keep event loop alive
+      if (this.pendingCallbacks.length === 0) {
+        this.client.acquire();
+      }
+
+      // Enqueue this promise's callbacks (FIFO order)
+      this.pendingCallbacks.push({ resolve, reject });
+
+      try {
+        // Send request to shared memory (synchronous write)
+        // C++ call() no longer returns a promise - we manage them here
+        this.client.call(Buffer.from(inputBuffer));
+      } catch (err: any) {
+        // Send failed - dequeue the callback we just added and reject
+        this.pendingCallbacks.pop();
+
+        // If queue is now empty, release ref to allow exit
+        if (this.pendingCallbacks.length === 0) {
+          this.client.release();
+        }
+
+        reject(new Error(`Shared memory async call failed: ${err.message}`));
+      }
+    });
+  }
+
+  async destroy(): Promise<void> {
+    // Kill the bb process
+    // Background thread and callbacks will be cleaned up by OS on process exit
+    this.process.kill('SIGTERM');
+    this.process.removeAllListeners();
+
+    // Close log file if open
+    if (this.logFd !== undefined) {
+      try {
+        closeSync(this.logFd);
+      } catch (e) {
+        // Ignore errors during cleanup
+      }
+    }
+  }
+}

package/src/bb_backends/node/native_socket.ts

@@ -4,7 +4,10 @@ import * as fs from 'fs';
 import * as os from 'os';
 import * as path from 'path';
 import { IMsgpackBackendAsync } from '../interface.js';
-import { findPackageRoot } from './platform.js';
+import readline from 'readline';
+import { threadId } from 'worker_threads';
+
+let instanceCounter = 0;
 
 /**
  * Asynchronous native backend that communicates with bb binary via Unix Domain Socket.
@@ -40,9 +43,9 @@ export class BarretenbergNativeSocketAsyncBackend implements IMsgpackBackendAsyn
   private responseBuffer: Buffer | null = null;
   private responseBytesRead: number = 0;
 
-  constructor(bbBinaryPath: string, threads?: number) {
+  constructor(bbBinaryPath: string, threads?: number, logger?: (msg: string) => void) {
     // Create a unique socket path in temp directory
-    this.socketPath = path.join(os.tmpdir(), `bb-${process.pid}-${Date.now()}.sock`);
+    this.socketPath = path.join(os.tmpdir(), `bb-${process.pid}-${threadId}-${instanceCounter++}.sock`);
 
     // Ensure socket path doesn't already exist (cleanup from previous crashes)
     if (fs.existsSync(this.socketPath)) {
@@ -57,18 +60,28 @@ export class BarretenbergNativeSocketAsyncBackend implements IMsgpackBackendAsyn
       connectionReject = reject;
     });
 
-    // Set HARDWARE_CONCURRENCY if threads specified
-    const env = threads !== undefined ? { ...process.env, HARDWARE_CONCURRENCY: threads.toString() } : process.env;
+    // If threads not set use num cpu cores, max 16.
+    const hwc = threads ? threads.toString() : Math.min(16, os.cpus().length).toString();
+    const env = { ...process.env, HARDWARE_CONCURRENCY: hwc };
 
     // Spawn bb process - it will create the socket server
-    const args = [bbBinaryPath, 'msgpack', 'run', '--input', this.socketPath];
-    this.process = spawn(findPackageRoot() + '/scripts/kill_wrapper.sh', args, {
-      stdio: ['ignore', 'ignore', 'ignore'],
+    const args = ['msgpack', 'run', '--input', this.socketPath];
+    this.process = spawn(bbBinaryPath, args, {
+      stdio: ['ignore', logger ? 'pipe' : 'ignore', logger ? 'pipe' : 'ignore'],
       env,
     });
-    // Disconnect from event loop so process can exit. The kill wrapper will reap bb once parent (node) dies.
+
+    // Disconnect from event loop so process can exit without waiting for bb
+    // The bb process has parent death monitoring (prctl on Linux, kqueue on macOS)
+    // so it will automatically exit when Node.js exits
     this.process.unref();
 
+    if (logger) {
+      logger("Logger attached to bb process. DON'T FORGET TO DESTROY THE BACKEND to allow Node.js to exit.");
+      readline.createInterface({ input: this.process.stdout! }).on('line', logger);
+      readline.createInterface({ input: this.process.stderr! }).on('line', logger);
+    }
+
     this.process.on('error', err => {
       if (connectionReject) {
         connectionReject(new Error(`Native backend process error: ${err.message}`));
@@ -157,7 +170,7 @@ export class BarretenbergNativeSocketAsyncBackend implements IMsgpackBackendAsyn
 
       // Set up event handlers
       this.socket.once('connect', () => {
-        this.socket!.unref();
+        // Socket starts referenced - will be unreferenced when no callbacks pending
 
         // Clear connection timeout on successful connection
         if (this.connectionTimeout) {
@@ -232,6 +245,11 @@ export class BarretenbergNativeSocketAsyncBackend implements IMsgpackBackendAsyn
             console.warn('Received response but no pending callback');
           }
 
+          // If no more pending callbacks, unref socket to allow process to exit
+          if (this.pendingCallbacks.length === 0 && this.socket) {
+            this.socket.unref();
+          }
+
           // Reset state for next message
           this.readingLength = true;
           this.lengthBytesRead = 0;
@@ -252,6 +270,11 @@ export class BarretenbergNativeSocketAsyncBackend implements IMsgpackBackendAsyn
     }
 
     return new Promise((resolve, reject) => {
+      // If this is the first pending callback, ref the socket to keep event loop alive
+      if (this.pendingCallbacks.length === 0) {
+        this.socket!.ref();
+      }
+
       // Enqueue this promise's callbacks (FIFO order)
       this.pendingCallbacks.push({ resolve, reject });
 
@@ -298,15 +321,8 @@ export class BarretenbergNativeSocketAsyncBackend implements IMsgpackBackendAsyn
   }
 
   async destroy(): Promise<void> {
-    // Cleanup first (closes socket, unrefs everything)
     this.cleanup();
-
-    // Send SIGTERM for graceful shutdown
-    // Process is unref'd so won't block event loop - just kill and return
-    try {
-      this.process.kill('SIGTERM');
-    } catch (e) {
-      // Already dead
-    }
+    this.process.kill('SIGTERM');
+    this.process.removeAllListeners();
   }
 }

package/src/bb_backends/node/platform.ts

@@ -24,7 +24,12 @@ export function findPackageRoot(): string | null {
   while (currentDir !== root) {
     const packageJsonPath = path.join(currentDir, 'package.json');
     if (fs.existsSync(packageJsonPath)) {
-      return currentDir;
+      // Check if this is the actual package root by verifying it has a 'build' directory
+      // This ensures we skip intermediate package.json files (e.g., in dest/node-cjs/)
+      const buildDir = path.join(currentDir, 'build');
+      if (fs.existsSync(buildDir)) {
+        return currentDir;
+      }
     }
     currentDir = path.dirname(currentDir);
   }

package/src/bb_backends/wasm.ts

@@ -3,7 +3,6 @@ import { fetchModuleAndThreads } from '../barretenberg_wasm/index.js';
 import { IMsgpackBackendSync, IMsgpackBackendAsync } from './interface.js';
 import { createMainWorker } from '../barretenberg_wasm/barretenberg_wasm_main/factory/node/index.js';
 import { getRemoteBarretenbergWasm } from '../barretenberg_wasm/helpers/index.js';
-import { createDebugLogger } from '../log/index.js';
 import { proxy } from 'comlink';
 
 /**
@@ -76,7 +75,7 @@ export class BarretenbergWasmAsyncBackend implements IMsgpackBackendAsync {
       await wasm.init(
         module,
         threads,
-        proxy(options.logger ?? createDebugLogger('bb_wasm_async')),
+        proxy(options.logger ?? (() => {})),
         options.memory?.initial,
         options.memory?.maximum,
       );

package/src/bbapi/exception_handling.test.ts

@@ -0,0 +1,54 @@
+import { BarretenbergWasmSyncBackend } from '../bb_backends/wasm.js';
+import { SyncApi } from '../cbind/generated/sync.js';
+import { BBApiException } from '../bbapi_exception.js';
+
+describe('BBApi Exception Handling from bb.js', () => {
+  let backend: BarretenbergWasmSyncBackend;
+  let api: SyncApi;
+
+  beforeAll(async () => {
+    backend = await BarretenbergWasmSyncBackend.new();
+    api = new SyncApi(backend);
+  }, 60000);
+
+  afterAll(() => {
+    backend.destroy();
+  });
+
+  it('should catch CRS initialization exceptions from WASM', () => {
+    // Create an SrsInitSrs command with invalid data that will cause an exception in C++
+    // We pass buffers that are too small, which will cause validation to fail
+    const invalidCommand = {
+      numPoints: 100, // Request 100 points (requires 6400 bytes)
+      pointsBuf: new Uint8Array(10), // Only 10 bytes - will cause exception
+      g2Point: new Uint8Array(10), // Only 10 bytes (needs 128) - will cause exception
+    };
+
+    // In WASM builds, throw_or_abort calls abort directly which throws a generic Error
+    // In native builds with exceptions, our try-catch in bbapi converts it to ErrorResponse
+    // This test verifies that errors are catchable from bb.js (even if as generic Error in WASM)
+    expect(() => {
+      api.srsInitSrs(invalidCommand);
+    }).toThrow();
+  });
+
+  it('should return error message from caught exception', () => {
+    const invalidCommand = {
+      numPoints: 100,
+      pointsBuf: new Uint8Array(10),
+      g2Point: new Uint8Array(10),
+    };
+
+    try {
+      api.srsInitSrs(invalidCommand);
+      fail('Expected exception to be thrown');
+    } catch (error) {
+      // Error is catchable and contains a useful message
+      expect(error).toBeInstanceOf(Error);
+      expect((error as Error).message).toBeTruthy();
+      expect((error as Error).message.length).toBeGreaterThan(0);
+      expect((error as Error).message).toContain('g1_identity');
+      console.log('Successfully caught exception from bb.js with message:', (error as Error).message);
+    }
+  });
+});

package/src/bin/index.ts

@@ -0,0 +1,14 @@
+#!/usr/bin/env node
+import { findBbBinary } from '../bb_backends/node/platform.js';
+import { spawnSync } from 'node:child_process';
+
+const bin = findBbBinary();
+
+if (!bin) {
+  console.error('Could not find bb binary. Please ensure it is built and accessible.');
+  process.exit(1);
+}
+
+const result = spawnSync(bin, process.argv.slice(2), { stdio: 'inherit' });
+
+process.exit(result.status ?? 1);

package/src/cbind/generate.ts

@@ -1,5 +1,8 @@
 /**
- * Generate TypeScript bindings from msgpack schema
+ * Multi-language code generation from BB msgpack schema
+ *
+ * Architecture:
+ *   Raw Schema → SchemaVisitor → CompiledSchema IR → Language Codegens → Files
  */
 
 import { writeFileSync, mkdirSync } from 'fs';
@@ -8,36 +11,42 @@ import { exec } from 'child_process';
 import { promisify } from 'util';
 import { fileURLToPath } from 'url';
 import { unpack } from 'msgpackr';
-import {
-  createSharedTypesCompiler,
-  createSyncApiCompiler,
-  createAsyncApiCompiler,
-  type SchemaCompiler,
-} from './schema_compiler.js';
+import { SchemaVisitor, type CompiledSchema } from './schema_visitor.js';
+import { TypeScriptCodegen } from './typescript_codegen.js';
+import { RustCodegen } from './rust_codegen.js';
 
 const execAsync = promisify(exec);
 
-interface GeneratorConfig {
+// Language generators - all use the same CompiledSchema IR
+interface LanguageGenerator {
   name: string;
-  outputFile: string;
-  createCompiler: () => SchemaCompiler;
+  enabled: boolean;
+  generate: (compiled: CompiledSchema) => Array<{ path: string; content: string }>;
 }
 
-const GENERATORS: GeneratorConfig[] = [
+const LANGUAGE_GENERATORS: LanguageGenerator[] = [
   {
-    name: 'Shared types',
-    outputFile: 'generated/api_types.ts',
-    createCompiler: createSharedTypesCompiler,
+    name: 'TypeScript',
+    enabled: true,
+    generate: (compiled) => {
+      const tsGen = new TypeScriptCodegen();
+      return [
+        { path: 'generated/api_types.ts', content: tsGen.generateTypes(compiled) },
+        { path: 'generated/sync.ts', content: tsGen.generateSyncApi(compiled) },
+        { path: 'generated/async.ts', content: tsGen.generateAsyncApi(compiled) },
+      ];
+    },
   },
   {
-    name: 'Sync API',
-    outputFile: 'generated/sync.ts',
-    createCompiler: createSyncApiCompiler,
-  },
-  {
-    name: 'Async API',
-    outputFile: 'generated/async.ts',
-    createCompiler: createAsyncApiCompiler,
+    name: 'Rust',
+    enabled: true,
+    generate: (compiled) => {
+      const rustGen = new RustCodegen();
+      return [
+        { path: '../../../rust/barretenberg-rs/src/generated_types.rs', content: rustGen.generateTypes(compiled) },
+        { path: '../../../rust/barretenberg-rs/src/api.rs', content: rustGen.generateApi(compiled) },
+      ];
+    },
   },
 ];
 
@@ -56,29 +65,39 @@ async function generate() {
     throw new Error('Invalid schema: missing commands or responses');
   }
 
-  console.log('Generating TypeScript bindings...\n');
+  // Compile schema once using visitor pattern
+  console.log('Compiling schema...');
+  const visitor = new SchemaVisitor();
+  const compiled = visitor.visit(schema.commands, schema.responses);
+
+  console.log(`Found ${compiled.commands.length} commands, ${compiled.structs.size} structs\n`);
 
   // Ensure output directory exists
   const outputDir = join(__dirname, 'generated');
   mkdirSync(outputDir, { recursive: true });
 
-  // Generate each output file
-  for (const config of GENERATORS) {
-    const compiler = config.createCompiler();
-    compiler.processApiSchema(schema.commands, schema.responses);
+  // Generate all language bindings from compiled IR
+  for (const generator of LANGUAGE_GENERATORS) {
+    if (!generator.enabled) {
+      console.log(`⊘ ${generator.name}: disabled`);
+      continue;
+    }
 
-    const outputPath = join(__dirname, config.outputFile);
-    const content = compiler.compile();
-    writeFileSync(outputPath, content);
+    const files = generator.generate(compiled);
 
-    console.log(`✓ ${config.name}: ${outputPath}`);
+    for (const file of files) {
+      const outputPath = join(__dirname, file.path);
+      mkdirSync(dirname(outputPath), { recursive: true });
+      writeFileSync(outputPath, file.content);
+      console.log(`✓ ${generator.name}: ${outputPath}`);
+    }
   }
 
   // Generate curve constants
   console.log('\nGenerating curve constants...');
   await generateCurveConstants(bbBuildPath, outputDir);
 
-  console.log('\nGeneration complete!');
+  console.log('\n✨ Generation complete! Clean, maintainable, multi-language architecture.');
 }
 
 async function generateCurveConstants(bbBuildPath: string, outputDir: string) {