@aztec/bb.js 0.0.1-alpha.6 → 0.0.1-fake-ceab37513c

package/src/barretenberg_wasm/helpers/browser/index.ts

@@ -0,0 +1,47 @@
+import { wrap } from 'comlink';
+
+export function getSharedMemoryAvailable() {
+  const globalScope = typeof window !== 'undefined' ? window : globalThis;
+  return typeof SharedArrayBuffer !== 'undefined' && globalScope.crossOriginIsolated;
+}
+
+export function getRemoteBarretenbergWasm<T>(worker: Worker) {
+  return wrap<T>(worker);
+}
+
+export function getNumCpu() {
+  return navigator.hardwareConcurrency;
+}
+
+export function threadLogger(): ((msg: string) => void) | undefined {
+  return console.log;
+}
+
+export function killSelf() {
+  self.close();
+}
+
+export function getAvailableThreads(logger: (msg: string) => void): number {
+  if (typeof navigator !== 'undefined' && navigator.hardwareConcurrency) {
+    return navigator.hardwareConcurrency;
+  } else {
+    logger(`Could not detect environment to query number of threads. Falling back to one thread.`);
+    return 1;
+  }
+}
+
+// Solution to async initialization of workers, taken from
+// https://github.com/GoogleChromeLabs/comlink/issues/635#issuecomment-1598913044
+
+/** The message expected by the `readinessListener`. */
+export const Ready = { ready: true };
+
+/** Listen for the readiness message from the Worker and call the `callback` once. */
+export function readinessListener(worker: Worker, callback: () => void) {
+  worker.addEventListener('message', function ready(event: MessageEvent<typeof Ready>) {
+    if (!!event.data && event.data.ready === true) {
+      worker.removeEventListener('message', ready);
+      callback();
+    }
+  });
+}

package/src/barretenberg_wasm/helpers/index.ts

@@ -0,0 +1 @@
+export * from './node/index.js';

package/src/barretenberg_wasm/{node → helpers/node}/index.ts

@@ -1,29 +1,27 @@
 import { Worker } from 'worker_threads';
-import { dirname } from 'path';
-import { fileURLToPath } from 'url';
-import { readFile } from 'fs/promises';
 import os from 'os';
-import { type BarretenbergWasm, type BarretenbergWasmWorker } from '../barretenberg_wasm.js';
 import { wrap } from 'comlink';
 import { nodeEndpoint } from './node_endpoint.js';
 import { writeSync } from 'fs';
 
-export async function fetchCode(name: string) {
-  const __dirname = dirname(fileURLToPath(import.meta.url));
-  return await readFile(__dirname + '/../../' + name);
+export function getSharedMemoryAvailable() {
+  return true;
 }
 
-export function createWorker() {
-  const __dirname = dirname(fileURLToPath(import.meta.url));
-  return new Worker(__dirname + `/worker.js`);
-}
-
-export function getRemoteBarretenbergWasm(worker: Worker): BarretenbergWasmWorker {
-  return wrap<BarretenbergWasm>(nodeEndpoint(worker)) as BarretenbergWasmWorker;
+/**
+ * Comlink allows you to produce a Proxy to the worker, enabling you to call methods as if it were a normal class.
+ * Note we give it the type information it needs so the returned Proxy object looks like that type.
+ * Node has a different implementation, needing this nodeEndpoint wrapper, hence this function exists here.
+ */
+export function getRemoteBarretenbergWasm<T>(worker: Worker) {
+  return wrap<T>(nodeEndpoint(worker));
 }
 
+/**
+ * Returns number of cpus as reported by the system, unless overriden by HARDWARE_CONCURRENCY env var.
+ */
 export function getNumCpu() {
-  return os.cpus().length;
+  return +process.env.HARDWARE_CONCURRENCY! || os.cpus().length;
 }
 
 /**
@@ -44,3 +42,14 @@ export function killSelf(): never {
   process.kill(process.pid);
   throw new Error();
 }
+
+export function getAvailableThreads(logger: (msg: string) => void): number {
+  try {
+    return os.cpus().length;
+  } catch (e: any) {
+    logger(
+      `Could not detect environment to query number of threads. Falling back to one thread. Error: ${e.message ?? e}`,
+    );
+    return 1;
+  }
+}

package/src/barretenberg_wasm/index.test.ts

@@ -0,0 +1,45 @@
+import { BarretenbergWasmMain, BarretenbergWasmMainWorker } from './barretenberg_wasm_main/index.js';
+import { Barretenberg } from '../index.js';
+
+describe('barretenberg wasm', () => {
+  let api: Barretenberg;
+  let wasm: BarretenbergWasmMainWorker;
+
+  beforeAll(async () => {
+    api = await Barretenberg.new({ threads: 2 });
+    wasm = api.getWasm();
+  }, 20000);
+
+  afterAll(async () => {
+    await api.destroy();
+  });
+
+  it('should new malloc, transfer and slice mem', async () => {
+    const length = 1024;
+    const ptr = await wasm.call('bbmalloc', length);
+    const buf = Buffer.alloc(length, 128);
+    await wasm.writeMemory(ptr, Uint8Array.from(buf));
+    const result = Buffer.from(await wasm.getMemorySlice(ptr, ptr + length));
+    await wasm.call('bbfree', ptr);
+    expect(result).toStrictEqual(buf);
+  });
+
+  it('test abort', async () => {
+    await expect(() => wasm.call('test_abort')).rejects.toThrow();
+  });
+
+  it('test c/c++ stdout/stderr', async () => {
+    // We're checking we don't crash, but you can manually confirm you see log lines handled by logstr.
+    await wasm.call('test_stdout_stderr');
+  });
+
+  it('should new malloc, transfer and slice mem', async () => {
+    const length = 1024;
+    const ptr = await wasm.call('bbmalloc', length);
+    const buf = Buffer.alloc(length, 128);
+    await wasm.writeMemory(ptr, Uint8Array.from(buf));
+    const result = Buffer.from(await wasm.getMemorySlice(ptr, ptr + length));
+    await wasm.call('bbfree', ptr);
+    expect(result).toStrictEqual(buf);
+  });
+});

package/src/barretenberg_wasm/index.ts

@@ -1 +1,22 @@
-export * from "./barretenberg_wasm.js";
+import { getSharedMemoryAvailable, getAvailableThreads } from './helpers/node/index.js';
+import { fetchCode } from './fetch_code/index.js';
+import { createDebugLogger } from '../log/index.js';
+
+export async function fetchModuleAndThreads(
+  desiredThreads = 32,
+  wasmPath?: string,
+  logger: (msg: string) => void = createDebugLogger('fetch_mat'),
+) {
+  const shared = getSharedMemoryAvailable();
+
+  const availableThreads = shared ? await getAvailableThreads(logger) : 1;
+  // We limit the number of threads to 32 as we do not benefit from greater numbers.
+  const limitedThreads = Math.min(desiredThreads, availableThreads, 32);
+
+  logger(`Fetching bb wasm from ${wasmPath ?? 'default location'}`);
+  const code = await fetchCode(shared, wasmPath);
+  logger(`Compiling bb wasm of ${code.byteLength} bytes`);
+  const module = await WebAssembly.compile(code);
+  logger('Compilation of bb wasm complete');
+  return { module, threads: limitedThreads };
+}

package/src/benchmark/index.ts

@@ -0,0 +1,26 @@
+import * as fs from 'fs';
+export * from './timer.js';
+
+const bfd = (() => {
+  const bfdStr = process.env.BENCHMARK_FD;
+  const bfd = bfdStr ? parseInt(bfdStr) : -1;
+  if (bfd >= 0 && !fs.fstatSync(bfd)) {
+    throw new Error('fd is not open. Did you redirect in your shell?');
+  }
+  return bfd;
+})();
+
+export function writeBenchmark<T>(name: string, value: T, labels: Record<string, any> = {}) {
+  if (bfd === -1) {
+    return;
+  }
+  const data = {
+    timestamp: new Date().toISOString(),
+    name,
+    type: typeof value,
+    value,
+    ...labels,
+  };
+  const jsonl = JSON.stringify(data) + '\n';
+  fs.writeSync(bfd, jsonl);
+}

package/src/benchmark/timer.ts

@@ -0,0 +1,45 @@
+/**
+ * Timer class to measure time intervals in milliseconds and seconds.
+ * Upon instantiation, it stores the current timestamp as the starting point.
+ * The 'ms()' method returns the elapsed time in milliseconds,
+ * while the 's()' method returns the elapsed time in seconds.
+ *
+ * @example
+ * const timer = new Timer();
+ * setTimeout(() =\> \{
+ *   console.log(`Elapsed time: ${timer.ms()} ms`);
+ * \}, 1000);
+ */
+export class Timer {
+  private start: number;
+
+  constructor() {
+    this.start = performance.now();
+  }
+
+  public us() {
+    return this.ms() * 1000;
+  }
+
+  /**
+   * Returns the elapsed time in milliseconds since the Timer instance was created.
+   * Provides a simple and convenient way to measure the time duration between two events
+   * or monitor performance of specific code sections.
+   *
+   * @returns The elapsed time in milliseconds.
+   */
+  public ms() {
+    return performance.now() - this.start;
+  }
+
+  /**
+   * Returns the time elapsed since the Timer instance was created, in seconds.
+   * The value is calculated by subtracting the initial start time from the current time
+   * and dividing the result by 1000 to convert milliseconds to seconds.
+   *
+   * @returns The elapsed time in seconds.
+   */
+  public s() {
+    return this.ms() / 1000;
+  }
+}

package/src/bigint-array/index.ts

@@ -1,21 +1,43 @@
-export function toBigIntBE(bytes: Uint8Array) {
-  // A Buffer in node, *is* a Uint8Array. We can't refuse it's type.
-  // However the algo below only works on an actual Uint8Array, hence we make a new one to be safe.
-  bytes = new Uint8Array(bytes);
-  let bigint = BigInt(0);
-  const view = new DataView(bytes.buffer);
-  for (let i = 0; i < bytes.byteLength; i++) {
-    bigint = (bigint << BigInt(8)) + BigInt(view.getUint8(i));
-  }
-  return bigint;
+/**
+ * Convert a 32-byte BE Buffer to a BigInt.
+ */
+export function buffer32BytesToBigIntBE(buf: Buffer): bigint {
+  return (
+    (buf.readBigUInt64BE(0) << 192n) +
+    (buf.readBigUInt64BE(8) << 128n) +
+    (buf.readBigUInt64BE(16) << 64n) +
+    buf.readBigUInt64BE(24)
+  );
+}
+
+/**
+ * Convert a BE Uint8Array to a BigInt.
+ */
+export function uint8ArrayToBigIntBE(bytes: Uint8Array): bigint {
+  const buffer = Buffer.from(bytes);
+  return buffer32BytesToBigIntBE(buffer);
 }
 
-export function toBufferBE(value: bigint, byteLength = 32) {
-  const bytes = new Uint8Array(byteLength);
-  const view = new DataView(bytes.buffer);
-  for (let i = 0; i < byteLength; i++) {
-    view.setUint8(byteLength - i - 1, Number(value & BigInt(0xff)));
-    value >>= BigInt(8);
+/**
+ * Convert a BigInt to a 32-byte BE Buffer.
+ */
+export function bigIntToBufferBE(value: bigint, byteLength = 32): Buffer {
+  if (byteLength != 32) {
+    throw new Error(
+      `Only 32 bytes supported for conversion from bigint to buffer, attempted byte length: ${byteLength}`,
+    );
   }
-  return bytes;
+  const buf = Buffer.alloc(byteLength);
+  buf.writeBigUInt64BE(value >> 192n, 0);
+  buf.writeBigUInt64BE((value >> 128n) & 0xffffffffffffffffn, 8);
+  buf.writeBigUInt64BE((value >> 64n) & 0xffffffffffffffffn, 16);
+  buf.writeBigUInt64BE(value & 0xffffffffffffffffn, 24);
+  return buf;
+}
+
+/**
+ * Convert a BigInt to a 32-byte BE Uint8Array.
+ */
+export function bigIntToUint8ArrayBE(value: bigint, byteLength = 32): Uint8Array {
+  return new Uint8Array(bigIntToBufferBE(value, byteLength));
 }

package/src/bindgen/index.ts

@@ -1,7 +1,7 @@
 import { generateRustCode } from './rust.js';
 import { generateTypeScriptCode } from './typescript.js';
 
-const [, , exports = '../exports.json', lang = 'ts'] = process.argv;
+const [, , exp = '../exports.json', lang = 'ts'] = process.argv;
 
 function generateCode(exports: string, lang: string) {
   switch (lang) {
@@ -14,4 +14,4 @@ function generateCode(exports: string, lang: string) {
   }
 }
 
-console.log(generateCode(exports, lang));
+console.log(generateCode(exp, lang));

package/src/bindgen/mappings.ts

@@ -2,8 +2,8 @@
 const typeMap: { [key: string]: string } = {
   in_ptr: 'Ptr',
   out_ptr: 'Ptr',
-  'barretenberg::fr::in_buf': 'Fr',
-  'barretenberg::fr::vec_in_buf': 'Fr[]',
+  'bb::fr::in_buf': 'Fr',
+  'bb::fr::vec_in_buf': 'Fr[]',
   'fr::in_buf': 'Fr',
   'fr::out_buf': 'Fr',
   'fr::vec_in_buf': 'Fr[]',
@@ -13,6 +13,7 @@ const typeMap: { [key: string]: string } = {
   'fq::vec_in_buf': 'Fq[]',
   'fq::vec_out_buf': 'Fq[]',
   'const uint8_t *': 'Uint8Array',
+  uint8_vec_vec_in_buf: 'Uint8Array[]',
   'uint8_t **': 'Uint8Array',
   in_str_buf: 'string',
   out_str_buf: 'string',

package/src/bindgen/typescript.ts

@@ -9,37 +9,52 @@ export function generateTypeScriptCode(filename: string) {
 
   let output = `// WARNING: FILE CODE GENERATED BY BINDGEN UTILITY. DO NOT EDIT!
 /* eslint-disable @typescript-eslint/no-unused-vars */
-import { BarretenbergBinder, BarretenbergBinderSync } from '../barretenberg_binder/index.js';
-import { BufferDeserializer, NumberDeserializer, VectorDeserializer, BoolDeserializer, StringDeserializer } from '../serialize/index.js';
-import { Fr, Fq, Point, Buffer32, Buffer128, Ptr } from '../types/index.js';
+import { BarretenbergWasmMain, BarretenbergWasmMainWorker } from '../barretenberg_wasm/barretenberg_wasm_main/index.js';
+import { BufferDeserializer, NumberDeserializer, VectorDeserializer, BoolDeserializer, StringDeserializer, serializeBufferable, OutputType } from '../serialize/index.js';
+import { Fr, Point, Buffer32, Ptr } from '../types/index.js';
 
+`;
+
+  output += generateClass(functionDeclarations);
+  output += generateSyncClass(functionDeclarations);
+
+  return output;
+}
+
+function generateClass(functionDeclarations: FunctionDeclaration[]) {
+  let output = `
 export class BarretenbergApi {
-  constructor(public binder: BarretenbergBinder) {}
+  constructor(protected wasm: BarretenbergWasmMainWorker) {}
 
-  async destroy() {
-    await this.binder.wasm.destroy();
-  }
 `;
 
   for (const { functionName, inArgs, outArgs } of functionDeclarations) {
     try {
       const parameters = inArgs.map(({ name, type }) => `${toCamelCase(name)}: ${mapType(type)}`).join(', ');
-      const inArgsVar = `[${inArgs.map(arg => toCamelCase(arg.name)).join(', ')}]`;
-      const outTypesVar = `[${outArgs.map(arg => mapDeserializer(arg.type)).join(', ')}]`;
-      const wasmCall = `const result = await this.binder.callWasmExport('${functionName}', ${inArgsVar}, ${outTypesVar});`;
+      const inArgsVar = `const inArgs = [${inArgs
+        .map(arg => toCamelCase(arg.name))
+        .join(', ')}].map(serializeBufferable);`;
+      const outTypesVar = `const outTypes: OutputType[] = [${outArgs
+        .map(arg => mapDeserializer(arg.type))
+        .join(', ')}];`;
+      const wasmCall = `const result = await this.wasm.callWasmExport('${functionName}', inArgs, outTypes.map(t=>t.SIZE_IN_BYTES));`;
+      const outVar = `const out = result.map((r, i) => outTypes[i].fromBuffer(r));`;
 
       const n = outArgs.length;
-      const returnStmt = n === 0 ? 'return;' : n === 1 ? 'return result[0];' : 'return result as any;';
+      const returnStmt = n === 0 ? 'return;' : n === 1 ? 'return out[0];' : 'return out as any;';
       const returnType =
         outArgs.length === 0
           ? 'void'
           : outArgs.length === 1
-          ? `${mapType(outArgs[0].type)}`
-          : `[${outArgs.map(a => mapType(a.type)).join(', ')}]`;
+            ? `${mapType(outArgs[0].type)}`
+            : `[${outArgs.map(a => mapType(a.type)).join(', ')}]`;
 
       output += `
   async ${toCamelCase(functionName)}(${parameters}): Promise<${returnType}> {
+    ${inArgsVar}
+    ${outTypesVar}
     ${wasmCall}
+    ${outVar}
     ${returnStmt}
   }
 `;
@@ -48,35 +63,45 @@ export class BarretenbergApi {
     }
   }
 
-  output += `}
+  output += `}`;
 
+  return output;
+}
+
+function generateSyncClass(functionDeclarations: FunctionDeclaration[]) {
+  let output = `
 export class BarretenbergApiSync {
-  constructor(public binder: BarretenbergBinderSync) {}
+  constructor(protected wasm: BarretenbergWasmMain) {}
 
-  async destroy() {
-    await this.binder.wasm.destroy();
-  }
 `;
 
   for (const { functionName, inArgs, outArgs } of functionDeclarations) {
     try {
       const parameters = inArgs.map(({ name, type }) => `${toCamelCase(name)}: ${mapType(type)}`).join(', ');
-      const inArgsVar = `[${inArgs.map(arg => toCamelCase(arg.name)).join(', ')}]`;
-      const outTypesVar = `[${outArgs.map(arg => mapDeserializer(arg.type)).join(', ')}]`;
-      const wasmCall = `const result = this.binder.callWasmExport('${functionName}', ${inArgsVar}, ${outTypesVar});`;
+      const inArgsVar = `const inArgs = [${inArgs
+        .map(arg => toCamelCase(arg.name))
+        .join(', ')}].map(serializeBufferable);`;
+      const outTypesVar = `const outTypes: OutputType[] = [${outArgs
+        .map(arg => mapDeserializer(arg.type))
+        .join(', ')}];`;
+      const wasmCall = `const result = this.wasm.callWasmExport('${functionName}', inArgs, outTypes.map(t=>t.SIZE_IN_BYTES));`;
+      const outVar = `const out = result.map((r, i) => outTypes[i].fromBuffer(r));`;
 
       const n = outArgs.length;
-      const returnStmt = n === 0 ? 'return;' : n === 1 ? 'return result[0];' : 'return result as any;';
+      const returnStmt = n === 0 ? 'return;' : n === 1 ? 'return out[0];' : 'return out as any;';
       const returnType =
         outArgs.length === 0
           ? 'void'
           : outArgs.length === 1
-          ? `${mapType(outArgs[0].type)}`
-          : `[${outArgs.map(a => mapType(a.type)).join(', ')}]`;
+            ? `${mapType(outArgs[0].type)}`
+            : `[${outArgs.map(a => mapType(a.type)).join(', ')}]`;
 
       output += `
   ${toCamelCase(functionName)}(${parameters}): ${returnType} {
+    ${inArgsVar}
+    ${outTypesVar}
     ${wasmCall}
+    ${outVar}
     ${returnStmt}
   }
 `;
@@ -85,7 +110,7 @@ export class BarretenbergApiSync {
     }
   }
 
-  output += '}';
+  output += `}`;
 
   return output;
 }

package/src/cbind/README.md

@@ -0,0 +1 @@
+Derive bindings from the reported scheme of msgpack from bb. Currently redundant with bindgen, until that is supplanted.

package/src/cbind/generate.ts

@@ -0,0 +1,89 @@
+/**
+ * Generate TypeScript bindings from msgpack schema
+ */
+
+import { writeFileSync, mkdirSync } from 'fs';
+import { dirname, join } from 'path';
+import { exec } from 'child_process';
+import { promisify } from 'util';
+import { fileURLToPath } from 'url';
+import {
+  createSharedTypesCompiler,
+  createSyncApiCompiler,
+  createAsyncApiCompiler,
+  createNativeApiCompiler,
+  type SchemaCompiler,
+} from './schema_compiler.js';
+
+const execAsync = promisify(exec);
+
+interface GeneratorConfig {
+  name: string;
+  outputFile: string;
+  createCompiler: () => SchemaCompiler;
+}
+
+const GENERATORS: GeneratorConfig[] = [
+  {
+    name: 'Shared types',
+    outputFile: 'generated/api_types.ts',
+    createCompiler: createSharedTypesCompiler,
+  },
+  {
+    name: 'Sync API',
+    outputFile: 'generated/sync.ts',
+    createCompiler: createSyncApiCompiler,
+  },
+  {
+    name: 'Async API',
+    outputFile: 'generated/async.ts',
+    createCompiler: createAsyncApiCompiler,
+  },
+  {
+    name: 'Native API',
+    outputFile: 'generated/native.ts',
+    createCompiler: createNativeApiCompiler,
+  },
+];
+
+// @ts-ignore
+const __dirname = dirname(fileURLToPath(import.meta.url));
+
+async function generate() {
+  const bbBuildPath = process.env.BB_BINARY_PATH || join(__dirname, '../../../cpp/build/bin/bb');
+
+  // Get schema from bb
+  console.log('Fetching msgpack schema from bb...');
+  const { stdout } = await execAsync(`${bbBuildPath} msgpack schema`);
+  const schema = JSON.parse(stdout.trim());
+
+  if (!schema.commands || !schema.responses) {
+    throw new Error('Invalid schema: missing commands or responses');
+  }
+
+  console.log('Generating TypeScript bindings...\n');
+
+  // Ensure output directory exists
+  const outputDir = join(__dirname, 'generated');
+  mkdirSync(outputDir, { recursive: true });
+
+  // Generate each output file
+  for (const config of GENERATORS) {
+    const compiler = config.createCompiler();
+    compiler.processApiSchema(schema.commands, schema.responses);
+
+    const outputPath = join(__dirname, config.outputFile);
+    const content = compiler.compile();
+    writeFileSync(outputPath, content);
+
+    console.log(`✓ ${config.name}: ${outputPath}`);
+  }
+
+  console.log('\nGeneration complete!');
+}
+
+// Run the generator
+generate().catch(error => {
+  console.error('Generation failed:', error);
+  process.exit(1);
+});