@aztec/bb.js 0.0.0-devnet → 0.0.0-test.0

package/src/barretenberg_wasm/barretenberg_wasm_main/heap_allocator.ts

@@ -15,17 +15,21 @@ export class HeapAllocator {
 
   constructor(private wasm: BarretenbergWasmMain) {}
 
-  copyToMemory(buffers: Uint8Array[]) {
-    return buffers.map(buf => {
-      if (buf.length <= this.inScratchRemaining) {
-        const ptr = (this.inScratchRemaining -= buf.length);
-        this.wasm.writeMemory(ptr, buf);
-        return ptr;
+  getInputs(buffers: (Uint8Array | number)[]) {
+    return buffers.map(bufOrNum => {
+      if (typeof bufOrNum === 'object') {
+        if (bufOrNum.length <= this.inScratchRemaining) {
+          const ptr = (this.inScratchRemaining -= bufOrNum.length);
+          this.wasm.writeMemory(ptr, bufOrNum);
+          return ptr;
+        } else {
+          const ptr = this.wasm.call('bbmalloc', bufOrNum.length);
+          this.wasm.writeMemory(ptr, bufOrNum);
+          this.allocs.push(ptr);
+          return ptr;
+        }
       } else {
-        const ptr = this.wasm.call('bbmalloc', buf.length);
-        this.wasm.writeMemory(ptr, buf);
-        this.allocs.push(ptr);
-        return ptr;
+        return bufOrNum;
       }
     });
   }

package/src/barretenberg_wasm/barretenberg_wasm_main/index.ts

@@ -7,8 +7,6 @@ import { type BarretenbergWasmThreadWorker } from '../barretenberg_wasm_thread/i
 import { BarretenbergWasmBase } from '../barretenberg_wasm_base/index.js';
 import { HeapAllocator } from './heap_allocator.js';
 
-const debug = createDebug('bb.js:wasm');
-
 /**
  * This is the "main thread" implementation of BarretenbergWasm.
  * It spawns a bunch of "child thread" implementations.
@@ -31,7 +29,7 @@ export class BarretenbergWasmMain extends BarretenbergWasmBase {
   public async init(
     module: WebAssembly.Module,
     threads = Math.min(getNumCpu(), BarretenbergWasmMain.MAX_THREADS),
-    logger: (msg: string) => void = debug,
+    logger: (msg: string) => void = createDebug('bb.js:bb_wasm'),
     initial = 32,
     maximum = 2 ** 16,
   ) {
@@ -42,9 +40,9 @@ export class BarretenbergWasmMain extends BarretenbergWasmBase {
     const shared = getSharedMemoryAvailable();
 
     this.logger(
-      `initial mem: ${initial} pages, ${initialMb}MiB. ` +
-        `max mem: ${maximum} pages, ${maxMb}MiB. ` +
-        `threads: ${threads}, shared: ${shared}`,
+      `Initializing bb wasm: initial memory ${initial} pages ${initialMb}MiB; ` +
+        `max memory: ${maximum} pages, ${maxMb}MiB; ` +
+        `threads: ${threads}; shared memory: ${shared}`,
     );
 
     this.memory = new WebAssembly.Memory({ initial, maximum, shared });
@@ -58,12 +56,11 @@ export class BarretenbergWasmMain extends BarretenbergWasmBase {
 
     // Create worker threads. Create 1 less than requested, as main thread counts as a thread.
     if (threads > 1) {
-      this.logger(`creating ${threads} worker threads...`);
+      this.logger(`Creating ${threads} worker threads`);
       this.workers = await Promise.all(Array.from({ length: threads - 1 }).map(createThreadWorker));
       this.remoteWasms = await Promise.all(this.workers.map(getRemoteBarretenbergWasm<BarretenbergWasmThreadWorker>));
       await Promise.all(this.remoteWasms.map(w => w.initThread(module, this.memory)));
     }
-    this.logger('init complete.');
   }
 
   /**
@@ -102,9 +99,9 @@ export class BarretenbergWasmMain extends BarretenbergWasmBase {
     /* eslint-enable camelcase */
   }
 
-  callWasmExport(funcName: string, inArgs: Uint8Array[], outLens: (number | undefined)[]) {
+  callWasmExport(funcName: string, inArgs: (Uint8Array | number)[], outLens: (number | undefined)[]) {
     const alloc = new HeapAllocator(this);
-    const inPtrs = alloc.copyToMemory(inArgs);
+    const inPtrs = alloc.getInputs(inArgs);
     const outPtrs = alloc.getOutputPtrs(outLens);
     this.call(funcName, ...inPtrs, ...outPtrs);
     const outArgs = this.getOutputArgs(outLens, outPtrs, alloc);

package/src/barretenberg_wasm/fetch_code/browser/barretenberg-threads.ts

@@ -0,0 +1,3 @@
+import barretenbergThreadsModule from '../../barretenberg-threads.wasm.gz';
+
+export default barretenbergThreadsModule;

package/src/barretenberg_wasm/fetch_code/browser/barretenberg.ts

@@ -0,0 +1,3 @@
+import barretenbergModule from '../../barretenberg.wasm.gz';
+
+export default barretenbergModule;

package/src/barretenberg_wasm/fetch_code/browser/index.ts

@@ -1,12 +1,33 @@
-import barretenbergModule from '../../barretenberg.wasm.gz';
-import barretenbergThreadsModule from '../../barretenberg-threads.wasm.gz';
 import pako from 'pako';
 
 // Annoyingly the wasm declares if it's memory is shared or not. So now we need two wasms if we want to be
 // able to fallback on "non shared memory" situations.
-export async function fetchCode(multithreaded: boolean) {
-  const res = await fetch(multithreaded ? barretenbergThreadsModule : barretenbergModule);
-  const compressedData = await res.arrayBuffer();
-  const decompressedData = pako.ungzip(new Uint8Array(compressedData));
-  return decompressedData.buffer;
+export async function fetchCode(multithreaded: boolean, wasmPath?: string) {
+  let url: string;
+  if (wasmPath) {
+    const suffix = multithreaded ? '-threads' : '';
+    const filePath = wasmPath.split('/').slice(0, -1).join('/');
+    const fileNameWithExtensions = wasmPath.split('/').pop();
+    const [fileName, ...extensions] = fileNameWithExtensions!.split('.');
+    url = `${filePath}/${fileName}${suffix}.${extensions.join('.')}`;
+  } else {
+    url = multithreaded
+      ? (await import(/* webpackIgnore: true */ './barretenberg-threads.js')).default
+      : (await import(/* webpackIgnore: true */ './barretenberg.js')).default;
+  }
+  const res = await fetch(url);
+  const maybeCompressedData = await res.arrayBuffer();
+  const buffer = new Uint8Array(maybeCompressedData);
+  const isGzip =
+    // Check magic number
+    buffer[0] === 0x1f &&
+    buffer[1] === 0x8b &&
+    // Check compression method:
+    buffer[2] === 0x08;
+  if (isGzip) {
+    const decompressedData = pako.ungzip(buffer);
+    return decompressedData.buffer;
+  } else {
+    return buffer;
+  }
 }

package/src/barretenberg_wasm/fetch_code/node/index.ts

@@ -14,8 +14,8 @@ function getCurrentDir() {
 }
 
 // eslint-disable-next-line @typescript-eslint/no-unused-vars
-export async function fetchCode(multithreaded: boolean) {
-  const path = getCurrentDir() + '/../../barretenberg-threads.wasm.gz';
+export async function fetchCode(multithreaded: boolean, wasmPath?: string) {
+  const path = wasmPath ?? getCurrentDir() + '/../../barretenberg-threads.wasm.gz';
   const compressedData = await readFile(path);
   const decompressedData = pako.ungzip(new Uint8Array(compressedData));
   return decompressedData.buffer;

package/src/barretenberg_wasm/index.test.ts

@@ -18,7 +18,7 @@ describe('barretenberg wasm', () => {
     const length = 1024;
     const ptr = await wasm.call('bbmalloc', length);
     const buf = Buffer.alloc(length, 128);
-    await wasm.writeMemory(ptr, buf);
+    await wasm.writeMemory(ptr, Uint8Array.from(buf));
     const result = Buffer.from(await wasm.getMemorySlice(ptr, ptr + length));
     await wasm.call('bbfree', ptr);
     expect(result).toStrictEqual(buf);
@@ -37,7 +37,7 @@ describe('barretenberg wasm', () => {
     const length = 1024;
     const ptr = await wasm.call('bbmalloc', length);
     const buf = Buffer.alloc(length, 128);
-    await wasm.writeMemory(ptr, buf);
+    await wasm.writeMemory(ptr, Uint8Array.from(buf));
     const result = Buffer.from(await wasm.getMemorySlice(ptr, ptr + length));
     await wasm.call('bbfree', ptr);
     expect(result).toStrictEqual(buf);

package/src/barretenberg_wasm/index.ts

@@ -5,29 +5,36 @@ import { getRemoteBarretenbergWasm, getSharedMemoryAvailable } from './helpers/n
 import { BarretenbergWasmMain, BarretenbergWasmMainWorker } from './barretenberg_wasm_main/index.js';
 import { fetchCode } from './fetch_code/index.js';
 
-const debug = createDebug('bb.js:wasm');
-
-export async function fetchModuleAndThreads(desiredThreads = 32) {
+export async function fetchModuleAndThreads(
+  desiredThreads = 32,
+  wasmPath?: string,
+  logger: (msg: string) => void = createDebug('bb.js:fetch_mat'),
+) {
   const shared = getSharedMemoryAvailable();
 
-  const availableThreads = shared ? await getAvailableThreads() : 1;
+  const availableThreads = shared ? await getAvailableThreads(logger) : 1;
   // We limit the number of threads to 32 as we do not benefit from greater numbers.
   const limitedThreads = Math.min(desiredThreads, availableThreads, 32);
 
-  const code = await fetchCode(shared);
+  logger(`Fetching bb wasm from ${wasmPath ?? 'default location'}`);
+  const code = await fetchCode(shared, wasmPath);
+  logger(`Compiling bb wasm of ${code.byteLength} bytes`);
   const module = await WebAssembly.compile(code);
+  logger('Compilation of bb wasm complete');
   return { module, threads: limitedThreads };
 }
 
-async function getAvailableThreads(): Promise<number> {
+async function getAvailableThreads(logger: (msg: string) => void): Promise<number> {
   if (typeof navigator !== 'undefined' && navigator.hardwareConcurrency) {
     return navigator.hardwareConcurrency;
   } else {
     try {
       const os = await import('os');
       return os.cpus().length;
-    } catch (e) {
-      debug(`Could not detect environment. Falling back to one thread.: {e}`);
+    } catch (e: any) {
+      logger(
+        `Could not detect environment to query number of threads. Falling back to one thread. Error: ${e.message ?? e}`,
+      );
       return 1;
     }
   }
@@ -38,11 +45,15 @@ export class BarretenbergWasm extends BarretenbergWasmMain {
    * Construct and initialize BarretenbergWasm within a Worker. Return both the worker and the wasm proxy.
    * Used when running in the browser, because we can't block the main thread.
    */
-  public static async new(desiredThreads?: number) {
+  public static async new(
+    desiredThreads?: number,
+    wasmPath?: string,
+    logger: (msg: string) => void = createDebug('bb.js:bb_wasm_main'),
+  ) {
     const worker = createMainWorker();
     const wasm = getRemoteBarretenbergWasm<BarretenbergWasmMainWorker>(worker);
-    const { module, threads } = await fetchModuleAndThreads(desiredThreads);
-    await wasm.init(module, threads, proxy(debug));
+    const { module, threads } = await fetchModuleAndThreads(desiredThreads, wasmPath, logger);
+    await wasm.init(module, threads, proxy(logger));
     return { worker, wasm };
   }
 }

package/src/bigint-array/index.ts

@@ -1,21 +1,43 @@
-export function toBigIntBE(bytes: Uint8Array) {
-  // A Buffer in node, *is* a Uint8Array. We can't refuse it's type.
-  // However the algo below only works on an actual Uint8Array, hence we make a new one to be safe.
-  bytes = new Uint8Array(bytes);
-  let bigint = BigInt(0);
-  const view = new DataView(bytes.buffer);
-  for (let i = 0; i < bytes.byteLength; i++) {
-    bigint = (bigint << BigInt(8)) + BigInt(view.getUint8(i));
-  }
-  return bigint;
+/**
+ * Convert a 32-byte BE Buffer to a BigInt.
+ */
+export function buffer32BytesToBigIntBE(buf: Buffer): bigint {
+  return (
+    (buf.readBigUInt64BE(0) << 192n) +
+    (buf.readBigUInt64BE(8) << 128n) +
+    (buf.readBigUInt64BE(16) << 64n) +
+    buf.readBigUInt64BE(24)
+  );
+}
+
+/**
+ * Convert a BE Uint8Array to a BigInt.
+ */
+export function uint8ArrayToBigIntBE(bytes: Uint8Array): bigint {
+  const buffer = Buffer.from(bytes);
+  return buffer32BytesToBigIntBE(buffer);
 }
 
-export function toBufferBE(value: bigint, byteLength = 32) {
-  const bytes = new Uint8Array(byteLength);
-  const view = new DataView(bytes.buffer);
-  for (let i = 0; i < byteLength; i++) {
-    view.setUint8(byteLength - i - 1, Number(value & BigInt(0xff)));
-    value >>= BigInt(8);
+/**
+ * Convert a BigInt to a 32-byte BE Buffer.
+ */
+export function bigIntToBufferBE(value: bigint, byteLength = 32): Buffer {
+  if (byteLength != 32) {
+    throw new Error(
+      `Only 32 bytes supported for conversion from bigint to buffer, attempted byte length: ${byteLength}`,
+    );
   }
-  return bytes;
+  const buf = Buffer.alloc(byteLength);
+  buf.writeBigUInt64BE(value >> 192n, 0);
+  buf.writeBigUInt64BE((value >> 128n) & 0xffffffffffffffffn, 8);
+  buf.writeBigUInt64BE((value >> 64n) & 0xffffffffffffffffn, 16);
+  buf.writeBigUInt64BE(value & 0xffffffffffffffffn, 24);
+  return buf;
+}
+
+/**
+ * Convert a BigInt to a 32-byte BE Uint8Array.
+ */
+export function bigIntToUint8ArrayBE(value: bigint, byteLength = 32): Uint8Array {
+  return new Uint8Array(bigIntToBufferBE(value, byteLength));
 }

package/src/crs/net_crs.ts

@@ -1,3 +1,4 @@
+import { retry, makeBackoff } from '../retry/index.js';
 /**
  * Downloader for CRS from the web or local.
  */
@@ -28,12 +29,16 @@ export class NetCrs {
 
     const g1End = this.numPoints * 64 - 1;
 
-    const response = await fetch('https://aztec-ignition.s3.amazonaws.com/MAIN%20IGNITION/flat/g1.dat', {
-      headers: {
-        Range: `bytes=0-${g1End}`,
-      },
-      cache: 'force-cache',
-    });
+    const response = await retry(
+      () =>
+        fetch('https://aztec-ignition.s3.amazonaws.com/MAIN%20IGNITION/flat/g1.dat', {
+          headers: {
+            Range: `bytes=0-${g1End}`,
+          },
+          cache: 'force-cache',
+        }),
+      makeBackoff([5, 5, 5]),
+    );
 
     return (this.data = new Uint8Array(await response.arrayBuffer()));
   }
@@ -42,9 +47,13 @@ export class NetCrs {
    * Download the G2 points data.
    */
   async downloadG2Data() {
-    const response2 = await fetch('https://aztec-ignition.s3.amazonaws.com/MAIN%20IGNITION/flat/g2.dat', {
-      cache: 'force-cache',
-    });
+    const response2 = await retry(
+      () =>
+        fetch('https://aztec-ignition.s3.amazonaws.com/MAIN%20IGNITION/flat/g2.dat', {
+          cache: 'force-cache',
+        }),
+      makeBackoff([5, 5, 5]),
+    );
 
     return (this.g2Data = new Uint8Array(await response2.arrayBuffer()));
   }

package/src/crs/node/index.ts

@@ -4,16 +4,22 @@ import { stat } from 'fs/promises';
 import createDebug from 'debug';
 import { homedir } from 'os';
 
-const debug = createDebug('bb.js:crs');
-
 /**
  * Generic CRS finder utility class.
  */
 export class Crs {
-  constructor(public readonly numPoints: number, public readonly path: string) {}
-
-  static async new(numPoints: number, crsPath = homedir() + '/.bb-crs') {
-    const crs = new Crs(numPoints, crsPath);
+  constructor(
+    public readonly numPoints: number,
+    public readonly path: string,
+    private readonly logger: (msg: string) => void = createDebug('bb.js:crs'),
+  ) {}
+
+  static async new(
+    numPoints: number,
+    crsPath = homedir() + '/.bb-crs',
+    logger: (msg: string) => void = createDebug('bb.js:crs'),
+  ) {
+    const crs = new Crs(numPoints, crsPath, logger);
     await crs.init();
     return crs;
   }
@@ -29,11 +35,11 @@ export class Crs {
       .catch(() => 0);
 
     if (g1FileSize >= this.numPoints * 64 && g1FileSize % 64 == 0 && g2FileSize == 128) {
-      debug(`using cached crs of size: ${g1FileSize / 64}`);
+      this.logger(`Using cached CRS of size ${g1FileSize / 64}`);
       return;
     }
 
-    debug(`downloading crs of size: ${this.numPoints}`);
+    this.logger(`Downloading CRS of size ${this.numPoints} into ${this.path}`);
     const crs = new NetCrs(this.numPoints);
     await crs.init();
     writeFileSync(this.path + '/bn254_g1.dat', crs.getG1Data());
@@ -45,7 +51,9 @@ export class Crs {
    * @returns The points data.
    */
   getG1Data(): Uint8Array {
-    const length = this.numPoints * 64;
+    // Ensure length > 0, otherwise we might read a huge file.
+    // This is a backup.
+    const length = Math.max(this.numPoints, 1) * 64;
     const fd = openSync(this.path + '/bn254_g1.dat', 'r');
     const buffer = new Uint8Array(length);
     readSync(fd, buffer, 0, length, 0);
@@ -66,10 +74,18 @@ export class Crs {
  * Generic Grumpkin CRS finder utility class.
  */
 export class GrumpkinCrs {
-  constructor(public readonly numPoints: number, public readonly path: string) {}
-
-  static async new(numPoints: number, crsPath = homedir() + '/.bb-crs') {
-    const crs = new GrumpkinCrs(numPoints, crsPath);
+  constructor(
+    public readonly numPoints: number,
+    public readonly path: string,
+    private readonly logger: (msg: string) => void = createDebug('bb.js:crs'),
+  ) {}
+
+  static async new(
+    numPoints: number,
+    crsPath = homedir() + '/.bb-crs',
+    logger: (msg: string) => void = createDebug('bb.js:crs'),
+  ) {
+    const crs = new GrumpkinCrs(numPoints, crsPath, logger);
     await crs.init();
     return crs;
   }
@@ -82,11 +98,11 @@ export class GrumpkinCrs {
       .catch(() => 0);
 
     if (g1FileSize >= this.numPoints * 64 && g1FileSize % 64 == 0) {
-      debug(`using cached crs of size: ${g1FileSize / 64}`);
+      this.logger(`Using cached Grumpkin CRS of size ${g1FileSize / 64}`);
       return;
     }
 
-    debug(`downloading crs of size: ${this.numPoints}`);
+    this.logger(`Downloading Grumpkin CRS of size ${this.numPoints} into ${this.path}`);
     const crs = new NetGrumpkinCrs(this.numPoints);
     await crs.init();
     writeFileSync(this.path + '/grumpkin_g1.dat', crs.getG1Data());

package/src/examples/simple.test.ts

@@ -19,7 +19,7 @@ describe('simple', () => {
 
   afterAll(async () => {
     await api.destroy();
-  });
+  }, 10000);
 
   it('should construct 512k gate proof', async () => {
     const valid = await api.examplesSimpleCreateAndVerifyProof();

package/src/index.ts

@@ -8,5 +8,6 @@ export {
   UltraHonkBackend,
   AztecClientBackend,
 } from './barretenberg/index.js';
+
 export { RawBuffer, Fr } from './types/index.js';
 export { splitHonkProof, reconstructHonkProof, ProofData } from './proof/index.js';