@aztec/bb.js 0.0.1-alpha.6 → 0.0.1-fake-ceab37513c

package/src/crs/browser/cached_net_crs.ts

@@ -1,5 +1,4 @@
-import { concatenateUint8Arrays, numToUInt32BE } from '../../serialize/index.js';
-import { NetCrs } from '../net_crs.js';
+import { NetCrs, NetGrumpkinCrs } from '../net_crs.js';
 import { get, set } from 'idb-keyval';
 
 /**
@@ -58,3 +57,43 @@ export class CachedNetCrs {
     return this.g2Data;
   }
 }
+
+/**
+ * Downloader for CRS from the web or local.
+ */
+export class CachedNetGrumpkinCrs {
+  private g1Data!: Uint8Array;
+
+  constructor(public readonly numPoints: number) {}
+
+  static async new(numPoints: number) {
+    const crs = new CachedNetGrumpkinCrs(numPoints);
+    await crs.init();
+    return crs;
+  }
+
+  /**
+   * Download the data.
+   */
+  async init() {
+    // Check if data is in IndexedDB
+    const g1Data = await get('grumpkinG1Data');
+    const netGrumpkinCrs = new NetGrumpkinCrs(this.numPoints);
+    const g1DataLength = this.numPoints * 64;
+
+    if (!g1Data || g1Data.length < g1DataLength) {
+      this.g1Data = await netGrumpkinCrs.downloadG1Data();
+      await set('grumpkinG1Data', this.g1Data);
+    } else {
+      this.g1Data = g1Data;
+    }
+  }
+
+  /**
+   * G1 points data for prover key.
+   * @returns The points data.
+   */
+  getG1Data(): Uint8Array {
+    return this.g1Data;
+  }
+}

package/src/crs/browser/index.ts

@@ -1 +1 @@
-export { CachedNetCrs as Crs } from './cached_net_crs.js';
+export { CachedNetCrs as Crs, CachedNetGrumpkinCrs as GrumpkinCrs } from './cached_net_crs.js';

package/src/crs/index.ts

@@ -1 +1 @@
-export { Crs } from './node/index.js';
+export { Crs, GrumpkinCrs } from './node/index.js';

package/src/crs/net_crs.ts

@@ -1,3 +1,4 @@
+import { retry, makeBackoff } from '../retry/index.js';
 /**
  * Downloader for CRS from the web or local.
  */
@@ -20,17 +21,24 @@ export class NetCrs {
     await this.downloadG2Data();
   }
 
-  async downloadG1Data() {
-    const g1Start = 28;
-    const g1End = g1Start + this.numPoints * 64 - 1;
+  /**
+   * Opens up a ReadableStream to the points data
+   */
+  async streamG1Data(): Promise<ReadableStream<Uint8Array>> {
+    const response = await this.fetchG1Data();
+    return response.body!;
+  }
 
-    const response = await fetch('https://aztec-ignition.s3.amazonaws.com/MAIN%20IGNITION/monomial/transcript00.dat', {
-      headers: {
-        Range: `bytes=${g1Start}-${g1End}`,
-      },
-      cache: 'force-cache',
-    });
+  /**
+   * Opens up a ReadableStream to the points data
+   */
+  async streamG2Data(): Promise<ReadableStream<Uint8Array>> {
+    const response = await this.fetchG2Data();
+    return response.body!;
+  }
 
+  async downloadG1Data() {
+    const response = await this.fetchG1Data();
     return (this.data = new Uint8Array(await response.arrayBuffer()));
   }
 
@@ -38,16 +46,7 @@ export class NetCrs {
    * Download the G2 points data.
    */
   async downloadG2Data() {
-    const g2Start = 28 + 5040000 * 64;
-    const g2End = g2Start + 128 - 1;
-
-    const response2 = await fetch('https://aztec-ignition.s3.amazonaws.com/MAIN%20IGNITION/sealed/transcript00.dat', {
-      headers: {
-        Range: `bytes=${g2Start}-${g2End}`,
-      },
-      cache: 'force-cache',
-    });
-
+    const response2 = await this.fetchG2Data();
     return (this.g2Data = new Uint8Array(await response2.arrayBuffer()));
   }
 
@@ -66,4 +65,100 @@ export class NetCrs {
   getG2Data(): Uint8Array {
     return this.g2Data;
   }
+
+  /**
+   * Fetches the appropriate range of points from a remote source
+   */
+  private async fetchG1Data(): Promise<Response> {
+    // Skip the download if numPoints is 0 (would download the entire file due to bad range header otherwise)
+    if (this.numPoints === 0) {
+      return new Response(new Uint8Array([]));
+    }
+
+    const g1End = this.numPoints * 64 - 1;
+    return await retry(
+      () =>
+        fetch('https://crs.aztec.network/g1.dat', {
+          headers: {
+            Range: `bytes=0-${g1End}`,
+          },
+          cache: 'force-cache',
+        }),
+      makeBackoff([5, 5, 5]),
+    );
+  }
+
+  /**
+   * Fetches the appropriate range of points from a remote source
+   */
+  private async fetchG2Data(): Promise<Response> {
+    return await retry(
+      () =>
+        fetch('https://crs.aztec.network/g2.dat', {
+          cache: 'force-cache',
+        }),
+      makeBackoff([5, 5, 5]),
+    );
+  }
+}
+
+/**
+ * Downloader for CRS from the web or local.
+ */
+export class NetGrumpkinCrs {
+  private data!: Uint8Array;
+
+  constructor(
+    /**
+     * The number of circuit gates.
+     */
+    public readonly numPoints: number,
+  ) {}
+
+  /**
+   * Download the data.
+   */
+  async init() {
+    await this.downloadG1Data();
+  }
+
+  async downloadG1Data() {
+    const response = await this.fetchG1Data();
+    return (this.data = new Uint8Array(await response.arrayBuffer()));
+  }
+
+  /**
+   * Opens up a ReadableStream to the points data
+   */
+  async streamG1Data(): Promise<ReadableStream<Uint8Array>> {
+    const response = await this.fetchG1Data();
+    return response.body!;
+  }
+
+  /**
+   * G1 points data for prover key.
+   * @returns The points data.
+   */
+  getG1Data(): Uint8Array {
+    return this.data;
+  }
+
+  /**
+   * Fetches the appropriate range of points from a remote source
+   */
+  private async fetchG1Data(): Promise<Response> {
+    // Skip the download if numPoints is 0 (would download the entire file due to bad range header otherwise)
+    if (this.numPoints === 0) {
+      return new Response(new Uint8Array([]));
+    }
+
+    const g1End = this.numPoints * 64 - 1;
+
+    return await fetch('https://crs.aztec.network/grumpkin_g1.dat', {
+      headers: {
+        Range: `bytes=0-${g1End}`,
+      },
+      cache: 'force-cache',
+    });
+  }
 }

package/src/crs/node/index.ts

@@ -1,33 +1,55 @@
-import { concatenateUint8Arrays, numToUInt32BE } from '../../serialize/serialize.js';
-import { NetCrs } from '../net_crs.js';
-import { FileCrs } from './file_crs.js';
+import { NetCrs, NetGrumpkinCrs } from '../net_crs.js';
+import { closeSync, mkdirSync, openSync, readFileSync, readSync, writeFileSync, createWriteStream } from 'fs';
+import { stat } from 'fs/promises';
+import { Readable } from 'stream';
+import { homedir } from 'os';
+import { finished } from 'stream/promises';
+import { createDebugLogger } from '../../log/index.js';
+import { join } from 'path';
 
 /**
  * Generic CRS finder utility class.
  */
 export class Crs {
-  private crs: FileCrs | NetCrs;
-
   constructor(
-    /**
-     * The number of circuit gates.
-     */
     public readonly numPoints: number,
-  ) {
-    this.crs = FileCrs.defaultExists() ? new FileCrs(numPoints) : new NetCrs(numPoints);
-  }
+    public readonly path: string,
+    private readonly logger: (msg: string) => void = createDebugLogger('crs'),
+  ) {}
 
-  static async new(numPoints: number) {
-    const crs = new Crs(numPoints);
+  static async new(
+    numPoints: number,
+    crsPath = process.env.CRS_PATH ?? join(homedir(), '.bb-crs'),
+    logger: (msg: string) => void = createDebugLogger('crs'),
+  ) {
+    const crs = new Crs(numPoints, crsPath, logger);
     await crs.init();
     return crs;
   }
 
-  /**
-   * Read CRS from our chosen source.
-   */
-  async init() {
-    await this.crs.init();
+  async init(): Promise<void> {
+    mkdirSync(this.path, { recursive: true });
+
+    const g1FileSize = await stat(this.path + '/bn254_g1.dat')
+      .then(stats => stats.size)
+      .catch(() => 0);
+    const g2FileSize = await stat(this.path + '/bn254_g2.dat')
+      .then(stats => stats.size)
+      .catch(() => 0);
+
+    if (g1FileSize >= this.numPoints * 64 && g1FileSize % 64 == 0 && g2FileSize == 128) {
+      this.logger(`Using cached CRS of size ${g1FileSize / 64}`);
+      return;
+    }
+
+    this.logger(`Downloading CRS of size ${this.numPoints} into ${this.path}`);
+    const crs = new NetCrs(this.numPoints);
+    const [g1, g2] = await Promise.all([crs.streamG1Data(), crs.streamG2Data()]);
+
+    await Promise.all([
+      finished(Readable.fromWeb(g1 as any).pipe(createWriteStream(this.path + '/bn254_g1.dat'))),
+      finished(Readable.fromWeb(g2 as any).pipe(createWriteStream(this.path + '/bn254_g2.dat'))),
+    ]);
   }
 
   /**
@@ -35,7 +57,14 @@ export class Crs {
    * @returns The points data.
    */
   getG1Data(): Uint8Array {
-    return this.crs.getG1Data();
+    // Ensure length > 0, otherwise we might read a huge file.
+    // This is a backup.
+    const length = Math.max(this.numPoints, 1) * 64;
+    const fd = openSync(this.path + '/bn254_g1.dat', 'r');
+    const buffer = new Uint8Array(length);
+    readSync(fd, buffer, 0, length, 0);
+    closeSync(fd);
+    return buffer;
   }
 
   /**
@@ -43,6 +72,55 @@ export class Crs {
    * @returns The points data.
    */
   getG2Data(): Uint8Array {
-    return this.crs.getG2Data();
+    return readFileSync(this.path + '/bn254_g2.dat');
+  }
+}
+
+/**
+ * Generic Grumpkin CRS finder utility class.
+ */
+export class GrumpkinCrs {
+  constructor(
+    public readonly numPoints: number,
+    public readonly path: string,
+    private readonly logger: (msg: string) => void = createDebugLogger('crs'),
+  ) {}
+
+  static async new(
+    numPoints: number,
+    crsPath = process.env.CRS_PATH ?? join(homedir(), '.bb-crs'),
+    logger: (msg: string) => void = createDebugLogger('crs'),
+  ) {
+    const crs = new GrumpkinCrs(numPoints, crsPath, logger);
+    await crs.init();
+    return crs;
+  }
+
+  async init(): Promise<void> {
+    mkdirSync(this.path, { recursive: true });
+
+    const g1FileSize = await stat(this.path + '/grumpkin_g1.flat.dat')
+      .then(stats => stats.size)
+      .catch(() => 0);
+
+    if (g1FileSize >= this.numPoints * 64 && g1FileSize % 64 == 0) {
+      this.logger(`Using cached Grumpkin CRS of size ${g1FileSize / 64}`);
+      return;
+    }
+
+    this.logger(`Downloading Grumpkin CRS of size ${this.numPoints} into ${this.path}`);
+    const crs = new NetGrumpkinCrs(this.numPoints);
+    const stream = await crs.streamG1Data();
+
+    await finished(Readable.fromWeb(stream as any).pipe(createWriteStream(this.path + '/grumpkin_g1.flat.dat')));
+    writeFileSync(this.path + '/grumpkin_size', String(crs.numPoints));
+  }
+
+  /**
+   * G1 points data for prover key.
+   * @returns The points data.
+   */
+  getG1Data(): Uint8Array {
+    return readFileSync(this.path + '/grumpkin_g1.flat.dat');
   }
 }

package/src/index.html

@@ -1,4 +1,4 @@
-<!DOCTYPE html>
+<!doctype html>
 <html>
   <head>
     <title>My App</title>

package/src/index.ts

@@ -1,5 +1,13 @@
-export * from './crs/index.js';
-export * from './barretenberg_wasm/index.js';
-export * from './barretenberg_api/index.js';
-export * from './factory/index.js';
-export { RawBuffer } from './types/index.js';
+export { Crs, GrumpkinCrs } from './crs/index.js';
+export {
+  type BackendOptions,
+  Barretenberg,
+  BarretenbergSync,
+  UltraHonkVerifierBackend,
+  UltraHonkBackend,
+  AztecClientBackend,
+} from './barretenberg/index.js';
+
+export { randomBytes } from './random/index.js';
+export { RawBuffer, Fr } from './types/index.js';
+export { splitHonkProof, reconstructHonkProof, deflattenFields, type ProofData } from './proof/index.js';

package/src/log/browser/index.ts

@@ -0,0 +1,35 @@
+import { pino } from 'pino';
+import { LogOptions } from '../types.js';
+import { Logger } from 'pino';
+
+const defaultOptions = {
+  name: 'bb.js',
+  useOnlyCustomLevels: false,
+  customLevels: { verbose: 25 },
+  browser: { asObject: false },
+};
+
+// Options must be exposed so they can be provided to threads upon creation
+// This way we ensure all loggers are spawned with the same options
+export let logOptions: LogOptions | undefined;
+
+let logger: Logger<'verbose'> | undefined;
+
+export function initLogger({ level = 'info' }: LogOptions = { level: 'info', useStdErr: false }) {
+  if (logger) {
+    return logger;
+  }
+  logOptions = { level, useStdErr: false };
+  logger = pino({ ...defaultOptions, level });
+}
+
+export function createDebugLogger(name: string) {
+  initLogger();
+
+  const sublogger = logger!.child({
+    name,
+  });
+  return (msg: string) => {
+    sublogger.debug(msg);
+  };
+}

package/src/log/index.ts

@@ -0,0 +1 @@
+export { initLogger, createDebugLogger, logOptions } from './node/index.js';

package/src/log/node/index.ts

@@ -0,0 +1,52 @@
+import { Logger, pino } from 'pino';
+import { LogLevels, LogOptions } from '../types.js';
+
+const defaultOptions = {
+  name: 'bb.js',
+  useOnlyCustomLevels: false,
+  customLevels: { verbose: 25 },
+};
+
+const defaultLevel = parseLogLevel(process.env.LOG_LEVEL);
+
+// Options must be exposed so they can be provided to threads upon creation
+// This way we ensure all loggers are spawned with the same options
+export let logOptions: LogOptions | undefined;
+
+let logger: Logger<'verbose'> | undefined;
+
+export function initLogger(
+  { level = defaultLevel, useStdErr = false }: LogOptions = { level: defaultLevel, useStdErr: false },
+) {
+  if (logger) {
+    return logger;
+  }
+  logOptions = { level, useStdErr };
+  const transport = pino.transport({
+    target: 'pino/file',
+    options: { destination: useStdErr ? 2 : 1 },
+  });
+  logger = pino({ ...defaultOptions, level }, transport);
+}
+
+export function createDebugLogger(name: string) {
+  initLogger();
+
+  const sublogger = logger!.child({
+    name,
+  });
+  return (msg: string) => {
+    sublogger.debug(msg);
+  };
+}
+
+function parseLogLevel(logLevel?: string): LogLevels {
+  if (!logLevel) {
+    return 'info';
+  }
+
+  const knownLogLevels: LogLevels[] = ['info', 'debug', 'warn', 'error', 'trace', 'silent', 'verbose'];
+  const [defaultLogLevel] = logLevel.split(';');
+
+  return knownLogLevels.indexOf(defaultLogLevel as LogLevels) !== -1 ? (defaultLogLevel as LogLevels) : 'info';
+}

package/src/log/types.ts

@@ -0,0 +1,6 @@
+export type LogLevels = 'silent' | 'info' | 'debug' | 'warn' | 'error' | 'trace' | 'verbose';
+
+export type LogOptions = {
+  level: LogLevels;
+  useStdErr: boolean;
+};