@aztec/blob-lib 0.0.0-test.0

package/src/encoding.ts

@@ -0,0 +1,138 @@
+import { Fr } from '@aztec/foundation/fields';
+import { BufferReader, FieldReader } from '@aztec/foundation/serialize';
+
+import type { Blob as BlobBuffer } from 'c-kzg';
+
+// Note duplicated from stdlib !
+// This will appear as 0x74785f7374617274 in logs
+export const TX_START_PREFIX = 8392562855083340404n;
+// These are helper constants to decode tx effects from blob encoded fields
+export const TX_START_PREFIX_BYTES_LENGTH = TX_START_PREFIX.toString(16).length / 2;
+// 7 bytes for: | 0 | txlen[0] | txlen[1] | 0 | REVERT_CODE_PREFIX | 0 | revertCode |
+export const TX_EFFECT_PREFIX_BYTE_LENGTH = TX_START_PREFIX_BYTES_LENGTH + 7;
+export const REVERT_CODE_PREFIX = 1;
+
+/**
+ * Deserializes a blob buffer into an array of field elements.
+ *
+ * Blobs are converted into BN254 fields to perform a poseidon2 hash on them (fieldHash).
+ * This method is sparse, meaning it does not include trailing zeros at the end of the blob.
+ *
+ * However, we cannot simply trim the zero's from the end of the blob, as some logs may include zero's
+ * within them.
+ * If we end on a set of zeros, such as the log below:
+ * length 7: [ a, b, c, d, e, 0, 0]
+ *
+ * we will end up with the incorrect hash if we trim the zeros from the end.
+ *
+ * Each transactions logs contains a TX start prefix, which includes a string followed
+ * by the length ( in field elements ) of the transaction's log.
+ *
+ * This function finds the end of the last transaction's logs, and returns the array up to this point.
+ *
+ * We search for a series of Tx Prefixes progressing the cursor in the field reader until we hit
+ * a field that is not a Tx Prefix, this indicates that we have reached the end of the last transaction's logs.
+ *
+ * +------------------+------------------+------------------+------------------+
+ * | TX1 Start Prefix | TX1 Log Fields   | TX2 Start Prefix | Padded zeros     |
+ * | [3 a,b,c]        | [3, a, b, c]     | [5 d,e,f,0,0]    | [0, 0, 0, .., 0] |
+ * +------------------+------------------+------------------+------------------+
+ *                                                          ^
+ *                                                          |
+ * Function reads until here --------------------------------
+ *
+ * @param blob - The blob buffer to deserialize.
+ * @returns An array of field elements.
+ */
+export function deserializeEncodedBlobToFields(blob: BlobBuffer): Fr[] {
+  // Convert blob buffer to array of field elements
+  const reader = BufferReader.asReader(blob);
+  const array = reader.readArray(blob.length >> 5, Fr); // >> 5 = / 32 (bytes per field)
+  const fieldReader = FieldReader.asReader(array);
+
+  // Read fields until we hit zeros at the end
+  while (!fieldReader.isFinished()) {
+    const currentField = fieldReader.peekField();
+
+    // Stop when we hit a zero field
+    if (!currentField || currentField.isZero()) {
+      break;
+    }
+
+    // Skip the remaining fields in this transaction
+    const len = getLengthFromFirstField(currentField);
+    fieldReader.skip(len);
+  }
+
+  // Return array up to last non-zero field
+  return array.slice(0, fieldReader.cursor);
+}
+
+/**
+ * Get the length of the transaction from the first field.
+ *
+ * @param firstField - The first field of the transaction.
+ * @returns The length of the transaction.
+ *
+ * @throws If the first field does not include the correct prefix - encoding invalid.
+ */
+export function getLengthFromFirstField(firstField: Fr): number {
+  // Check that the first field includes the correct prefix
+  if (!isValidFirstField(firstField)) {
+    throw new Error('Invalid prefix');
+  }
+  const buf = firstField.toBuffer().subarray(-TX_EFFECT_PREFIX_BYTE_LENGTH);
+  return new Fr(buf.subarray(TX_START_PREFIX_BYTES_LENGTH + 1, TX_START_PREFIX_BYTES_LENGTH + 3)).toNumber();
+}
+
+/**
+ * Determines whether a field is the first field of a tx effect
+ */
+export function isValidFirstField(field: Fr): boolean {
+  const buf = field.toBuffer();
+  if (
+    !buf
+      .subarray(0, field.size - TX_EFFECT_PREFIX_BYTE_LENGTH)
+      .equals(Buffer.alloc(field.size - TX_EFFECT_PREFIX_BYTE_LENGTH))
+  ) {
+    return false;
+  }
+  const sliced = buf.subarray(-TX_EFFECT_PREFIX_BYTE_LENGTH);
+  if (
+    // Checking we start with the correct prefix...
+    !new Fr(sliced.subarray(0, TX_START_PREFIX_BYTES_LENGTH)).equals(new Fr(TX_START_PREFIX)) ||
+    // ...and include the revert code prefix..
+    sliced[sliced.length - 3] !== REVERT_CODE_PREFIX ||
+    // ...and the following revert code is valid.
+    sliced[sliced.length - 1] > 4
+  ) {
+    return false;
+  }
+  return true;
+}
+
+/**
+ * Extract the fields from a blob buffer, but do not take into account encoding
+ * that will include trailing zeros.
+ *
+ * +------------------+------------------+------------------+------------------+
+ * |                  |                  |                  | Padded zeros     |
+ * | [3 a,b,c]        | [3, a, b, c]     | [5 d,e,f,0,0]    | [0, 0, 0, .., 0] |
+ * +------------------+------------------+------------------+------------------+
+ *                                                ^
+ *                                                |
+ * Function reads until here ----------------------
+ */
+export function extractBlobFieldsFromBuffer(blob: BlobBuffer): Fr[] {
+  const reader = BufferReader.asReader(blob);
+  const array = reader.readArray(blob.length >> 5, Fr);
+
+  // Find the index of the last non-zero field
+  let lastNonZeroIndex = array.length - 1;
+  while (lastNonZeroIndex >= 0 && array[lastNonZeroIndex].isZero()) {
+    lastNonZeroIndex--;
+  }
+
+  // Return the trimmed array
+  return array.slice(0, lastNonZeroIndex + 1);
+}

package/src/errors.ts

@@ -0,0 +1,6 @@
+export class BlobDeserializationError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = 'BlobDeserializationError';
+  }
+}

package/src/index.ts

@@ -0,0 +1,23 @@
+import cKzg from 'c-kzg';
+
+/* eslint-disable import/no-named-as-default-member */
+const { loadTrustedSetup } = cKzg;
+
+export * from './blob.js';
+export * from './encoding.js';
+export * from './interface.js';
+export * from './errors.js';
+export * from './blob_public_inputs.js';
+export * from './sponge_blob.js';
+
+try {
+  loadTrustedSetup();
+} catch (error: any) {
+  if (error.message.includes('trusted setup is already loaded')) {
+    // NB: The c-kzg lib has no way of checking whether the setup is loaded or not,
+    // and it throws an error if it's already loaded, even though nothing is wrong.
+    // This is a rudimentary way of ensuring we load the trusted setup if we need it.
+  } else {
+    throw new Error(error);
+  }
+}

package/src/interface.ts

@@ -0,0 +1,11 @@
+/**
+ * The relevant parts of a response from https://ethereum.github.io/beacon-APIs/?urls.primaryName=dev#/Beacon/getBlobSidecars
+ */
+export interface BlobJson {
+  blob: string;
+  index?: number;
+  // eslint-disable-next-line camelcase
+  kzg_commitment: string;
+  // eslint-disable-next-line camelcase
+  kzg_proof: string;
+}

package/src/sponge_blob.ts

@@ -0,0 +1,178 @@
+import { type FieldsOf, makeTuple } from '@aztec/foundation/array';
+import { poseidon2Permutation } from '@aztec/foundation/crypto';
+import { Fr } from '@aztec/foundation/fields';
+import {
+  BufferReader,
+  FieldReader,
+  type Tuple,
+  serializeToBuffer,
+  serializeToFields,
+} from '@aztec/foundation/serialize';
+
+/**
+ * A Poseidon2 sponge used to accumulate data that will be added to a blob.
+ * See noir-projects/noir-protocol-circuits/crates/types/src/abis/sponge_blob.nr.
+ */
+export class SpongeBlob {
+  constructor(
+    /** Sponge with absorbed tx effects that will go into a blob. */
+    public readonly sponge: Poseidon2Sponge,
+    /** Number of effects absorbed so far. */
+    public fields: number,
+    /** Number of effects that will be absorbed. */
+    public readonly expectedFields: number,
+  ) {}
+
+  static fromBuffer(buffer: Buffer | BufferReader): SpongeBlob {
+    const reader = BufferReader.asReader(buffer);
+    return new SpongeBlob(reader.readObject(Poseidon2Sponge), reader.readNumber(), reader.readNumber());
+  }
+
+  toBuffer() {
+    return serializeToBuffer(this.sponge, this.fields, this.expectedFields);
+  }
+
+  static getFields(fields: FieldsOf<SpongeBlob>) {
+    return [fields.sponge, fields.fields, fields.expectedFields];
+  }
+
+  toFields(): Fr[] {
+    return serializeToFields(...SpongeBlob.getFields(this));
+  }
+
+  static fromFields(fields: Fr[] | FieldReader): SpongeBlob {
+    const reader = FieldReader.asReader(fields);
+    return new SpongeBlob(
+      reader.readObject(Poseidon2Sponge),
+      reader.readField().toNumber(),
+      reader.readField().toNumber(),
+    );
+  }
+
+  clone() {
+    return SpongeBlob.fromBuffer(this.toBuffer());
+  }
+
+  async absorb(fields: Fr[]) {
+    if (this.fields + fields.length > this.expectedFields) {
+      throw new Error(
+        `Attempted to fill spongeblob with ${this.fields + fields.length}, but it has a max of ${this.expectedFields}`,
+      );
+    }
+    await this.sponge.absorb(fields);
+    this.fields += fields.length;
+  }
+
+  async squeeze(): Promise<Fr> {
+    // If the blob sponge is not 'full', we append 1 to match Poseidon2::hash_internal()
+    // NB: There is currently no use case in which we don't 'fill' a blob sponge, but adding for completeness
+    if (this.fields != this.expectedFields) {
+      await this.sponge.absorb([Fr.ONE]);
+    }
+    return this.sponge.squeeze();
+  }
+
+  static empty(): SpongeBlob {
+    return new SpongeBlob(Poseidon2Sponge.empty(), 0, 0);
+  }
+
+  static init(expectedFields: number): SpongeBlob {
+    return new SpongeBlob(Poseidon2Sponge.init(expectedFields), 0, expectedFields);
+  }
+}
+
+// This is just noir's stdlib version of the poseidon2 sponge. We use it for a blob-specific implmentation of the hasher.
+export class Poseidon2Sponge {
+  constructor(
+    public cache: Tuple<Fr, 3>,
+    public state: Tuple<Fr, 4>,
+    public cacheSize: number,
+    public squeezeMode: boolean,
+  ) {}
+
+  static fromBuffer(buffer: Buffer | BufferReader): Poseidon2Sponge {
+    const reader = BufferReader.asReader(buffer);
+    return new Poseidon2Sponge(
+      reader.readArray(3, Fr),
+      reader.readArray(4, Fr),
+      reader.readNumber(),
+      reader.readBoolean(),
+    );
+  }
+
+  toBuffer() {
+    return serializeToBuffer(this.cache, this.state, this.cacheSize, this.squeezeMode);
+  }
+
+  static getFields(fields: FieldsOf<Poseidon2Sponge>) {
+    return [fields.cache, fields.state, fields.cacheSize, fields.squeezeMode];
+  }
+
+  toFields(): Fr[] {
+    return serializeToFields(...Poseidon2Sponge.getFields(this));
+  }
+
+  static fromFields(fields: Fr[] | FieldReader): Poseidon2Sponge {
+    const reader = FieldReader.asReader(fields);
+    return new Poseidon2Sponge(
+      reader.readFieldArray(3),
+      reader.readFieldArray(4),
+      reader.readField().toNumber(),
+      reader.readBoolean(),
+    );
+  }
+
+  static empty(): Poseidon2Sponge {
+    return new Poseidon2Sponge(
+      makeTuple(3, () => Fr.ZERO),
+      makeTuple(4, () => Fr.ZERO),
+      0,
+      false,
+    );
+  }
+
+  static init(expectedFields: number): Poseidon2Sponge {
+    const iv = new Fr(expectedFields).mul(new Fr(BigInt('18446744073709551616')));
+    const sponge = Poseidon2Sponge.empty();
+    sponge.state[3] = iv;
+    return sponge;
+  }
+
+  // Note: there isn't currently an impl in ts that allows for a custom aborption via an
+  // existing sponge.
+  // A custom blob-based impl of noir/noir-repo/noir_stdlib/src/hash/poseidon2.nr
+  async performDuplex() {
+    for (let i = 0; i < this.cache.length; i++) {
+      if (i < this.cacheSize) {
+        this.state[i] = this.state[i].add(this.cache[i]);
+      }
+    }
+    const perm = await poseidon2Permutation(this.state);
+    // ts doesn't understand that the above always gives 4
+    this.state = [perm[0], perm[1], perm[2], perm[3]];
+  }
+
+  async absorb(fields: Fr[]) {
+    if (this.squeezeMode) {
+      throw new Error(`Poseidon sponge is not able to absorb more inputs.`);
+    }
+    for (const field of fields) {
+      if (this.cacheSize == this.cache.length) {
+        await this.performDuplex();
+        this.cache[0] = field;
+        this.cacheSize = 1;
+      } else {
+        this.cache[this.cacheSize++] = field;
+      }
+    }
+  }
+
+  async squeeze(): Promise<Fr> {
+    if (this.squeezeMode) {
+      throw new Error(`Poseidon sponge has already been squeezed.`);
+    }
+    await this.performDuplex();
+    this.squeezeMode = true;
+    return this.state[0];
+  }
+}

package/src/testing.ts

@@ -0,0 +1,95 @@
+import { BLOBS_PER_BLOCK } from '@aztec/constants';
+import { makeTuple } from '@aztec/foundation/array';
+import { toBufferBE } from '@aztec/foundation/bigint-buffer';
+import { Fr } from '@aztec/foundation/fields';
+
+import { Blob } from './blob.js';
+import { BlobPublicInputs, BlockBlobPublicInputs } from './blob_public_inputs.js';
+import { TX_START_PREFIX, TX_START_PREFIX_BYTES_LENGTH } from './encoding.js';
+import { Poseidon2Sponge, SpongeBlob } from './sponge_blob.js';
+
+/**
+ * Makes arbitrary poseidon sponge for blob inputs.
+ * Note: will not verify inside the circuit.
+ * @param seed - The seed to use for generating the sponge.
+ * @returns A sponge blob instance.
+ */
+export function makeSpongeBlob(seed = 1): SpongeBlob {
+  return new SpongeBlob(
+    new Poseidon2Sponge(
+      makeTuple(3, i => new Fr(i)),
+      makeTuple(4, i => new Fr(i)),
+      1,
+      false,
+    ),
+    seed,
+    seed + 1,
+  );
+}
+
+/**
+ * Makes arbitrary blob public inputs.
+ * Note: will not verify inside the circuit.
+ * @param seed - The seed to use for generating the blob inputs.
+ * @returns A blob public inputs instance.
+ */
+export function makeBlobPublicInputs(seed = 1): BlobPublicInputs {
+  return new BlobPublicInputs(
+    new Fr(seed),
+    BigInt(seed + 1),
+    makeTuple(2, i => new Fr(i)),
+  );
+}
+
+/**
+ * Makes arbitrary block blob public inputs.
+ * Note: will not verify inside the circuit.
+ * @param seed - The seed to use for generating the blob inputs.
+ * @returns A block blob public inputs instance.
+ */
+export function makeBlockBlobPublicInputs(seed = 1): BlockBlobPublicInputs {
+  return new BlockBlobPublicInputs(makeTuple(BLOBS_PER_BLOCK, () => makeBlobPublicInputs(seed)));
+}
+
+// TODO: copied form stdlib tx effect
+function encodeFirstField(length: number): Fr {
+  const lengthBuf = Buffer.alloc(2);
+  lengthBuf.writeUInt16BE(length, 0);
+  return new Fr(
+    Buffer.concat([
+      toBufferBE(TX_START_PREFIX, TX_START_PREFIX_BYTES_LENGTH),
+      Buffer.alloc(1),
+      lengthBuf,
+      Buffer.alloc(1),
+      Buffer.from([1]),
+      Buffer.alloc(1),
+      Buffer.alloc(1),
+    ]),
+  );
+}
+
+/**
+ * Make an encoded blob with the given length
+ *
+ * This will deserialise correctly in the archiver
+ * @param length
+ * @returns
+ */
+export function makeEncodedBlob(length: number): Promise<Blob> {
+  return Blob.fromFields([encodeFirstField(length + 1), ...Array.from({ length: length }, () => Fr.random())]);
+}
+
+/**
+ * Make an unencoded blob with the given length
+ *
+ * This will fail deserialisation in the archiver
+ * @param length
+ * @returns
+ */
+export function makeUnencodedBlob(length: number): Promise<Blob> {
+  return Blob.fromFields([...Array.from({ length: length }, () => Fr.random())]);
+}
+
+export function makeEncodedBlobFields(fields: Fr[]): Promise<Blob> {
+  return Blob.fromFields([encodeFirstField(fields.length + 1), ...fields]);
+}