@aztec/stdlib 0.0.0-test.1 → 0.0.1-commit.b655e406

package/src/slashing/types.ts

@@ -0,0 +1,152 @@
+import { EthAddress } from '@aztec/foundation/eth-address';
+
+import { z } from 'zod';
+
+import { type ZodFor, schemas } from '../schemas/index.js';
+
+export enum OffenseType {
+  UNKNOWN = 0,
+  /** The data for proving an epoch was not publicly available, we slash its committee */
+  DATA_WITHHOLDING = 1,
+  /** An epoch was not successfully proven in time, we slash its committee */
+  VALID_EPOCH_PRUNED = 2,
+  /** A proposer failed to attest or propose during an epoch according to the Sentinel */
+  INACTIVITY = 3,
+  /** A proposer sent an invalid block proposal over the p2p network to the committee */
+  BROADCASTED_INVALID_BLOCK_PROPOSAL = 4,
+  /** A proposer pushed to L1 a block with insufficient committee attestations */
+  PROPOSED_INSUFFICIENT_ATTESTATIONS = 5,
+  /** A proposer pushed to L1 a block with incorrect committee attestations (ie signature from a non-committee member) */
+  PROPOSED_INCORRECT_ATTESTATIONS = 6,
+  /** A committee member attested to a block that was built as a descendent of an invalid block (as in a block with invalid attestations) */
+  ATTESTED_DESCENDANT_OF_INVALID = 7,
+}
+
+export function getOffenseTypeName(offense: OffenseType) {
+  switch (offense) {
+    case OffenseType.UNKNOWN:
+      return 'unknown';
+    case OffenseType.DATA_WITHHOLDING:
+      return 'data_withholding';
+    case OffenseType.VALID_EPOCH_PRUNED:
+      return 'valid_epoch_pruned';
+    case OffenseType.INACTIVITY:
+      return 'inactivity';
+    case OffenseType.BROADCASTED_INVALID_BLOCK_PROPOSAL:
+      return 'broadcasted_invalid_block_proposal';
+    case OffenseType.PROPOSED_INSUFFICIENT_ATTESTATIONS:
+      return 'proposed_insufficient_attestations';
+    case OffenseType.PROPOSED_INCORRECT_ATTESTATIONS:
+      return 'proposed_incorrect_attestations';
+    case OffenseType.ATTESTED_DESCENDANT_OF_INVALID:
+      return 'attested_descendant_of_invalid';
+    default:
+      throw new Error(`Unknown offense type: ${offense}`);
+  }
+}
+
+export const OffenseTypeSchema = z.nativeEnum(OffenseType);
+
+export const OffenseToBigInt: Record<OffenseType, bigint> = {
+  [OffenseType.UNKNOWN]: 0n,
+  [OffenseType.DATA_WITHHOLDING]: 1n,
+  [OffenseType.VALID_EPOCH_PRUNED]: 2n,
+  [OffenseType.INACTIVITY]: 3n,
+  [OffenseType.BROADCASTED_INVALID_BLOCK_PROPOSAL]: 4n,
+  [OffenseType.PROPOSED_INSUFFICIENT_ATTESTATIONS]: 5n,
+  [OffenseType.PROPOSED_INCORRECT_ATTESTATIONS]: 6n,
+  [OffenseType.ATTESTED_DESCENDANT_OF_INVALID]: 7n,
+};
+
+export function bigIntToOffense(offense: bigint): OffenseType {
+  switch (offense) {
+    case 0n:
+      return OffenseType.UNKNOWN;
+    case 1n:
+      return OffenseType.DATA_WITHHOLDING;
+    case 2n:
+      return OffenseType.VALID_EPOCH_PRUNED;
+    case 3n:
+      return OffenseType.INACTIVITY;
+    case 4n:
+      return OffenseType.BROADCASTED_INVALID_BLOCK_PROPOSAL;
+    case 5n:
+      return OffenseType.PROPOSED_INSUFFICIENT_ATTESTATIONS;
+    case 6n:
+      return OffenseType.PROPOSED_INCORRECT_ATTESTATIONS;
+    case 7n:
+      return OffenseType.ATTESTED_DESCENDANT_OF_INVALID;
+    default:
+      throw new Error(`Unknown offense: ${offense}`);
+  }
+}
+
+export type Offense = {
+  validator: EthAddress;
+  amount: bigint;
+  offenseType: OffenseType;
+  epochOrSlot: bigint;
+};
+
+export type OffenseIdentifier = Pick<Offense, 'validator' | 'offenseType' | 'epochOrSlot'>;
+
+export const OffenseSchema = z.object({
+  validator: schemas.EthAddress,
+  amount: schemas.BigInt,
+  offenseType: OffenseTypeSchema,
+  epochOrSlot: schemas.BigInt,
+}) satisfies ZodFor<Offense>;
+
+/** Offense by a validator in the context of a slash payload */
+export type ValidatorSlashOffense = {
+  epochOrSlot: bigint;
+  offenseType: OffenseType;
+};
+
+/** Slashed amount and total offenses by a validator in the context of a slash payload */
+export type ValidatorSlash = {
+  validator: EthAddress;
+  amount: bigint;
+  offenses: ValidatorSlashOffense[];
+};
+
+/** Slash payload as published by the empire slash proposer */
+export type SlashPayload = {
+  address: EthAddress;
+  slashes: ValidatorSlash[];
+  timestamp: bigint;
+};
+
+/** Slash payload with round information from empire slash proposer */
+export type SlashPayloadRound = SlashPayload & { votes: bigint; round: bigint };
+
+export const SlashPayloadRoundSchema = z.object({
+  address: schemas.EthAddress,
+  timestamp: schemas.BigInt,
+  votes: schemas.BigInt,
+  round: schemas.BigInt,
+  slashes: z.array(
+    z.object({
+      validator: schemas.EthAddress,
+      amount: schemas.BigInt,
+      offenses: z.array(z.object({ offenseType: OffenseTypeSchema, epochOrSlot: schemas.BigInt })),
+    }),
+  ),
+}) satisfies ZodFor<SlashPayloadRound>;
+
+/** Votes for a validator slash in the consensus slash proposer */
+export type ValidatorSlashVote = number;
+
+export type ProposerSlashAction =
+  /** Create a new slash payload on an empire-based slash proposer */
+  | { type: 'create-empire-payload'; data: ValidatorSlash[] }
+  /** Vote for a slashing payload on an empire-based slash proposer */
+  | { type: 'vote-empire-payload'; payload: EthAddress }
+  /** Execute a slashing payload on an empire-based slash proposer */
+  | { type: 'execute-empire-payload'; round: bigint }
+  /** Vote for offenses on a consensus slashing proposer */
+  | { type: 'vote-offenses'; votes: ValidatorSlashVote[]; committees: EthAddress[][]; round: bigint }
+  /** Execute a slashing round on a consensus slashing proposer */
+  | { type: 'execute-slash'; committees: EthAddress[][]; round: bigint };
+
+export type ProposerSlashActionType = ProposerSlashAction['type'];

package/src/snapshots/download.ts

@@ -0,0 +1,123 @@
+import { fromEntries, getEntries, maxBy } from '@aztec/foundation/collection';
+import { jsonParseWithSchema } from '@aztec/foundation/json-rpc';
+import type { ReadOnlyFileStore } from '@aztec/stdlib/file-store';
+
+import { createReadStream, createWriteStream } from 'fs';
+import fs from 'fs/promises';
+import pathMod from 'path';
+import { pipeline } from 'stream/promises';
+import { createGunzip, gunzipSync } from 'zlib';
+
+import {
+  SnapshotDataKeys,
+  type SnapshotDataUrls,
+  type SnapshotMetadata,
+  type SnapshotsIndex,
+  type SnapshotsIndexMetadata,
+  SnapshotsIndexSchema,
+} from './types.js';
+
+export async function getSnapshotIndex(
+  metadata: SnapshotsIndexMetadata,
+  store: ReadOnlyFileStore,
+): Promise<SnapshotsIndex | undefined> {
+  const basePath = getBasePath(metadata);
+  const snapshotIndexPath = `${basePath}/index.json`;
+  try {
+    if (await store.exists(snapshotIndexPath)) {
+      const snapshotIndexData = await store.read(snapshotIndexPath);
+      const buf = maybeGunzip(snapshotIndexData);
+      return jsonParseWithSchema(buf.toString('utf-8'), SnapshotsIndexSchema);
+    } else {
+      return undefined;
+    }
+  } catch (err) {
+    throw new Error(`Error reading snapshot index from ${snapshotIndexPath}: ${err}`);
+  }
+}
+
+export async function getLatestSnapshotMetadata(
+  metadata: SnapshotsIndexMetadata,
+  store: ReadOnlyFileStore,
+): Promise<SnapshotMetadata | undefined> {
+  const snapshotsIndex = await getSnapshotIndex(metadata, store);
+  return snapshotsIndex?.snapshots && maxBy(snapshotsIndex?.snapshots, s => s.l1BlockNumber);
+}
+
+export function getBasePath(metadata: SnapshotsIndexMetadata): string {
+  return `aztec-${metadata.l1ChainId}-${metadata.rollupVersion}-${metadata.rollupAddress}`;
+}
+
+export function getSnapshotIndexPath(metadata: SnapshotsIndexMetadata): string {
+  return `${getBasePath(metadata)}/index.json`;
+}
+
+export function makeSnapshotPaths(baseDir: string): SnapshotDataUrls {
+  // We do not use path.join since that screws up protocol prefixes
+  return fromEntries(SnapshotDataKeys.map(key => [key, `${baseDir}/${key}.db`]));
+}
+
+function isGzipMagic(data: Buffer): boolean {
+  return data.length >= 2 && data[0] === 0x1f && data[1] === 0x8b;
+}
+
+function maybeGunzip(data: Buffer): Buffer {
+  const magicNumberIndicatesGzip = isGzipMagic(data);
+
+  if (magicNumberIndicatesGzip) {
+    try {
+      const out = gunzipSync(data);
+      return out;
+    } catch (err) {
+      throw new Error(`Decompression of gzipped data failed: ${(err as Error).message}`);
+    }
+  }
+  return data;
+}
+
+async function detectGzip(localFilePathToPeek: string): Promise<boolean> {
+  // Peek the actual bytes we downloaded.
+  try {
+    const fd = await fs.open(localFilePathToPeek, 'r');
+    try {
+      const header = Buffer.alloc(2);
+      const { bytesRead } = await fd.read(header, 0, 2, 0);
+      return bytesRead >= 2 && isGzipMagic(header);
+    } finally {
+      await fd.close();
+    }
+  } catch {
+    return false;
+  }
+}
+
+export async function downloadSnapshot(
+  snapshot: Pick<SnapshotMetadata, 'dataUrls'>,
+  localPaths: Record<SnapshotDataKeys, string>,
+  store: ReadOnlyFileStore,
+): Promise<void> {
+  await Promise.all(
+    getEntries(localPaths).map(async ([key, path]) => {
+      await fs.mkdir(pathMod.dirname(path), { recursive: true });
+
+      const tmpPath = `${path}.download`;
+      try {
+        const url = snapshot.dataUrls[key];
+        await store.download(url, tmpPath);
+
+        const isGzip = await detectGzip(tmpPath);
+
+        const read = createReadStream(tmpPath);
+        const write = createWriteStream(path);
+        if (isGzip) {
+          const gunzip = createGunzip();
+          await pipeline(read, gunzip, write);
+        } else {
+          await pipeline(read, write);
+        }
+      } finally {
+        await fs.unlink(tmpPath).catch(() => undefined);
+      }
+    }),
+  );
+}

package/src/snapshots/index.ts

@@ -0,0 +1,3 @@
+export * from './types.js';
+export * from './upload.js';
+export { getBasePath, downloadSnapshot, getLatestSnapshotMetadata, makeSnapshotPaths } from './download.js';

package/src/snapshots/types.ts

@@ -0,0 +1,70 @@
+import type { EthAddress } from '@aztec/foundation/eth-address';
+import { type ZodFor, schemas } from '@aztec/foundation/schemas';
+
+import { z } from 'zod';
+
+export const SnapshotDataKeys = [
+  'archiver',
+  'nullifier-tree',
+  'public-data-tree',
+  'note-hash-tree',
+  'archive-tree',
+  'l1-to-l2-message-tree',
+] as const;
+
+export type SnapshotDataKeys = (typeof SnapshotDataKeys)[number];
+
+export type SnapshotDataUrls = Record<SnapshotDataKeys, string>;
+
+export type SnapshotMetadata = {
+  l2BlockNumber: number;
+  l2BlockHash: string;
+  l1BlockNumber: number;
+  timestamp: number;
+  dataUrls: SnapshotDataUrls;
+  schemaVersions: { archiver: number; worldState: number };
+};
+
+export type SnapshotsIndexMetadata = {
+  l1ChainId: number;
+  rollupVersion: number;
+  rollupAddress: EthAddress;
+};
+
+export type SnapshotsIndex = SnapshotsIndexMetadata & {
+  snapshots: SnapshotMetadata[];
+};
+
+export type UploadSnapshotMetadata = Pick<SnapshotMetadata, 'l2BlockNumber' | 'l2BlockHash' | 'l1BlockNumber'> &
+  Pick<SnapshotsIndex, 'l1ChainId' | 'rollupVersion' | 'rollupAddress'>;
+
+export const SnapshotsIndexSchema = z.object({
+  l1ChainId: z.number(),
+  rollupVersion: z.number(),
+  rollupAddress: schemas.EthAddress,
+  snapshots: z.array(
+    z.object({
+      l2BlockNumber: z.number(),
+      l2BlockHash: z.string(),
+      l1BlockNumber: z.number(),
+      timestamp: z.number(),
+      schemaVersions: z.object({
+        archiver: z.number(),
+        worldState: z.number(),
+      }),
+      dataUrls: z
+        .record(z.enum(SnapshotDataKeys), z.string())
+        // See https://stackoverflow.com/questions/77958464/zod-record-with-required-keys
+        .refine((obj): obj is Required<typeof obj> => SnapshotDataKeys.every(key => !!obj[key])),
+    }),
+  ),
+}) satisfies ZodFor<SnapshotsIndex>;
+
+export const UploadSnapshotMetadataSchema = z.object({
+  l2BlockNumber: z.number(),
+  l2BlockHash: z.string(),
+  l1BlockNumber: z.number(),
+  l1ChainId: z.number(),
+  rollupVersion: z.number(),
+  rollupAddress: schemas.EthAddress,
+}) satisfies ZodFor<UploadSnapshotMetadata>;

package/src/snapshots/upload.ts

@@ -0,0 +1,64 @@
+import { fromEntries, getEntries, pick } from '@aztec/foundation/collection';
+import { jsonStringify } from '@aztec/foundation/json-rpc';
+import { isoDate } from '@aztec/foundation/string';
+import type { FileStore } from '@aztec/stdlib/file-store';
+
+import { getBasePath, getSnapshotIndex, getSnapshotIndexPath } from './download.js';
+import type { SnapshotDataKeys, SnapshotMetadata, SnapshotsIndex, UploadSnapshotMetadata } from './types.js';
+
+export async function uploadSnapshotData(
+  localPaths: Record<SnapshotDataKeys, string>,
+  schemaVersions: SnapshotMetadata['schemaVersions'],
+  metadata: UploadSnapshotMetadata,
+  store: FileStore,
+  opts: { pathFor?: (key: SnapshotDataKeys) => string; private?: boolean } = {},
+): Promise<SnapshotMetadata> {
+  const timestamp = Date.now();
+  const date = isoDate();
+  const basePath = getBasePath(metadata);
+  const targetPathFor =
+    opts.pathFor ?? ((key: SnapshotDataKeys) => `${basePath}/${key}-${date}-${metadata.l2BlockHash}.db`);
+
+  const dataUrls = fromEntries(
+    await Promise.all(
+      getEntries(localPaths).map(
+        async ([key, path]) =>
+          [key, await store.upload(targetPathFor(key), path, { compress: true, public: !opts.private })] as const,
+      ),
+    ),
+  );
+
+  return {
+    ...pick(metadata, 'l1BlockNumber', 'l2BlockHash', 'l2BlockNumber'),
+    schemaVersions,
+    timestamp,
+    dataUrls,
+  };
+}
+
+export async function uploadSnapshotToIndex(
+  localPaths: Record<SnapshotDataKeys, string>,
+  schemaVersions: SnapshotMetadata['schemaVersions'],
+  metadata: UploadSnapshotMetadata,
+  store: FileStore,
+): Promise<SnapshotMetadata> {
+  const newSnapshotMetadata = await uploadSnapshotData(localPaths, schemaVersions, metadata, store);
+  const snapshotsIndex = (await getSnapshotIndex(metadata, store)) ?? createEmptyIndex(metadata);
+  snapshotsIndex.snapshots.unshift(newSnapshotMetadata);
+
+  await store.save(getSnapshotIndexPath(metadata), Buffer.from(jsonStringify(snapshotsIndex, true)), {
+    public: true, // Make the index publicly accessible
+    compress: false, // Ensure index.json is not gzipped
+    metadata: { ['Cache-control']: 'no-store' }, // Do not cache object versions
+  });
+  return newSnapshotMetadata;
+}
+
+function createEmptyIndex(
+  metadata: Pick<SnapshotsIndex, 'l1ChainId' | 'rollupVersion' | 'rollupAddress'>,
+): SnapshotsIndex {
+  return {
+    ...pick(metadata, 'l1ChainId', 'rollupVersion', 'rollupAddress'),
+    snapshots: [],
+  };
+}

package/src/stats/stats.ts

@@ -81,22 +81,31 @@ export type ClientCircuitName =
   | 'private-kernel-reset'
   | 'private-kernel-tail'
   | 'private-kernel-tail-to-public'
+  | 'hiding-kernel-to-rollup'
+  | 'hiding-kernel-to-public'
   | 'app-circuit';
 
 export type ServerCircuitName =
-  | 'base-parity'
-  | 'root-parity'
-  | 'private-base-rollup'
-  | 'public-base-rollup'
-  | 'merge-rollup'
-  | 'block-root-rollup'
-  | 'single-tx-block-root-rollup'
-  | 'empty-block-root-rollup'
-  | 'block-merge-rollup'
-  | 'root-rollup'
-  | 'avm-circuit'
-  | 'tube-circuit';
+  | 'parity-base'
+  | 'parity-root'
+  | 'chonk-verifier-public'
+  | 'rollup-tx-base-private'
+  | 'rollup-tx-base-public'
+  | 'rollup-tx-merge'
+  | 'rollup-block-root-first'
+  | 'rollup-block-root-first-single-tx'
+  | 'rollup-block-root-first-empty-tx'
+  | 'rollup-block-root'
+  | 'rollup-block-root-single-tx'
+  | 'rollup-block-merge'
+  | 'rollup-checkpoint-root'
+  | 'rollup-checkpoint-root-single-block'
+  | 'rollup-checkpoint-padding'
+  | 'rollup-checkpoint-merge'
+  | 'rollup-root'
+  | 'avm-circuit';
 
+// Circuit names should match the corresponding names in noir-protocol-circuits.
 export type CircuitName = ClientCircuitName | ServerCircuitName;
 
 /** Stats for circuit simulation. */
@@ -172,8 +181,8 @@ export type CircuitVerificationStats = {
   eventName: 'circuit-verification';
   /** Name of the circuit. */
   circuitName: CircuitName;
-  /** Type of proof (client-ivc, honk, etc) */
-  proofType: 'client-ivc' | 'ultra-honk';
+  /** Type of proof (chonk, honk, etc) */
+  proofType: 'chonk' | 'ultra-honk';
   /** Duration in ms. */
   duration: number;
 };
@@ -199,9 +208,9 @@ export type L2BlockHandledStats = {
   /** Total duration in ms. */
   duration: number;
   /** Pending block number. */
-  unfinalisedBlockNumber: bigint;
+  unfinalizedBlockNumber: bigint;
   /** Proven block number. */
-  finalisedBlockNumber: bigint;
+  finalizedBlockNumber: bigint;
   /** Oldest historic block number. */
   oldestHistoricBlock: bigint;
 } & L2BlockStats;
@@ -220,12 +229,12 @@ export type TxStats = {
   nullifierCount: number;
   /** Number of private logs */
   privateLogCount: number;
-  /** How many classes were registered through the canonical class registerer. */
-  classRegisteredCount: number;
+  /** How many classes were published to the canonical class registry. */
+  classPublishedCount: number;
   /** Serialized size of contract class logs in fields. */
   contractClassLogSize: number;
   /** How this tx pays for its fee */
-  feePaymentMethod: 'fee_juice' | 'fpc_public' | 'fpc_private';
+  feePaymentMethod: 'fee_juice' | 'fpc';
 };
 
 /**