@aztec/stdlib 0.0.0-test.1 → 0.0.1-commit.017a351

package/src/slashing/serialization.ts

@@ -0,0 +1,23 @@
+import { EthAddress } from '@aztec/foundation/eth-address';
+import { BufferReader, bigintToUInt64BE, bigintToUInt128BE, serializeToBuffer } from '@aztec/foundation/serialize';
+
+import type { Offense, OffenseType } from './types.js';
+
+export function serializeOffense(offense: Offense): Buffer {
+  return serializeToBuffer(
+    offense.validator,
+    bigintToUInt128BE(offense.amount),
+    offense.offenseType,
+    bigintToUInt64BE(offense.epochOrSlot),
+  );
+}
+
+export function deserializeOffense(buffer: Buffer): Offense {
+  const reader = BufferReader.asReader(buffer);
+  const validator = reader.readObject(EthAddress);
+  const amount = reader.readUInt128();
+  const offense = reader.readNumber() as OffenseType;
+  const epochOrSlot = reader.readUInt64();
+
+  return { validator, amount, offenseType: offense, epochOrSlot };
+}

package/src/slashing/types.ts

@@ -0,0 +1,145 @@
+import { EthAddress } from '@aztec/foundation/eth-address';
+
+import { z } from 'zod';
+
+import { schemas, zodFor } from '../schemas/index.js';
+
+export enum OffenseType {
+  UNKNOWN = 0,
+  /** The data for the txs in a published checkpoint was not available within the tolerance window, we slash the checkpoint's attesters */
+  DATA_WITHHOLDING = 1,
+  /** A proposer failed to attest or propose during an epoch according to the Sentinel */
+  INACTIVITY = 3,
+  /** A proposer sent an invalid block proposal over the p2p network to the committee */
+  BROADCASTED_INVALID_BLOCK_PROPOSAL = 4,
+  /** A proposer pushed to L1 a block with insufficient committee attestations */
+  PROPOSED_INSUFFICIENT_ATTESTATIONS = 5,
+  /** A proposer pushed to L1 a block with incorrect committee attestations (ie signature from a non-committee member) */
+  PROPOSED_INCORRECT_ATTESTATIONS = 6,
+  /** A proposer published a checkpoint to L1 that builds on an invalid checkpoint (as in a checkpoint with invalid or insufficient attestations) */
+  PROPOSED_DESCENDANT_OF_CHECKPOINT_WITH_INVALID_ATTESTATIONS = 7,
+  /** A proposer sent duplicate proposals for the same position (slot, indexWithinCheckpoint for blocks or slot for checkpoints) */
+  DUPLICATE_PROPOSAL = 8,
+  /** A validator signed attestations for different proposals at the same slot (equivocation) */
+  DUPLICATE_ATTESTATION = 9,
+  /** A committee member attested to a checkpoint proposal in a slot with an invalid block proposal */
+  ATTESTED_TO_INVALID_CHECKPOINT_PROPOSAL = 10,
+  /** A proposer broadcast an invalid checkpoint proposal, detected by retained evidence or deterministic recomputation */
+  BROADCASTED_INVALID_CHECKPOINT_PROPOSAL = 11,
+}
+
+export function getOffenseTypeName(offense: OffenseType) {
+  switch (offense) {
+    case OffenseType.UNKNOWN:
+      return 'unknown';
+    case OffenseType.DATA_WITHHOLDING:
+      return 'data_withholding';
+    case OffenseType.INACTIVITY:
+      return 'inactivity';
+    case OffenseType.BROADCASTED_INVALID_BLOCK_PROPOSAL:
+      return 'broadcasted_invalid_block_proposal';
+    case OffenseType.PROPOSED_INSUFFICIENT_ATTESTATIONS:
+      return 'proposed_insufficient_attestations';
+    case OffenseType.PROPOSED_INCORRECT_ATTESTATIONS:
+      return 'proposed_incorrect_attestations';
+    case OffenseType.PROPOSED_DESCENDANT_OF_CHECKPOINT_WITH_INVALID_ATTESTATIONS:
+      return 'proposed_descendant_of_checkpoint_with_invalid_attestations';
+    case OffenseType.DUPLICATE_PROPOSAL:
+      return 'duplicate_proposal';
+    case OffenseType.DUPLICATE_ATTESTATION:
+      return 'duplicate_attestation';
+    case OffenseType.ATTESTED_TO_INVALID_CHECKPOINT_PROPOSAL:
+      return 'attested_to_invalid_checkpoint_proposal';
+    case OffenseType.BROADCASTED_INVALID_CHECKPOINT_PROPOSAL:
+      return 'broadcasted_invalid_checkpoint_proposal';
+    default:
+      throw new Error(`Unknown offense type: ${offense}`);
+  }
+}
+
+export const OffenseTypeSchema = z.nativeEnum(OffenseType);
+
+export const OffenseToBigInt: Record<OffenseType, bigint> = {
+  [OffenseType.UNKNOWN]: 0n,
+  [OffenseType.DATA_WITHHOLDING]: 1n,
+  [OffenseType.INACTIVITY]: 3n,
+  [OffenseType.BROADCASTED_INVALID_BLOCK_PROPOSAL]: 4n,
+  [OffenseType.PROPOSED_INSUFFICIENT_ATTESTATIONS]: 5n,
+  [OffenseType.PROPOSED_INCORRECT_ATTESTATIONS]: 6n,
+  [OffenseType.PROPOSED_DESCENDANT_OF_CHECKPOINT_WITH_INVALID_ATTESTATIONS]: 7n,
+  [OffenseType.DUPLICATE_PROPOSAL]: 8n,
+  [OffenseType.DUPLICATE_ATTESTATION]: 9n,
+  [OffenseType.ATTESTED_TO_INVALID_CHECKPOINT_PROPOSAL]: 10n,
+  [OffenseType.BROADCASTED_INVALID_CHECKPOINT_PROPOSAL]: 11n,
+};
+
+export function bigIntToOffense(offense: bigint): OffenseType {
+  switch (offense) {
+    case 0n:
+      return OffenseType.UNKNOWN;
+    case 1n:
+      return OffenseType.DATA_WITHHOLDING;
+    case 3n:
+      return OffenseType.INACTIVITY;
+    case 4n:
+      return OffenseType.BROADCASTED_INVALID_BLOCK_PROPOSAL;
+    case 5n:
+      return OffenseType.PROPOSED_INSUFFICIENT_ATTESTATIONS;
+    case 6n:
+      return OffenseType.PROPOSED_INCORRECT_ATTESTATIONS;
+    case 7n:
+      return OffenseType.PROPOSED_DESCENDANT_OF_CHECKPOINT_WITH_INVALID_ATTESTATIONS;
+    case 8n:
+      return OffenseType.DUPLICATE_PROPOSAL;
+    case 9n:
+      return OffenseType.DUPLICATE_ATTESTATION;
+    case 10n:
+      return OffenseType.ATTESTED_TO_INVALID_CHECKPOINT_PROPOSAL;
+    case 11n:
+      return OffenseType.BROADCASTED_INVALID_CHECKPOINT_PROPOSAL;
+    default:
+      throw new Error(`Unknown offense: ${offense}`);
+  }
+}
+
+export type Offense = {
+  validator: EthAddress;
+  amount: bigint;
+  offenseType: OffenseType;
+  epochOrSlot: bigint;
+};
+
+export type OffenseIdentifier = Pick<Offense, 'validator' | 'offenseType' | 'epochOrSlot'>;
+
+export const OffenseSchema = zodFor<Offense>()(
+  z.object({
+    validator: schemas.EthAddress,
+    amount: schemas.BigInt,
+    offenseType: OffenseTypeSchema,
+    epochOrSlot: schemas.BigInt,
+  }),
+);
+
+/** Offense by a validator in the context of a slash payload */
+export type ValidatorSlashOffense = {
+  epochOrSlot: bigint;
+  offenseType: OffenseType;
+};
+
+/** Slashed amount and total offenses by a validator in the context of a slash payload */
+export type ValidatorSlash = {
+  validator: EthAddress;
+  amount: bigint;
+  offenses: ValidatorSlashOffense[];
+};
+
+/** Votes for a validator slash in the slashing proposer */
+export type ValidatorSlashVote = number;
+
+export type ProposerSlashAction =
+  /** Vote for offenses on the slashing proposer */
+  | { type: 'vote-offenses'; votes: ValidatorSlashVote[]; committees: EthAddress[][]; round: bigint }
+  /** Execute a slashing round on the slashing proposer */
+  | { type: 'execute-slash'; committees: EthAddress[][]; round: bigint };
+
+export type ProposerSlashActionType = ProposerSlashAction['type'];

package/src/slashing/votes.ts

@@ -0,0 +1,126 @@
+import { sumBigint } from '@aztec/foundation/bigint';
+import { padArrayEnd } from '@aztec/foundation/collection';
+import { EthAddress } from '@aztec/foundation/eth-address';
+import { type Logger, createLogger } from '@aztec/foundation/log';
+import type { PartialBy } from '@aztec/foundation/types';
+
+import { getEpochForOffense } from './helpers.js';
+import type { Offense, ValidatorSlashVote } from './types.js';
+
+/**
+ * Creates a consensus-slash vote for a given set of committees based on a set of Offenses
+ * @param offenses - Array of offenses to consider
+ * @param committees - Array of committees (each containing array of validator addresses)
+ * @param epochsForCommittees - Array of epochs corresponding to each committee
+ * @param settings - Settings including slashingAmounts and optional validator override lists
+ * @param settings.maxSlashedValidators - If set, limits the total number of [validator, epoch] pairs
+ *   with non-zero votes. The lowest-vote pairs are zeroed out to stay within the limit.
+ * @param logger - Logger, logs which validators were dropped.
+ * @returns Array of ValidatorSlashVote, where each vote is how many slash units the validator in that position should be slashed
+ */
+export function getSlashConsensusVotesFromOffenses(
+  offenses: PartialBy<Offense, 'epochOrSlot'>[],
+  committees: EthAddress[][],
+  epochsForCommittees: bigint[],
+  settings: {
+    slashingAmounts: [bigint, bigint, bigint];
+    epochDuration: number;
+    targetCommitteeSize: number;
+    maxSlashedValidators?: number;
+  },
+  logger: Logger = createLogger('slasher:votes'),
+): ValidatorSlashVote[] {
+  const { slashingAmounts, targetCommitteeSize, maxSlashedValidators } = settings;
+
+  if (committees.length !== epochsForCommittees.length) {
+    throw new Error('committees and epochsForCommittees must have the same length');
+  }
+
+  const votes = committees.flatMap((committee, committeeIndex) => {
+    const committeeEpoch = epochsForCommittees[committeeIndex];
+
+    // Map over actual committee members, then pad to targetCommitteeSize.
+    // Padding handles cases where committees may be empty (e.g., when there aren't enough validators to fill the committee size during network startup).
+    const votes = committee.map(validator => {
+      // Find offenses for this validator in this specific epoch.
+      // If an offense has no epoch, it is considered for all epochs due to a slashAlways setting.
+      const validatorOffenses = offenses.filter(
+        o =>
+          o.validator.equals(validator) &&
+          (o.epochOrSlot === undefined || getEpochForOffense(o, settings) === committeeEpoch),
+      );
+
+      // Sum up the penalties for this validator in this epoch
+      const slashAmount = sumBigint(validatorOffenses.map(o => o.amount));
+      const slashUnits = getSlashUnitsForAmount(slashAmount, slashingAmounts);
+      return Number(slashUnits);
+    });
+
+    return padArrayEnd(votes, 0, targetCommitteeSize);
+  });
+
+  // if a cap is set, zero out the lowest-vote [validator, epoch] pairs so that the most severe slashes stay.
+  if (maxSlashedValidators === undefined) {
+    return votes;
+  }
+
+  const nonZeroByDescendingVote = [...votes.entries()].filter(([, vote]) => vote > 0).sort(([, a], [, b]) => b - a);
+
+  const toTruncate = nonZeroByDescendingVote.slice(maxSlashedValidators);
+  for (const [idx] of toTruncate) {
+    votes[idx] = 0;
+  }
+
+  if (toTruncate.length > 0) {
+    const truncated = toTruncate.map(([idx]) => {
+      const committeeIndex = Math.floor(idx / targetCommitteeSize);
+      const positionInCommittee = idx % targetCommitteeSize;
+      return {
+        validator: committees[committeeIndex][positionInCommittee].toString(),
+        epoch: epochsForCommittees[committeeIndex],
+      };
+    });
+    logger.warn(
+      `Truncated ${toTruncate.length} validator-epoch pairs to stay within limit of ${maxSlashedValidators}`,
+      { truncated },
+    );
+  }
+
+  return votes;
+}
+
+/** Returns the slash vote for the given amount to slash. */
+function getSlashUnitsForAmount(amountToSlash: bigint, slashingAmounts: [bigint, bigint, bigint]): number {
+  if (amountToSlash >= slashingAmounts[2]) {
+    return 3;
+  } else if (amountToSlash >= slashingAmounts[1]) {
+    return 2;
+  } else if (amountToSlash >= slashingAmounts[0]) {
+    return 1;
+  } else {
+    return 0;
+  }
+}
+
+/**
+ * Encodes a set of slash votes into a Buffer for use in a consensus slashing vote transaction.
+ * Each vote is represented as a 2-bit value, which represents how many slashing units the validator should be slashed.
+ * @param votes - The array of slash votes to encode
+ * @returns A Buffer containing the encoded slash votes
+ */
+export function encodeSlashConsensusVotes(votes: ValidatorSlashVote[]): Buffer {
+  if (votes.length % 4 !== 0) {
+    throw new Error('Votes array must have a length that is a multiple of 4');
+  }
+  const buffer = Buffer.alloc(votes.length / 4);
+  for (let i = 0; i < votes.length; i += 4) {
+    // Encode votes to match Solidity's bit order (LSB to MSB)
+    // Bits 0-1: validator at index i
+    // Bits 2-3: validator at index i+1
+    // Bits 4-5: validator at index i+2
+    // Bits 6-7: validator at index i+3
+    const voteByte = votes[i] | (votes[i + 1] << 2) | (votes[i + 2] << 4) | (votes[i + 3] << 6);
+    buffer.writeUInt8(voteByte, i / 4);
+  }
+  return buffer;
+}

package/src/snapshots/download.ts

@@ -0,0 +1,123 @@
+import { fromEntries, getEntries, maxBy } from '@aztec/foundation/collection';
+import { jsonParseWithSchema } from '@aztec/foundation/json-rpc';
+import type { ReadOnlyFileStore } from '@aztec/stdlib/file-store';
+
+import { createReadStream, createWriteStream } from 'fs';
+import fs from 'fs/promises';
+import pathMod from 'path';
+import { pipeline } from 'stream/promises';
+import { createGunzip, gunzipSync } from 'zlib';
+
+import {
+  SnapshotDataKeys,
+  type SnapshotDataUrls,
+  type SnapshotMetadata,
+  type SnapshotsIndex,
+  type SnapshotsIndexMetadata,
+  SnapshotsIndexSchema,
+} from './types.js';
+
+export async function getSnapshotIndex(
+  metadata: SnapshotsIndexMetadata,
+  store: ReadOnlyFileStore,
+): Promise<SnapshotsIndex | undefined> {
+  const basePath = getBasePath(metadata);
+  const snapshotIndexPath = `${basePath}/index.json`;
+  try {
+    if (await store.exists(snapshotIndexPath)) {
+      const snapshotIndexData = await store.read(snapshotIndexPath);
+      const buf = maybeGunzip(snapshotIndexData);
+      return jsonParseWithSchema(buf.toString('utf-8'), SnapshotsIndexSchema);
+    } else {
+      return undefined;
+    }
+  } catch (err) {
+    throw new Error(`Error reading snapshot index from ${snapshotIndexPath}: ${err}`);
+  }
+}
+
+export async function getLatestSnapshotMetadata(
+  metadata: SnapshotsIndexMetadata,
+  store: ReadOnlyFileStore,
+): Promise<SnapshotMetadata | undefined> {
+  const snapshotsIndex = await getSnapshotIndex(metadata, store);
+  return snapshotsIndex?.snapshots && maxBy(snapshotsIndex?.snapshots, s => s.l1BlockNumber);
+}
+
+export function getBasePath(metadata: SnapshotsIndexMetadata): string {
+  return `aztec-${metadata.l1ChainId}-${metadata.rollupVersion}-${metadata.rollupAddress}`;
+}
+
+export function getSnapshotIndexPath(metadata: SnapshotsIndexMetadata): string {
+  return `${getBasePath(metadata)}/index.json`;
+}
+
+export function makeSnapshotPaths(baseDir: string): SnapshotDataUrls {
+  // We do not use path.join since that screws up protocol prefixes
+  return fromEntries(SnapshotDataKeys.map(key => [key, `${baseDir}/${key}.db`]));
+}
+
+function isGzipMagic(data: Buffer): boolean {
+  return data.length >= 2 && data[0] === 0x1f && data[1] === 0x8b;
+}
+
+function maybeGunzip(data: Buffer): Buffer {
+  const magicNumberIndicatesGzip = isGzipMagic(data);
+
+  if (magicNumberIndicatesGzip) {
+    try {
+      const out = gunzipSync(data);
+      return out;
+    } catch (err) {
+      throw new Error(`Decompression of gzipped data failed: ${(err as Error).message}`);
+    }
+  }
+  return data;
+}
+
+async function detectGzip(localFilePathToPeek: string): Promise<boolean> {
+  // Peek the actual bytes we downloaded.
+  try {
+    const fd = await fs.open(localFilePathToPeek, 'r');
+    try {
+      const header = Buffer.alloc(2);
+      const { bytesRead } = await fd.read(header, 0, 2, 0);
+      return bytesRead >= 2 && isGzipMagic(header);
+    } finally {
+      await fd.close();
+    }
+  } catch {
+    return false;
+  }
+}
+
+export async function downloadSnapshot(
+  snapshot: Pick<SnapshotMetadata, 'dataUrls'>,
+  localPaths: Record<SnapshotDataKeys, string>,
+  store: ReadOnlyFileStore,
+): Promise<void> {
+  await Promise.all(
+    getEntries(localPaths).map(async ([key, path]) => {
+      await fs.mkdir(pathMod.dirname(path), { recursive: true });
+
+      const tmpPath = `${path}.download`;
+      try {
+        const url = snapshot.dataUrls[key];
+        await store.download(url, tmpPath);
+
+        const isGzip = await detectGzip(tmpPath);
+
+        const read = createReadStream(tmpPath);
+        const write = createWriteStream(path);
+        if (isGzip) {
+          const gunzip = createGunzip();
+          await pipeline(read, gunzip, write);
+        } else {
+          await pipeline(read, write);
+        }
+      } finally {
+        await fs.unlink(tmpPath).catch(() => undefined);
+      }
+    }),
+  );
+}

package/src/snapshots/index.ts

@@ -0,0 +1,3 @@
+export * from './types.js';
+export * from './upload.js';
+export { getBasePath, downloadSnapshot, getLatestSnapshotMetadata, makeSnapshotPaths } from './download.js';

package/src/snapshots/types.ts

@@ -0,0 +1,74 @@
+import type { EthAddress } from '@aztec/foundation/eth-address';
+import { schemas, zodFor } from '@aztec/foundation/schemas';
+
+import { z } from 'zod';
+
+export const SnapshotDataKeys = [
+  'archiver',
+  'nullifier-tree',
+  'public-data-tree',
+  'note-hash-tree',
+  'archive-tree',
+  'l1-to-l2-message-tree',
+] as const;
+
+export type SnapshotDataKeys = (typeof SnapshotDataKeys)[number];
+
+export type SnapshotDataUrls = Record<SnapshotDataKeys, string>;
+
+export type SnapshotMetadata = {
+  l2BlockNumber: number;
+  l2BlockHash: string;
+  l1BlockNumber: number;
+  timestamp: number;
+  dataUrls: SnapshotDataUrls;
+  schemaVersions: { archiver: number; worldState: number };
+};
+
+export type SnapshotsIndexMetadata = {
+  l1ChainId: number;
+  rollupVersion: number;
+  rollupAddress: EthAddress;
+};
+
+export type SnapshotsIndex = SnapshotsIndexMetadata & {
+  snapshots: SnapshotMetadata[];
+};
+
+export type UploadSnapshotMetadata = Pick<SnapshotMetadata, 'l2BlockNumber' | 'l2BlockHash' | 'l1BlockNumber'> &
+  Pick<SnapshotsIndex, 'l1ChainId' | 'rollupVersion' | 'rollupAddress'>;
+
+export const SnapshotsIndexSchema = zodFor<SnapshotsIndex>()(
+  z.object({
+    l1ChainId: z.number(),
+    rollupVersion: z.number(),
+    rollupAddress: schemas.EthAddress,
+    snapshots: z.array(
+      z.object({
+        l2BlockNumber: z.number(),
+        l2BlockHash: z.string(),
+        l1BlockNumber: z.number(),
+        timestamp: z.number(),
+        schemaVersions: z.object({
+          archiver: z.number(),
+          worldState: z.number(),
+        }),
+        dataUrls: z
+          .record(z.enum(SnapshotDataKeys), z.string())
+          // See https://stackoverflow.com/questions/77958464/zod-record-with-required-keys
+          .refine((obj): obj is Required<typeof obj> => SnapshotDataKeys.every(key => !!obj[key])),
+      }),
+    ),
+  }),
+);
+
+export const UploadSnapshotMetadataSchema = zodFor<UploadSnapshotMetadata>()(
+  z.object({
+    l2BlockNumber: z.number(),
+    l2BlockHash: z.string(),
+    l1BlockNumber: z.number(),
+    l1ChainId: z.number(),
+    rollupVersion: z.number(),
+    rollupAddress: schemas.EthAddress,
+  }),
+);

package/src/snapshots/upload.ts

@@ -0,0 +1,64 @@
+import { fromEntries, getEntries, pick } from '@aztec/foundation/collection';
+import { jsonStringify } from '@aztec/foundation/json-rpc';
+import { isoDate } from '@aztec/foundation/string';
+import type { FileStore } from '@aztec/stdlib/file-store';
+
+import { getBasePath, getSnapshotIndex, getSnapshotIndexPath } from './download.js';
+import type { SnapshotDataKeys, SnapshotMetadata, SnapshotsIndex, UploadSnapshotMetadata } from './types.js';
+
+export async function uploadSnapshotData(
+  localPaths: Record<SnapshotDataKeys, string>,
+  schemaVersions: SnapshotMetadata['schemaVersions'],
+  metadata: UploadSnapshotMetadata,
+  store: FileStore,
+  opts: { pathFor?: (key: SnapshotDataKeys) => string; private?: boolean } = {},
+): Promise<SnapshotMetadata> {
+  const timestamp = Date.now();
+  const date = isoDate();
+  const basePath = getBasePath(metadata);
+  const targetPathFor =
+    opts.pathFor ?? ((key: SnapshotDataKeys) => `${basePath}/${key}-${date}-${metadata.l2BlockHash}.db`);
+
+  const dataUrls = fromEntries(
+    await Promise.all(
+      getEntries(localPaths).map(
+        async ([key, path]) =>
+          [key, await store.upload(targetPathFor(key), path, { compress: true, public: !opts.private })] as const,
+      ),
+    ),
+  );
+
+  return {
+    ...pick(metadata, 'l1BlockNumber', 'l2BlockHash', 'l2BlockNumber'),
+    schemaVersions,
+    timestamp,
+    dataUrls,
+  };
+}
+
+export async function uploadSnapshotToIndex(
+  localPaths: Record<SnapshotDataKeys, string>,
+  schemaVersions: SnapshotMetadata['schemaVersions'],
+  metadata: UploadSnapshotMetadata,
+  store: FileStore,
+): Promise<SnapshotMetadata> {
+  const newSnapshotMetadata = await uploadSnapshotData(localPaths, schemaVersions, metadata, store);
+  const snapshotsIndex = (await getSnapshotIndex(metadata, store)) ?? createEmptyIndex(metadata);
+  snapshotsIndex.snapshots.unshift(newSnapshotMetadata);
+
+  await store.save(getSnapshotIndexPath(metadata), Buffer.from(jsonStringify(snapshotsIndex, true)), {
+    public: true, // Make the index publicly accessible
+    compress: false, // Ensure index.json is not gzipped
+    metadata: { ['Cache-control']: 'no-store' }, // Do not cache object versions
+  });
+  return newSnapshotMetadata;
+}
+
+function createEmptyIndex(
+  metadata: Pick<SnapshotsIndex, 'l1ChainId' | 'rollupVersion' | 'rollupAddress'>,
+): SnapshotsIndex {
+  return {
+    ...pick(metadata, 'l1ChainId', 'rollupVersion', 'rollupAddress'),
+    snapshots: [],
+  };
+}