@aztec-labs/stdlib 6.0.0-nightly.20260829

package/src/snapshots/download.ts

@@ -0,0 +1,122 @@
+import { fromEntries, getEntries, maxBy } from '@aztec-labs/foundation/collection';
+import { jsonParseWithSchema } from '@aztec-labs/foundation/json-rpc';
+import type { ReadOnlyFileStore } from '@aztec-labs/stdlib/file-store';
+import { createReadStream, createWriteStream } from 'fs';
+import fs from 'fs/promises';
+import pathMod from 'path';
+import { pipeline } from 'stream/promises';
+import { createGunzip, gunzipSync } from 'zlib';
+
+import {
+  SnapshotDataKeys,
+  type SnapshotDataUrls,
+  type SnapshotMetadata,
+  type SnapshotsIndex,
+  type SnapshotsIndexMetadata,
+  SnapshotsIndexSchema,
+} from './types.js';
+
+export async function getSnapshotIndex(
+  metadata: SnapshotsIndexMetadata,
+  store: ReadOnlyFileStore,
+): Promise<SnapshotsIndex | undefined> {
+  const basePath = getBasePath(metadata);
+  const snapshotIndexPath = `${basePath}/index.json`;
+  try {
+    if (await store.exists(snapshotIndexPath)) {
+      const snapshotIndexData = await store.read(snapshotIndexPath);
+      const buf = maybeGunzip(snapshotIndexData);
+      return jsonParseWithSchema(buf.toString('utf-8'), SnapshotsIndexSchema);
+    } else {
+      return undefined;
+    }
+  } catch (err) {
+    throw new Error(`Error reading snapshot index from ${snapshotIndexPath}: ${err}`);
+  }
+}
+
+export async function getLatestSnapshotMetadata(
+  metadata: SnapshotsIndexMetadata,
+  store: ReadOnlyFileStore,
+): Promise<SnapshotMetadata | undefined> {
+  const snapshotsIndex = await getSnapshotIndex(metadata, store);
+  return snapshotsIndex?.snapshots && maxBy(snapshotsIndex?.snapshots, s => s.l1BlockNumber);
+}
+
+export function getBasePath(metadata: SnapshotsIndexMetadata): string {
+  return `aztec-${metadata.l1ChainId}-${metadata.rollupVersion}-${metadata.rollupAddress}`;
+}
+
+export function getSnapshotIndexPath(metadata: SnapshotsIndexMetadata): string {
+  return `${getBasePath(metadata)}/index.json`;
+}
+
+export function makeSnapshotPaths(baseDir: string): SnapshotDataUrls {
+  // We do not use path.join since that screws up protocol prefixes
+  return fromEntries(SnapshotDataKeys.map(key => [key, `${baseDir}/${key}.db`]));
+}
+
+function isGzipMagic(data: Buffer): boolean {
+  return data.length >= 2 && data[0] === 0x1f && data[1] === 0x8b;
+}
+
+function maybeGunzip(data: Buffer): Buffer {
+  const magicNumberIndicatesGzip = isGzipMagic(data);
+
+  if (magicNumberIndicatesGzip) {
+    try {
+      const out = gunzipSync(data);
+      return out;
+    } catch (err) {
+      throw new Error(`Decompression of gzipped data failed: ${(err as Error).message}`);
+    }
+  }
+  return data;
+}
+
+async function detectGzip(localFilePathToPeek: string): Promise<boolean> {
+  // Peek the actual bytes we downloaded.
+  try {
+    const fd = await fs.open(localFilePathToPeek, 'r');
+    try {
+      const header = Buffer.alloc(2);
+      const { bytesRead } = await fd.read(header, 0, 2, 0);
+      return bytesRead >= 2 && isGzipMagic(header);
+    } finally {
+      await fd.close();
+    }
+  } catch {
+    return false;
+  }
+}
+
+export async function downloadSnapshot(
+  snapshot: Pick<SnapshotMetadata, 'dataUrls'>,
+  localPaths: Record<SnapshotDataKeys, string>,
+  store: ReadOnlyFileStore,
+): Promise<void> {
+  await Promise.all(
+    getEntries(localPaths).map(async ([key, path]) => {
+      await fs.mkdir(pathMod.dirname(path), { recursive: true });
+
+      const tmpPath = `${path}.download`;
+      try {
+        const url = snapshot.dataUrls[key];
+        await store.download(url, tmpPath);
+
+        const isGzip = await detectGzip(tmpPath);
+
+        const read = createReadStream(tmpPath);
+        const write = createWriteStream(path);
+        if (isGzip) {
+          const gunzip = createGunzip();
+          await pipeline(read, gunzip, write);
+        } else {
+          await pipeline(read, write);
+        }
+      } finally {
+        await fs.unlink(tmpPath).catch(() => undefined);
+      }
+    }),
+  );
+}

package/src/snapshots/index.ts

@@ -0,0 +1,3 @@
+export * from './types.js';
+export * from './upload.js';
+export { getBasePath, downloadSnapshot, getLatestSnapshotMetadata, makeSnapshotPaths } from './download.js';

package/src/snapshots/types.ts

@@ -0,0 +1,73 @@
+import type { EthAddress } from '@aztec-labs/foundation/eth-address';
+import { schemas, zodFor } from '@aztec-labs/foundation/schemas';
+import { z } from 'zod';
+
+export const SnapshotDataKeys = [
+  'archiver',
+  'nullifier-tree',
+  'public-data-tree',
+  'note-hash-tree',
+  'archive-tree',
+  'l1-to-l2-message-tree',
+] as const;
+
+export type SnapshotDataKeys = (typeof SnapshotDataKeys)[number];
+
+export type SnapshotDataUrls = Record<SnapshotDataKeys, string>;
+
+export type SnapshotMetadata = {
+  l2BlockNumber: number;
+  l2BlockHash: string;
+  l1BlockNumber: number;
+  timestamp: number;
+  dataUrls: SnapshotDataUrls;
+  schemaVersions: { archiver: number; worldState: number };
+};
+
+export type SnapshotsIndexMetadata = {
+  l1ChainId: number;
+  rollupVersion: number;
+  rollupAddress: EthAddress;
+};
+
+export type SnapshotsIndex = SnapshotsIndexMetadata & {
+  snapshots: SnapshotMetadata[];
+};
+
+export type UploadSnapshotMetadata = Pick<SnapshotMetadata, 'l2BlockNumber' | 'l2BlockHash' | 'l1BlockNumber'> &
+  Pick<SnapshotsIndex, 'l1ChainId' | 'rollupVersion' | 'rollupAddress'>;
+
+export const SnapshotsIndexSchema = zodFor<SnapshotsIndex>()(
+  z.object({
+    l1ChainId: z.number(),
+    rollupVersion: z.number(),
+    rollupAddress: schemas.EthAddress,
+    snapshots: z.array(
+      z.object({
+        l2BlockNumber: z.number(),
+        l2BlockHash: z.string(),
+        l1BlockNumber: z.number(),
+        timestamp: z.number(),
+        schemaVersions: z.object({
+          archiver: z.number(),
+          worldState: z.number(),
+        }),
+        dataUrls: z
+          .record(z.enum(SnapshotDataKeys), z.string())
+          // See https://stackoverflow.com/questions/77958464/zod-record-with-required-keys
+          .refine((obj): obj is Required<typeof obj> => SnapshotDataKeys.every(key => !!obj[key])),
+      }),
+    ),
+  }),
+);
+
+export const UploadSnapshotMetadataSchema = zodFor<UploadSnapshotMetadata>()(
+  z.object({
+    l2BlockNumber: z.number(),
+    l2BlockHash: z.string(),
+    l1BlockNumber: z.number(),
+    l1ChainId: z.number(),
+    rollupVersion: z.number(),
+    rollupAddress: schemas.EthAddress,
+  }),
+);

package/src/snapshots/upload.ts

@@ -0,0 +1,64 @@
+import { fromEntries, getEntries, pick } from '@aztec-labs/foundation/collection';
+import { jsonStringify } from '@aztec-labs/foundation/json-rpc';
+import { isoDate } from '@aztec-labs/foundation/string';
+import type { FileStore } from '@aztec-labs/stdlib/file-store';
+
+import { getBasePath, getSnapshotIndex, getSnapshotIndexPath } from './download.js';
+import type { SnapshotDataKeys, SnapshotMetadata, SnapshotsIndex, UploadSnapshotMetadata } from './types.js';
+
+export async function uploadSnapshotData(
+  localPaths: Record<SnapshotDataKeys, string>,
+  schemaVersions: SnapshotMetadata['schemaVersions'],
+  metadata: UploadSnapshotMetadata,
+  store: FileStore,
+  opts: { pathFor?: (key: SnapshotDataKeys) => string; private?: boolean } = {},
+): Promise<SnapshotMetadata> {
+  const timestamp = Date.now();
+  const date = isoDate();
+  const basePath = getBasePath(metadata);
+  const targetPathFor =
+    opts.pathFor ?? ((key: SnapshotDataKeys) => `${basePath}/${key}-${date}-${metadata.l2BlockHash}.db`);
+
+  const dataUrls = fromEntries(
+    await Promise.all(
+      getEntries(localPaths).map(
+        async ([key, path]) =>
+          [key, await store.upload(targetPathFor(key), path, { compress: true, public: !opts.private })] as const,
+      ),
+    ),
+  );
+
+  return {
+    ...pick(metadata, 'l1BlockNumber', 'l2BlockHash', 'l2BlockNumber'),
+    schemaVersions,
+    timestamp,
+    dataUrls,
+  };
+}
+
+export async function uploadSnapshotToIndex(
+  localPaths: Record<SnapshotDataKeys, string>,
+  schemaVersions: SnapshotMetadata['schemaVersions'],
+  metadata: UploadSnapshotMetadata,
+  store: FileStore,
+): Promise<SnapshotMetadata> {
+  const newSnapshotMetadata = await uploadSnapshotData(localPaths, schemaVersions, metadata, store);
+  const snapshotsIndex = (await getSnapshotIndex(metadata, store)) ?? createEmptyIndex(metadata);
+  snapshotsIndex.snapshots.unshift(newSnapshotMetadata);
+
+  await store.save(getSnapshotIndexPath(metadata), Buffer.from(jsonStringify(snapshotsIndex, true)), {
+    public: true, // Make the index publicly accessible
+    compress: false, // Ensure index.json is not gzipped
+    metadata: { ['Cache-control']: 'no-store' }, // Do not cache object versions
+  });
+  return newSnapshotMetadata;
+}
+
+function createEmptyIndex(
+  metadata: Pick<SnapshotsIndex, 'l1ChainId' | 'rollupVersion' | 'rollupAddress'>,
+): SnapshotsIndex {
+  return {
+    ...pick(metadata, 'l1ChainId', 'rollupVersion', 'rollupAddress'),
+    snapshots: [],
+  };
+}

package/src/stats/index.ts

@@ -0,0 +1 @@
+export * from './stats.js';

package/src/stats/stats.ts

@@ -0,0 +1,295 @@
+/** Stats associated with an ACIR proof generation.*/
+export type ProofConstructed = {
+  /** Name of the event for metrics purposes */
+  eventName: 'proof_construction_time';
+  /** Name of the program being proven */
+  acir_test: string;
+  /** Number of threads used for proving */
+  threads: number;
+  /** Time spent proving */
+  value: number;
+};
+
+/** Stats associated with an L2 block. */
+export type L2BlockStats = {
+  /** Number of txs in the L2 block. */
+  txCount: number;
+  /** Number of the L2 block. */
+  blockNumber: number;
+  /** Number of public logs. */
+  publicLogCount?: number;
+};
+
+export type CheckpointStats = {
+  /** Number of transactions in the checkpoint */
+  txCount: number;
+  /** Number of blocks in the checkpoint */
+  blockCount: number;
+};
+
+/** Stats logged for each L1 publish tx.*/
+export type L1PublishStats = {
+  /** Address of the sender. */
+  sender: string;
+  /** Effective gas price of the tx. */
+  gasPrice: bigint;
+  /** Effective gas used in the tx. */
+  gasUsed: bigint;
+  /** Hash of the L1 tx. */
+  transactionHash: string;
+  /** Gas cost of the calldata. */
+  calldataGas: number;
+  /** Size in bytes of the calldata. */
+  calldataSize: number;
+  /** Gas cost of the blob data */
+  blobDataGas: bigint;
+  /** Amount of blob gas used. */
+  blobGasUsed: bigint;
+  /** Number of blobs in the tx */
+  blobCount?: number;
+  /** Number of L1 blocks between tx submission and inclusion */
+  inclusionBlocks?: number;
+};
+
+/** Stats logged for each L1 rollup publish tx.*/
+export type L1PublishCheckpointStats = {
+  /** Name of the event for metrics purposes */
+  eventName: 'rollup-published-to-l1';
+} & L1PublishStats &
+  CheckpointStats;
+
+/** Stats logged for each L1 rollup publish tx.*/
+export type L1PublishProofStats = {
+  /** Name of the event for metrics purposes */
+  eventName: 'proof-published-to-l1';
+} & L1PublishStats;
+
+/** Stats logged for synching node chain history.  */
+export type NodeSyncedChainHistoryStats = {
+  /** Name of the event. */
+  eventName: 'node-synced-chain-history';
+  /** Number of txs in the L2 block.. */
+  txCount: number;
+  /** Number of txs in each block. */
+  txsPerBlock: number;
+  /** Duration in ms. */
+  duration: number;
+  /** Id of the L2 block. */
+  blockNumber: number;
+  /** Number of blocks processed. */
+  blockCount: number;
+  /** Size of the db in bytes. */
+  dbSize: number;
+};
+
+export type ClientCircuitName =
+  | 'private-kernel-init'
+  | 'private-kernel-init-2'
+  | 'private-kernel-init-3'
+  | 'private-kernel-init-4'
+  | 'private-kernel-init-5'
+  | 'private-kernel-inner'
+  | 'private-kernel-inner-2'
+  | 'private-kernel-inner-3'
+  | 'private-kernel-inner-4'
+  | 'private-kernel-inner-5'
+  | 'private-kernel-reset'
+  | 'private-kernel-reset-tail'
+  | 'private-kernel-reset-tail-to-public'
+  | 'hiding-kernel-to-rollup'
+  | 'hiding-kernel-to-public'
+  | 'app-circuit';
+
+export type ServerCircuitName =
+  | 'inbox-parity-64'
+  | 'inbox-parity-256'
+  | 'inbox-parity-1024'
+  | 'chonk-verifier-public'
+  | 'rollup-tx-base-private'
+  | 'rollup-tx-base-public'
+  | 'rollup-tx-merge'
+  | 'rollup-block-root'
+  | 'rollup-block-root-single-tx'
+  | 'rollup-block-root-no-txs'
+  | 'rollup-block-merge'
+  | 'rollup-checkpoint-root'
+  | 'rollup-checkpoint-root-single-block'
+  | 'rollup-checkpoint-padding'
+  | 'rollup-checkpoint-merge'
+  | 'rollup-root'
+  | 'avm-circuit';
+
+// Circuit names should match the corresponding names in noir-protocol-circuits.
+export type CircuitName = ClientCircuitName | ServerCircuitName;
+
+/** Stats for circuit simulation. */
+export type CircuitSimulationStats = {
+  /** name of the event. */
+  eventName: 'circuit-simulation';
+  /** Name of the circuit. */
+  circuitName: CircuitName;
+  /** Optional. The function name that's being simulated */
+  appCircuitName?: string;
+  /** Duration in ms. */
+  duration: number;
+  /** Size in bytes of circuit inputs. */
+  inputSize: number;
+  /** Size in bytes of circuit outputs (aka public inputs). */
+  outputSize: number;
+};
+
+export type PublicDBAccessStats = {
+  eventName: 'public-db-access';
+  duration: number;
+  operation: string;
+};
+
+export type AvmSimulationStats = {
+  /** name of the event. */
+  eventName: 'avm-simulation';
+  /** Name of the circuit. */
+  appCircuitName: string;
+  /** Duration in ms. */
+  duration: number;
+};
+
+/** Stats for witness generation. */
+export type CircuitWitnessGenerationStats = {
+  /** name of the event. */
+  eventName: 'circuit-witness-generation';
+  /** Name of the circuit. */
+  circuitName: CircuitName;
+  /** Optional. The function name that's being proven */
+  appCircuitName?: string;
+  /** Duration in ms. */
+  duration: number;
+  /** Size in bytes of circuit inputs. */
+  inputSize: number;
+  /** Size in bytes of circuit outputs (aka public inputs). */
+  outputSize: number;
+};
+
+/** Stats for proving a circuit */
+export type CircuitProvingStats = {
+  /** Name of the event. */
+  eventName: 'circuit-proving';
+  /** Name of the circuit. */
+  circuitName: CircuitName;
+  /** Optional. The function name that was proven */
+  appCircuitName?: string;
+  /** Duration in ms. */
+  duration: number;
+  /** The size of the circuit (in gates) */
+  circuitSize: number;
+  /** Size in bytes of circuit inputs. */
+  inputSize: number;
+  /** Size in bytes of the proof. */
+  proofSize: number;
+  /** The number of public inputs */
+  numPublicInputs: number;
+};
+
+/** Stats for verifying a circuit */
+export type CircuitVerificationStats = {
+  /** Name of the event. */
+  eventName: 'circuit-verification';
+  /** Name of the circuit. */
+  circuitName: CircuitName;
+  /** Type of proof (chonk, honk, etc) */
+  proofType: 'chonk' | 'ultra-honk';
+  /** Duration in ms. */
+  duration: number;
+};
+
+/** Stats for an L2 block built by a sequencer. */
+export type L2BlockBuiltStats = {
+  /** Name of the event. */
+  eventName: 'l2-block-built';
+  /** Total duration in ms. */
+  duration: number;
+  /** Time for processing public txs in ms. */
+  publicProcessDuration: number;
+} & L2BlockStats;
+
+/** Stats for an L2 block processed by the world state synchronizer. */
+export type L2BlockHandledStats = {
+  /** Name of the event. */
+  eventName: 'l2-block-handled';
+  /** Total duration in ms. */
+  duration: number;
+  /** Pending block number. */
+  unfinalizedBlockNumber: bigint;
+  /** Proven block number. */
+  finalizedBlockNumber: bigint;
+  /** Oldest historic block number. */
+  oldestHistoricBlock: bigint;
+} & L2BlockStats;
+
+/** Stats for a tx. */
+export type TxStats = {
+  /** Hash of the tx. */
+  txHash: string;
+  /** Total size in bytes. */
+  size: number;
+  /** Size of the proof. */
+  proofSize: number;
+  /** Number of note hashes */
+  noteHashCount: number;
+  /** Number of nullifiers */
+  nullifierCount: number;
+  /** Number of private logs */
+  privateLogCount: number;
+  /** How many classes were published to the canonical class registry. */
+  classPublishedCount: number;
+  /** Serialized size of contract class logs in fields. */
+  contractClassLogSize: number;
+  /** How this tx pays for its fee */
+  feePaymentMethod: 'fee_juice' | 'fpc';
+};
+
+/**
+ * Stats for tree insertions
+ */
+export type TreeInsertionStats = {
+  /** Name of the event. */
+  eventName: 'tree-insertion';
+  /** Duration in ms. */
+  duration: number;
+  /** The size of the insertion batch */
+  batchSize: number;
+  /** The tree name */
+  treeName: string;
+  /** The tree depth */
+  treeDepth: number;
+  /** Tree type */
+  treeType: 'append-only' | 'indexed';
+  /** Number of hashes performed */
+  hashCount: number;
+  /** Average duration of a hash operation */
+  hashDuration: number;
+};
+
+/** A new tx was added to the tx pool. */
+export type TxAddedToPoolStats = {
+  /** Name of the event. */
+  eventName: 'tx-added-to-pool';
+} & TxStats;
+
+/** Stats emitted in structured logs with an `eventName` for tracking. */
+export type Stats =
+  | AvmSimulationStats
+  | CircuitProvingStats
+  | CircuitSimulationStats
+  | CircuitWitnessGenerationStats
+  | PublicDBAccessStats
+  | L1PublishCheckpointStats
+  | L1PublishProofStats
+  | L2BlockBuiltStats
+  | L2BlockHandledStats
+  | NodeSyncedChainHistoryStats
+  | ProofConstructed
+  | TreeInsertionStats
+  | TxAddedToPoolStats;
+
+/** Set of event names across emitted stats. */
+export type StatsEventName = Stats['eventName'];