@aztec/stdlib 0.82.2 → 0.82.3

package/src/file-store/http.ts

@@ -0,0 +1,58 @@
+import { type Logger, createLogger } from '@aztec/foundation/log';
+import { makeBackoff, retry } from '@aztec/foundation/retry';
+
+import { createWriteStream } from 'fs';
+import { mkdir } from 'fs/promises';
+import { dirname } from 'path';
+import { Readable } from 'stream';
+import { finished } from 'stream/promises';
+
+import type { ReadOnlyFileStore } from './interface.js';
+
+export class HttpFileStore implements ReadOnlyFileStore {
+  private readonly fetch: typeof fetch;
+
+  constructor(private readonly baseUrl: string, private readonly log: Logger = createLogger('stdlib:http-file-store')) {
+    this.fetch = async (...args: Parameters<typeof fetch>): Promise<Response> => {
+      return await retry(
+        () => fetch(...args),
+        `Fetching ${args[0]}`,
+        makeBackoff([1, 1, 3]),
+        this.log,
+        /*failSilently=*/ true,
+      );
+    };
+  }
+
+  public async read(pathOrUrl: string): Promise<Buffer> {
+    const url = this.getUrl(pathOrUrl);
+    const response = await this.fetch(url);
+    if (response.ok) {
+      return Buffer.from(await response.arrayBuffer());
+    } else {
+      throw new Error(`Error fetching file from ${url}: ${response.statusText}`);
+    }
+  }
+
+  public async download(pathOrUrl: string, destPath: string): Promise<void> {
+    const url = this.getUrl(pathOrUrl);
+    const response = await this.fetch(url);
+    if (response.ok) {
+      await mkdir(dirname(destPath), { recursive: true });
+      // Typescript complains about Readable.fromWeb, hence the cast
+      await finished(Readable.fromWeb(response.body! as any).pipe(createWriteStream(destPath)));
+    } else {
+      throw new Error(`Error fetching file from ${url}: ${response.statusText}`);
+    }
+  }
+
+  public async exists(pathOrUrl: string): Promise<boolean> {
+    const url = this.getUrl(pathOrUrl);
+    const response = await this.fetch(url);
+    return response.ok;
+  }
+
+  private getUrl(path: string): string {
+    return URL.canParse(path) ? path : `${this.baseUrl.replace(/\/$/, '')}/${path}`;
+  }
+}

package/src/file-store/index.ts

@@ -0,0 +1,2 @@
+export * from './interface.js';
+export * from './factory.js';

package/src/file-store/interface.ts

@@ -0,0 +1,19 @@
+/** Simple read-only file store. */
+export interface ReadOnlyFileStore {
+  /** Reads a file given a path, or an URI as returned by calling `save`. Returns file contents. */
+  read(pathOrUrl: string): Promise<Buffer>;
+  /** Downloads a file given a path, or an URI as returned by calling `save`. Saves file to local path. */
+  download(pathOrUrlStr: string, destPath: string): Promise<void>;
+  /** Returns whether a file at the given path or URI exists. */
+  exists(pathOrUrl: string): Promise<boolean>;
+}
+
+export type FileStoreSaveOptions = { public?: boolean; metadata?: Record<string, string>; compress?: boolean };
+
+/** Simple file store. */
+export interface FileStore extends ReadOnlyFileStore {
+  /** Saves contents to the given path. Returns an URI that can be used later to `read` the file. */
+  save(path: string, data: Buffer, opts?: FileStoreSaveOptions): Promise<string>;
+  /** Uploads contents from a local file. Returns an URI that can be used later to `read` the file. */
+  upload(destPath: string, srcPath: string, opts?: FileStoreSaveOptions): Promise<string>;
+}

package/src/file-store/local.ts

@@ -0,0 +1,46 @@
+import { access, mkdir, readFile, writeFile } from 'fs/promises';
+import { dirname, resolve } from 'path';
+
+import type { FileStore } from './interface.js';
+
+export class LocalFileStore implements FileStore {
+  constructor(private readonly basePath: string) {}
+
+  public async save(path: string, data: Buffer): Promise<string> {
+    const fullPath = this.getFullPath(path);
+    await mkdir(dirname(fullPath), { recursive: true });
+    await writeFile(fullPath, data);
+    return `file://${fullPath}`;
+  }
+
+  public async upload(destPath: string, srcPath: string, _opts: { compress: boolean }): Promise<string> {
+    const data = await readFile(srcPath);
+    return this.save(destPath, data);
+  }
+
+  public read(pathOrUrlStr: string): Promise<Buffer> {
+    const fullPath = this.getFullPath(pathOrUrlStr);
+    return readFile(fullPath);
+  }
+
+  public async download(pathOrUrlStr: string, destPath: string): Promise<void> {
+    const data = await this.read(pathOrUrlStr);
+    const fullPath = this.getFullPath(destPath);
+    await writeFile(fullPath, data);
+  }
+
+  public exists(pathOrUrlStr: string): Promise<boolean> {
+    const fullPath = this.getFullPath(pathOrUrlStr);
+    return access(fullPath)
+      .then(() => true)
+      .catch(() => false);
+  }
+
+  private getFullPath(pathOrUrl: string): string {
+    if (URL.canParse(pathOrUrl)) {
+      return new URL(pathOrUrl).pathname;
+    } else {
+      return resolve(this.basePath, pathOrUrl);
+    }
+  }
+}

package/src/interfaces/aztec-node-admin.ts

@@ -17,13 +17,23 @@ export interface AztecNodeAdmin {
    */
   setConfig(config: Partial<SequencerConfig & ProverConfig>): Promise<void>;
 
-  /** Forces the next block to be built bypassing all time and pending checks. Useful for testing. */
+  /**
+   * Forces the next block to be built bypassing all time and pending checks.
+   * Useful for testing.
+   */
   flushTxs(): Promise<void>;
+
+  /**
+   * Pauses syncing, creates a backup of archiver and world-state databases, and uploads them. Returns immediately.
+   * @param location - The location to upload the snapshot to.
+   */
+  startSnapshotUpload(location: string): Promise<void>;
 }
 
 export const AztecNodeAdminApiSchema: ApiSchemaFor<AztecNodeAdmin> = {
   setConfig: z.function().args(SequencerConfigSchema.merge(ProverConfigSchema).partial()).returns(z.void()),
   flushTxs: z.function().returns(z.void()),
+  startSnapshotUpload: z.function().args(z.string()).returns(z.void()),
 };
 
 export function createAztecNodeAdminClient(

package/src/interfaces/aztec-node.ts

@@ -46,6 +46,8 @@ import {
   TxValidationResultSchema,
 } from '../tx/index.js';
 import { TxEffect } from '../tx/tx_effect.js';
+import { ValidatorsStatsSchema } from '../validators/schemas.js';
+import type { ValidatorsStats } from '../validators/types.js';
 import { type ComponentsVersions, getVersioningResponseHandler } from '../versioning/index.js';
 import {
   type GetContractClassLogsResponse,
@@ -379,6 +381,9 @@ export interface AztecNode
    */
   getBlockHeader(blockNumber?: L2BlockNumber): Promise<BlockHeader | undefined>;
 
+  /** Returns stats for validators if enabled. */
+  getValidatorsStats(): Promise<ValidatorsStats>;
+
   /**
    * Simulates the public part of a transaction with the current state.
    * This currently just checks that the transaction execution succeeds.
@@ -525,6 +530,8 @@ export const AztecNodeApiSchema: ApiSchemaFor<AztecNode> = {
 
   getBlockHeader: z.function().args(optional(L2BlockNumberSchema)).returns(BlockHeader.schema.optional()),
 
+  getValidatorsStats: z.function().returns(ValidatorsStatsSchema),
+
   simulatePublicCalls: z.function().args(Tx.schema, optional(z.boolean())).returns(PublicSimulationOutput.schema),
 
   isValidTx: z

package/src/interfaces/p2p.ts

@@ -50,6 +50,9 @@ export interface P2PClient extends P2PApiWithoutAttestations {
    * @returns BlockAttestations
    */
   getAttestationsForSlot(slot: bigint, proposalId?: string): Promise<BlockAttestation[]>;
+
+  /** Manually adds an attestation to the p2p client attestation pool. */
+  addAttestation(attestation: BlockAttestation): Promise<void>;
 }
 
 export type P2PApi<T extends P2PClientType = P2PClientType.Full> = T extends P2PClientType.Full
@@ -64,4 +67,5 @@ export const P2PApiSchema: ApiSchemaFor<P2PApi> = {
   getPendingTxs: z.function().returns(z.array(Tx.schema)),
   getEncodedEnr: z.function().returns(z.string().optional()),
   getPeers: z.function().args(optional(z.boolean())).returns(z.array(PeerInfoSchema)),
+  addAttestation: z.function().args(BlockAttestation.schema).returns(z.void()),
 };

package/src/interfaces/prover-node.ts

@@ -1,6 +1,8 @@
 import { z } from 'zod';
 
+import { type L2Tips, L2TipsSchema } from '../block/l2_block_source.js';
 import { type ApiSchemaFor, schemas } from '../schemas/index.js';
+import { type WorldStateSyncStatus, WorldStateSyncStatusSchema } from './world_state.js';
 
 const EpochProvingJobState = [
   'initialized',
@@ -31,6 +33,10 @@ export interface ProverNodeApi {
   getJobs(): Promise<{ uuid: string; status: EpochProvingJobState; epochNumber: number }[]>;
 
   startProof(epochNumber: number): Promise<void>;
+
+  getL2Tips(): Promise<L2Tips>;
+
+  getWorldStateSyncStatus(): Promise<WorldStateSyncStatus>;
 }
 
 /** Schemas for prover node API functions. */
@@ -41,4 +47,8 @@ export const ProverNodeApiSchema: ApiSchemaFor<ProverNodeApi> = {
     .returns(z.array(z.object({ uuid: z.string(), status: z.enum(EpochProvingJobState), epochNumber: z.number() }))),
 
   startProof: z.function().args(schemas.Integer).returns(z.void()),
+
+  getL2Tips: z.function().args().returns(L2TipsSchema),
+
+  getWorldStateSyncStatus: z.function().args().returns(WorldStateSyncStatusSchema),
 };

package/src/interfaces/service.ts

@@ -11,6 +11,9 @@ export interface Service {
 
   /** Stops the service. */
   stop(): Promise<void>;
+
+  /** Resumes the service after it was stopped */
+  resume(): void;
 }
 
 /** Tries to call stop on a given object and awaits it. Logs any errors and does not rethrow. */
@@ -23,3 +26,13 @@ export async function tryStop(service: Maybe<Service>, logger?: Logger): Promise
     logger?.error(`Error stopping service ${(service as object).constructor?.name}: ${err}`);
   }
 }
+
+export function tryRestart(service: Maybe<Service>, logger?: Logger) {
+  try {
+    return typeof service === 'object' && service && 'restart' in service && typeof service.restart === 'function'
+      ? service.restart()
+      : Promise.resolve();
+  } catch (err) {
+    logger?.error(`Error restarting service ${(service as object).constructor?.name}: ${err}`);
+  }
+}

package/src/interfaces/world_state.ts

@@ -1,7 +1,10 @@
 import { z } from 'zod';
 
+import type { SnapshotDataKeys } from '../snapshots/types.js';
 import type { MerkleTreeReadOperations, MerkleTreeWriteOperations } from './merkle_tree_operations.js';
 
+export type { SnapshotDataKeys };
+
 /**
  * Defines the possible states of the world state synchronizer.
  */
@@ -41,37 +44,36 @@ export interface ForkMerkleTreeOperations {
 
   /** Gets a handle that allows reading the state as it was at the given block number. */
   getSnapshot(blockNumber: number): MerkleTreeReadOperations;
+
+  /** Backups the db to the target path. */
+  backupTo(dstPath: string, compact?: boolean): Promise<Record<Exclude<SnapshotDataKeys, 'archiver'>, string>>;
 }
 
 /** Defines the interface for a world state synchronizer. */
 export interface WorldStateSynchronizer extends ForkMerkleTreeOperations {
-  /**
-   * Starts the synchronizer.
-   * @returns A promise that resolves once the initial sync is completed.
-   */
+  /** Starts the synchronizer. */
   start(): void;
 
-  /**
-   * Returns the current status of the synchronizer.
-   * @returns The current status of the synchronizer.
-   */
+  /** Returns the current status of the synchronizer. */
   status(): Promise<WorldStateSynchronizerStatus>;
 
-  /**
-   * Stops the synchronizer.
-   */
+  /** Stops the synchronizer and its database. */
   stop(): Promise<void>;
 
+  /** Stops the synchronizer from syncing, but keeps the database online. */
+  stopSync(): Promise<void>;
+
+  /** Resumes synching after a stopSync call. */
+  resumeSync(): void;
+
   /**
    * Forces an immediate sync to an optionally provided minimum block number
-   * @param minBlockNumber - The minimum block number that we must sync to
+   * @param minBlockNumber - The minimum block number that we must sync to (may sync further)
    * @returns A promise that resolves with the block number the world state was synced to
    */
   syncImmediate(minBlockNumber?: number): Promise<number>;
 
-  /**
-   * Returns an instance of MerkleTreeAdminOperations that will not include uncommitted data.
-   */
+  /** Returns an instance of MerkleTreeAdminOperations that will not include uncommitted data. */
   getCommitted(): MerkleTreeReadOperations;
 }
 

package/src/snapshots/download.ts

@@ -0,0 +1,60 @@
+import { fromEntries, getEntries, maxBy } from '@aztec/foundation/collection';
+import { jsonParseWithSchemaSync } from '@aztec/foundation/json-rpc';
+import type { ReadOnlyFileStore } from '@aztec/stdlib/file-store';
+
+import { join } from 'path';
+
+import {
+  SnapshotDataKeys,
+  type SnapshotDataUrls,
+  type SnapshotMetadata,
+  type SnapshotsIndex,
+  type SnapshotsIndexMetadata,
+  SnapshotsIndexSchema,
+} from './types.js';
+
+export async function getSnapshotIndex(
+  metadata: SnapshotsIndexMetadata,
+  store: ReadOnlyFileStore,
+): Promise<SnapshotsIndex | undefined> {
+  const basePath = getBasePath(metadata);
+  const snapshotIndexPath = `${basePath}/index.json`;
+  try {
+    if (await store.exists(snapshotIndexPath)) {
+      const snapshotIndexData = await store.read(snapshotIndexPath);
+      return jsonParseWithSchemaSync(snapshotIndexData.toString(), SnapshotsIndexSchema);
+    } else {
+      return undefined;
+    }
+  } catch (err) {
+    throw new Error(`Error reading snapshot index from ${snapshotIndexPath}: ${err}`);
+  }
+}
+
+export async function getLatestSnapshotMetadata(
+  metadata: SnapshotsIndexMetadata,
+  store: ReadOnlyFileStore,
+): Promise<SnapshotMetadata | undefined> {
+  const snapshotsIndex = await getSnapshotIndex(metadata, store);
+  return snapshotsIndex?.snapshots && maxBy(snapshotsIndex?.snapshots, s => s.l1BlockNumber);
+}
+
+export function getBasePath(metadata: SnapshotsIndexMetadata): string {
+  return `aztec-${metadata.l1ChainId}-${metadata.l2Version}-${metadata.rollupAddress}`;
+}
+
+export function getSnapshotIndexPath(metadata: SnapshotsIndexMetadata): string {
+  return `${getBasePath(metadata)}/index.json`;
+}
+
+export function makeSnapshotLocalPaths(baseDir: string): SnapshotDataUrls {
+  return fromEntries(SnapshotDataKeys.map(key => [key, join(baseDir, `${key}.db`)]));
+}
+
+export async function downloadSnapshot(
+  snapshot: Pick<SnapshotMetadata, 'dataUrls'>,
+  localPaths: Record<SnapshotDataKeys, string>,
+  store: ReadOnlyFileStore,
+): Promise<void> {
+  await Promise.all(getEntries(localPaths).map(([key, path]) => store.download(snapshot.dataUrls[key], path)));
+}

package/src/snapshots/index.ts

@@ -0,0 +1,3 @@
+export * from './types.js';
+export * from './upload.js';
+export { downloadSnapshot, getLatestSnapshotMetadata, makeSnapshotLocalPaths } from './download.js';

package/src/snapshots/types.ts

@@ -0,0 +1,58 @@
+import type { EthAddress } from '@aztec/foundation/eth-address';
+import { type ZodFor, schemas } from '@aztec/foundation/schemas';
+
+import { z } from 'zod';
+
+export const SnapshotDataKeys = [
+  'archiver',
+  'nullifier-tree',
+  'public-data-tree',
+  'note-hash-tree',
+  'archive-tree',
+  'l1-to-l2-message-tree',
+] as const;
+
+export type SnapshotDataKeys = (typeof SnapshotDataKeys)[number];
+
+export type SnapshotDataUrls = Record<SnapshotDataKeys, string>;
+
+export type SnapshotMetadata = {
+  l2BlockNumber: number;
+  l2BlockHash: string;
+  l1BlockNumber: number;
+  timestamp: number;
+  dataUrls: SnapshotDataUrls;
+  schemaVersions: { archiver: number; worldState: number };
+};
+
+export type SnapshotsIndexMetadata = {
+  l1ChainId: number;
+  l2Version: number;
+  rollupAddress: EthAddress;
+};
+
+export type SnapshotsIndex = SnapshotsIndexMetadata & {
+  snapshots: SnapshotMetadata[];
+};
+
+export const SnapshotsIndexSchema = z.object({
+  l1ChainId: z.number(),
+  l2Version: z.number(),
+  rollupAddress: schemas.EthAddress,
+  snapshots: z.array(
+    z.object({
+      l2BlockNumber: z.number(),
+      l2BlockHash: z.string(),
+      l1BlockNumber: z.number(),
+      timestamp: z.number(),
+      schemaVersions: z.object({
+        archiver: z.number(),
+        worldState: z.number(),
+      }),
+      dataUrls: z
+        .record(z.enum(SnapshotDataKeys), z.string())
+        // See https://stackoverflow.com/questions/77958464/zod-record-with-required-keys
+        .refine((obj): obj is Required<typeof obj> => SnapshotDataKeys.every(key => !!obj[key])),
+    }),
+  ),
+}) satisfies ZodFor<SnapshotsIndex>;

package/src/snapshots/upload.ts

@@ -0,0 +1,53 @@
+import { fromEntries, getEntries, pick } from '@aztec/foundation/collection';
+import { jsonStringify } from '@aztec/foundation/json-rpc';
+import type { FileStore } from '@aztec/stdlib/file-store';
+
+import { getBasePath, getSnapshotIndex, getSnapshotIndexPath } from './download.js';
+import type { SnapshotDataKeys, SnapshotMetadata, SnapshotsIndex } from './types.js';
+
+export type UploadSnapshotMetadata = Pick<SnapshotMetadata, 'l2BlockNumber' | 'l2BlockHash' | 'l1BlockNumber'> &
+  Pick<SnapshotsIndex, 'l1ChainId' | 'l2Version' | 'rollupAddress'>;
+
+export async function uploadSnapshot(
+  localPaths: Record<SnapshotDataKeys, string>,
+  schemaVersions: SnapshotMetadata['schemaVersions'],
+  metadata: UploadSnapshotMetadata,
+  store: FileStore,
+): Promise<SnapshotMetadata> {
+  const timestamp = Date.now();
+  const date = new Date().toISOString().replace(/[-:T]/g, '').replace(/\..+$/, '');
+  const basePath = getBasePath(metadata);
+  const targetPathFor = (key: SnapshotDataKeys) => `${basePath}/${key}-${date}-${metadata.l2BlockHash}.db`;
+
+  const dataUrls = fromEntries(
+    await Promise.all(
+      getEntries(localPaths).map(
+        async ([key, path]) =>
+          [key, await store.upload(targetPathFor(key), path, { compress: true, public: true })] as const,
+      ),
+    ),
+  );
+
+  const snapshotsIndex = (await getSnapshotIndex(metadata, store)) ?? createEmptyIndex(metadata);
+
+  const newSnapshotMetadata: SnapshotMetadata = {
+    ...pick(metadata, 'l1BlockNumber', 'l2BlockHash', 'l2BlockNumber'),
+    schemaVersions,
+    timestamp,
+    dataUrls,
+  };
+  snapshotsIndex.snapshots.unshift(newSnapshotMetadata);
+
+  await store.save(getSnapshotIndexPath(metadata), Buffer.from(jsonStringify(snapshotsIndex, true)), {
+    public: true, // Make the index publicly accessible
+    metadata: { ['Cache-control']: 'no-store' }, // Do not cache object versions
+  });
+  return newSnapshotMetadata;
+}
+
+function createEmptyIndex(metadata: Pick<SnapshotsIndex, 'l1ChainId' | 'l2Version' | 'rollupAddress'>): SnapshotsIndex {
+  return {
+    ...pick(metadata, 'l1ChainId', 'l2Version', 'rollupAddress'),
+    snapshots: [],
+  };
+}