@aztec/prover-node 0.86.0 → 0.87.0

package/dest/test/index.js

@@ -1,3 +1,3 @@
 import { ProverNode } from '../prover-node.js';
-class TestProverNode_ extends ProverNode {
+class TestProverNodeClass extends ProverNode {
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aztec/prover-node",
-  "version": "0.86.0",
+  "version": "0.87.0",
   "type": "module",
   "exports": {
     ".": "./dest/index.js",
@@ -15,7 +15,8 @@
     "build:dev": "tsc -b --watch",
     "clean": "rm -rf ./dest .tsbuildinfo",
     "bb": "node --no-warnings ./dest/bb/index.js",
-    "test": "NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=${JEST_MAX_WORKERS:-8}"
+    "test": "NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=${JEST_MAX_WORKERS:-8}",
+    "run-failed-epoch": "node --no-warnings ./dest/bin/run-failed-epoch.js"
   },
   "jest": {
     "moduleNameMapper": {
@@ -51,25 +52,25 @@
     ]
   },
   "dependencies": {
-    "@aztec/archiver": "0.86.0",
-    "@aztec/bb-prover": "0.86.0",
-    "@aztec/blob-sink": "0.86.0",
-    "@aztec/constants": "0.86.0",
-    "@aztec/epoch-cache": "0.86.0",
-    "@aztec/ethereum": "0.86.0",
-    "@aztec/foundation": "0.86.0",
-    "@aztec/kv-store": "0.86.0",
-    "@aztec/l1-artifacts": "0.86.0",
-    "@aztec/node-lib": "0.86.0",
-    "@aztec/noir-protocol-circuits-types": "0.86.0",
-    "@aztec/p2p": "0.86.0",
-    "@aztec/protocol-contracts": "0.86.0",
-    "@aztec/prover-client": "0.86.0",
-    "@aztec/sequencer-client": "0.86.0",
-    "@aztec/simulator": "0.86.0",
-    "@aztec/stdlib": "0.86.0",
-    "@aztec/telemetry-client": "0.86.0",
-    "@aztec/world-state": "0.86.0",
+    "@aztec/archiver": "0.87.0",
+    "@aztec/bb-prover": "0.87.0",
+    "@aztec/blob-sink": "0.87.0",
+    "@aztec/constants": "0.87.0",
+    "@aztec/epoch-cache": "0.87.0",
+    "@aztec/ethereum": "0.87.0",
+    "@aztec/foundation": "0.87.0",
+    "@aztec/kv-store": "0.87.0",
+    "@aztec/l1-artifacts": "0.87.0",
+    "@aztec/node-lib": "0.87.0",
+    "@aztec/noir-protocol-circuits-types": "0.87.0",
+    "@aztec/p2p": "0.87.0",
+    "@aztec/protocol-contracts": "0.87.0",
+    "@aztec/prover-client": "0.87.0",
+    "@aztec/sequencer-client": "0.87.0",
+    "@aztec/simulator": "0.87.0",
+    "@aztec/stdlib": "0.87.0",
+    "@aztec/telemetry-client": "0.87.0",
+    "@aztec/world-state": "0.87.0",
     "source-map-support": "^0.5.21",
     "tslib": "^2.4.0",
     "viem": "2.23.7"
@@ -77,12 +78,12 @@
   "devDependencies": {
     "@jest/globals": "^29.5.0",
     "@types/jest": "^29.5.0",
-    "@types/node": "^18.7.23",
+    "@types/node": "^22.15.17",
     "@types/source-map-support": "^0.5.10",
     "jest": "^29.5.0",
     "jest-mock-extended": "^3.0.3",
     "ts-node": "^10.9.1",
-    "typescript": "^5.0.4"
+    "typescript": "^5.3.3"
   },
   "files": [
     "dest",
@@ -91,6 +92,6 @@
   ],
   "types": "./dest/index.d.ts",
   "engines": {
-    "node": ">=18"
+    "node": ">=20.10"
   }
 }

package/src/actions/download-epoch-proving-job.ts

@@ -0,0 +1,44 @@
+import { jsonParseWithSchemaSync } from '@aztec/foundation/json-rpc';
+import type { Logger } from '@aztec/foundation/log';
+import { urlJoin } from '@aztec/foundation/string';
+import { snapshotSync } from '@aztec/node-lib/actions';
+import { createReadOnlyFileStore } from '@aztec/stdlib/file-store';
+import { UploadSnapshotMetadataSchema, makeSnapshotPaths } from '@aztec/stdlib/snapshots';
+
+import { readFileSync } from 'fs';
+
+import { deserializeEpochProvingJobData } from '../job/epoch-proving-job-data.js';
+
+/**
+ * Given a location returned by `uploadEpochProofFailure`, downloads the world state and archiver snapshots
+ * and the proving job data, so we can re-run the job later using `rerunEpochProvingJob`. This is decoupled
+ * from actually proving so we can download once and run multiple times.
+ */
+export async function downloadEpochProvingJob(
+  location: string,
+  log: Logger,
+  config: {
+    dataDirectory: string;
+    jobDataDownloadPath: string;
+  },
+) {
+  log.info(`Downloading epoch proving job data from ${location}`);
+  const fileStore = await createReadOnlyFileStore(location);
+  const metadataUrl = urlJoin(location, 'metadata.json');
+  const metadataRaw = await fileStore.read(metadataUrl);
+  const metadata = jsonParseWithSchemaSync(metadataRaw.toString(), UploadSnapshotMetadataSchema);
+
+  const dataUrls = makeSnapshotPaths(location);
+  log.info(`Downloading state snapshot from ${location} to local data directory`, { metadata, dataUrls });
+  await snapshotSync({ dataUrls }, log, { ...config, ...metadata, snapshotsUrl: location });
+
+  const dataPath = urlJoin(location, 'data.bin');
+  const localPath = config.jobDataDownloadPath;
+  log.info(`Downloading epoch proving job data from ${dataPath} to ${localPath}`);
+  await fileStore.download(dataPath, localPath);
+
+  const jobData = deserializeEpochProvingJobData(readFileSync(localPath));
+  log.info(`Epoch proving job data for epoch ${jobData.epochNumber} downloaded successfully`);
+
+  return metadata;
+}

package/src/actions/index.ts

@@ -0,0 +1,2 @@
+export * from './download-epoch-proving-job.js';
+export * from './rerun-epoch-proving-job.js';

package/src/actions/rerun-epoch-proving-job.ts

@@ -0,0 +1,61 @@
+import { createArchiverStore } from '@aztec/archiver';
+import type { Logger } from '@aztec/foundation/log';
+import type { DataStoreConfig } from '@aztec/kv-store/config';
+import { type ProverClientConfig, createProverClient } from '@aztec/prover-client';
+import { ProverBrokerConfig, createAndStartProvingBroker } from '@aztec/prover-client/broker';
+import { PublicProcessorFactory } from '@aztec/simulator/server';
+import { getTelemetryClient } from '@aztec/telemetry-client';
+import { createWorldState } from '@aztec/world-state';
+
+import { readFileSync } from 'fs';
+
+import { deserializeEpochProvingJobData } from '../job/epoch-proving-job-data.js';
+import { EpochProvingJob } from '../job/epoch-proving-job.js';
+import { ProverNodeJobMetrics } from '../metrics.js';
+
+/**
+ * Given a local folder where `downloadEpochProvingJob` was called, creates a new archiver and world state
+ * using the state snapshots, and creates a new epoch proving job to prove the downloaded proving job.
+ * Proving is done with a local proving broker and agents as specified by the config.
+ */
+export async function rerunEpochProvingJob(
+  localPath: string,
+  log: Logger,
+  config: DataStoreConfig & ProverBrokerConfig & ProverClientConfig,
+) {
+  const jobData = deserializeEpochProvingJobData(readFileSync(localPath));
+  log.info(`Loaded proving job data for epoch ${jobData.epochNumber}`);
+
+  const telemetry = getTelemetryClient();
+  const metrics = new ProverNodeJobMetrics(telemetry.getMeter('prover-job'), telemetry.getTracer('prover-job'));
+  const worldState = await createWorldState(config);
+  const archiver = await createArchiverStore(config);
+  const publicProcessorFactory = new PublicProcessorFactory(archiver);
+
+  const publisher = { submitEpochProof: () => Promise.resolve(true) };
+  const l2BlockSourceForReorgDetection = undefined;
+  const deadline = undefined;
+
+  // This starts a local proving broker that does not get exposed as a service. This should be good enough for
+  // smallish epochs to be proven if we run on a large machine, but as epochs grow larger, we may want to switch
+  // this out for a live proving broker with multiple agents that we can connect to.
+  const broker = await createAndStartProvingBroker(config, telemetry);
+  const prover = await createProverClient(config, worldState, broker, telemetry);
+
+  const provingJob = new EpochProvingJob(
+    jobData,
+    worldState,
+    prover.createEpochProver(),
+    publicProcessorFactory,
+    publisher,
+    l2BlockSourceForReorgDetection,
+    metrics,
+    deadline,
+    { skipEpochCheck: true },
+  );
+
+  log.info(`Rerunning epoch proving job for epoch ${jobData.epochNumber}`);
+  await provingJob.run();
+  log.info(`Completed job for epoch ${jobData.epochNumber} with status ${provingJob.getState()}`);
+  return provingJob.getState();
+}

package/src/actions/upload-epoch-proof-failure.ts

@@ -0,0 +1,88 @@
+import { ARCHIVER_DB_VERSION, type Archiver } from '@aztec/archiver';
+import { tryRmDir } from '@aztec/foundation/fs';
+import { jsonStringify } from '@aztec/foundation/json-rpc';
+import type { Logger } from '@aztec/foundation/log';
+import { isoDate } from '@aztec/foundation/string';
+import type { DataStoreConfig } from '@aztec/kv-store/config';
+import { buildSnapshotMetadata, createBackups } from '@aztec/node-lib/actions';
+import type { ChainConfig } from '@aztec/stdlib/config';
+import { type FileStore, createFileStore } from '@aztec/stdlib/file-store';
+import type { WorldStateSynchronizer } from '@aztec/stdlib/interfaces/server';
+import { type UploadSnapshotMetadata, getBasePath, uploadSnapshotData } from '@aztec/stdlib/snapshots';
+import { WORLD_STATE_DB_VERSION } from '@aztec/world-state';
+
+import { mkdtemp } from 'fs/promises';
+import { tmpdir } from 'os';
+import { dirname, join } from 'path';
+
+import { type EpochProvingJobData, serializeEpochProvingJobData } from '../job/epoch-proving-job-data.js';
+
+type UploadEpochProofConfig = Pick<ChainConfig, 'l1ChainId' | 'rollupVersion'> & Pick<DataStoreConfig, 'dataDirectory'>;
+
+/** Whether uploaded data to the file store should be of public access. */
+const PUBLIC_UPLOADS = true;
+
+/**
+ * Uploads a snapshot of world state and archiver (requires pausing them) along with the proving job data,
+ * so we can download and re-run the job later under the same conditions.
+ * @param location The location to upload the data to (used to create the `FileStore`).
+ */
+export async function uploadEpochProofFailure(
+  location: string,
+  jobId: string,
+  jobData: EpochProvingJobData,
+  archiver: Archiver,
+  worldState: WorldStateSynchronizer,
+  config: UploadEpochProofConfig,
+  log: Logger,
+) {
+  const epochNumber = jobData.epochNumber;
+  log.warn(`Uploading epoch proof failure for ${epochNumber} to ${location}`, { epochNumber, jobId, location });
+
+  const backupDir = await mkdtemp(join(config.dataDirectory ?? tmpdir(), 'epoch-proof-data-'));
+  const store = await createFileStore(location);
+  if (!store) {
+    throw new Error(`Failed to create file store for epoch proof failure upload for location ${location}.`);
+  }
+
+  try {
+    const versions = { archiver: ARCHIVER_DB_VERSION, worldState: WORLD_STATE_DB_VERSION };
+    const uploadMetadata = await buildSnapshotMetadata(archiver, config);
+    const paths = await createBackups(backupDir, archiver, worldState, log);
+
+    const basePath = `${getBasePath(uploadMetadata)}/${epochNumber}-${isoDate()}-${jobId}`;
+    const pathFor = (key: string) => `${basePath}/${key}.db`;
+    const [metadata, dataUrl, metadataUrl] = await Promise.all([
+      uploadSnapshotData(paths, versions, uploadMetadata, store, { pathFor, private: !PUBLIC_UPLOADS }),
+      uploadJobData(jobData, store, basePath),
+      uploadSnapshotMetadata(uploadMetadata, store, basePath),
+    ] as const);
+
+    const baseUrl = dirname(metadataUrl);
+    log.warn(`Uploaded epoch ${epochNumber} proof failure data to ${baseUrl}`, {
+      epochNumber,
+      location,
+      basePath,
+      metadataUrl,
+      dataUrl,
+      metadata,
+      jobId,
+    });
+    return baseUrl;
+  } finally {
+    log.info(`Cleaning up backup dir ${backupDir}`);
+    await tryRmDir(backupDir, log);
+  }
+}
+
+async function uploadJobData(jobData: EpochProvingJobData, store: FileStore, basePath: string) {
+  const data = serializeEpochProvingJobData(jobData);
+  const path = `${basePath}/data.bin`;
+  return await store.save(path, data, { compress: true, public: PUBLIC_UPLOADS });
+}
+
+async function uploadSnapshotMetadata(metadata: UploadSnapshotMetadata, store: FileStore, basePath: string) {
+  const data = Buffer.from(jsonStringify(metadata), 'utf-8');
+  const path = `${basePath}/metadata.json`;
+  return await store.save(path, data, { compress: false, public: PUBLIC_UPLOADS });
+}

package/src/bin/run-failed-epoch.ts

@@ -0,0 +1,77 @@
+/* eslint-disable no-console */
+import type { L1ContractAddresses } from '@aztec/ethereum';
+import { Fr } from '@aztec/foundation/fields';
+import { jsonParseWithSchemaSync, jsonStringify } from '@aztec/foundation/json-rpc';
+import { createLogger } from '@aztec/foundation/log';
+import { downloadEpochProvingJob, getProverNodeConfigFromEnv, rerunEpochProvingJob } from '@aztec/prover-node';
+import { type UploadSnapshotMetadata, UploadSnapshotMetadataSchema } from '@aztec/stdlib/snapshots';
+
+import { existsSync, mkdirSync } from 'fs';
+import { readFile, writeFile } from 'fs/promises';
+import { basename, join } from 'path';
+
+const logger = createLogger('prover-node:run-failed-epoch');
+
+function printUsage() {
+  console.error('Usage: run-failed-epoch <proof-uri> [out-dir=./data]');
+}
+
+async function rerunFailedEpoch(provingJobUrl: string, baseLocalDir: string) {
+  const localDir = join(baseLocalDir, basename(provingJobUrl));
+  const jobPath = join(localDir, 'data.bin');
+  const dataDir = join(localDir, 'state');
+
+  const env = getProverNodeConfigFromEnv();
+  const config = {
+    ...getProverNodeConfigFromEnv(),
+    dataDirectory: dataDir,
+    dataStoreMapSizeKB: env.dataStoreMapSizeKB ?? 1024 * 1024,
+    proverId: env.proverId ?? Fr.random(),
+  };
+
+  let metadata: UploadSnapshotMetadata;
+  const metadataPath = join(localDir, 'metadata.json');
+  if (existsSync(metadataPath)) {
+    logger.info(`Using downloaded data`);
+    metadata = jsonParseWithSchemaSync(await readFile(metadataPath, 'utf-8'), UploadSnapshotMetadataSchema);
+  } else {
+    logger.info(`Downloading epoch proving job data and state from ${provingJobUrl} to ${localDir}`);
+    metadata = await downloadEpochProvingJob(provingJobUrl!, logger, {
+      jobDataDownloadPath: jobPath,
+      dataDirectory: dataDir,
+    });
+    await writeFile(metadataPath, jsonStringify(metadata, true));
+    logger.info(`Download to ${localDir} complete`);
+  }
+
+  logger.info(`Rerunning proving job from ${jobPath} with state from ${dataDir}`, metadata);
+  const result = await rerunEpochProvingJob(jobPath, logger, {
+    ...config,
+    l1Contracts: { rollupAddress: metadata.rollupAddress } as L1ContractAddresses,
+    rollupVersion: metadata.rollupVersion,
+  });
+
+  console.error(`Epoch proving job complete with result ${result}`);
+}
+
+async function main() {
+  if (process.argv[2] === '--help') {
+    printUsage();
+    return;
+  }
+
+  const uri = process.argv[2];
+  const outDir = process.argv[3] || './data';
+  if (!uri) {
+    printUsage();
+    throw new Error('Missing URL to epoch proving job');
+  }
+
+  mkdirSync(outDir, { recursive: true });
+  await rerunFailedEpoch(uri, outDir);
+}
+
+main().catch(err => {
+  console.error(err);
+  process.exit(1);
+});

package/src/config.ts

@@ -40,10 +40,11 @@ export type ProverNodeConfig = ArchiverConfig &
   SpecificProverNodeConfig &
   GenesisStateConfig;
 
-type SpecificProverNodeConfig = {
+export type SpecificProverNodeConfig = {
   proverNodeMaxPendingJobs: number;
   proverNodePollingIntervalMs: number;
   proverNodeMaxParallelBlocksPerEpoch: number;
+  proverNodeFailedEpochStore: string | undefined;
   txGatheringIntervalMs: number;
   txGatheringBatchSize: number;
   txGatheringMaxParallelRequestsPerNode: number;
@@ -65,6 +66,11 @@ const specificProverNodeConfigMappings: ConfigMappingsType<SpecificProverNodeCon
     description: 'The Maximum number of blocks to process in parallel while proving an epoch',
     ...numberConfigHelper(32),
   },
+  proverNodeFailedEpochStore: {
+    env: 'PROVER_NODE_FAILED_EPOCH_STORE',
+    description: 'File store where to upload node state when an epoch fails to be proven',
+    defaultValue: undefined,
+  },
   txGatheringIntervalMs: {
     env: 'PROVER_NODE_TX_GATHERING_INTERVAL_MS',
     description: 'How often to check that tx data is available',

package/src/factory.ts

@@ -2,6 +2,7 @@ import { type Archiver, createArchiver } from '@aztec/archiver';
 import { type BlobSinkClientInterface, createBlobSinkClient } from '@aztec/blob-sink/client';
 import { EpochCache } from '@aztec/epoch-cache';
 import { L1TxUtils, RollupContract, createEthereumChain, createExtendedL1Client } from '@aztec/ethereum';
+import { pick } from '@aztec/foundation/collection';
 import { type Logger, createLogger } from '@aztec/foundation/log';
 import type { DataStoreConfig } from '@aztec/kv-store/config';
 import { trySnapshotSync } from '@aztec/node-lib/actions';
@@ -17,7 +18,7 @@ import { EpochMonitor } from './monitors/epoch-monitor.js';
 import type { TxSource } from './prover-coordination/combined-prover-coordination.js';
 import { createProverCoordination } from './prover-coordination/factory.js';
 import { ProverNodePublisher } from './prover-node-publisher.js';
-import { ProverNode, type ProverNodeOptions } from './prover-node.js';
+import { ProverNode } from './prover-node.js';
 
 /** Creates a new prover node given a config. */
 export async function createProverNode(
@@ -80,14 +81,27 @@ export async function createProverNode(
     telemetry,
   });
 
-  const proverNodeConfig: ProverNodeOptions = {
-    maxPendingJobs: config.proverNodeMaxPendingJobs,
-    pollingIntervalMs: config.proverNodePollingIntervalMs,
-    maxParallelBlocksPerEpoch: config.proverNodeMaxParallelBlocksPerEpoch,
-    txGatheringIntervalMs: config.txGatheringIntervalMs,
+  const proverNodeConfig = {
+    ...pick(
+      config,
+      'proverNodeMaxPendingJobs',
+      'proverNodeMaxParallelBlocksPerEpoch',
+      'proverNodePollingIntervalMs',
+      'txGatheringMaxParallelRequests',
+      'txGatheringIntervalMs',
+      'txGatheringTimeoutMs',
+      'proverNodeFailedEpochStore',
+      'dataDirectory',
+      'l1ChainId',
+      'rollupVersion',
+    ),
   };
 
-  const epochMonitor = await EpochMonitor.create(archiver, proverNodeConfig, telemetry);
+  const epochMonitor = await EpochMonitor.create(
+    archiver,
+    { pollingIntervalMs: config.proverNodePollingIntervalMs },
+    telemetry,
+  );
 
   return new ProverNode(
     prover,

package/src/index.ts

@@ -1,3 +1,4 @@
+export * from './actions/index.js';
 export * from './config.js';
 export * from './factory.js';
 export * from './http.js';

package/src/job/epoch-proving-job-data.ts

@@ -0,0 +1,68 @@
+import { Fr } from '@aztec/foundation/fields';
+import { BufferReader, serializeToBuffer } from '@aztec/foundation/serialize';
+import { L2Block } from '@aztec/stdlib/block';
+import { BlockHeader, Tx } from '@aztec/stdlib/tx';
+
+/** All data from an epoch used in proving. */
+export type EpochProvingJobData = {
+  epochNumber: bigint;
+  blocks: L2Block[];
+  txs: Tx[];
+  l1ToL2Messages: Record<number, Fr[]>;
+  previousBlockHeader: BlockHeader;
+};
+
+export function validateEpochProvingJobData(data: EpochProvingJobData) {
+  if (data.blocks.length > 0 && data.previousBlockHeader.getBlockNumber() + 1 !== data.blocks[0].number) {
+    throw new Error(
+      `Initial block number ${
+        data.blocks[0].number
+      } does not match previous block header ${data.previousBlockHeader.getBlockNumber()}`,
+    );
+  }
+
+  for (const blockNumber of data.blocks.map(block => block.number)) {
+    if (!(blockNumber in data.l1ToL2Messages)) {
+      throw new Error(`Missing L1 to L2 messages for block number ${blockNumber}`);
+    }
+  }
+}
+
+export function serializeEpochProvingJobData(data: EpochProvingJobData): Buffer {
+  const blocks = data.blocks.map(block => block.toBuffer());
+  const txs = data.txs.map(tx => tx.toBuffer());
+  const l1ToL2Messages = Object.entries(data.l1ToL2Messages).map(([blockNumber, messages]) => [
+    Number(blockNumber),
+    messages.length,
+    ...messages,
+  ]);
+
+  return serializeToBuffer(
+    Number(data.epochNumber),
+    data.previousBlockHeader,
+    blocks.length,
+    ...blocks,
+    txs.length,
+    ...txs,
+    l1ToL2Messages.length,
+    ...l1ToL2Messages,
+  );
+}
+
+export function deserializeEpochProvingJobData(buf: Buffer): EpochProvingJobData {
+  const reader = BufferReader.asReader(buf);
+  const epochNumber = BigInt(reader.readNumber());
+  const previousBlockHeader = reader.readObject(BlockHeader);
+  const blocks = reader.readVector(L2Block);
+  const txs = reader.readVector(Tx);
+
+  const l1ToL2MessageBlockCount = reader.readNumber();
+  const l1ToL2Messages: Record<number, Fr[]> = {};
+  for (let i = 0; i < l1ToL2MessageBlockCount; i++) {
+    const blockNumber = reader.readNumber();
+    const messages = reader.readVector(Fr);
+    l1ToL2Messages[blockNumber] = messages;
+  }
+
+  return { epochNumber, previousBlockHeader, blocks, txs, l1ToL2Messages };
+}