@aztec/prover-node 0.0.0-test.1 → 0.0.1-commit.017a351

package/dest/prover-publisher-factory.d.ts

@@ -0,0 +1,25 @@
+import type { RollupContract } from '@aztec/ethereum/contracts';
+import type { L1TxUtils } from '@aztec/ethereum/l1-tx-utils';
+import type { PublisherManager } from '@aztec/ethereum/publisher-manager';
+import type { LoggerBindings } from '@aztec/foundation/log';
+import type { ProverPublisherConfig, ProverTxSenderConfig } from '@aztec/sequencer-client';
+import type { TelemetryClient } from '@aztec/telemetry-client';
+import { ProverNodePublisher } from './prover-node-publisher.js';
+export declare class ProverPublisherFactory {
+    private config;
+    private deps;
+    private bindings?;
+    constructor(config: ProverTxSenderConfig & ProverPublisherConfig, deps: {
+        rollupContract: RollupContract;
+        publisherManager: PublisherManager<L1TxUtils>;
+        telemetry?: TelemetryClient;
+    }, bindings?: LoggerBindings | undefined);
+    start(): Promise<void>;
+    stop(): Promise<void>;
+    /**
+     * Creates a new Prover Publisher instance.
+     * @returns A new ProverNodePublisher instance.
+     */
+    create(): Promise<ProverNodePublisher>;
+}
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoicHJvdmVyLXB1Ymxpc2hlci1mYWN0b3J5LmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi9zcmMvcHJvdmVyLXB1Ymxpc2hlci1mYWN0b3J5LnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLE9BQU8sS0FBSyxFQUFFLGNBQWMsRUFBRSxNQUFNLDJCQUEyQixDQUFDO0FBQ2hFLE9BQU8sS0FBSyxFQUFFLFNBQVMsRUFBRSxNQUFNLDZCQUE2QixDQUFDO0FBQzdELE9BQU8sS0FBSyxFQUFFLGdCQUFnQixFQUFFLE1BQU0sbUNBQW1DLENBQUM7QUFDMUUsT0FBTyxLQUFLLEVBQUUsY0FBYyxFQUFFLE1BQU0sdUJBQXVCLENBQUM7QUFDNUQsT0FBTyxLQUFLLEVBQUUscUJBQXFCLEVBQUUsb0JBQW9CLEVBQUUsTUFBTSx5QkFBeUIsQ0FBQztBQUMzRixPQUFPLEtBQUssRUFBRSxlQUFlLEVBQUUsTUFBTSx5QkFBeUIsQ0FBQztBQUUvRCxPQUFPLEVBQUUsbUJBQW1CLEVBQUUsTUFBTSw0QkFBNEIsQ0FBQztBQUVqRSxxQkFBYSxzQkFBc0I7SUFFL0IsT0FBTyxDQUFDLE1BQU07SUFDZCxPQUFPLENBQUMsSUFBSTtJQUtaLE9BQU8sQ0FBQyxRQUFRLENBQUM7SUFQbkIsWUFDVSxNQUFNLEVBQUUsb0JBQW9CLEdBQUcscUJBQXFCLEVBQ3BELElBQUksRUFBRTtRQUNaLGNBQWMsRUFBRSxjQUFjLENBQUM7UUFDL0IsZ0JBQWdCLEVBQUUsZ0JBQWdCLENBQUMsU0FBUyxDQUFDLENBQUM7UUFDOUMsU0FBUyxDQUFDLEVBQUUsZUFBZSxDQUFDO0tBQzdCLEVBQ08sUUFBUSxDQUFDLDRCQUFnQixFQUMvQjtJQUVTLEtBQUssa0JBRWpCO0lBRVksSUFBSSxrQkFFaEI7SUFFRDs7O09BR0c7SUFDVSxNQUFNLElBQUksT0FBTyxDQUFDLG1CQUFtQixDQUFDLENBV2xEO0NBQ0YifQ==

package/dest/prover-publisher-factory.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"prover-publisher-factory.d.ts","sourceRoot":"","sources":["../src/prover-publisher-factory.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAChE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,6BAA6B,CAAC;AAC7D,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,mCAAmC,CAAC;AAC1E,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAC5D,OAAO,KAAK,EAAE,qBAAqB,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAC3F,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAE/D,OAAO,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAC;AAEjE,qBAAa,sBAAsB;IAE/B,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,IAAI;IAKZ,OAAO,CAAC,QAAQ,CAAC;IAPnB,YACU,MAAM,EAAE,oBAAoB,GAAG,qBAAqB,EACpD,IAAI,EAAE;QACZ,cAAc,EAAE,cAAc,CAAC;QAC/B,gBAAgB,EAAE,gBAAgB,CAAC,SAAS,CAAC,CAAC;QAC9C,SAAS,CAAC,EAAE,eAAe,CAAC;KAC7B,EACO,QAAQ,CAAC,4BAAgB,EAC/B;IAES,KAAK,kBAEjB;IAEY,IAAI,kBAEhB;IAED;;;OAGG;IACU,MAAM,IAAI,OAAO,CAAC,mBAAmB,CAAC,CAWlD;CACF"}

package/dest/prover-publisher-factory.js

@@ -0,0 +1,28 @@
+import { ProverNodePublisher } from './prover-node-publisher.js';
+export class ProverPublisherFactory {
+    config;
+    deps;
+    bindings;
+    constructor(config, deps, bindings){
+        this.config = config;
+        this.deps = deps;
+        this.bindings = bindings;
+    }
+    async start() {
+        await this.deps.publisherManager.start();
+    }
+    async stop() {
+        await this.deps.publisherManager.stop();
+    }
+    /**
+   * Creates a new Prover Publisher instance.
+   * @returns A new ProverNodePublisher instance.
+   */ async create() {
+        const l1Publisher = await this.deps.publisherManager.getAvailablePublisher();
+        return new ProverNodePublisher(this.config, {
+            rollupContract: this.deps.rollupContract,
+            l1TxUtils: l1Publisher,
+            telemetry: this.deps.telemetry
+        }, this.bindings);
+    }
+}

package/dest/test/index.d.ts

@@ -1,10 +1,12 @@
 import type { EpochProverManager } from '@aztec/stdlib/interfaces/server';
+import type { EpochProvingJob } from '../job/epoch-proving-job.js';
 import type { ProverNodePublisher } from '../prover-node-publisher.js';
 import { ProverNode } from '../prover-node.js';
-declare class TestProverNode_ extends ProverNode {
+declare abstract class TestProverNodeClass extends ProverNode {
     prover: EpochProverManager;
     publisher: ProverNodePublisher;
+    abstract tryUploadEpochFailure(job: EpochProvingJob): Promise<string | undefined>;
 }
-export type TestProverNode = TestProverNode_;
+export type TestProverNode = TestProverNodeClass;
 export {};
-//# sourceMappingURL=index.d.ts.map
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiaW5kZXguZC50cyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uL3NyYy90ZXN0L2luZGV4LnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLE9BQU8sS0FBSyxFQUFFLGtCQUFrQixFQUFFLE1BQU0saUNBQWlDLENBQUM7QUFFMUUsT0FBTyxLQUFLLEVBQUUsZUFBZSxFQUFFLE1BQU0sNkJBQTZCLENBQUM7QUFDbkUsT0FBTyxLQUFLLEVBQUUsbUJBQW1CLEVBQUUsTUFBTSw2QkFBNkIsQ0FBQztBQUN2RSxPQUFPLEVBQUUsVUFBVSxFQUFFLE1BQU0sbUJBQW1CLENBQUM7QUFFL0MsdUJBQWUsbUJBQW9CLFNBQVEsVUFBVTtJQUNwQyxNQUFNLEVBQUUsa0JBQWtCLENBQUM7SUFDM0IsU0FBUyxFQUFFLG1CQUFtQixDQUFDO0lBRTlDLFNBQXlCLHFCQUFxQixDQUFDLEdBQUcsRUFBRSxlQUFlLEdBQUcsT0FBTyxDQUFDLE1BQU0sR0FBRyxTQUFTLENBQUMsQ0FBQztDQUNuRztBQUVELE1BQU0sTUFBTSxjQUFjLEdBQUcsbUJBQW1CLENBQUMifQ==

package/dest/test/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/test/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,iCAAiC,CAAC;AAE1E,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,6BAA6B,CAAC;AACvE,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAE/C,cAAM,eAAgB,SAAQ,UAAU;IACtB,MAAM,EAAG,kBAAkB,CAAC;IAC5B,SAAS,EAAG,mBAAmB,CAAC;CACjD;AAED,MAAM,MAAM,cAAc,GAAG,eAAe,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/test/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,iCAAiC,CAAC;AAE1E,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,6BAA6B,CAAC;AACnE,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,6BAA6B,CAAC;AACvE,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAE/C,uBAAe,mBAAoB,SAAQ,UAAU;IACpC,MAAM,EAAE,kBAAkB,CAAC;IAC3B,SAAS,EAAE,mBAAmB,CAAC;IAE9C,SAAyB,qBAAqB,CAAC,GAAG,EAAE,eAAe,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CAAC;CACnG;AAED,MAAM,MAAM,cAAc,GAAG,mBAAmB,CAAC"}

package/dest/test/index.js

@@ -1,5 +1,3 @@
 import { ProverNode } from '../prover-node.js';
-class TestProverNode_ extends ProverNode {
-    prover;
-    publisher;
+class TestProverNodeClass extends ProverNode {
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aztec/prover-node",
-  "version": "0.0.0-test.1",
+  "version": "0.0.1-commit.017a351",
   "type": "module",
   "exports": {
     ".": "./dest/index.js",
@@ -11,13 +11,12 @@
     "../package.common.json"
   ],
   "scripts": {
-    "build": "yarn clean && tsc -b",
-    "build:dev": "tsc -b --watch",
+    "build": "yarn clean && ../scripts/tsc.sh",
+    "build:dev": "../scripts/tsc.sh --watch",
     "clean": "rm -rf ./dest .tsbuildinfo",
-    "formatting": "run -T prettier --check ./src && run -T eslint ./src",
-    "formatting:fix": "run -T eslint --fix ./src && run -T prettier -w ./src",
     "bb": "node --no-warnings ./dest/bb/index.js",
-    "test": "NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=${JEST_MAX_WORKERS:-8}"
+    "test": "NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=${JEST_MAX_WORKERS:-8}",
+    "run-failed-epoch": "node --no-warnings ./dest/bin/run-failed-epoch.js"
   },
   "jest": {
     "moduleNameMapper": {
@@ -50,41 +49,49 @@
     "testTimeout": 120000,
     "setupFiles": [
       "../../foundation/src/jest/setup.mjs"
+    ],
+    "testEnvironment": "../../foundation/src/jest/env.mjs",
+    "setupFilesAfterEnv": [
+      "../../foundation/src/jest/setupAfterEnv.mjs"
     ]
   },
   "dependencies": {
-    "@aztec/archiver": "0.0.0-test.1",
-    "@aztec/bb-prover": "0.0.0-test.1",
-    "@aztec/blob-sink": "0.0.0-test.1",
-    "@aztec/constants": "0.0.0-test.1",
-    "@aztec/epoch-cache": "0.0.0-test.1",
-    "@aztec/ethereum": "0.0.0-test.1",
-    "@aztec/foundation": "0.0.0-test.1",
-    "@aztec/kv-store": "0.0.0-test.1",
-    "@aztec/l1-artifacts": "0.0.0-test.1",
-    "@aztec/noir-protocol-circuits-types": "0.0.0-test.1",
-    "@aztec/p2p": "0.0.0-test.1",
-    "@aztec/protocol-contracts": "0.0.0-test.1",
-    "@aztec/prover-client": "0.0.0-test.1",
-    "@aztec/sequencer-client": "0.0.0-test.1",
-    "@aztec/simulator": "0.0.0-test.1",
-    "@aztec/stdlib": "0.0.0-test.1",
-    "@aztec/telemetry-client": "0.0.0-test.1",
-    "@aztec/world-state": "0.0.0-test.1",
+    "@aztec/archiver": "0.0.1-commit.017a351",
+    "@aztec/bb-prover": "0.0.1-commit.017a351",
+    "@aztec/blob-client": "0.0.1-commit.017a351",
+    "@aztec/blob-lib": "0.0.1-commit.017a351",
+    "@aztec/constants": "0.0.1-commit.017a351",
+    "@aztec/epoch-cache": "0.0.1-commit.017a351",
+    "@aztec/ethereum": "0.0.1-commit.017a351",
+    "@aztec/foundation": "0.0.1-commit.017a351",
+    "@aztec/kv-store": "0.0.1-commit.017a351",
+    "@aztec/l1-artifacts": "0.0.1-commit.017a351",
+    "@aztec/native": "0.0.1-commit.017a351",
+    "@aztec/node-keystore": "0.0.1-commit.017a351",
+    "@aztec/node-lib": "0.0.1-commit.017a351",
+    "@aztec/noir-protocol-circuits-types": "0.0.1-commit.017a351",
+    "@aztec/p2p": "0.0.1-commit.017a351",
+    "@aztec/protocol-contracts": "0.0.1-commit.017a351",
+    "@aztec/prover-client": "0.0.1-commit.017a351",
+    "@aztec/sequencer-client": "0.0.1-commit.017a351",
+    "@aztec/simulator": "0.0.1-commit.017a351",
+    "@aztec/stdlib": "0.0.1-commit.017a351",
+    "@aztec/telemetry-client": "0.0.1-commit.017a351",
+    "@aztec/world-state": "0.0.1-commit.017a351",
     "source-map-support": "^0.5.21",
     "tslib": "^2.4.0",
-    "viem": "2.22.8"
+    "viem": "npm:@aztec/viem@2.38.2"
   },
   "devDependencies": {
-    "@jest/globals": "^29.5.0",
-    "@types/jest": "^29.5.0",
-    "@types/memdown": "^3.0.0",
-    "@types/node": "^18.7.23",
+    "@jest/globals": "^30.0.0",
+    "@types/jest": "^30.0.0",
+    "@types/node": "^22.15.17",
     "@types/source-map-support": "^0.5.10",
-    "jest": "^29.5.0",
-    "jest-mock-extended": "^3.0.3",
+    "@typescript/native-preview": "7.0.0-dev.20260113.1",
+    "jest": "^30.0.0",
+    "jest-mock-extended": "^4.0.0",
     "ts-node": "^10.9.1",
-    "typescript": "^5.0.4"
+    "typescript": "^5.3.3"
   },
   "files": [
     "dest",
@@ -93,6 +100,6 @@
   ],
   "types": "./dest/index.d.ts",
   "engines": {
-    "node": ">=18"
+    "node": ">=20.10"
   }
 }

package/src/actions/download-epoch-proving-job.ts

@@ -0,0 +1,44 @@
+import { jsonParseWithSchema } from '@aztec/foundation/json-rpc';
+import type { Logger } from '@aztec/foundation/log';
+import { urlJoin } from '@aztec/foundation/string';
+import { snapshotSync } from '@aztec/node-lib/actions';
+import { createReadOnlyFileStore } from '@aztec/stdlib/file-store';
+import { UploadSnapshotMetadataSchema, makeSnapshotPaths } from '@aztec/stdlib/snapshots';
+
+import { readFileSync } from 'fs';
+
+import { deserializeEpochProvingJobData } from '../job/epoch-proving-job-data.js';
+
+/**
+ * Given a location returned by `uploadEpochProofFailure`, downloads the world state and archiver snapshots
+ * and the proving job data, so we can re-run the job later using `rerunEpochProvingJob`. This is decoupled
+ * from actually proving so we can download once and run multiple times.
+ */
+export async function downloadEpochProvingJob(
+  location: string,
+  log: Logger,
+  config: {
+    dataDirectory: string;
+    jobDataDownloadPath: string;
+  },
+) {
+  log.info(`Downloading epoch proving job data from ${location}`);
+  const fileStore = await createReadOnlyFileStore(location);
+  const metadataUrl = urlJoin(location, 'metadata.json');
+  const metadataRaw = await fileStore.read(metadataUrl);
+  const metadata = jsonParseWithSchema(metadataRaw.toString(), UploadSnapshotMetadataSchema);
+
+  const dataUrls = makeSnapshotPaths(location);
+  log.info(`Downloading state snapshot from ${location} to local data directory`, { metadata, dataUrls });
+  await snapshotSync({ dataUrls }, log, { ...config, ...metadata, fileStore });
+
+  const dataPath = urlJoin(location, 'data.bin');
+  const localPath = config.jobDataDownloadPath;
+  log.info(`Downloading epoch proving job data from ${dataPath} to ${localPath}`);
+  await fileStore.download(dataPath, localPath);
+
+  const jobData = deserializeEpochProvingJobData(readFileSync(localPath));
+  log.info(`Epoch proving job data for epoch ${jobData.epochNumber} downloaded successfully`);
+
+  return metadata;
+}

package/src/actions/index.ts

@@ -0,0 +1,2 @@
+export * from './download-epoch-proving-job.js';
+export * from './rerun-epoch-proving-job.js';

package/src/actions/rerun-epoch-proving-job.ts

@@ -0,0 +1,74 @@
+import { createArchiverStore, createContractDataSource } from '@aztec/archiver';
+import type { L1ContractsConfig } from '@aztec/ethereum/config';
+import type { Logger } from '@aztec/foundation/log';
+import { type ProverClientConfig, createProverClient } from '@aztec/prover-client';
+import { ProverBrokerConfig, createAndStartProvingBroker } from '@aztec/prover-client/broker';
+import { PublicProcessorFactory } from '@aztec/simulator/server';
+import type { DataStoreConfig } from '@aztec/stdlib/kv-store';
+import type { GenesisData } from '@aztec/stdlib/world-state';
+import { getTelemetryClient } from '@aztec/telemetry-client';
+import { createWorldState } from '@aztec/world-state';
+
+import { readFileSync } from 'fs';
+
+import { deserializeEpochProvingJobData } from '../job/epoch-proving-job-data.js';
+import { EpochProvingJob } from '../job/epoch-proving-job.js';
+import { ProverNodeJobMetrics } from '../metrics.js';
+
+/**
+ * Given a local folder where `downloadEpochProvingJob` was called, creates a new archiver and world state
+ * using the state snapshots, and creates a new epoch proving job to prove the downloaded proving job.
+ * Proving is done with a local proving broker and agents as specified by the config.
+ */
+export async function rerunEpochProvingJob(
+  localPath: string,
+  log: Logger,
+  config: DataStoreConfig & ProverBrokerConfig & ProverClientConfig & Pick<L1ContractsConfig, 'aztecEpochDuration'>,
+  genesis?: GenesisData,
+) {
+  const jobData = deserializeEpochProvingJobData(readFileSync(localPath));
+  log.info(`Loaded proving job data for epoch ${jobData.epochNumber}`);
+
+  const telemetry = getTelemetryClient();
+  const metrics = new ProverNodeJobMetrics(telemetry.getMeter('prover-job'), telemetry.getTracer('prover-job'));
+  await using worldState = await createWorldState(config, genesis);
+  const initialBlockHash = await worldState.getInitialHeader().hash();
+  const archiver = await createArchiverStore(config, initialBlockHash);
+  const publicProcessorFactory = new PublicProcessorFactory(
+    createContractDataSource(archiver),
+    undefined,
+    undefined,
+    log.getBindings(),
+  );
+
+  const publisher = {
+    submitEpochProof: () => Promise.resolve(true),
+    analyzeEpochProofSubmission: () => Promise.resolve(),
+  };
+  const l2BlockSourceForReorgDetection = undefined;
+  const deadline = undefined;
+
+  // This starts a local proving broker that does not get exposed as a service. This should be good enough for
+  // smallish epochs to be proven if we run on a large machine, but as epochs grow larger, we may want to switch
+  // this out for a live proving broker with multiple agents that we can connect to.
+  const broker = await createAndStartProvingBroker(config, telemetry);
+  const prover = await createProverClient(config, worldState, broker, telemetry);
+
+  const provingJob = new EpochProvingJob(
+    jobData,
+    worldState,
+    prover.createEpochProver(),
+    publicProcessorFactory,
+    publisher,
+    l2BlockSourceForReorgDetection,
+    metrics,
+    deadline,
+    { skipEpochCheck: true },
+    log.getBindings(),
+  );
+
+  log.info(`Rerunning epoch proving job for epoch ${jobData.epochNumber}`);
+  await provingJob.run();
+  log.info(`Completed job for epoch ${jobData.epochNumber} with status ${provingJob.getState()}`);
+  return provingJob.getState();
+}

package/src/actions/upload-epoch-proof-failure.ts

@@ -0,0 +1,88 @@
+import { ARCHIVER_DB_VERSION, type Archiver } from '@aztec/archiver';
+import { tryRmDir } from '@aztec/foundation/fs';
+import { jsonStringify } from '@aztec/foundation/json-rpc';
+import type { Logger } from '@aztec/foundation/log';
+import { isoDate } from '@aztec/foundation/string';
+import { buildSnapshotMetadata, createBackups } from '@aztec/node-lib/actions';
+import type { ChainConfig } from '@aztec/stdlib/config';
+import { type FileStore, createFileStore } from '@aztec/stdlib/file-store';
+import type { WorldStateSynchronizer } from '@aztec/stdlib/interfaces/server';
+import type { DataStoreConfig } from '@aztec/stdlib/kv-store';
+import { type UploadSnapshotMetadata, getBasePath, uploadSnapshotData } from '@aztec/stdlib/snapshots';
+import { WORLD_STATE_DB_VERSION } from '@aztec/world-state';
+
+import { mkdtemp } from 'fs/promises';
+import { tmpdir } from 'os';
+import { dirname, join } from 'path';
+
+import { type EpochProvingJobData, serializeEpochProvingJobData } from '../job/epoch-proving-job-data.js';
+
+type UploadEpochProofConfig = Pick<ChainConfig, 'l1ChainId' | 'rollupVersion'> & Pick<DataStoreConfig, 'dataDirectory'>;
+
+/** Whether uploaded data to the file store should be of public access. */
+const PUBLIC_UPLOADS = true;
+
+/**
+ * Uploads a snapshot of world state and archiver (requires pausing them) along with the proving job data,
+ * so we can download and re-run the job later under the same conditions.
+ * @param location The location to upload the data to (used to create the `FileStore`).
+ */
+export async function uploadEpochProofFailure(
+  location: string,
+  jobId: string,
+  jobData: EpochProvingJobData,
+  archiver: Archiver,
+  worldState: WorldStateSynchronizer,
+  config: UploadEpochProofConfig,
+  log: Logger,
+) {
+  const epochNumber = jobData.epochNumber;
+  log.warn(`Uploading epoch proof failure for ${epochNumber} to ${location}`, { epochNumber, jobId, location });
+
+  const backupDir = await mkdtemp(join(config.dataDirectory ?? tmpdir(), 'epoch-proof-data-'));
+  const store = await createFileStore(location);
+  if (!store) {
+    throw new Error(`Failed to create file store for epoch proof failure upload for location ${location}.`);
+  }
+
+  try {
+    const versions = { archiver: ARCHIVER_DB_VERSION, worldState: WORLD_STATE_DB_VERSION };
+    const uploadMetadata = await buildSnapshotMetadata(archiver, config);
+    const paths = await createBackups(backupDir, archiver, worldState, log);
+
+    const basePath = `${getBasePath(uploadMetadata)}/${epochNumber}-${isoDate()}-${jobId}`;
+    const pathFor = (key: string) => `${basePath}/${key}.db`;
+    const [metadata, dataUrl, metadataUrl] = await Promise.all([
+      uploadSnapshotData(paths, versions, uploadMetadata, store, { pathFor, private: !PUBLIC_UPLOADS }),
+      uploadJobData(jobData, store, basePath),
+      uploadSnapshotMetadata(uploadMetadata, store, basePath),
+    ] as const);
+
+    const baseUrl = dirname(metadataUrl);
+    log.warn(`Uploaded epoch ${epochNumber} proof failure data to ${baseUrl}`, {
+      epochNumber,
+      location,
+      basePath,
+      metadataUrl,
+      dataUrl,
+      metadata,
+      jobId,
+    });
+    return baseUrl;
+  } finally {
+    log.info(`Cleaning up backup dir ${backupDir}`);
+    await tryRmDir(backupDir, log);
+  }
+}
+
+async function uploadJobData(jobData: EpochProvingJobData, store: FileStore, basePath: string) {
+  const data = serializeEpochProvingJobData(jobData);
+  const path = `${basePath}/data.bin`;
+  return await store.save(path, data, { compress: true, public: PUBLIC_UPLOADS });
+}
+
+async function uploadSnapshotMetadata(metadata: UploadSnapshotMetadata, store: FileStore, basePath: string) {
+  const data = Buffer.from(jsonStringify(metadata), 'utf-8');
+  const path = `${basePath}/metadata.json`;
+  return await store.save(path, data, { compress: false, public: PUBLIC_UPLOADS });
+}

package/src/bin/run-failed-epoch.ts

@@ -0,0 +1,79 @@
+/* eslint-disable no-console */
+import { getL1ContractsConfigEnvVars } from '@aztec/ethereum/config';
+import { EthAddress } from '@aztec/foundation/eth-address';
+import { jsonParseWithSchema, jsonStringify } from '@aztec/foundation/json-rpc';
+import { createLogger } from '@aztec/foundation/log';
+import { downloadEpochProvingJob, getProverNodeConfigFromEnv, rerunEpochProvingJob } from '@aztec/prover-node';
+import { type UploadSnapshotMetadata, UploadSnapshotMetadataSchema } from '@aztec/stdlib/snapshots';
+
+import { existsSync, mkdirSync } from 'fs';
+import { readFile, writeFile } from 'fs/promises';
+import { basename, join } from 'path';
+
+const logger = createLogger('prover-node:run-failed-epoch');
+
+function printUsage() {
+  console.error('Usage: run-failed-epoch <proof-uri> [out-dir=./data]');
+}
+
+async function rerunFailedEpoch(provingJobUrl: string, baseLocalDir: string) {
+  const localDir = join(baseLocalDir, basename(provingJobUrl));
+  const jobPath = join(localDir, 'data.bin');
+  const dataDir = join(localDir, 'state');
+
+  const env = getProverNodeConfigFromEnv();
+  const l1Config = getL1ContractsConfigEnvVars();
+  const config = {
+    ...env,
+    ...l1Config,
+    dataDirectory: dataDir,
+    dataStoreMapSizeKb: env.dataStoreMapSizeKb ?? 1024 * 1024,
+    proverId: env.proverId ?? EthAddress.random(),
+  };
+
+  let metadata: UploadSnapshotMetadata;
+  const metadataPath = join(localDir, 'metadata.json');
+  if (existsSync(metadataPath)) {
+    logger.info(`Using downloaded data`);
+    metadata = jsonParseWithSchema(await readFile(metadataPath, 'utf-8'), UploadSnapshotMetadataSchema);
+  } else {
+    logger.info(`Downloading epoch proving job data and state from ${provingJobUrl} to ${localDir}`);
+    metadata = await downloadEpochProvingJob(provingJobUrl!, logger, {
+      jobDataDownloadPath: jobPath,
+      dataDirectory: dataDir,
+    });
+    await writeFile(metadataPath, jsonStringify(metadata, true));
+    logger.info(`Download to ${localDir} complete`);
+  }
+
+  logger.info(`Rerunning proving job from ${jobPath} with state from ${dataDir}`, metadata);
+  const result = await rerunEpochProvingJob(jobPath, logger, {
+    ...config,
+    rollupAddress: metadata.rollupAddress,
+    rollupVersion: metadata.rollupVersion,
+  });
+
+  console.error(`Epoch proving job complete with result ${result}`);
+}
+
+async function main() {
+  if (process.argv[2] === '--help') {
+    printUsage();
+    return;
+  }
+
+  const uri = process.argv[2];
+  const outDir = process.argv[3] || './data';
+  if (!uri) {
+    printUsage();
+    throw new Error('Missing URL to epoch proving job');
+  }
+
+  mkdirSync(outDir, { recursive: true });
+  await rerunFailedEpoch(uri, outDir);
+}
+
+main().catch(err => {
+  console.error(err);
+  process.exit(1);
+});