@aztec-labs/prover-node 6.0.0-nightly.20260829

package/dest/actions/rerun-epoch-proving-job.d.ts

@@ -0,0 +1,23 @@
+import type { L1ContractsConfig } from '@aztec-labs/ethereum/config';
+import type { Logger } from '@aztec-labs/foundation/log';
+import { type ProverClientConfig } from '@aztec-labs/prover-client';
+import { ProverBrokerConfig } from '@aztec-labs/prover-client/broker';
+import { type CheckpointSubTreeProofs } from '@aztec-labs/prover-client/orchestrator';
+import type { DataStoreConfig } from '@aztec-labs/stdlib/kv-store';
+import type { GenesisData } from '@aztec-labs/stdlib/world-state';
+type RerunConfig = DataStoreConfig & ProverBrokerConfig & ProverClientConfig & Pick<L1ContractsConfig, 'aztecEpochDuration'>;
+/**
+ * Given a local folder where `downloadEpochProvingJob` was called, creates a new archiver and world state
+ * using the state snapshots, and creates a new epoch proving session to prove the downloaded proving job.
+ * Proving is done with a local proving broker and agents as specified by the config.
+ */
+export declare function rerunEpochProvingJob(localPath: string, log: Logger, config: RerunConfig, genesis?: GenesisData): Promise<"awaiting-checkpoints" | "awaiting-predecessor" | "awaiting-root" | "cancelled" | "completed" | "failed" | "initialized" | "publishing-proof" | "stopped" | "superseded" | "timed-out">;
+/**
+ * Re-proves a single downloaded checkpoint proving job (as uploaded by a `CheckpointProver` failure).
+ * Reconstructs just that checkpoint's sub-tree prover from the snapshot and awaits its block proofs — no
+ * epoch top-tree, no L1 submission — so a checkpoint-level failure can be reproduced offline in isolation.
+ * Returns the block-rollup proof outputs on success; throws if the checkpoint fails to prove again.
+ */
+export declare function rerunCheckpointProvingJob(localPath: string, log: Logger, config: RerunConfig, genesis?: GenesisData): Promise<CheckpointSubTreeProofs['blockProofOutputs']>;
+export {};
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoicmVydW4tZXBvY2gtcHJvdmluZy1qb2IuZC50cyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uL3NyYy9hY3Rpb25zL3JlcnVuLWVwb2NoLXByb3Zpbmctam9iLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUNBLE9BQU8sS0FBSyxFQUFFLGlCQUFpQixFQUFFLE1BQU0sNkJBQTZCLENBQUM7QUFFckUsT0FBTyxLQUFLLEVBQUUsTUFBTSxFQUFFLE1BQU0sNEJBQTRCLENBQUM7QUFFekQsT0FBTyxFQUFFLEtBQUssa0JBQWtCLEVBQXNCLE1BQU0sMkJBQTJCLENBQUM7QUFDeEYsT0FBTyxFQUFFLGtCQUFrQixFQUErQixNQUFNLGtDQUFrQyxDQUFDO0FBRW5HLE9BQU8sRUFBRSxLQUFLLHVCQUF1QixFQUFjLE1BQU0sd0NBQXdDLENBQUM7QUFLbEcsT0FBTyxLQUFLLEVBQUUsZUFBZSxFQUFFLE1BQU0sNkJBQTZCLENBQUM7QUFHbkUsT0FBTyxLQUFLLEVBQUUsV0FBVyxFQUFFLE1BQU0sZ0NBQWdDLENBQUM7QUFVbEUsS0FBSyxXQUFXLEdBQUcsZUFBZSxHQUNoQyxrQkFBa0IsR0FDbEIsa0JBQWtCLEdBQ2xCLElBQUksQ0FBQyxpQkFBaUIsRUFBRSxvQkFBb0IsQ0FBQyxDQUFDO0FBRWhEOzs7O0dBSUc7QUFDSCx3QkFBc0Isb0JBQW9CLENBQUMsU0FBUyxFQUFFLE1BQU0sRUFBRSxHQUFHLEVBQUUsTUFBTSxFQUFFLE1BQU0sRUFBRSxXQUFXLEVBQUUsT0FBTyxDQUFDLEVBQUUsV0FBVyxtTUFzQ3BIO0FBRUQ7Ozs7O0dBS0c7QUFDSCx3QkFBc0IseUJBQXlCLENBQzdDLFNBQVMsRUFBRSxNQUFNLEVBQ2pCLEdBQUcsRUFBRSxNQUFNLEVBQ1gsTUFBTSxFQUFFLFdBQVcsRUFDbkIsT0FBTyxDQUFDLEVBQUUsV0FBVyxHQUNwQixPQUFPLENBQUMsdUJBQXVCLENBQUMsbUJBQW1CLENBQUMsQ0FBQyxDQWdCdkQifQ==

package/dest/actions/rerun-epoch-proving-job.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"rerun-epoch-proving-job.d.ts","sourceRoot":"","sources":["../../src/actions/rerun-epoch-proving-job.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,6BAA6B,CAAC;AAErE,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,4BAA4B,CAAC;AAEzD,OAAO,EAAE,KAAK,kBAAkB,EAAsB,MAAM,2BAA2B,CAAC;AACxF,OAAO,EAAE,kBAAkB,EAA+B,MAAM,kCAAkC,CAAC;AAEnG,OAAO,EAAE,KAAK,uBAAuB,EAAc,MAAM,wCAAwC,CAAC;AAKlG,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,6BAA6B,CAAC;AAGnE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,gCAAgC,CAAC;AAUlE,KAAK,WAAW,GAAG,eAAe,GAChC,kBAAkB,GAClB,kBAAkB,GAClB,IAAI,CAAC,iBAAiB,EAAE,oBAAoB,CAAC,CAAC;AAEhD;;;;GAIG;AACH,wBAAsB,oBAAoB,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,OAAO,CAAC,EAAE,WAAW,mMAsCpH;AAED;;;;;GAKG;AACH,wBAAsB,yBAAyB,CAC7C,SAAS,EAAE,MAAM,EACjB,GAAG,EAAE,MAAM,EACX,MAAM,EAAE,WAAW,EACnB,OAAO,CAAC,EAAE,WAAW,GACpB,OAAO,CAAC,uBAAuB,CAAC,mBAAmB,CAAC,CAAC,CAgBvD"}

package/dest/actions/rerun-epoch-proving-job.js

@@ -0,0 +1,264 @@
+function _ts_add_disposable_resource(env, value, async) {
+    if (value !== null && value !== void 0) {
+        if (typeof value !== "object" && typeof value !== "function") throw new TypeError("Object expected.");
+        var dispose, inner;
+        if (async) {
+            if (!Symbol.asyncDispose) throw new TypeError("Symbol.asyncDispose is not defined.");
+            dispose = value[Symbol.asyncDispose];
+        }
+        if (dispose === void 0) {
+            if (!Symbol.dispose) throw new TypeError("Symbol.dispose is not defined.");
+            dispose = value[Symbol.dispose];
+            if (async) inner = dispose;
+        }
+        if (typeof dispose !== "function") throw new TypeError("Object not disposable.");
+        if (inner) dispose = function() {
+            try {
+                inner.call(this);
+            } catch (e) {
+                return Promise.reject(e);
+            }
+        };
+        env.stack.push({
+            value: value,
+            dispose: dispose,
+            async: async
+        });
+    } else if (async) {
+        env.stack.push({
+            async: true
+        });
+    }
+    return value;
+}
+function _ts_dispose_resources(env) {
+    var _SuppressedError = typeof SuppressedError === "function" ? SuppressedError : function(error, suppressed, message) {
+        var e = new Error(message);
+        return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
+    };
+    return (_ts_dispose_resources = function _ts_dispose_resources(env) {
+        function fail(e) {
+            env.error = env.hasError ? new _SuppressedError(e, env.error, "An error was suppressed during disposal.") : e;
+            env.hasError = true;
+        }
+        var r, s = 0;
+        function next() {
+            while(r = env.stack.pop()){
+                try {
+                    if (!r.async && s === 1) return s = 0, env.stack.push(r), Promise.resolve().then(next);
+                    if (r.dispose) {
+                        var result = r.dispose.call(r.value);
+                        if (r.async) return s |= 2, Promise.resolve(result).then(next, function(e) {
+                            fail(e);
+                            return next();
+                        });
+                    } else s |= 1;
+                } catch (e) {
+                    fail(e);
+                }
+            }
+            if (s === 1) return env.hasError ? Promise.reject(env.error) : Promise.resolve();
+            if (env.hasError) throw env.error;
+        }
+        return next();
+    })(env);
+}
+import { createArchiverStore, createContractDataSource } from '@aztec-labs/archiver';
+import { BlockNumber } from '@aztec-labs/foundation/branded-types';
+import { DateProvider } from '@aztec-labs/foundation/timer';
+import { createProverClient } from '@aztec-labs/prover-client';
+import { createAndStartProvingBroker } from '@aztec-labs/prover-client/broker';
+import { getLastSiblingPath } from '@aztec-labs/prover-client/helpers';
+import { ChonkCache } from '@aztec-labs/prover-client/orchestrator';
+import { AvmSimulatorPool, PublicProcessorFactory } from '@aztec-labs/simulator/server';
+import { getEpochAtSlot, getSlotRangeForEpoch } from '@aztec-labs/stdlib/epoch-helpers';
+import { MerkleTreeId } from '@aztec-labs/stdlib/trees';
+import { getTelemetryClient } from '@aztec-labs/telemetry-client';
+import { createWorldState } from '@aztec-labs/world-state';
+import { readFileSync } from 'fs';
+import { CheckpointProver } from '../job/checkpoint-prover.js';
+import { deserializeEpochProvingJobData } from '../job/epoch-proving-job-data.js';
+import { EpochSession } from '../job/epoch-session.js';
+import { ProverNodeJobMetrics } from '../metrics.js';
+/**
+ * Given a local folder where `downloadEpochProvingJob` was called, creates a new archiver and world state
+ * using the state snapshots, and creates a new epoch proving session to prove the downloaded proving job.
+ * Proving is done with a local proving broker and agents as specified by the config.
+ */ export async function rerunEpochProvingJob(localPath, log, config, genesis) {
+    const env = {
+        stack: [],
+        error: void 0,
+        hasError: false
+    };
+    try {
+        const ctx = _ts_add_disposable_resource(env, await createRerunContext(localPath, log, config, genesis), true);
+        const { jobData, prover, metrics } = ctx;
+        log.info(`Rerunning epoch proving for epoch ${jobData.epochNumber}`);
+        const provers = [];
+        for(let i = 0; i < jobData.checkpoints.length; i++){
+            provers.push(await buildCheckpointProver(ctx, i, log));
+        }
+        // Local rerun never publishes — stub the service so submit() always resolves 'published'
+        // and withdraw is a no-op.
+        const publishingService = {
+            submit: ()=>Promise.resolve('published'),
+            withdraw: ()=>{}
+        };
+        const l1Constants = {
+            epochDuration: config.aztecEpochDuration
+        };
+        const [fromSlot, toSlot] = getSlotRangeForEpoch(jobData.epochNumber, l1Constants);
+        const spec = {
+            kind: 'full',
+            epochNumber: jobData.epochNumber,
+            fromSlot,
+            toSlot
+        };
+        const session = new EpochSession(spec, provers, {
+            proverFactory: prover,
+            proverId: prover.getProverId(),
+            publishingService,
+            metrics,
+            dateProvider: new DateProvider(),
+            deadline: undefined,
+            config: {},
+            bindings: log.getBindings()
+        });
+        const finalState = await session.start();
+        log.info(`Completed proving for epoch ${jobData.epochNumber} with status ${finalState}`, {
+            derivedEpoch: getEpochAtSlot(provers[0].slotNumber, l1Constants)
+        });
+        return finalState;
+    } catch (e) {
+        env.error = e;
+        env.hasError = true;
+    } finally{
+        const result = _ts_dispose_resources(env);
+        if (result) await result;
+    }
+}
+/**
+ * Re-proves a single downloaded checkpoint proving job (as uploaded by a `CheckpointProver` failure).
+ * Reconstructs just that checkpoint's sub-tree prover from the snapshot and awaits its block proofs — no
+ * epoch top-tree, no L1 submission — so a checkpoint-level failure can be reproduced offline in isolation.
+ * Returns the block-rollup proof outputs on success; throws if the checkpoint fails to prove again.
+ */ export async function rerunCheckpointProvingJob(localPath, log, config, genesis) {
+    const env = {
+        stack: [],
+        error: void 0,
+        hasError: false
+    };
+    try {
+        const ctx = _ts_add_disposable_resource(env, await createRerunContext(localPath, log, config, genesis), true);
+        const { jobData } = ctx;
+        const checkpointNumber = jobData.checkpoints[0].number;
+        log.info(`Rerunning checkpoint proving for checkpoint ${checkpointNumber} (epoch ${jobData.epochNumber})`);
+        const prover = await buildCheckpointProver(ctx, 0, log);
+        try {
+            const { blockProofOutputs } = await prover.whenSubTreeProofsReady();
+            log.info(`Completed proving for checkpoint ${checkpointNumber} with ${blockProofOutputs.length} block proof(s)`);
+            return blockProofOutputs;
+        } finally{
+            prover.cancel({
+                routine: true
+            });
+            await prover.whenDone();
+        }
+    } catch (e) {
+        env.error = e;
+        env.hasError = true;
+    } finally{
+        const result = _ts_dispose_resources(env);
+        if (result) await result;
+    }
+}
+/**
+ * Rebuilds the offline proving environment from a downloaded job: world state + archiver from the
+ * snapshots, a local proving broker + client, a chonk cache, and a tx provider that replays the job's txs.
+ */ async function createRerunContext(localPath, log, config, genesis) {
+    const jobData = deserializeEpochProvingJobData(readFileSync(localPath));
+    log.info(`Loaded proving job data for epoch ${jobData.epochNumber}`);
+    const telemetry = getTelemetryClient();
+    const metrics = new ProverNodeJobMetrics(telemetry.getMeter('prover-job'), telemetry.getTracer('prover-job'));
+    const worldState = await createWorldState(config, genesis);
+    const initialBlockHash = await worldState.getInitialHeader().hash();
+    const archiver = await createArchiverStore(config, initialBlockHash);
+    const avmSimulator = await AvmSimulatorPool.spawn({
+        wsdbIpcPath: worldState.getIpcPath()
+    });
+    const publicProcessorFactory = new PublicProcessorFactory(createContractDataSource(archiver), avmSimulator, undefined, undefined, log.getBindings());
+    const broker = await createAndStartProvingBroker(config, telemetry);
+    const prover = await createProverClient(config, worldState, broker, telemetry);
+    const chonkCache = new ChonkCache(log.getBindings());
+    const txProvider = makeReplayingTxProvider(jobData.txs);
+    return {
+        jobData,
+        metrics,
+        worldState,
+        publicProcessorFactory,
+        prover,
+        chonkCache,
+        txProvider,
+        async [Symbol.asyncDispose] () {
+            await avmSimulator[Symbol.asyncDispose]();
+            await worldState[Symbol.asyncDispose]();
+        }
+    };
+}
+/** Reconstructs the `CheckpointProver` for the checkpoint at `index` in the job, ready to prove. */ async function buildCheckpointProver(ctx, index, log) {
+    const { jobData, worldState, prover, chonkCache, publicProcessorFactory, txProvider, metrics } = ctx;
+    const checkpoint = jobData.checkpoints[index];
+    const previousBlockHeader = index === 0 ? jobData.previousBlockHeader : jobData.checkpoints[index - 1].blocks.at(-1).header;
+    const previousInboxRollingHash = index === 0 ? jobData.previousInboxRollingHash : jobData.checkpoints[index - 1].header.inboxRollingHash;
+    const l1ToL2Messages = jobData.l1ToL2Messages[checkpoint.number] ?? [];
+    const previousArchiveSiblingPath = await getLastSiblingPath(MerkleTreeId.ARCHIVE, worldState.getSnapshot(BlockNumber(checkpoint.blocks[0].number - 1)));
+    const attestations = checkpoint.number === jobData.checkpoints.at(-1).number ? jobData.attestations : [];
+    return new CheckpointProver({
+        checkpoint,
+        epochNumber: jobData.epochNumber,
+        attestations,
+        previousBlockHeader,
+        l1ToL2Messages,
+        previousInboxRollingHash,
+        previousArchiveSiblingPath
+    }, {
+        proverFactory: prover,
+        chonkCache,
+        publicProcessorFactory,
+        dbProvider: worldState,
+        txProvider,
+        dateProvider: new DateProvider(),
+        proverId: prover.getProverId(),
+        metrics,
+        txGatheringTimeoutMs: 120_000,
+        deadline: undefined,
+        log
+    });
+}
+/** Build a synthetic ITxProvider that returns the supplied txs map by lookup. */ function makeReplayingTxProvider(txs) {
+    const lookup = (hashes)=>{
+        const found = [];
+        const missing = [];
+        for (const hash of hashes){
+            const tx = txs.get(hash.toString());
+            if (tx) {
+                found.push(tx);
+            } else {
+                missing.push(hash);
+            }
+        }
+        return {
+            txs: found,
+            missingTxs: missing
+        };
+    };
+    return {
+        getAvailableTxs: (hashes)=>Promise.resolve(lookup(hashes)),
+        hasTxs: (hashes)=>Promise.resolve(hashes.map((h)=>txs.has(h.toString()))),
+        getTxsForBlockProposal: ()=>Promise.resolve({
+                txs: [],
+                missingTxs: []
+            }),
+        getTxsForBlock: (block)=>Promise.resolve(lookup(block.body.txEffects.map((e)=>e.txHash)))
+    };
+}

package/dest/actions/upload-epoch-proof-failure.d.ts

@@ -0,0 +1,15 @@
+import { type Archiver } from '@aztec-labs/archiver';
+import type { Logger } from '@aztec-labs/foundation/log';
+import type { ChainConfig } from '@aztec-labs/stdlib/config';
+import type { WorldStateSynchronizer } from '@aztec-labs/stdlib/interfaces/server';
+import type { DataStoreConfig } from '@aztec-labs/stdlib/kv-store';
+import { type EpochProvingJobData } from '../job/epoch-proving-job-data.js';
+type UploadEpochProofConfig = Pick<ChainConfig, 'l1ChainId' | 'rollupVersion'> & Pick<DataStoreConfig, 'dataDirectory'>;
+/**
+ * Uploads a snapshot of world state and archiver (requires pausing them) along with the proving job data,
+ * so we can download and re-run the job later under the same conditions.
+ * @param location The location to upload the data to (used to create the `FileStore`).
+ */
+export declare function uploadEpochProofFailure(location: string, jobId: string, jobData: EpochProvingJobData, archiver: Archiver, worldState: WorldStateSynchronizer, config: UploadEpochProofConfig, log: Logger): Promise<string>;
+export {};
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoidXBsb2FkLWVwb2NoLXByb29mLWZhaWx1cmUuZC50cyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uL3NyYy9hY3Rpb25zL3VwbG9hZC1lcG9jaC1wcm9vZi1mYWlsdXJlLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLE9BQU8sRUFBdUIsS0FBSyxRQUFRLEVBQUUsTUFBTSxzQkFBc0IsQ0FBQztBQUcxRSxPQUFPLEtBQUssRUFBRSxNQUFNLEVBQUUsTUFBTSw0QkFBNEIsQ0FBQztBQUd6RCxPQUFPLEtBQUssRUFBRSxXQUFXLEVBQUUsTUFBTSwyQkFBMkIsQ0FBQztBQUU3RCxPQUFPLEtBQUssRUFBRSxzQkFBc0IsRUFBRSxNQUFNLHNDQUFzQyxDQUFDO0FBQ25GLE9BQU8sS0FBSyxFQUFFLGVBQWUsRUFBRSxNQUFNLDZCQUE2QixDQUFDO0FBT25FLE9BQU8sRUFBRSxLQUFLLG1CQUFtQixFQUFnQyxNQUFNLGtDQUFrQyxDQUFDO0FBRTFHLEtBQUssc0JBQXNCLEdBQUcsSUFBSSxDQUFDLFdBQVcsRUFBRSxXQUFXLEdBQUcsZUFBZSxDQUFDLEdBQUcsSUFBSSxDQUFDLGVBQWUsRUFBRSxlQUFlLENBQUMsQ0FBQztBQUt4SDs7OztHQUlHO0FBQ0gsd0JBQXNCLHVCQUF1QixDQUMzQyxRQUFRLEVBQUUsTUFBTSxFQUNoQixLQUFLLEVBQUUsTUFBTSxFQUNiLE9BQU8sRUFBRSxtQkFBbUIsRUFDNUIsUUFBUSxFQUFFLFFBQVEsRUFDbEIsVUFBVSxFQUFFLHNCQUFzQixFQUNsQyxNQUFNLEVBQUUsc0JBQXNCLEVBQzlCLEdBQUcsRUFBRSxNQUFNLG1CQXVDWiJ9

package/dest/actions/upload-epoch-proof-failure.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"upload-epoch-proof-failure.d.ts","sourceRoot":"","sources":["../../src/actions/upload-epoch-proof-failure.ts"],"names":[],"mappings":"AAAA,OAAO,EAAuB,KAAK,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAG1E,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,4BAA4B,CAAC;AAGzD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAC;AAE7D,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,sCAAsC,CAAC;AACnF,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,6BAA6B,CAAC;AAOnE,OAAO,EAAE,KAAK,mBAAmB,EAAgC,MAAM,kCAAkC,CAAC;AAE1G,KAAK,sBAAsB,GAAG,IAAI,CAAC,WAAW,EAAE,WAAW,GAAG,eAAe,CAAC,GAAG,IAAI,CAAC,eAAe,EAAE,eAAe,CAAC,CAAC;AAKxH;;;;GAIG;AACH,wBAAsB,uBAAuB,CAC3C,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,mBAAmB,EAC5B,QAAQ,EAAE,QAAQ,EAClB,UAAU,EAAE,sBAAsB,EAClC,MAAM,EAAE,sBAAsB,EAC9B,GAAG,EAAE,MAAM,mBAuCZ"}

package/dest/actions/upload-epoch-proof-failure.js

@@ -0,0 +1,78 @@
+import { ARCHIVER_DB_VERSION } from '@aztec-labs/archiver';
+import { tryRmDir } from '@aztec-labs/foundation/fs';
+import { jsonStringify } from '@aztec-labs/foundation/json-rpc';
+import { isoDate } from '@aztec-labs/foundation/string';
+import { buildSnapshotMetadata, createBackups } from '@aztec-labs/node-lib/actions';
+import { createFileStore } from '@aztec-labs/stdlib/file-store';
+import { getBasePath, uploadSnapshotData } from '@aztec-labs/stdlib/snapshots';
+import { WORLD_STATE_DB_VERSION } from '@aztec-labs/world-state';
+import { mkdtemp } from 'fs/promises';
+import { tmpdir } from 'os';
+import { dirname, join } from 'path';
+import { serializeEpochProvingJobData } from '../job/epoch-proving-job-data.js';
+/** Whether uploaded data to the file store should be of public access. */ const PUBLIC_UPLOADS = true;
+/**
+ * Uploads a snapshot of world state and archiver (requires pausing them) along with the proving job data,
+ * so we can download and re-run the job later under the same conditions.
+ * @param location The location to upload the data to (used to create the `FileStore`).
+ */ export async function uploadEpochProofFailure(location, jobId, jobData, archiver, worldState, config, log) {
+    const epochNumber = jobData.epochNumber;
+    log.warn(`Uploading epoch proof failure for ${epochNumber} to ${location}`, {
+        epochNumber,
+        jobId,
+        location
+    });
+    const backupDir = await mkdtemp(join(config.dataDirectory ?? tmpdir(), 'epoch-proof-data-'));
+    const store = await createFileStore(location);
+    if (!store) {
+        throw new Error(`Failed to create file store for epoch proof failure upload for location ${location}.`);
+    }
+    try {
+        const versions = {
+            archiver: ARCHIVER_DB_VERSION,
+            worldState: WORLD_STATE_DB_VERSION
+        };
+        const uploadMetadata = await buildSnapshotMetadata(archiver, config);
+        const paths = await createBackups(backupDir, archiver, worldState, log);
+        const basePath = `${getBasePath(uploadMetadata)}/${epochNumber}-${isoDate()}-${jobId}`;
+        const pathFor = (key)=>`${basePath}/${key}.db`;
+        const [metadata, dataUrl, metadataUrl] = await Promise.all([
+            uploadSnapshotData(paths, versions, uploadMetadata, store, {
+                pathFor,
+                private: !PUBLIC_UPLOADS
+            }),
+            uploadJobData(jobData, store, basePath),
+            uploadSnapshotMetadata(uploadMetadata, store, basePath)
+        ]);
+        const baseUrl = dirname(metadataUrl);
+        log.warn(`Uploaded epoch ${epochNumber} proof failure data to ${baseUrl}`, {
+            epochNumber,
+            location,
+            basePath,
+            metadataUrl,
+            dataUrl,
+            metadata,
+            jobId
+        });
+        return baseUrl;
+    } finally{
+        log.info(`Cleaning up backup dir ${backupDir}`);
+        await tryRmDir(backupDir, log);
+    }
+}
+async function uploadJobData(jobData, store, basePath) {
+    const data = serializeEpochProvingJobData(jobData);
+    const path = `${basePath}/data.bin`;
+    return await store.save(path, data, {
+        compress: true,
+        public: PUBLIC_UPLOADS
+    });
+}
+async function uploadSnapshotMetadata(metadata, store, basePath) {
+    const data = Buffer.from(jsonStringify(metadata), 'utf-8');
+    const path = `${basePath}/metadata.json`;
+    return await store.save(path, data, {
+        compress: false,
+        public: PUBLIC_UPLOADS
+    });
+}

package/dest/bin/run-failed-epoch.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoicnVuLWZhaWxlZC1lcG9jaC5kLnRzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vLi4vc3JjL2Jpbi9ydW4tZmFpbGVkLWVwb2NoLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiIifQ==

package/dest/bin/run-failed-epoch.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"run-failed-epoch.d.ts","sourceRoot":"","sources":["../../src/bin/run-failed-epoch.ts"],"names":[],"mappings":""}

package/dest/bin/run-failed-epoch.js

@@ -0,0 +1,68 @@
+/* eslint-disable no-console */ import { getL1ContractsConfigEnvVars } from '@aztec-labs/ethereum/config';
+import { EthAddress } from '@aztec-labs/foundation/eth-address';
+import { jsonParseWithSchema, jsonStringify } from '@aztec-labs/foundation/json-rpc';
+import { createLogger } from '@aztec-labs/foundation/log';
+import { downloadEpochProvingJob, getProverNodeConfigFromEnv, rerunEpochProvingJob } from '@aztec-labs/prover-node';
+import { UploadSnapshotMetadataSchema } from '@aztec-labs/stdlib/snapshots';
+import { existsSync, mkdirSync } from 'fs';
+import { readFile, writeFile } from 'fs/promises';
+import { basename, join } from 'path';
+const logger = createLogger('prover-node:run-failed-epoch');
+function printUsage() {
+    console.error('Usage: run-failed-epoch <proof-uri> [out-dir=./data]');
+}
+async function rerunFailedEpoch(provingJobUrl, baseLocalDir) {
+    const localDir = join(baseLocalDir, basename(provingJobUrl));
+    const jobPath = join(localDir, 'data.bin');
+    const dataDir = join(localDir, 'state');
+    const env = getProverNodeConfigFromEnv();
+    const l1Config = getL1ContractsConfigEnvVars();
+    const config = {
+        ...env,
+        ...l1Config,
+        dataDirectory: dataDir,
+        dataStoreMapSizeKb: env.dataStoreMapSizeKb ?? 1024 * 1024,
+        proverId: env.proverId ?? EthAddress.random()
+    };
+    let metadata;
+    const metadataPath = join(localDir, 'metadata.json');
+    if (existsSync(metadataPath)) {
+        logger.info(`Using downloaded data`);
+        metadata = jsonParseWithSchema(await readFile(metadataPath, 'utf-8'), UploadSnapshotMetadataSchema);
+    } else {
+        logger.info(`Downloading epoch proving job data and state from ${provingJobUrl} to ${localDir}`);
+        metadata = await downloadEpochProvingJob(provingJobUrl, logger, {
+            jobDataDownloadPath: jobPath,
+            dataDirectory: dataDir
+        });
+        await writeFile(metadataPath, jsonStringify(metadata, true));
+        logger.info(`Download to ${localDir} complete`);
+    }
+    logger.info(`Rerunning proving job from ${jobPath} with state from ${dataDir}`, metadata);
+    const result = await rerunEpochProvingJob(jobPath, logger, {
+        ...config,
+        rollupAddress: metadata.rollupAddress,
+        rollupVersion: metadata.rollupVersion
+    });
+    console.error(`Epoch proving job complete with result ${result}`);
+}
+async function main() {
+    if (process.argv[2] === '--help') {
+        printUsage();
+        return;
+    }
+    const uri = process.argv[2];
+    const outDir = process.argv[3] || './data';
+    if (!uri) {
+        printUsage();
+        throw new Error('Missing URL to epoch proving job');
+    }
+    mkdirSync(outDir, {
+        recursive: true
+    });
+    await rerunFailedEpoch(uri, outDir);
+}
+main().catch((err)=>{
+    console.error(err);
+    process.exit(1);
+});

package/dest/checkpoint-store.d.ts

@@ -0,0 +1,95 @@
+import type { BlockNumber, EpochNumber, SlotNumber } from '@aztec-labs/foundation/branded-types';
+import { type LoggerBindings } from '@aztec-labs/foundation/log';
+import type { L2BlockSource } from '@aztec-labs/stdlib/block';
+import type { Checkpoint } from '@aztec-labs/stdlib/checkpoint';
+import { type L1RollupConstants } from '@aztec-labs/stdlib/epoch-helpers';
+import { CheckpointProver, type CheckpointProverArgs, type CheckpointProverDeps, type CheckpointProverTestHooks } from './job/checkpoint-prover.js';
+/** Register-time data needed to construct a `CheckpointProver` (everything except the checkpoint + epoch). */
+export type RegisterCheckpointData = Omit<CheckpointProverArgs, 'checkpoint' | 'epochNumber'>;
+/** Factory used by the store to construct new provers. Tests can inject a stub. */
+export type CheckpointProverFactory = (args: CheckpointProverArgs, deps: CheckpointProverDeps) => CheckpointProver;
+/**
+ * Prover-node-wide registry of `CheckpointProver` instances, content-addressed by
+ * `(checkpoint number, slot, checkpoint archive root)`.
+ *
+ * The store survives every epoch / session boundary. A prover lives from its first
+ * `addOrUpdate` call until either:
+ *  - its checkpoint is pruned by an L1 reorg (`cancelAndRemoveAboveBlock`), or
+ *  - its epoch's proof-submission window has closed (`reapExpired`), so the proof could no
+ *    longer be accepted on L1 even if produced.
+ *
+ * A prover's sub-tree work forks world-state per block and does not survive a prune of a base
+ * block, so there is nothing to preserve across a reorg: a pruned prover is cancelled and
+ * dropped, and a re-add (even of identical content) constructs a fresh prover.
+ */
+export declare class CheckpointStore {
+    private readonly l2BlockSource;
+    private readonly proverDeps;
+    private readonly proverFactoryFn;
+    private readonly provers;
+    /**
+     * Teardowns of provers already removed from `provers` (by prune or reap), awaited on `stop()`.
+     * Keyed by a monotonic id rather than the prover's content id: a prune-then-re-add can leave two
+     * teardowns for the same content id in flight at once, which a content-id key would clobber.
+     */
+    private readonly pendingTeardowns;
+    private nextTeardownId;
+    /** Test-only hooks injected into every prover this store constructs. */
+    private testHooks;
+    private readonly log;
+    constructor(l2BlockSource: Pick<L2BlockSource, 'getL1Constants'>, proverDeps: Omit<CheckpointProverDeps, 'log'>, bindings?: LoggerBindings, proverFactoryFn?: CheckpointProverFactory);
+    start(): Promise<void>;
+    stop(): Promise<void>;
+    /**
+     * Tracks the teardown of a prover just removed from the store so `stop()` can await it. The entry
+     * removes itself once teardown settles, so the map stays bounded by the number in flight.
+     */
+    private trackTeardown;
+    /**
+     * Registers a checkpoint with the store. If a prover already exists for the
+     * `(number, slot, archive root)` content key it is reused (an at-least-once re-registration of
+     * still-canonical content); otherwise a new prover is constructed.
+     */
+    addOrUpdate(checkpoint: Checkpoint, data: RegisterCheckpointData): Promise<CheckpointProver>;
+    /**
+     * Installs test-only hooks applied to every prover constructed from now on. Used by the e2e harness to
+     * force a checkpoint sub-tree failure without monkey-patching the prover factory.
+     */
+    setTestHooks(hooks: CheckpointProverTestHooks): void;
+    /**
+     * Cancels and removes every prover that holds a block above the prune target. A checkpoint is orphaned by a prune to
+     * block `targetBlockNumber` iff its last block sits above the target — including a checkpoint whose range straddles
+     * the target (partially orphaned), which block-range marking catches without boundary ambiguity. Keying off the
+     * surviving block number (rather than a checkpoint number) is correct even when the source has already
+     * re-checkpointed past the divergence: the prune event reports the highest surviving block, which by construction
+     * survives on the source, whereas the source's current checkpointed tip can sit above the prune target.
+     *
+     * The prover's in-flight sub-tree work forks world-state per block and faults once its base block is pruned, so it
+     * cannot be reused; it is cancelled (aborting the fork reads) and dropped. A subsequent re-add constructs a fresh
+     * prover. Returns the removed provers.
+     */
+    cancelAndRemoveAboveBlock(targetBlockNumber: BlockNumber): CheckpointProver[];
+    /**
+     * Drops provers whose epoch is at or below the supplied expired epoch. Once an epoch's
+     * proof-submission window has closed, its proof can no longer be accepted on L1, so the
+     * prover is no longer needed.
+     */
+    reapExpired(expiredEpoch: EpochNumber): void;
+    /** Returns the prover with the supplied id, or undefined. */
+    get(id: string): CheckpointProver | undefined;
+    /** Returns the prover for the supplied checkpoint (by its content-addressed id), or undefined. */
+    getByCheckpoint(checkpoint: Checkpoint): CheckpointProver | undefined;
+    /** Every prover currently in the store, in insertion order. */
+    listAll(): CheckpointProver[];
+    /** Provers in the store, sorted by checkpoint number. */
+    list(): CheckpointProver[];
+    /**
+     * Provers whose slot is in the supplied epoch's slot range, sorted by checkpoint number.
+     */
+    listForEpoch(epoch: EpochNumber): Promise<CheckpointProver[]>;
+    /** Provers whose slot falls within `[fromSlot, toSlot]`, sorted by checkpoint number. */
+    listInSlotRange(fromSlot: SlotNumber, toSlot: SlotNumber): CheckpointProver[];
+}
+/** Sub-set of `L1RollupConstants` actually consumed by the store's slot helpers. */
+export type CheckpointStoreL1Constants = Pick<L1RollupConstants, 'epochDuration'>;
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiY2hlY2twb2ludC1zdG9yZS5kLnRzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vc3JjL2NoZWNrcG9pbnQtc3RvcmUudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsT0FBTyxLQUFLLEVBQUUsV0FBVyxFQUFvQixXQUFXLEVBQUUsVUFBVSxFQUFFLE1BQU0sc0NBQXNDLENBQUM7QUFDbkgsT0FBTyxFQUFlLEtBQUssY0FBYyxFQUFnQixNQUFNLDRCQUE0QixDQUFDO0FBQzVGLE9BQU8sS0FBSyxFQUFFLGFBQWEsRUFBRSxNQUFNLDBCQUEwQixDQUFDO0FBQzlELE9BQU8sS0FBSyxFQUFFLFVBQVUsRUFBRSxNQUFNLCtCQUErQixDQUFDO0FBQ2hFLE9BQU8sRUFBRSxLQUFLLGlCQUFpQixFQUF3QyxNQUFNLGtDQUFrQyxDQUFDO0FBRWhILE9BQU8sRUFDTCxnQkFBZ0IsRUFDaEIsS0FBSyxvQkFBb0IsRUFDekIsS0FBSyxvQkFBb0IsRUFDekIsS0FBSyx5QkFBeUIsRUFDL0IsTUFBTSw0QkFBNEIsQ0FBQztBQUVwQyw4R0FBOEc7QUFDOUcsTUFBTSxNQUFNLHNCQUFzQixHQUFHLElBQUksQ0FBQyxvQkFBb0IsRUFBRSxZQUFZLEdBQUcsYUFBYSxDQUFDLENBQUM7QUFFOUYsbUZBQW1GO0FBQ25GLE1BQU0sTUFBTSx1QkFBdUIsR0FBRyxDQUFDLElBQUksRUFBRSxvQkFBb0IsRUFBRSxJQUFJLEVBQUUsb0JBQW9CLEtBQUssZ0JBQWdCLENBQUM7QUFFbkg7Ozs7Ozs7Ozs7Ozs7R0FhRztBQUNILHFCQUFhLGVBQWU7SUFjeEIsT0FBTyxDQUFDLFFBQVEsQ0FBQyxhQUFhO0lBQzlCLE9BQU8sQ0FBQyxRQUFRLENBQUMsVUFBVTtJQUUzQixPQUFPLENBQUMsUUFBUSxDQUFDLGVBQWU7SUFoQmxDLE9BQU8sQ0FBQyxRQUFRLENBQUMsT0FBTyxDQUF1QztJQUMvRDs7OztPQUlHO0lBQ0gsT0FBTyxDQUFDLFFBQVEsQ0FBQyxnQkFBZ0IsQ0FBb0M7SUFDckUsT0FBTyxDQUFDLGNBQWMsQ0FBSztJQUMzQix3RUFBd0U7SUFDeEUsT0FBTyxDQUFDLFNBQVMsQ0FBaUM7SUFDbEQsT0FBTyxDQUFDLFFBQVEsQ0FBQyxHQUFHLENBQVM7SUFFN0IsWUFDbUIsYUFBYSxFQUFFLElBQUksQ0FBQyxhQUFhLEVBQUUsZ0JBQWdCLENBQUMsRUFDcEQsVUFBVSxFQUFFLElBQUksQ0FBQyxvQkFBb0IsRUFBRSxLQUFLLENBQUMsRUFDOUQsUUFBUSxDQUFDLEVBQUUsY0FBYyxFQUNSLGVBQWUsR0FBRSx1QkFBMEUsRUFHN0c7SUFFTSxLQUFLLElBQUksT0FBTyxDQUFDLElBQUksQ0FBQyxDQUU1QjtJQUVZLElBQUksSUFBSSxPQUFPLENBQUMsSUFBSSxDQUFDLENBU2pDO0lBRUQ7OztPQUdHO0lBQ0gsT0FBTyxDQUFDLGFBQWE7SUFPckI7Ozs7T0FJRztJQUNVLFdBQVcsQ0FBQyxVQUFVLEVBQUUsVUFBVSxFQUFFLElBQUksRUFBRSxzQkFBc0IsR0FBRyxPQUFPLENBQUMsZ0JBQWdCLENBQUMsQ0E2QnhHO0lBRUQ7OztPQUdHO0lBQ0ksWUFBWSxDQUFDLEtBQUssRUFBRSx5QkFBeUIsR0FBRyxJQUFJLENBRTFEO0lBRUQ7Ozs7Ozs7Ozs7O09BV0c7SUFDSSx5QkFBeUIsQ0FBQyxpQkFBaUIsRUFBRSxXQUFXLEdBQUcsZ0JBQWdCLEVBQUUsQ0FZbkY7SUFFRDs7OztPQUlHO0lBQ0ksV0FBVyxDQUFDLFlBQVksRUFBRSxXQUFXLEdBQUcsSUFBSSxDQWlCbEQ7SUFFRCw2REFBNkQ7SUFDdEQsR0FBRyxDQUFDLEVBQUUsRUFBRSxNQUFNLEdBQUcsZ0JBQWdCLEdBQUcsU0FBUyxDQUVuRDtJQUVELGtHQUFrRztJQUMzRixlQUFlLENBQUMsVUFBVSxFQUFFLFVBQVUsR0FBRyxnQkFBZ0IsR0FBRyxTQUFTLENBRTNFO0lBRUQsK0RBQStEO0lBQ3hELE9BQU8sSUFBSSxnQkFBZ0IsRUFBRSxDQUVuQztJQUVELHlEQUF5RDtJQUNsRCxJQUFJLElBQUksZ0JBQWdCLEVBQUUsQ0FFaEM7SUFFRDs7T0FFRztJQUNVLFlBQVksQ0FBQyxLQUFLLEVBQUUsV0FBVyxHQUFHLE9BQU8sQ0FBQyxnQkFBZ0IsRUFBRSxDQUFDLENBSXpFO0lBRUQseUZBQXlGO0lBQ2xGLGVBQWUsQ0FBQyxRQUFRLEVBQUUsVUFBVSxFQUFFLE1BQU0sRUFBRSxVQUFVLEdBQUcsZ0JBQWdCLEVBQUUsQ0FFbkY7Q0FDRjtBQUVELG9GQUFvRjtBQUNwRixNQUFNLE1BQU0sMEJBQTBCLEdBQUcsSUFBSSxDQUFDLGlCQUFpQixFQUFFLGVBQWUsQ0FBQyxDQUFDIn0=

package/dest/checkpoint-store.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"checkpoint-store.d.ts","sourceRoot":"","sources":["../src/checkpoint-store.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAoB,WAAW,EAAE,UAAU,EAAE,MAAM,sCAAsC,CAAC;AACnH,OAAO,EAAe,KAAK,cAAc,EAAgB,MAAM,4BAA4B,CAAC;AAC5F,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AAC9D,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,+BAA+B,CAAC;AAChE,OAAO,EAAE,KAAK,iBAAiB,EAAwC,MAAM,kCAAkC,CAAC;AAEhH,OAAO,EACL,gBAAgB,EAChB,KAAK,oBAAoB,EACzB,KAAK,oBAAoB,EACzB,KAAK,yBAAyB,EAC/B,MAAM,4BAA4B,CAAC;AAEpC,8GAA8G;AAC9G,MAAM,MAAM,sBAAsB,GAAG,IAAI,CAAC,oBAAoB,EAAE,YAAY,GAAG,aAAa,CAAC,CAAC;AAE9F,mFAAmF;AACnF,MAAM,MAAM,uBAAuB,GAAG,CAAC,IAAI,EAAE,oBAAoB,EAAE,IAAI,EAAE,oBAAoB,KAAK,gBAAgB,CAAC;AAEnH;;;;;;;;;;;;;GAaG;AACH,qBAAa,eAAe;IAcxB,OAAO,CAAC,QAAQ,CAAC,aAAa;IAC9B,OAAO,CAAC,QAAQ,CAAC,UAAU;IAE3B,OAAO,CAAC,QAAQ,CAAC,eAAe;IAhBlC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAuC;IAC/D;;;;OAIG;IACH,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAoC;IACrE,OAAO,CAAC,cAAc,CAAK;IAC3B,wEAAwE;IACxE,OAAO,CAAC,SAAS,CAAiC;IAClD,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAS;IAE7B,YACmB,aAAa,EAAE,IAAI,CAAC,aAAa,EAAE,gBAAgB,CAAC,EACpD,UAAU,EAAE,IAAI,CAAC,oBAAoB,EAAE,KAAK,CAAC,EAC9D,QAAQ,CAAC,EAAE,cAAc,EACR,eAAe,GAAE,uBAA0E,EAG7G;IAEM,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAE5B;IAEY,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CASjC;IAED;;;OAGG;IACH,OAAO,CAAC,aAAa;IAOrB;;;;OAIG;IACU,WAAW,CAAC,UAAU,EAAE,UAAU,EAAE,IAAI,EAAE,sBAAsB,GAAG,OAAO,CAAC,gBAAgB,CAAC,CA6BxG;IAED;;;OAGG;IACI,YAAY,CAAC,KAAK,EAAE,yBAAyB,GAAG,IAAI,CAE1D;IAED;;;;;;;;;;;OAWG;IACI,yBAAyB,CAAC,iBAAiB,EAAE,WAAW,GAAG,gBAAgB,EAAE,CAYnF;IAED;;;;OAIG;IACI,WAAW,CAAC,YAAY,EAAE,WAAW,GAAG,IAAI,CAiBlD;IAED,6DAA6D;IACtD,GAAG,CAAC,EAAE,EAAE,MAAM,GAAG,gBAAgB,GAAG,SAAS,CAEnD;IAED,kGAAkG;IAC3F,eAAe,CAAC,UAAU,EAAE,UAAU,GAAG,gBAAgB,GAAG,SAAS,CAE3E;IAED,+DAA+D;IACxD,OAAO,IAAI,gBAAgB,EAAE,CAEnC;IAED,yDAAyD;IAClD,IAAI,IAAI,gBAAgB,EAAE,CAEhC;IAED;;OAEG;IACU,YAAY,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAIzE;IAED,yFAAyF;IAClF,eAAe,CAAC,QAAQ,EAAE,UAAU,EAAE,MAAM,EAAE,UAAU,GAAG,gBAAgB,EAAE,CAEnF;CACF;AAED,oFAAoF;AACpF,MAAM,MAAM,0BAA0B,GAAG,IAAI,CAAC,iBAAiB,EAAE,eAAe,CAAC,CAAC"}