@aztec/prover-client 0.0.1-commit.e6bd8901 → 0.0.1-commit.ec7ac5448

package/dest/proving_broker/proving_agent.js

@@ -13,16 +13,16 @@ import { ProvingJobController, ProvingJobControllerStatus } from './proving_job_
     circuitProver;
     proofAllowList;
     pollIntervalMs;
-    log;
     currentJobController;
     runningPromise;
-    constructor(/** The source of proving jobs */ broker, /** Database holding proof inputs and outputs */ proofStore, /** The prover implementation to defer jobs to */ circuitProver, /** Optional list of allowed proof types to build */ proofAllowList = [], /** How long to wait between jobs */ pollIntervalMs = 1000, log = createLogger('prover-client:proving-agent')){
+    log;
+    constructor(/** The source of proving jobs */ broker, /** Database holding proof inputs and outputs */ proofStore, /** The prover implementation to defer jobs to */ circuitProver, /** Optional list of allowed proof types to build */ proofAllowList = [], /** How long to wait between jobs */ pollIntervalMs = 1000, bindings){
         this.broker = broker;
         this.proofStore = proofStore;
         this.circuitProver = circuitProver;
         this.proofAllowList = proofAllowList;
         this.pollIntervalMs = pollIntervalMs;
-        this.log = log;
+        this.log = createLogger('prover-client:proving-agent', bindings);
         this.runningPromise = new RunningPromise(this.work.bind(this), this.log, this.pollIntervalMs);
     }
     setCircuitProver(circuitProver) {
@@ -124,7 +124,7 @@ import { ProvingJobController, ProvingJobControllerStatus } from './proving_job_
             // trigger a run of the main work loop when proving completes
             // no need to await this here. The controller will stay alive (in DONE state) until the result is send to the broker
             void this.runningPromise.trigger();
-        });
+        }, this.log.getBindings());
         if (abortedProofJobId) {
             this.log.info(`Aborting job id=${abortedProofJobId} type=${abortedProofName} to start new job id=${this.currentJobController.getJobId()} type=${this.currentJobController.getProofTypeName()} inputsUri=${truncate(job.inputsUri)}`);
         } else {

package/dest/proving_broker/proving_broker.d.ts

@@ -1,4 +1,5 @@
-import { type GetProvingJobResponse, type ProofUri, type ProvingJob, type ProvingJobConsumer, type ProvingJobFilter, type ProvingJobId, type ProvingJobProducer, type ProvingJobStatus } from '@aztec/stdlib/interfaces/server';
+import { EpochNumber } from '@aztec/foundation/branded-types';
+import { type GetProvingJobResponse, type ProofUri, type ProvingJob, type ProvingJobBrokerDebug, type ProvingJobConsumer, type ProvingJobFilter, type ProvingJobId, type ProvingJobProducer, type ProvingJobStatus } from '@aztec/stdlib/interfaces/server';
 import { ProvingRequestType } from '@aztec/stdlib/proofs';
 import { type TelemetryClient, type Traceable, type Tracer } from '@aztec/telemetry-client';
 import { type ProverBrokerConfig } from './config.js';
@@ -7,7 +8,7 @@ import type { ProvingBrokerDatabase } from './proving_broker_database.js';
  * A broker that manages proof requests and distributes them to workers based on their priority.
  * It takes a backend that is responsible for storing and retrieving proof requests and results.
  */
-export declare class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Traceable {
+export declare class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, ProvingJobBrokerDebug, Traceable {
     #private;
     private database;
     private logger;
@@ -38,7 +39,8 @@ export declare class ProvingBroker implements ProvingJobProducer, ProvingJobCons
     private epochHeight;
     private maxEpochsToKeepResultsFor;
     private started;
-    constructor(database: ProvingBrokerDatabase, { proverBrokerJobTimeoutMs, proverBrokerPollIntervalMs, proverBrokerJobMaxRetries, proverBrokerMaxEpochsToKeepResultsFor }?: Required<Pick<ProverBrokerConfig, 'proverBrokerJobTimeoutMs' | 'proverBrokerPollIntervalMs' | 'proverBrokerJobMaxRetries' | 'proverBrokerMaxEpochsToKeepResultsFor'>>, client?: TelemetryClient, logger?: import("@aztec/foundation/log").Logger);
+    private debugReplayEnabled;
+    constructor(database: ProvingBrokerDatabase, { proverBrokerJobTimeoutMs, proverBrokerPollIntervalMs, proverBrokerJobMaxRetries, proverBrokerMaxEpochsToKeepResultsFor, proverBrokerDebugReplayEnabled }?: Required<Pick<ProverBrokerConfig, 'proverBrokerJobTimeoutMs' | 'proverBrokerPollIntervalMs' | 'proverBrokerJobMaxRetries' | 'proverBrokerMaxEpochsToKeepResultsFor' | 'proverBrokerDebugReplayEnabled'>>, client?: TelemetryClient, logger?: import("@aztec/foundation/log").Logger);
     private measureQueueDepth;
     private countActiveJobs;
     start(): Promise<void>;
@@ -54,6 +56,7 @@ export declare class ProvingBroker implements ProvingJobProducer, ProvingJobCons
         job: ProvingJob;
         time: number;
     } | undefined>;
+    replayProvingJob(jobId: ProvingJobId, type: ProvingRequestType, epochNumber: EpochNumber, inputsUri: ProofUri): Promise<ProvingJobStatus>;
     private cleanUpProvingJobState;
     private cleanupPass;
     private cleanupStaleJobs;
@@ -70,4 +73,4 @@ export declare class ProvingBroker implements ProvingJobProducer, ProvingJobCons
  * is to get picked up by agents
  */
 export declare const PROOF_TYPES_IN_PRIORITY_ORDER: ProvingRequestType[];
-//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoicHJvdmluZ19icm9rZXIuZC50cyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uL3NyYy9wcm92aW5nX2Jyb2tlci9wcm92aW5nX2Jyb2tlci50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFLQSxPQUFPLEVBQ0wsS0FBSyxxQkFBcUIsRUFDMUIsS0FBSyxRQUFRLEVBQ2IsS0FBSyxVQUFVLEVBQ2YsS0FBSyxrQkFBa0IsRUFDdkIsS0FBSyxnQkFBZ0IsRUFDckIsS0FBSyxZQUFZLEVBQ2pCLEtBQUssa0JBQWtCLEVBRXZCLEtBQUssZ0JBQWdCLEVBRXRCLE1BQU0saUNBQWlDLENBQUM7QUFDekMsT0FBTyxFQUFFLGtCQUFrQixFQUFFLE1BQU0sc0JBQXNCLENBQUM7QUFDMUQsT0FBTyxFQUFFLEtBQUssZUFBZSxFQUFFLEtBQUssU0FBUyxFQUFFLEtBQUssTUFBTSxFQUFzQixNQUFNLHlCQUF5QixDQUFDO0FBSWhILE9BQU8sRUFBRSxLQUFLLGtCQUFrQixFQUE2QixNQUFNLGFBQWEsQ0FBQztBQUNqRixPQUFPLEtBQUssRUFBRSxxQkFBcUIsRUFBRSxNQUFNLDhCQUE4QixDQUFDO0FBVzFFOzs7R0FHRztBQUNILHFCQUFhLGFBQWMsWUFBVyxrQkFBa0IsRUFBRSxrQkFBa0IsRUFBRSxTQUFTOztJQStFbkYsT0FBTyxDQUFDLFFBQVE7SUFnQmhCLE9BQU8sQ0FBQyxNQUFNO0lBOUZoQixPQUFPLENBQUMsTUFBTSxDQTZCWjtJQUlGLE9BQU8sQ0FBQyxTQUFTLENBQXVDO0lBRXhELE9BQU8sQ0FBQyxZQUFZLENBQW9EO0lBR3hFLE9BQU8sQ0FBQyxVQUFVLENBQWtDO0lBTXBELE9BQU8sQ0FBQyxVQUFVLENBQStDO0lBR2pFLE9BQU8sQ0FBQyxPQUFPLENBQW1DO0lBR2xELE9BQU8sQ0FBQyxRQUFRLENBQTBFO0lBRTFGLE9BQU8sQ0FBQyxjQUFjLENBQWlCO0lBQ3ZDLE9BQU8sQ0FBQyxZQUFZLENBQW9CO0lBQ3hDLE9BQU8sQ0FBQyxZQUFZLENBQVM7SUFDN0IsT0FBTyxDQUFDLFVBQVUsQ0FBUztJQUUzQixPQUFPLENBQUMsZUFBZSxDQUErQjtJQUN0RCxTQUFnQixNQUFNLEVBQUUsTUFBTSxDQUFDO0lBRS9CLE9BQU8sQ0FBQyx5QkFBeUIsQ0FBc0I7SUFFdkQ7Ozs7Ozs7OztPQVNHO0lBQ0gsT0FBTyxDQUFDLFdBQVcsQ0FBSztJQUN4QixPQUFPLENBQUMseUJBQXlCLENBQUs7SUFFdEMsT0FBTyxDQUFDLE9BQU8sQ0FBUztJQUV4QixZQUNVLFFBQVEsRUFBRSxxQkFBcUIsRUFDdkMsRUFDRSx3QkFBd0IsRUFDeEIsMEJBQTBCLEVBQzFCLHlCQUF5QixFQUN6QixxQ0FBcUMsRUFDdEMsR0FBRSxRQUFRLENBQ1QsSUFBSSxDQUNGLGtCQUFrQixFQUNoQiwwQkFBMEIsR0FDMUIsNEJBQTRCLEdBQzVCLDJCQUEyQixHQUMzQix1Q0FBdUMsQ0FDMUMsQ0FDMEIsRUFDN0IsTUFBTSxHQUFFLGVBQXNDLEVBQ3RDLE1BQU0seUNBQStDLEVBUTlEO0lBRUQsT0FBTyxDQUFDLGlCQUFpQixDQUV2QjtJQUVGLE9BQU8sQ0FBQyxlQUFlLENBVXJCO0lBRVcsS0FBSyxJQUFJLE9BQU8sQ0FBQyxJQUFJLENBQUMsQ0E2QmxDO0lBRVksSUFBSSxJQUFJLE9BQU8sQ0FBQyxJQUFJLENBQUMsQ0FNakM7SUFFTSxpQkFBaUIsQ0FBQyxHQUFHLEVBQUUsVUFBVSxHQUFHLE9BQU8sQ0FBQyxnQkFBZ0IsQ0FBQyxDQUVuRTtJQUVNLGdCQUFnQixDQUFDLEVBQUUsRUFBRSxZQUFZLEdBQUcsT0FBTyxDQUFDLElBQUksQ0FBQyxDQUV2RDtJQUVNLG1CQUFtQixDQUFDLEVBQUUsRUFBRSxZQUFZLEdBQUcsT0FBTyxDQUFDLGdCQUFnQixDQUFDLENBRXRFO0lBRU0sZ0JBQWdCLENBQUMsR0FBRyxFQUFFLFlBQVksRUFBRSxHQUFHLE9BQU8sQ0FBQyxZQUFZLEVBQUUsQ0FBQyxDQUVwRTtJQUVNLGFBQWEsQ0FBQyxNQUFNLENBQUMsRUFBRSxnQkFBZ0IsR0FBRyxPQUFPLENBQUMscUJBQXFCLEdBQUcsU0FBUyxDQUFDLENBRTFGO0lBRU0sdUJBQXVCLENBQzVCLEVBQUUsRUFBRSxZQUFZLEVBQ2hCLEtBQUssRUFBRSxRQUFRLEVBQ2YsTUFBTSxDQUFDLEVBQUUsZ0JBQWdCLEdBQ3hCLE9BQU8sQ0FBQyxxQkFBcUIsR0FBRyxTQUFTLENBQUMsQ0FFNUM7SUFFTSxxQkFBcUIsQ0FDMUIsRUFBRSxFQUFFLFlBQVksRUFDaEIsR0FBRyxFQUFFLE1BQU0sRUFDWCxLQUFLLFVBQVEsRUFDYixNQUFNLENBQUMsRUFBRSxnQkFBZ0IsR0FDeEIsT0FBTyxDQUFDLHFCQUFxQixHQUFHLFNBQVMsQ0FBQyxDQUU1QztJQUVNLHdCQUF3QixDQUM3QixFQUFFLEVBQUUsWUFBWSxFQUNoQixTQUFTLEVBQUUsTUFBTSxFQUNqQixNQUFNLENBQUMsRUFBRSxnQkFBZ0IsR0FDeEIsT0FBTyxDQUFDO1FBQUUsR0FBRyxFQUFFLFVBQVUsQ0FBQztRQUFDLElBQUksRUFBRSxNQUFNLENBQUE7S0FBRSxHQUFHLFNBQVMsQ0FBQyxDQUV4RDtJQWtERCxPQUFPLENBQUMsc0JBQXNCO1lBOFFoQixXQUFXO0lBVXpCLE9BQU8sQ0FBQyxnQkFBZ0I7SUFnQnhCLE9BQU8sQ0FBQyxvQkFBb0I7SUFxQjVCLE9BQU8sQ0FBQyxrQkFBa0I7SUFZMUIsT0FBTyxDQUFDLFVBQVU7SUFJbEIsT0FBTyxDQUFDLGlCQUFpQjtDQUcxQjtBQWlERDs7Ozs7O0dBTUc7QUFDSCxlQUFPLE1BQU0sNkJBQTZCLEVBQUUsa0JBQWtCLEVBbUI3RCxDQUFDIn0=
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoicHJvdmluZ19icm9rZXIuZC50cyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uL3NyYy9wcm92aW5nX2Jyb2tlci9wcm92aW5nX2Jyb2tlci50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSxPQUFPLEVBQUUsV0FBVyxFQUFFLE1BQU0saUNBQWlDLENBQUM7QUFLOUQsT0FBTyxFQUNMLEtBQUsscUJBQXFCLEVBQzFCLEtBQUssUUFBUSxFQUNiLEtBQUssVUFBVSxFQUNmLEtBQUsscUJBQXFCLEVBQzFCLEtBQUssa0JBQWtCLEVBQ3ZCLEtBQUssZ0JBQWdCLEVBQ3JCLEtBQUssWUFBWSxFQUNqQixLQUFLLGtCQUFrQixFQUV2QixLQUFLLGdCQUFnQixFQUV0QixNQUFNLGlDQUFpQyxDQUFDO0FBQ3pDLE9BQU8sRUFBRSxrQkFBa0IsRUFBRSxNQUFNLHNCQUFzQixDQUFDO0FBQzFELE9BQU8sRUFBRSxLQUFLLGVBQWUsRUFBRSxLQUFLLFNBQVMsRUFBRSxLQUFLLE1BQU0sRUFBc0IsTUFBTSx5QkFBeUIsQ0FBQztBQUloSCxPQUFPLEVBQUUsS0FBSyxrQkFBa0IsRUFBNkIsTUFBTSxhQUFhLENBQUM7QUFDakYsT0FBTyxLQUFLLEVBQUUscUJBQXFCLEVBQUUsTUFBTSw4QkFBOEIsQ0FBQztBQVcxRTs7O0dBR0c7QUFDSCxxQkFBYSxhQUFjLFlBQVcsa0JBQWtCLEVBQUUsa0JBQWtCLEVBQUUscUJBQXFCLEVBQUUsU0FBUzs7SUFpRjFHLE9BQU8sQ0FBQyxRQUFRO0lBa0JoQixPQUFPLENBQUMsTUFBTTtJQWxHaEIsT0FBTyxDQUFDLE1BQU0sQ0E2Qlo7SUFJRixPQUFPLENBQUMsU0FBUyxDQUF1QztJQUV4RCxPQUFPLENBQUMsWUFBWSxDQUFvRDtJQUd4RSxPQUFPLENBQUMsVUFBVSxDQUFrQztJQU1wRCxPQUFPLENBQUMsVUFBVSxDQUErQztJQUdqRSxPQUFPLENBQUMsT0FBTyxDQUFtQztJQUdsRCxPQUFPLENBQUMsUUFBUSxDQUEwRTtJQUUxRixPQUFPLENBQUMsY0FBYyxDQUFpQjtJQUN2QyxPQUFPLENBQUMsWUFBWSxDQUFvQjtJQUN4QyxPQUFPLENBQUMsWUFBWSxDQUFTO0lBQzdCLE9BQU8sQ0FBQyxVQUFVLENBQVM7SUFFM0IsT0FBTyxDQUFDLGVBQWUsQ0FBK0I7SUFDdEQsU0FBZ0IsTUFBTSxFQUFFLE1BQU0sQ0FBQztJQUUvQixPQUFPLENBQUMseUJBQXlCLENBQXNCO0lBRXZEOzs7Ozs7Ozs7T0FTRztJQUNILE9BQU8sQ0FBQyxXQUFXLENBQUs7SUFDeEIsT0FBTyxDQUFDLHlCQUF5QixDQUFLO0lBRXRDLE9BQU8sQ0FBQyxPQUFPLENBQVM7SUFFeEIsT0FBTyxDQUFDLGtCQUFrQixDQUFVO0lBRXBDLFlBQ1UsUUFBUSxFQUFFLHFCQUFxQixFQUN2QyxFQUNFLHdCQUF3QixFQUN4QiwwQkFBMEIsRUFDMUIseUJBQXlCLEVBQ3pCLHFDQUFxQyxFQUNyQyw4QkFBOEIsRUFDL0IsR0FBRSxRQUFRLENBQ1QsSUFBSSxDQUNGLGtCQUFrQixFQUNoQiwwQkFBMEIsR0FDMUIsNEJBQTRCLEdBQzVCLDJCQUEyQixHQUMzQix1Q0FBdUMsR0FDdkMsZ0NBQWdDLENBQ25DLENBQzBCLEVBQzdCLE1BQU0sR0FBRSxlQUFzQyxFQUN0QyxNQUFNLHlDQUErQyxFQVM5RDtJQUVELE9BQU8sQ0FBQyxpQkFBaUIsQ0FFdkI7SUFFRixPQUFPLENBQUMsZUFBZSxDQVVyQjtJQUVXLEtBQUssSUFBSSxPQUFPLENBQUMsSUFBSSxDQUFDLENBNkJsQztJQUVZLElBQUksSUFBSSxPQUFPLENBQUMsSUFBSSxDQUFDLENBTWpDO0lBRU0saUJBQWlCLENBQUMsR0FBRyxFQUFFLFVBQVUsR0FBRyxPQUFPLENBQUMsZ0JBQWdCLENBQUMsQ0FFbkU7SUFFTSxnQkFBZ0IsQ0FBQyxFQUFFLEVBQUUsWUFBWSxHQUFHLE9BQU8sQ0FBQyxJQUFJLENBQUMsQ0FFdkQ7SUFFTSxtQkFBbUIsQ0FBQyxFQUFFLEVBQUUsWUFBWSxHQUFHLE9BQU8sQ0FBQyxnQkFBZ0IsQ0FBQyxDQUV0RTtJQUVNLGdCQUFnQixDQUFDLEdBQUcsRUFBRSxZQUFZLEVBQUUsR0FBRyxPQUFPLENBQUMsWUFBWSxFQUFFLENBQUMsQ0FFcEU7SUFFTSxhQUFhLENBQUMsTUFBTSxDQUFDLEVBQUUsZ0JBQWdCLEdBQUcsT0FBTyxDQUFDLHFCQUFxQixHQUFHLFNBQVMsQ0FBQyxDQUUxRjtJQUVNLHVCQUF1QixDQUM1QixFQUFFLEVBQUUsWUFBWSxFQUNoQixLQUFLLEVBQUUsUUFBUSxFQUNmLE1BQU0sQ0FBQyxFQUFFLGdCQUFnQixHQUN4QixPQUFPLENBQUMscUJBQXFCLEdBQUcsU0FBUyxDQUFDLENBRTVDO0lBRU0scUJBQXFCLENBQzFCLEVBQUUsRUFBRSxZQUFZLEVBQ2hCLEdBQUcsRUFBRSxNQUFNLEVBQ1gsS0FBSyxVQUFRLEVBQ2IsTUFBTSxDQUFDLEVBQUUsZ0JBQWdCLEdBQ3hCLE9BQU8sQ0FBQyxxQkFBcUIsR0FBRyxTQUFTLENBQUMsQ0FFNUM7SUFFTSx3QkFBd0IsQ0FDN0IsRUFBRSxFQUFFLFlBQVksRUFDaEIsU0FBUyxFQUFFLE1BQU0sRUFDakIsTUFBTSxDQUFDLEVBQUUsZ0JBQWdCLEdBQ3hCLE9BQU8sQ0FBQztRQUFFLEdBQUcsRUFBRSxVQUFVLENBQUM7UUFBQyxJQUFJLEVBQUUsTUFBTSxDQUFBO0tBQUUsR0FBRyxTQUFTLENBQUMsQ0FFeEQ7SUFFWSxnQkFBZ0IsQ0FDM0IsS0FBSyxFQUFFLFlBQVksRUFDbkIsSUFBSSxFQUFFLGtCQUFrQixFQUN4QixXQUFXLEVBQUUsV0FBVyxFQUN4QixTQUFTLEVBQUUsUUFBUSxHQUNsQixPQUFPLENBQUMsZ0JBQWdCLENBQUMsQ0FnQjNCO0lBa0RELE9BQU8sQ0FBQyxzQkFBc0I7WUFtUmhCLFdBQVc7SUFVekIsT0FBTyxDQUFDLGdCQUFnQjtJQWdCeEIsT0FBTyxDQUFDLG9CQUFvQjtJQXFDNUIsT0FBTyxDQUFDLGtCQUFrQjtJQVkxQixPQUFPLENBQUMsVUFBVTtJQUlsQixPQUFPLENBQUMsaUJBQWlCO0NBRzFCO0FBaUREOzs7Ozs7R0FNRztBQUNILGVBQU8sTUFBTSw2QkFBNkIsRUFBRSxrQkFBa0IsRUFtQjdELENBQUMifQ==

package/dest/proving_broker/proving_broker.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"proving_broker.d.ts","sourceRoot":"","sources":["../../src/proving_broker/proving_broker.ts"],"names":[],"mappings":"AAKA,OAAO,EACL,KAAK,qBAAqB,EAC1B,KAAK,QAAQ,EACb,KAAK,UAAU,EACf,KAAK,kBAAkB,EACvB,KAAK,gBAAgB,EACrB,KAAK,YAAY,EACjB,KAAK,kBAAkB,EAEvB,KAAK,gBAAgB,EAEtB,MAAM,iCAAiC,CAAC;AACzC,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAC1D,OAAO,EAAE,KAAK,eAAe,EAAE,KAAK,SAAS,EAAE,KAAK,MAAM,EAAsB,MAAM,yBAAyB,CAAC;AAIhH,OAAO,EAAE,KAAK,kBAAkB,EAA6B,MAAM,aAAa,CAAC;AACjF,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,8BAA8B,CAAC;AAW1E;;;GAGG;AACH,qBAAa,aAAc,YAAW,kBAAkB,EAAE,kBAAkB,EAAE,SAAS;;IA+EnF,OAAO,CAAC,QAAQ;IAgBhB,OAAO,CAAC,MAAM;IA9FhB,OAAO,CAAC,MAAM,CA6BZ;IAIF,OAAO,CAAC,SAAS,CAAuC;IAExD,OAAO,CAAC,YAAY,CAAoD;IAGxE,OAAO,CAAC,UAAU,CAAkC;IAMpD,OAAO,CAAC,UAAU,CAA+C;IAGjE,OAAO,CAAC,OAAO,CAAmC;IAGlD,OAAO,CAAC,QAAQ,CAA0E;IAE1F,OAAO,CAAC,cAAc,CAAiB;IACvC,OAAO,CAAC,YAAY,CAAoB;IACxC,OAAO,CAAC,YAAY,CAAS;IAC7B,OAAO,CAAC,UAAU,CAAS;IAE3B,OAAO,CAAC,eAAe,CAA+B;IACtD,SAAgB,MAAM,EAAE,MAAM,CAAC;IAE/B,OAAO,CAAC,yBAAyB,CAAsB;IAEvD;;;;;;;;;OASG;IACH,OAAO,CAAC,WAAW,CAAK;IACxB,OAAO,CAAC,yBAAyB,CAAK;IAEtC,OAAO,CAAC,OAAO,CAAS;IAExB,YACU,QAAQ,EAAE,qBAAqB,EACvC,EACE,wBAAwB,EACxB,0BAA0B,EAC1B,yBAAyB,EACzB,qCAAqC,EACtC,GAAE,QAAQ,CACT,IAAI,CACF,kBAAkB,EAChB,0BAA0B,GAC1B,4BAA4B,GAC5B,2BAA2B,GAC3B,uCAAuC,CAC1C,CAC0B,EAC7B,MAAM,GAAE,eAAsC,EACtC,MAAM,yCAA+C,EAQ9D;IAED,OAAO,CAAC,iBAAiB,CAEvB;IAEF,OAAO,CAAC,eAAe,CAUrB;IAEW,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CA6BlC;IAEY,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAMjC;IAEM,iBAAiB,CAAC,GAAG,EAAE,UAAU,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAEnE;IAEM,gBAAgB,CAAC,EAAE,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAEvD;IAEM,mBAAmB,CAAC,EAAE,EAAE,YAAY,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAEtE;IAEM,gBAAgB,CAAC,GAAG,EAAE,YAAY,EAAE,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC,CAEpE;IAEM,aAAa,CAAC,MAAM,CAAC,EAAE,gBAAgB,GAAG,OAAO,CAAC,qBAAqB,GAAG,SAAS,CAAC,CAE1F;IAEM,uBAAuB,CAC5B,EAAE,EAAE,YAAY,EAChB,KAAK,EAAE,QAAQ,EACf,MAAM,CAAC,EAAE,gBAAgB,GACxB,OAAO,CAAC,qBAAqB,GAAG,SAAS,CAAC,CAE5C;IAEM,qBAAqB,CAC1B,EAAE,EAAE,YAAY,EAChB,GAAG,EAAE,MAAM,EACX,KAAK,UAAQ,EACb,MAAM,CAAC,EAAE,gBAAgB,GACxB,OAAO,CAAC,qBAAqB,GAAG,SAAS,CAAC,CAE5C;IAEM,wBAAwB,CAC7B,EAAE,EAAE,YAAY,EAChB,SAAS,EAAE,MAAM,EACjB,MAAM,CAAC,EAAE,gBAAgB,GACxB,OAAO,CAAC;QAAE,GAAG,EAAE,UAAU,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,SAAS,CAAC,CAExD;IAkDD,OAAO,CAAC,sBAAsB;YA8QhB,WAAW;IAUzB,OAAO,CAAC,gBAAgB;IAgBxB,OAAO,CAAC,oBAAoB;IAqB5B,OAAO,CAAC,kBAAkB;IAY1B,OAAO,CAAC,UAAU;IAIlB,OAAO,CAAC,iBAAiB;CAG1B;AAiDD;;;;;;GAMG;AACH,eAAO,MAAM,6BAA6B,EAAE,kBAAkB,EAmB7D,CAAC"}
+{"version":3,"file":"proving_broker.d.ts","sourceRoot":"","sources":["../../src/proving_broker/proving_broker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iCAAiC,CAAC;AAK9D,OAAO,EACL,KAAK,qBAAqB,EAC1B,KAAK,QAAQ,EACb,KAAK,UAAU,EACf,KAAK,qBAAqB,EAC1B,KAAK,kBAAkB,EACvB,KAAK,gBAAgB,EACrB,KAAK,YAAY,EACjB,KAAK,kBAAkB,EAEvB,KAAK,gBAAgB,EAEtB,MAAM,iCAAiC,CAAC;AACzC,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAC1D,OAAO,EAAE,KAAK,eAAe,EAAE,KAAK,SAAS,EAAE,KAAK,MAAM,EAAsB,MAAM,yBAAyB,CAAC;AAIhH,OAAO,EAAE,KAAK,kBAAkB,EAA6B,MAAM,aAAa,CAAC;AACjF,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,8BAA8B,CAAC;AAW1E;;;GAGG;AACH,qBAAa,aAAc,YAAW,kBAAkB,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,SAAS;;IAiF1G,OAAO,CAAC,QAAQ;IAkBhB,OAAO,CAAC,MAAM;IAlGhB,OAAO,CAAC,MAAM,CA6BZ;IAIF,OAAO,CAAC,SAAS,CAAuC;IAExD,OAAO,CAAC,YAAY,CAAoD;IAGxE,OAAO,CAAC,UAAU,CAAkC;IAMpD,OAAO,CAAC,UAAU,CAA+C;IAGjE,OAAO,CAAC,OAAO,CAAmC;IAGlD,OAAO,CAAC,QAAQ,CAA0E;IAE1F,OAAO,CAAC,cAAc,CAAiB;IACvC,OAAO,CAAC,YAAY,CAAoB;IACxC,OAAO,CAAC,YAAY,CAAS;IAC7B,OAAO,CAAC,UAAU,CAAS;IAE3B,OAAO,CAAC,eAAe,CAA+B;IACtD,SAAgB,MAAM,EAAE,MAAM,CAAC;IAE/B,OAAO,CAAC,yBAAyB,CAAsB;IAEvD;;;;;;;;;OASG;IACH,OAAO,CAAC,WAAW,CAAK;IACxB,OAAO,CAAC,yBAAyB,CAAK;IAEtC,OAAO,CAAC,OAAO,CAAS;IAExB,OAAO,CAAC,kBAAkB,CAAU;IAEpC,YACU,QAAQ,EAAE,qBAAqB,EACvC,EACE,wBAAwB,EACxB,0BAA0B,EAC1B,yBAAyB,EACzB,qCAAqC,EACrC,8BAA8B,EAC/B,GAAE,QAAQ,CACT,IAAI,CACF,kBAAkB,EAChB,0BAA0B,GAC1B,4BAA4B,GAC5B,2BAA2B,GAC3B,uCAAuC,GACvC,gCAAgC,CACnC,CAC0B,EAC7B,MAAM,GAAE,eAAsC,EACtC,MAAM,yCAA+C,EAS9D;IAED,OAAO,CAAC,iBAAiB,CAEvB;IAEF,OAAO,CAAC,eAAe,CAUrB;IAEW,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CA6BlC;IAEY,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAMjC;IAEM,iBAAiB,CAAC,GAAG,EAAE,UAAU,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAEnE;IAEM,gBAAgB,CAAC,EAAE,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAEvD;IAEM,mBAAmB,CAAC,EAAE,EAAE,YAAY,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAEtE;IAEM,gBAAgB,CAAC,GAAG,EAAE,YAAY,EAAE,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC,CAEpE;IAEM,aAAa,CAAC,MAAM,CAAC,EAAE,gBAAgB,GAAG,OAAO,CAAC,qBAAqB,GAAG,SAAS,CAAC,CAE1F;IAEM,uBAAuB,CAC5B,EAAE,EAAE,YAAY,EAChB,KAAK,EAAE,QAAQ,EACf,MAAM,CAAC,EAAE,gBAAgB,GACxB,OAAO,CAAC,qBAAqB,GAAG,SAAS,CAAC,CAE5C;IAEM,qBAAqB,CAC1B,EAAE,EAAE,YAAY,EAChB,GAAG,EAAE,MAAM,EACX,KAAK,UAAQ,EACb,MAAM,CAAC,EAAE,gBAAgB,GACxB,OAAO,CAAC,qBAAqB,GAAG,SAAS,CAAC,CAE5C;IAEM,wBAAwB,CAC7B,EAAE,EAAE,YAAY,EAChB,SAAS,EAAE,MAAM,EACjB,MAAM,CAAC,EAAE,gBAAgB,GACxB,OAAO,CAAC;QAAE,GAAG,EAAE,UAAU,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,SAAS,CAAC,CAExD;IAEY,gBAAgB,CAC3B,KAAK,EAAE,YAAY,EACnB,IAAI,EAAE,kBAAkB,EACxB,WAAW,EAAE,WAAW,EACxB,SAAS,EAAE,QAAQ,GAClB,OAAO,CAAC,gBAAgB,CAAC,CAgB3B;IAkDD,OAAO,CAAC,sBAAsB;YAmRhB,WAAW;IAUzB,OAAO,CAAC,gBAAgB;IAgBxB,OAAO,CAAC,oBAAoB;IAqC5B,OAAO,CAAC,kBAAkB;IAY1B,OAAO,CAAC,UAAU;IAIlB,OAAO,CAAC,iBAAiB;CAG1B;AAiDD;;;;;;GAMG;AACH,eAAO,MAAM,6BAA6B,EAAE,kBAAkB,EAmB7D,CAAC"}

package/dest/proving_broker/proving_broker.js

@@ -51,7 +51,8 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
    */ epochHeight;
     maxEpochsToKeepResultsFor;
     started;
-    constructor(database, { proverBrokerJobTimeoutMs, proverBrokerPollIntervalMs, proverBrokerJobMaxRetries, proverBrokerMaxEpochsToKeepResultsFor } = defaultProverBrokerConfig, client = getTelemetryClient(), logger = createLogger('prover-client:proving-broker')){
+    debugReplayEnabled;
+    constructor(database, { proverBrokerJobTimeoutMs, proverBrokerPollIntervalMs, proverBrokerJobMaxRetries, proverBrokerMaxEpochsToKeepResultsFor, proverBrokerDebugReplayEnabled } = defaultProverBrokerConfig, client = getTelemetryClient(), logger = createLogger('prover-client:proving-broker')){
         this.database = database;
         this.logger = logger;
         this.queues = {
@@ -104,6 +105,7 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
         this.jobTimeoutMs = proverBrokerJobTimeoutMs;
         this.maxRetries = proverBrokerJobMaxRetries;
         this.maxEpochsToKeepResultsFor = proverBrokerMaxEpochsToKeepResultsFor;
+        this.debugReplayEnabled = proverBrokerDebugReplayEnabled ?? false;
     }
     measureQueueDepth;
     countActiveJobs;
@@ -163,6 +165,32 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
     reportProvingJobProgress(id, startedAt, filter) {
         return Promise.resolve(this.#reportProvingJobProgress(id, startedAt, filter));
     }
+    async replayProvingJob(jobId, type, epochNumber, inputsUri) {
+        if (!this.debugReplayEnabled) {
+            throw new Error('Debug replay not enabled. Set PROVER_BROKER_DEBUG_REPLAY_ENABLED=true');
+        }
+        this.logger.info(`Replaying proving job`, {
+            provingJobId: jobId,
+            epochNumber,
+            inputsUri
+        });
+        // Clear existing state and enqueue
+        this.cleanUpProvingJobState([
+            jobId
+        ]);
+        const job = {
+            id: jobId,
+            type,
+            epochNumber,
+            inputsUri
+        };
+        this.jobsCache.set(jobId, job);
+        await this.database.addProvingJob(job);
+        this.enqueueJobInternal(job);
+        return {
+            status: 'in-queue'
+        };
+    }
     async #enqueueProvingJob(job) {
         // We return the job status at the start of this call
         const jobStatus = this.#getProvingJobStatus(job.id);
@@ -211,7 +239,7 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
             this.logger.info(`Cancelling job id=${id}`, {
                 provingJobId: id
             });
-            await this.#reportProvingJobError(id, 'Aborted', false);
+            await this.#reportProvingJobError(id, 'Aborted', false, undefined, true);
         }
     }
     cleanUpProvingJobState(ids) {
@@ -285,7 +313,7 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
         }
         return undefined;
     }
-    async #reportProvingJobError(id, err, retry = false, filter) {
+    async #reportProvingJobError(id, err, retry = false, filter, aborted = false) {
         const info = this.inProgress.get(id);
         const item = this.jobsCache.get(id);
         const retries = this.retries.get(id) ?? 0;
@@ -332,7 +360,11 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
         this.resultsCache.set(id, result);
         this.promises.get(id).resolve(result);
         this.completedJobNotifications.push(id);
-        this.instrumentation.incRejectedJobs(item.type);
+        if (aborted) {
+            this.instrumentation.incAbortedJobs(item.type);
+        } else {
+            this.instrumentation.incRejectedJobs(item.type);
+        }
         if (info) {
             const duration = this.msTimeSource() - info.startedAt;
             this.instrumentation.recordJobDuration(item.type, duration);
@@ -483,12 +515,28 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
             const now = this.msTimeSource();
             const msSinceLastUpdate = now - metadata.lastUpdatedAt;
             if (msSinceLastUpdate >= this.jobTimeoutMs) {
-                this.logger.warn(`Proving job id=${id} timed out. Adding it back to the queue.`, {
-                    provingJobId: id
-                });
                 this.inProgress.delete(id);
-                this.enqueueJobInternal(item);
                 this.instrumentation.incTimedOutJobs(item.type);
+                const retries = this.retries.get(id) ?? 0;
+                if (retries + 1 < this.maxRetries && !this.isJobStale(item)) {
+                    this.logger.warn(`Proving job id=${id} timed out. Re-enqueueing (retry ${retries + 1}/${this.maxRetries}).`, {
+                        provingJobId: id
+                    });
+                    this.retries.set(id, retries + 1);
+                    this.enqueueJobInternal(item);
+                } else {
+                    this.logger.error(`Proving job id=${id} timed out after ${retries + 1} attempts. Marking as failed.`, {
+                        provingJobId: id
+                    });
+                    const result = {
+                        status: 'rejected',
+                        reason: 'Timed out'
+                    };
+                    this.resultsCache.set(id, result);
+                    this.promises.get(id)?.resolve(result);
+                    this.completedJobNotifications.push(id);
+                    this.instrumentation.incRejectedJobs(item.type);
+                }
             }
         }
     }

package/dest/proving_broker/proving_broker_instrumentation.d.ts

@@ -7,6 +7,7 @@ export declare class ProvingBrokerInstrumentation {
     private activeJobs;
     private resolvedJobs;
     private rejectedJobs;
+    private abortedJobs;
     private timedOutJobs;
     private cachedJobs;
     private totalJobs;
@@ -18,6 +19,7 @@ export declare class ProvingBrokerInstrumentation {
     monitorActiveJobs(fn: MonitorCallback): void;
     incResolvedJobs(proofType: ProvingRequestType): void;
     incRejectedJobs(proofType: ProvingRequestType): void;
+    incAbortedJobs(proofType: ProvingRequestType): void;
     incRetriedJobs(proofType: ProvingRequestType): void;
     incTimedOutJobs(proofType: ProvingRequestType): void;
     incCachedJobs(proofType: ProvingRequestType): void;
@@ -26,4 +28,4 @@ export declare class ProvingBrokerInstrumentation {
     recordJobDuration(proofType: ProvingRequestType, msOrTimer: Timer | number): void;
     private observe;
 }
-//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoicHJvdmluZ19icm9rZXJfaW5zdHJ1bWVudGF0aW9uLmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi9zcmMvcHJvdmluZ19icm9rZXIvcHJvdmluZ19icm9rZXJfaW5zdHJ1bWVudGF0aW9uLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLE9BQU8sS0FBSyxFQUFFLEtBQUssRUFBRSxNQUFNLHlCQUF5QixDQUFDO0FBQ3JELE9BQU8sRUFBRSxrQkFBa0IsRUFBRSxNQUFNLHNCQUFzQixDQUFDO0FBQzFELE9BQU8sRUFNTCxLQUFLLGVBQWUsRUFFckIsTUFBTSx5QkFBeUIsQ0FBQztBQUVqQyxNQUFNLE1BQU0sZUFBZSxHQUFHLENBQUMsU0FBUyxFQUFFLGtCQUFrQixLQUFLLE1BQU0sQ0FBQztBQUV4RSxxQkFBYSw0QkFBNEI7SUFDdkMsT0FBTyxDQUFDLFNBQVMsQ0FBa0I7SUFDbkMsT0FBTyxDQUFDLFVBQVUsQ0FBa0I7SUFDcEMsT0FBTyxDQUFDLFlBQVksQ0FBZ0I7SUFDcEMsT0FBTyxDQUFDLFlBQVksQ0FBZ0I7SUFDcEMsT0FBTyxDQUFDLFlBQVksQ0FBZ0I7SUFDcEMsT0FBTyxDQUFDLFVBQVUsQ0FBZ0I7SUFDbEMsT0FBTyxDQUFDLFNBQVMsQ0FBZ0I7SUFDakMsT0FBTyxDQUFDLE9BQU8sQ0FBWTtJQUMzQixPQUFPLENBQUMsV0FBVyxDQUFZO0lBQy9CLE9BQU8sQ0FBQyxXQUFXLENBQWdCO0lBRW5DLFlBQVksTUFBTSxFQUFFLGVBQWUsRUFBRSxJQUFJLFNBQWtCLEVBc0IxRDtJQUVELGlCQUFpQixDQUFDLEVBQUUsRUFBRSxlQUFlLFFBRXBDO0lBRUQsaUJBQWlCLENBQUMsRUFBRSxFQUFFLGVBQWUsUUFFcEM7SUFFRCxlQUFlLENBQUMsU0FBUyxFQUFFLGtCQUFrQixRQUk1QztJQUVELGVBQWUsQ0FBQyxTQUFTLEVBQUUsa0JBQWtCLFFBSTVDO0lBRUQsY0FBYyxDQUFDLFNBQVMsRUFBRSxrQkFBa0IsUUFJM0M7SUFFRCxlQUFlLENBQUMsU0FBUyxFQUFFLGtCQUFrQixRQUk1QztJQUVELGFBQWEsQ0FBQyxTQUFTLEVBQUUsa0JBQWtCLFFBSTFDO0lBRUQsWUFBWSxDQUFDLFNBQVMsRUFBRSxrQkFBa0IsUUFJekM7SUFFRCxhQUFhLENBQUMsU0FBUyxFQUFFLGtCQUFrQixFQUFFLFNBQVMsRUFBRSxLQUFLLEdBQUcsTUFBTSxRQUtyRTtJQUVELGlCQUFpQixDQUFDLFNBQVMsRUFBRSxrQkFBa0IsRUFBRSxTQUFTLEVBQUUsS0FBSyxHQUFHLE1BQU0sUUFLekU7SUFFRCxPQUFPLENBQUMsT0FBTztDQVdoQiJ9
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoicHJvdmluZ19icm9rZXJfaW5zdHJ1bWVudGF0aW9uLmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi9zcmMvcHJvdmluZ19icm9rZXIvcHJvdmluZ19icm9rZXJfaW5zdHJ1bWVudGF0aW9uLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLE9BQU8sS0FBSyxFQUFFLEtBQUssRUFBRSxNQUFNLHlCQUF5QixDQUFDO0FBQ3JELE9BQU8sRUFBRSxrQkFBa0IsRUFBRSxNQUFNLHNCQUFzQixDQUFDO0FBQzFELE9BQU8sRUFNTCxLQUFLLGVBQWUsRUFHckIsTUFBTSx5QkFBeUIsQ0FBQztBQUVqQyxNQUFNLE1BQU0sZUFBZSxHQUFHLENBQUMsU0FBUyxFQUFFLGtCQUFrQixLQUFLLE1BQU0sQ0FBQztBQUV4RSxxQkFBYSw0QkFBNEI7SUFDdkMsT0FBTyxDQUFDLFNBQVMsQ0FBa0I7SUFDbkMsT0FBTyxDQUFDLFVBQVUsQ0FBa0I7SUFDcEMsT0FBTyxDQUFDLFlBQVksQ0FBZ0I7SUFDcEMsT0FBTyxDQUFDLFlBQVksQ0FBZ0I7SUFDcEMsT0FBTyxDQUFDLFdBQVcsQ0FBZ0I7SUFDbkMsT0FBTyxDQUFDLFlBQVksQ0FBZ0I7SUFDcEMsT0FBTyxDQUFDLFVBQVUsQ0FBZ0I7SUFDbEMsT0FBTyxDQUFDLFNBQVMsQ0FBZ0I7SUFDakMsT0FBTyxDQUFDLE9BQU8sQ0FBWTtJQUMzQixPQUFPLENBQUMsV0FBVyxDQUFZO0lBQy9CLE9BQU8sQ0FBQyxXQUFXLENBQWdCO0lBRW5DLFlBQVksTUFBTSxFQUFFLGVBQWUsRUFBRSxJQUFJLFNBQWtCLEVBMkIxRDtJQUVELGlCQUFpQixDQUFDLEVBQUUsRUFBRSxlQUFlLFFBRXBDO0lBRUQsaUJBQWlCLENBQUMsRUFBRSxFQUFFLGVBQWUsUUFFcEM7SUFFRCxlQUFlLENBQUMsU0FBUyxFQUFFLGtCQUFrQixRQUk1QztJQUVELGVBQWUsQ0FBQyxTQUFTLEVBQUUsa0JBQWtCLFFBSTVDO0lBRUQsY0FBYyxDQUFDLFNBQVMsRUFBRSxrQkFBa0IsUUFJM0M7SUFFRCxjQUFjLENBQUMsU0FBUyxFQUFFLGtCQUFrQixRQUkzQztJQUVELGVBQWUsQ0FBQyxTQUFTLEVBQUUsa0JBQWtCLFFBSTVDO0lBRUQsYUFBYSxDQUFDLFNBQVMsRUFBRSxrQkFBa0IsUUFJMUM7SUFFRCxZQUFZLENBQUMsU0FBUyxFQUFFLGtCQUFrQixRQUl6QztJQUVELGFBQWEsQ0FBQyxTQUFTLEVBQUUsa0JBQWtCLEVBQUUsU0FBUyxFQUFFLEtBQUssR0FBRyxNQUFNLFFBS3JFO0lBRUQsaUJBQWlCLENBQUMsU0FBUyxFQUFFLGtCQUFrQixFQUFFLFNBQVMsRUFBRSxLQUFLLEdBQUcsTUFBTSxRQUt6RTtJQUVELE9BQU8sQ0FBQyxPQUFPO0NBV2hCIn0=

package/dest/proving_broker/proving_broker_instrumentation.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"proving_broker_instrumentation.d.ts","sourceRoot":"","sources":["../../src/proving_broker/proving_broker_instrumentation.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AACrD,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAC1D,OAAO,EAML,KAAK,eAAe,EAErB,MAAM,yBAAyB,CAAC;AAEjC,MAAM,MAAM,eAAe,GAAG,CAAC,SAAS,EAAE,kBAAkB,KAAK,MAAM,CAAC;AAExE,qBAAa,4BAA4B;IACvC,OAAO,CAAC,SAAS,CAAkB;IACnC,OAAO,CAAC,UAAU,CAAkB;IACpC,OAAO,CAAC,YAAY,CAAgB;IACpC,OAAO,CAAC,YAAY,CAAgB;IACpC,OAAO,CAAC,YAAY,CAAgB;IACpC,OAAO,CAAC,UAAU,CAAgB;IAClC,OAAO,CAAC,SAAS,CAAgB;IACjC,OAAO,CAAC,OAAO,CAAY;IAC3B,OAAO,CAAC,WAAW,CAAY;IAC/B,OAAO,CAAC,WAAW,CAAgB;IAEnC,YAAY,MAAM,EAAE,eAAe,EAAE,IAAI,SAAkB,EAsB1D;IAED,iBAAiB,CAAC,EAAE,EAAE,eAAe,QAEpC;IAED,iBAAiB,CAAC,EAAE,EAAE,eAAe,QAEpC;IAED,eAAe,CAAC,SAAS,EAAE,kBAAkB,QAI5C;IAED,eAAe,CAAC,SAAS,EAAE,kBAAkB,QAI5C;IAED,cAAc,CAAC,SAAS,EAAE,kBAAkB,QAI3C;IAED,eAAe,CAAC,SAAS,EAAE,kBAAkB,QAI5C;IAED,aAAa,CAAC,SAAS,EAAE,kBAAkB,QAI1C;IAED,YAAY,CAAC,SAAS,EAAE,kBAAkB,QAIzC;IAED,aAAa,CAAC,SAAS,EAAE,kBAAkB,EAAE,SAAS,EAAE,KAAK,GAAG,MAAM,QAKrE;IAED,iBAAiB,CAAC,SAAS,EAAE,kBAAkB,EAAE,SAAS,EAAE,KAAK,GAAG,MAAM,QAKzE;IAED,OAAO,CAAC,OAAO;CAWhB"}
+{"version":3,"file":"proving_broker_instrumentation.d.ts","sourceRoot":"","sources":["../../src/proving_broker/proving_broker_instrumentation.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AACrD,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAC1D,OAAO,EAML,KAAK,eAAe,EAGrB,MAAM,yBAAyB,CAAC;AAEjC,MAAM,MAAM,eAAe,GAAG,CAAC,SAAS,EAAE,kBAAkB,KAAK,MAAM,CAAC;AAExE,qBAAa,4BAA4B;IACvC,OAAO,CAAC,SAAS,CAAkB;IACnC,OAAO,CAAC,UAAU,CAAkB;IACpC,OAAO,CAAC,YAAY,CAAgB;IACpC,OAAO,CAAC,YAAY,CAAgB;IACpC,OAAO,CAAC,WAAW,CAAgB;IACnC,OAAO,CAAC,YAAY,CAAgB;IACpC,OAAO,CAAC,UAAU,CAAgB;IAClC,OAAO,CAAC,SAAS,CAAgB;IACjC,OAAO,CAAC,OAAO,CAAY;IAC3B,OAAO,CAAC,WAAW,CAAY;IAC/B,OAAO,CAAC,WAAW,CAAgB;IAEnC,YAAY,MAAM,EAAE,eAAe,EAAE,IAAI,SAAkB,EA2B1D;IAED,iBAAiB,CAAC,EAAE,EAAE,eAAe,QAEpC;IAED,iBAAiB,CAAC,EAAE,EAAE,eAAe,QAEpC;IAED,eAAe,CAAC,SAAS,EAAE,kBAAkB,QAI5C;IAED,eAAe,CAAC,SAAS,EAAE,kBAAkB,QAI5C;IAED,cAAc,CAAC,SAAS,EAAE,kBAAkB,QAI3C;IAED,cAAc,CAAC,SAAS,EAAE,kBAAkB,QAI3C;IAED,eAAe,CAAC,SAAS,EAAE,kBAAkB,QAI5C;IAED,aAAa,CAAC,SAAS,EAAE,kBAAkB,QAI1C;IAED,YAAY,CAAC,SAAS,EAAE,kBAAkB,QAIzC;IAED,aAAa,CAAC,SAAS,EAAE,kBAAkB,EAAE,SAAS,EAAE,KAAK,GAAG,MAAM,QAKrE;IAED,iBAAiB,CAAC,SAAS,EAAE,kBAAkB,EAAE,SAAS,EAAE,KAAK,GAAG,MAAM,QAKzE;IAED,OAAO,CAAC,OAAO;CAWhB"}

package/dest/proving_broker/proving_broker_instrumentation.js

@@ -1,10 +1,11 @@
 import { ProvingRequestType } from '@aztec/stdlib/proofs';
-import { Attributes, Metrics } from '@aztec/telemetry-client';
+import { Attributes, Metrics, createUpDownCounterWithDefault } from '@aztec/telemetry-client';
 export class ProvingBrokerInstrumentation {
     queueSize;
     activeJobs;
     resolvedJobs;
     rejectedJobs;
+    abortedJobs;
     timedOutJobs;
     cachedJobs;
     totalJobs;
@@ -15,12 +16,17 @@ export class ProvingBrokerInstrumentation {
         const meter = client.getMeter(name);
         this.queueSize = meter.createObservableGauge(Metrics.PROVING_QUEUE_SIZE);
         this.activeJobs = meter.createObservableGauge(Metrics.PROVING_QUEUE_ACTIVE_JOBS);
-        this.resolvedJobs = meter.createUpDownCounter(Metrics.PROVING_QUEUE_RESOLVED_JOBS);
-        this.rejectedJobs = meter.createUpDownCounter(Metrics.PROVING_QUEUE_REJECTED_JOBS);
-        this.retriedJobs = meter.createUpDownCounter(Metrics.PROVING_QUEUE_RETRIED_JOBS);
-        this.timedOutJobs = meter.createUpDownCounter(Metrics.PROVING_QUEUE_TIMED_OUT_JOBS);
-        this.cachedJobs = meter.createUpDownCounter(Metrics.PROVING_QUEUE_CACHED_JOBS);
-        this.totalJobs = meter.createUpDownCounter(Metrics.PROVING_QUEUE_TOTAL_JOBS);
+        const provingJobTypes = Object.values(ProvingRequestType).filter((v)=>typeof v === 'string');
+        const provingJobAttrs = {
+            [Attributes.PROVING_JOB_TYPE]: provingJobTypes
+        };
+        this.resolvedJobs = createUpDownCounterWithDefault(meter, Metrics.PROVING_QUEUE_RESOLVED_JOBS, provingJobAttrs);
+        this.rejectedJobs = createUpDownCounterWithDefault(meter, Metrics.PROVING_QUEUE_REJECTED_JOBS, provingJobAttrs);
+        this.abortedJobs = createUpDownCounterWithDefault(meter, Metrics.PROVING_QUEUE_ABORTED_JOBS, provingJobAttrs);
+        this.retriedJobs = createUpDownCounterWithDefault(meter, Metrics.PROVING_QUEUE_RETRIED_JOBS, provingJobAttrs);
+        this.timedOutJobs = createUpDownCounterWithDefault(meter, Metrics.PROVING_QUEUE_TIMED_OUT_JOBS, provingJobAttrs);
+        this.cachedJobs = createUpDownCounterWithDefault(meter, Metrics.PROVING_QUEUE_CACHED_JOBS, provingJobAttrs);
+        this.totalJobs = createUpDownCounterWithDefault(meter, Metrics.PROVING_QUEUE_TOTAL_JOBS, provingJobAttrs);
         this.jobWait = meter.createHistogram(Metrics.PROVING_QUEUE_JOB_WAIT);
         this.jobDuration = meter.createHistogram(Metrics.PROVING_QUEUE_JOB_DURATION);
     }
@@ -40,6 +46,11 @@ export class ProvingBrokerInstrumentation {
             [Attributes.PROVING_JOB_TYPE]: ProvingRequestType[proofType]
         });
     }
+    incAbortedJobs(proofType) {
+        this.abortedJobs.add(1, {
+            [Attributes.PROVING_JOB_TYPE]: ProvingRequestType[proofType]
+        });
+    }
     incRetriedJobs(proofType) {
         this.retriedJobs.add(1, {
             [Attributes.PROVING_JOB_TYPE]: ProvingRequestType[proofType]

package/dest/proving_broker/proving_job_controller.d.ts

@@ -1,4 +1,5 @@
 import { EpochNumber } from '@aztec/foundation/branded-types';
+import { type LoggerBindings } from '@aztec/foundation/log';
 import type { ProvingJobId, ProvingJobInputs, ProvingJobResultsMap, ServerCircuitProver } from '@aztec/stdlib/interfaces/server';
 import { ProvingRequestType } from '@aztec/stdlib/proofs';
 export declare enum ProvingJobControllerStatus {
@@ -13,12 +14,12 @@ export declare class ProvingJobController {
     private startedAt;
     private circuitProver;
     private onComplete;
-    private log;
     private status;
     private promise?;
     private abortController;
     private result?;
-    constructor(jobId: ProvingJobId, inputs: ProvingJobInputs, epochNumber: EpochNumber, startedAt: number, circuitProver: ServerCircuitProver, onComplete: () => void, log?: import("@aztec/foundation/log").Logger);
+    private log;
+    constructor(jobId: ProvingJobId, inputs: ProvingJobInputs, epochNumber: EpochNumber, startedAt: number, circuitProver: ServerCircuitProver, onComplete: () => void, bindings?: LoggerBindings);
     start(): void;
     getStatus(): ProvingJobControllerStatus;
     getResult(): ProvingJobResultsMap[ProvingRequestType] | Error | undefined;
@@ -30,4 +31,4 @@ export declare class ProvingJobController {
     private run;
     private generateProof;
 }
-//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoicHJvdmluZ19qb2JfY29udHJvbGxlci5kLnRzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vLi4vc3JjL3Byb3ZpbmdfYnJva2VyL3Byb3Zpbmdfam9iX2NvbnRyb2xsZXIudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsT0FBTyxFQUFFLFdBQVcsRUFBRSxNQUFNLGlDQUFpQyxDQUFDO0FBSTlELE9BQU8sS0FBSyxFQUNWLFlBQVksRUFDWixnQkFBZ0IsRUFDaEIsb0JBQW9CLEVBQ3BCLG1CQUFtQixFQUNwQixNQUFNLGlDQUFpQyxDQUFDO0FBQ3pDLE9BQU8sRUFBRSxrQkFBa0IsRUFBRSxNQUFNLHNCQUFzQixDQUFDO0FBRTFELG9CQUFZLDBCQUEwQjtJQUNwQyxJQUFJLFNBQVM7SUFDYixPQUFPLFlBQVk7SUFDbkIsSUFBSSxTQUFTO0NBQ2Q7QUFFRCxxQkFBYSxvQkFBb0I7SUFPN0IsT0FBTyxDQUFDLEtBQUs7SUFDYixPQUFPLENBQUMsTUFBTTtJQUNkLE9BQU8sQ0FBQyxXQUFXO0lBQ25CLE9BQU8sQ0FBQyxTQUFTO0lBQ2pCLE9BQU8sQ0FBQyxhQUFhO0lBQ3JCLE9BQU8sQ0FBQyxVQUFVO0lBQ2xCLE9BQU8sQ0FBQyxHQUFHO0lBWmIsT0FBTyxDQUFDLE1BQU0sQ0FBK0Q7SUFDN0UsT0FBTyxDQUFDLE9BQU8sQ0FBQyxDQUFnQjtJQUNoQyxPQUFPLENBQUMsZUFBZSxDQUF5QjtJQUNoRCxPQUFPLENBQUMsTUFBTSxDQUFDLENBQW1EO0lBRWxFLFlBQ1UsS0FBSyxFQUFFLFlBQVksRUFDbkIsTUFBTSxFQUFFLGdCQUFnQixFQUN4QixXQUFXLEVBQUUsV0FBVyxFQUN4QixTQUFTLEVBQUUsTUFBTSxFQUNqQixhQUFhLEVBQUUsbUJBQW1CLEVBQ2xDLFVBQVUsRUFBRSxNQUFNLElBQUksRUFDdEIsR0FBRyx5Q0FBK0YsRUFDeEc7SUFFRyxLQUFLLElBQUksSUFBSSxDQWlCbkI7SUFFTSxTQUFTLElBQUksMEJBQTBCLENBRTdDO0lBRU0sU0FBUyxJQUFJLG9CQUFvQixDQUFDLGtCQUFrQixDQUFDLEdBQUcsS0FBSyxHQUFHLFNBQVMsQ0FFL0U7SUFFTSxLQUFLLElBQUksSUFBSSxDQWFuQjtJQUVNLFFBQVEsSUFBSSxZQUFZLENBRTlCO0lBRU0sWUFBWSxJQUFJLGtCQUFrQixDQUV4QztJQUVNLFlBQVksSUFBSSxNQUFNLENBRTVCO0lBRU0sZ0JBQWdCLElBQUksTUFBTSxDQUVoQztJQUVELE9BQU8sQ0FBQyxHQUFHLENBNEJUO1lBRVksYUFBYTtDQWtGNUIifQ==
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoicHJvdmluZ19qb2JfY29udHJvbGxlci5kLnRzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vLi4vc3JjL3Byb3ZpbmdfYnJva2VyL3Byb3Zpbmdfam9iX2NvbnRyb2xsZXIudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsT0FBTyxFQUFFLFdBQVcsRUFBRSxNQUFNLGlDQUFpQyxDQUFDO0FBRzlELE9BQU8sRUFBZSxLQUFLLGNBQWMsRUFBZ0IsTUFBTSx1QkFBdUIsQ0FBQztBQUN2RixPQUFPLEtBQUssRUFDVixZQUFZLEVBQ1osZ0JBQWdCLEVBQ2hCLG9CQUFvQixFQUNwQixtQkFBbUIsRUFDcEIsTUFBTSxpQ0FBaUMsQ0FBQztBQUN6QyxPQUFPLEVBQUUsa0JBQWtCLEVBQUUsTUFBTSxzQkFBc0IsQ0FBQztBQUUxRCxvQkFBWSwwQkFBMEI7SUFDcEMsSUFBSSxTQUFTO0lBQ2IsT0FBTyxZQUFZO0lBQ25CLElBQUksU0FBUztDQUNkO0FBRUQscUJBQWEsb0JBQW9CO0lBUTdCLE9BQU8sQ0FBQyxLQUFLO0lBQ2IsT0FBTyxDQUFDLE1BQU07SUFDZCxPQUFPLENBQUMsV0FBVztJQUNuQixPQUFPLENBQUMsU0FBUztJQUNqQixPQUFPLENBQUMsYUFBYTtJQUNyQixPQUFPLENBQUMsVUFBVTtJQVpwQixPQUFPLENBQUMsTUFBTSxDQUErRDtJQUM3RSxPQUFPLENBQUMsT0FBTyxDQUFDLENBQWdCO0lBQ2hDLE9BQU8sQ0FBQyxlQUFlLENBQXlCO0lBQ2hELE9BQU8sQ0FBQyxNQUFNLENBQUMsQ0FBbUQ7SUFDbEUsT0FBTyxDQUFDLEdBQUcsQ0FBUztJQUVwQixZQUNVLEtBQUssRUFBRSxZQUFZLEVBQ25CLE1BQU0sRUFBRSxnQkFBZ0IsRUFDeEIsV0FBVyxFQUFFLFdBQVcsRUFDeEIsU0FBUyxFQUFFLE1BQU0sRUFDakIsYUFBYSxFQUFFLG1CQUFtQixFQUNsQyxVQUFVLEVBQUUsTUFBTSxJQUFJLEVBQzlCLFFBQVEsQ0FBQyxFQUFFLGNBQWMsRUFNMUI7SUFFTSxLQUFLLElBQUksSUFBSSxDQWlCbkI7SUFFTSxTQUFTLElBQUksMEJBQTBCLENBRTdDO0lBRU0sU0FBUyxJQUFJLG9CQUFvQixDQUFDLGtCQUFrQixDQUFDLEdBQUcsS0FBSyxHQUFHLFNBQVMsQ0FFL0U7SUFFTSxLQUFLLElBQUksSUFBSSxDQWFuQjtJQUVNLFFBQVEsSUFBSSxZQUFZLENBRTlCO0lBRU0sWUFBWSxJQUFJLGtCQUFrQixDQUV4QztJQUVNLFlBQVksSUFBSSxNQUFNLENBRTVCO0lBRU0sZ0JBQWdCLElBQUksTUFBTSxDQUVoQztJQUVELE9BQU8sQ0FBQyxHQUFHLENBNEJUO1lBRVksYUFBYTtDQWtGNUIifQ==

package/dest/proving_broker/proving_job_controller.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"proving_job_controller.d.ts","sourceRoot":"","sources":["../../src/proving_broker/proving_job_controller.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iCAAiC,CAAC;AAI9D,OAAO,KAAK,EACV,YAAY,EACZ,gBAAgB,EAChB,oBAAoB,EACpB,mBAAmB,EACpB,MAAM,iCAAiC,CAAC;AACzC,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAE1D,oBAAY,0BAA0B;IACpC,IAAI,SAAS;IACb,OAAO,YAAY;IACnB,IAAI,SAAS;CACd;AAED,qBAAa,oBAAoB;IAO7B,OAAO,CAAC,KAAK;IACb,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,WAAW;IACnB,OAAO,CAAC,SAAS;IACjB,OAAO,CAAC,aAAa;IACrB,OAAO,CAAC,UAAU;IAClB,OAAO,CAAC,GAAG;IAZb,OAAO,CAAC,MAAM,CAA+D;IAC7E,OAAO,CAAC,OAAO,CAAC,CAAgB;IAChC,OAAO,CAAC,eAAe,CAAyB;IAChD,OAAO,CAAC,MAAM,CAAC,CAAmD;IAElE,YACU,KAAK,EAAE,YAAY,EACnB,MAAM,EAAE,gBAAgB,EACxB,WAAW,EAAE,WAAW,EACxB,SAAS,EAAE,MAAM,EACjB,aAAa,EAAE,mBAAmB,EAClC,UAAU,EAAE,MAAM,IAAI,EACtB,GAAG,yCAA+F,EACxG;IAEG,KAAK,IAAI,IAAI,CAiBnB;IAEM,SAAS,IAAI,0BAA0B,CAE7C;IAEM,SAAS,IAAI,oBAAoB,CAAC,kBAAkB,CAAC,GAAG,KAAK,GAAG,SAAS,CAE/E;IAEM,KAAK,IAAI,IAAI,CAanB;IAEM,QAAQ,IAAI,YAAY,CAE9B;IAEM,YAAY,IAAI,kBAAkB,CAExC;IAEM,YAAY,IAAI,MAAM,CAE5B;IAEM,gBAAgB,IAAI,MAAM,CAEhC;IAED,OAAO,CAAC,GAAG,CA4BT;YAEY,aAAa;CAkF5B"}
+{"version":3,"file":"proving_job_controller.d.ts","sourceRoot":"","sources":["../../src/proving_broker/proving_job_controller.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iCAAiC,CAAC;AAG9D,OAAO,EAAe,KAAK,cAAc,EAAgB,MAAM,uBAAuB,CAAC;AACvF,OAAO,KAAK,EACV,YAAY,EACZ,gBAAgB,EAChB,oBAAoB,EACpB,mBAAmB,EACpB,MAAM,iCAAiC,CAAC;AACzC,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAE1D,oBAAY,0BAA0B;IACpC,IAAI,SAAS;IACb,OAAO,YAAY;IACnB,IAAI,SAAS;CACd;AAED,qBAAa,oBAAoB;IAQ7B,OAAO,CAAC,KAAK;IACb,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,WAAW;IACnB,OAAO,CAAC,SAAS;IACjB,OAAO,CAAC,aAAa;IACrB,OAAO,CAAC,UAAU;IAZpB,OAAO,CAAC,MAAM,CAA+D;IAC7E,OAAO,CAAC,OAAO,CAAC,CAAgB;IAChC,OAAO,CAAC,eAAe,CAAyB;IAChD,OAAO,CAAC,MAAM,CAAC,CAAmD;IAClE,OAAO,CAAC,GAAG,CAAS;IAEpB,YACU,KAAK,EAAE,YAAY,EACnB,MAAM,EAAE,gBAAgB,EACxB,WAAW,EAAE,WAAW,EACxB,SAAS,EAAE,MAAM,EACjB,aAAa,EAAE,mBAAmB,EAClC,UAAU,EAAE,MAAM,IAAI,EAC9B,QAAQ,CAAC,EAAE,cAAc,EAM1B;IAEM,KAAK,IAAI,IAAI,CAiBnB;IAEM,SAAS,IAAI,0BAA0B,CAE7C;IAEM,SAAS,IAAI,oBAAoB,CAAC,kBAAkB,CAAC,GAAG,KAAK,GAAG,SAAS,CAE/E;IAEM,KAAK,IAAI,IAAI,CAanB;IAEM,QAAQ,IAAI,YAAY,CAE9B;IAEM,YAAY,IAAI,kBAAkB,CAExC;IAEM,YAAY,IAAI,MAAM,CAE5B;IAEM,gBAAgB,IAAI,MAAM,CAEhC;IAED,OAAO,CAAC,GAAG,CA4BT;YAEY,aAAa;CAkF5B"}

package/dest/proving_broker/proving_job_controller.js

@@ -15,19 +15,18 @@ export class ProvingJobController {
     startedAt;
     circuitProver;
     onComplete;
-    log;
     status;
     promise;
     abortController;
     result;
-    constructor(jobId, inputs, epochNumber, startedAt, circuitProver, onComplete, log = createLogger('prover-client:proving-agent:job-controller-' + randomBytes(4).toString('hex'))){
+    log;
+    constructor(jobId, inputs, epochNumber, startedAt, circuitProver, onComplete, bindings){
         this.jobId = jobId;
         this.inputs = inputs;
         this.epochNumber = epochNumber;
         this.startedAt = startedAt;
         this.circuitProver = circuitProver;
         this.onComplete = onComplete;
-        this.log = log;
         this.status = "idle";
         this.abortController = new AbortController();
         this.run = async ()=>{
@@ -61,6 +60,10 @@ export class ProvingJobController {
                 });
             }
         };
+        this.log = createLogger('prover-client:proving-agent:job-controller', {
+            instanceId: randomBytes(4).toString('hex'),
+            ...bindings
+        });
     }
     start() {
         if (this.status !== "idle") {

package/dest/proving_broker/rpc.d.ts

@@ -1,9 +1,13 @@
-import { type ProvingJobBroker, type ProvingJobConsumer, type ProvingJobProducer } from '@aztec/stdlib/interfaces/server';
+import { type ProvingJobBroker, type ProvingJobBrokerDebug, type ProvingJobConsumer, type ProvingJobProducer } from '@aztec/stdlib/interfaces/server';
 import { type ApiSchemaFor } from '@aztec/stdlib/schemas';
 import { type ComponentsVersions } from '@aztec/stdlib/versioning';
+/** Indefinite backoff for broker communication: 1, 1, 1, 2, 4, 4, 4, ... seconds. */
+export declare function proverBrokerBackoff(): Generator<number, void, unknown>;
 export declare const ProvingJobProducerSchema: ApiSchemaFor<ProvingJobProducer>;
 export declare const ProvingJobConsumerSchema: ApiSchemaFor<ProvingJobConsumer>;
 export declare const ProvingJobBrokerSchema: ApiSchemaFor<ProvingJobBroker>;
+export declare const ProvingJobBrokerDebugSchema: ApiSchemaFor<ProvingJobBrokerDebug>;
+export declare const ProvingJobBrokerSchemaWithDebug: ApiSchemaFor<ProvingJobBroker & ProvingJobBrokerDebug>;
 export declare function createProvingJobBrokerClient(url: string, versions: Partial<ComponentsVersions>, fetch?: (host: string, body: unknown, extraHeaders?: Record<string, string> | undefined, noRetry?: boolean | undefined) => Promise<{
     response: any;
     headers: {
@@ -22,4 +26,4 @@ export declare function createProvingJobConsumerClient(url: string, versions: Pa
         get: (header: string) => string | null | undefined;
     };
 }>): ProvingJobConsumer;
-//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoicnBjLmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi9zcmMvcHJvdmluZ19icm9rZXIvcnBjLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUNBLE9BQU8sRUFJTCxLQUFLLGdCQUFnQixFQUNyQixLQUFLLGtCQUFrQixFQUV2QixLQUFLLGtCQUFrQixFQUV4QixNQUFNLGlDQUFpQyxDQUFDO0FBRXpDLE9BQU8sRUFBRSxLQUFLLFlBQVksRUFBWSxNQUFNLHVCQUF1QixDQUFDO0FBQ3BFLE9BQU8sRUFBRSxLQUFLLGtCQUFrQixFQUFnQyxNQUFNLDBCQUEwQixDQUFDO0FBY2pHLGVBQU8sTUFBTSx3QkFBd0IsRUFBRSxZQUFZLENBQUMsa0JBQWtCLENBS3JFLENBQUM7QUFFRixlQUFPLE1BQU0sd0JBQXdCLEVBQUUsWUFBWSxDQUFDLGtCQUFrQixDQWNyRSxDQUFDO0FBRUYsZUFBTyxNQUFNLHNCQUFzQixFQUFFLFlBQVksQ0FBQyxnQkFBZ0IsQ0FHakUsQ0FBQztBQUVGLHdCQUFnQiw0QkFBNEIsQ0FDMUMsR0FBRyxFQUFFLE1BQU0sRUFDWCxRQUFRLEVBQUUsT0FBTyxDQUFDLGtCQUFrQixDQUFDLEVBQ3JDLEtBQUs7Ozs7O0VBQW9DLEdBQ3hDLGdCQUFnQixDQU1sQjtBQUVELHdCQUFnQiw4QkFBOEIsQ0FDNUMsR0FBRyxFQUFFLE1BQU0sRUFDWCxRQUFRLEVBQUUsT0FBTyxDQUFDLGtCQUFrQixDQUFDLEVBQ3JDLEtBQUs7Ozs7O0VBQW9DLEdBQ3hDLGtCQUFrQixDQU1wQjtBQUVELHdCQUFnQiw4QkFBOEIsQ0FDNUMsR0FBRyxFQUFFLE1BQU0sRUFDWCxRQUFRLEVBQUUsT0FBTyxDQUFDLGtCQUFrQixDQUFDLEVBQ3JDLEtBQUs7Ozs7O0VBQW9DLEdBQ3hDLGtCQUFrQixDQU1wQiJ9
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoicnBjLmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi9zcmMvcHJvdmluZ19icm9rZXIvcnBjLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUVBLE9BQU8sRUFJTCxLQUFLLGdCQUFnQixFQUNyQixLQUFLLHFCQUFxQixFQUMxQixLQUFLLGtCQUFrQixFQUV2QixLQUFLLGtCQUFrQixFQUV4QixNQUFNLGlDQUFpQyxDQUFDO0FBRXpDLE9BQU8sRUFBRSxLQUFLLFlBQVksRUFBWSxNQUFNLHVCQUF1QixDQUFDO0FBQ3BFLE9BQU8sRUFBRSxLQUFLLGtCQUFrQixFQUFnQyxNQUFNLDBCQUEwQixDQUFDO0FBS2pHLHFGQUFxRjtBQUNyRix3QkFBaUIsbUJBQW1CLHFDQU1uQztBQVdELGVBQU8sTUFBTSx3QkFBd0IsRUFBRSxZQUFZLENBQUMsa0JBQWtCLENBS3JFLENBQUM7QUFFRixlQUFPLE1BQU0sd0JBQXdCLEVBQUUsWUFBWSxDQUFDLGtCQUFrQixDQWNyRSxDQUFDO0FBRUYsZUFBTyxNQUFNLHNCQUFzQixFQUFFLFlBQVksQ0FBQyxnQkFBZ0IsQ0FHakUsQ0FBQztBQUVGLGVBQU8sTUFBTSwyQkFBMkIsRUFBRSxZQUFZLENBQUMscUJBQXFCLENBSzNFLENBQUM7QUFFRixlQUFPLE1BQU0sK0JBQStCLEVBQUUsWUFBWSxDQUFDLGdCQUFnQixHQUFHLHFCQUFxQixDQUdsRyxDQUFDO0FBRUYsd0JBQWdCLDRCQUE0QixDQUMxQyxHQUFHLEVBQUUsTUFBTSxFQUNYLFFBQVEsRUFBRSxPQUFPLENBQUMsa0JBQWtCLENBQUMsRUFDckMsS0FBSzs7Ozs7RUFBOEMsR0FDbEQsZ0JBQWdCLENBTWxCO0FBRUQsd0JBQWdCLDhCQUE4QixDQUM1QyxHQUFHLEVBQUUsTUFBTSxFQUNYLFFBQVEsRUFBRSxPQUFPLENBQUMsa0JBQWtCLENBQUMsRUFDckMsS0FBSzs7Ozs7RUFBOEMsR0FDbEQsa0JBQWtCLENBTXBCO0FBRUQsd0JBQWdCLDhCQUE4QixDQUM1QyxHQUFHLEVBQUUsTUFBTSxFQUNYLFFBQVEsRUFBRSxPQUFPLENBQUMsa0JBQWtCLENBQUMsRUFDckMsS0FBSzs7Ozs7RUFBOEMsR0FDbEQsa0JBQWtCLENBTXBCIn0=

package/dest/proving_broker/rpc.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"rpc.d.ts","sourceRoot":"","sources":["../../src/proving_broker/rpc.ts"],"names":[],"mappings":"AACA,OAAO,EAIL,KAAK,gBAAgB,EACrB,KAAK,kBAAkB,EAEvB,KAAK,kBAAkB,EAExB,MAAM,iCAAiC,CAAC;AAEzC,OAAO,EAAE,KAAK,YAAY,EAAY,MAAM,uBAAuB,CAAC;AACpE,OAAO,EAAE,KAAK,kBAAkB,EAAgC,MAAM,0BAA0B,CAAC;AAcjG,eAAO,MAAM,wBAAwB,EAAE,YAAY,CAAC,kBAAkB,CAKrE,CAAC;AAEF,eAAO,MAAM,wBAAwB,EAAE,YAAY,CAAC,kBAAkB,CAcrE,CAAC;AAEF,eAAO,MAAM,sBAAsB,EAAE,YAAY,CAAC,gBAAgB,CAGjE,CAAC;AAEF,wBAAgB,4BAA4B,CAC1C,GAAG,EAAE,MAAM,EACX,QAAQ,EAAE,OAAO,CAAC,kBAAkB,CAAC,EACrC,KAAK;;;;;EAAoC,GACxC,gBAAgB,CAMlB;AAED,wBAAgB,8BAA8B,CAC5C,GAAG,EAAE,MAAM,EACX,QAAQ,EAAE,OAAO,CAAC,kBAAkB,CAAC,EACrC,KAAK;;;;;EAAoC,GACxC,kBAAkB,CAMpB;AAED,wBAAgB,8BAA8B,CAC5C,GAAG,EAAE,MAAM,EACX,QAAQ,EAAE,OAAO,CAAC,kBAAkB,CAAC,EACrC,KAAK;;;;;EAAoC,GACxC,kBAAkB,CAMpB"}
+{"version":3,"file":"rpc.d.ts","sourceRoot":"","sources":["../../src/proving_broker/rpc.ts"],"names":[],"mappings":"AAEA,OAAO,EAIL,KAAK,gBAAgB,EACrB,KAAK,qBAAqB,EAC1B,KAAK,kBAAkB,EAEvB,KAAK,kBAAkB,EAExB,MAAM,iCAAiC,CAAC;AAEzC,OAAO,EAAE,KAAK,YAAY,EAAY,MAAM,uBAAuB,CAAC;AACpE,OAAO,EAAE,KAAK,kBAAkB,EAAgC,MAAM,0BAA0B,CAAC;AAKjG,qFAAqF;AACrF,wBAAiB,mBAAmB,qCAMnC;AAWD,eAAO,MAAM,wBAAwB,EAAE,YAAY,CAAC,kBAAkB,CAKrE,CAAC;AAEF,eAAO,MAAM,wBAAwB,EAAE,YAAY,CAAC,kBAAkB,CAcrE,CAAC;AAEF,eAAO,MAAM,sBAAsB,EAAE,YAAY,CAAC,gBAAgB,CAGjE,CAAC;AAEF,eAAO,MAAM,2BAA2B,EAAE,YAAY,CAAC,qBAAqB,CAK3E,CAAC;AAEF,eAAO,MAAM,+BAA+B,EAAE,YAAY,CAAC,gBAAgB,GAAG,qBAAqB,CAGlG,CAAC;AAEF,wBAAgB,4BAA4B,CAC1C,GAAG,EAAE,MAAM,EACX,QAAQ,EAAE,OAAO,CAAC,kBAAkB,CAAC,EACrC,KAAK;;;;;EAA8C,GAClD,gBAAgB,CAMlB;AAED,wBAAgB,8BAA8B,CAC5C,GAAG,EAAE,MAAM,EACX,QAAQ,EAAE,OAAO,CAAC,kBAAkB,CAAC,EACrC,KAAK;;;;;EAA8C,GAClD,kBAAkB,CAMpB;AAED,wBAAgB,8BAA8B,CAC5C,GAAG,EAAE,MAAM,EACX,QAAQ,EAAE,OAAO,CAAC,kBAAkB,CAAC,EACrC,KAAK;;;;;EAA8C,GAClD,kBAAkB,CAMpB"}

package/dest/proving_broker/rpc.js

@@ -1,3 +1,4 @@
+import { EpochNumberSchema } from '@aztec/foundation/branded-types';
 import { createSafeJsonRpcClient } from '@aztec/foundation/json-rpc/client';
 import { ProofUri, ProvingJob, ProvingJobId, ProvingJobStatus } from '@aztec/stdlib/interfaces/server';
 import { ProvingRequestType } from '@aztec/stdlib/proofs';
@@ -5,6 +6,19 @@ import { optional } from '@aztec/stdlib/schemas';
 import { getVersioningResponseHandler } from '@aztec/stdlib/versioning';
 import { makeTracedFetch } from '@aztec/telemetry-client';
 import { z } from 'zod';
+/** Indefinite backoff for broker communication: 1, 1, 1, 2, 4, 4, 4, ... seconds. */ export function* proverBrokerBackoff() {
+    const v = [
+        1,
+        1,
+        1,
+        2,
+        4
+    ];
+    let i = 0;
+    while(true){
+        yield v[Math.min(i++, v.length - 1)];
+    }
+}
 const ProvingJobFilterSchema = z.object({
     allowList: z.array(z.nativeEnum(ProvingRequestType))
 });
@@ -28,33 +42,28 @@ export const ProvingJobBrokerSchema = {
     ...ProvingJobConsumerSchema,
     ...ProvingJobProducerSchema
 };
-export function createProvingJobBrokerClient(url, versions, fetch = makeTracedFetch([
-    1,
-    2,
-    3
-], false)) {
+export const ProvingJobBrokerDebugSchema = {
+    replayProvingJob: z.function().args(ProvingJobId, z.nativeEnum(ProvingRequestType), EpochNumberSchema, ProofUri).returns(ProvingJobStatus)
+};
+export const ProvingJobBrokerSchemaWithDebug = {
+    ...ProvingJobBrokerSchema,
+    ...ProvingJobBrokerDebugSchema
+};
+export function createProvingJobBrokerClient(url, versions, fetch = makeTracedFetch(proverBrokerBackoff, false)) {
     return createSafeJsonRpcClient(url, ProvingJobBrokerSchema, {
         namespaceMethods: 'proverBroker',
         fetch,
         onResponse: getVersioningResponseHandler(versions)
     });
 }
-export function createProvingJobProducerClient(url, versions, fetch = makeTracedFetch([
-    1,
-    2,
-    3
-], false)) {
+export function createProvingJobProducerClient(url, versions, fetch = makeTracedFetch(proverBrokerBackoff, false)) {
     return createSafeJsonRpcClient(url, ProvingJobProducerSchema, {
         namespaceMethods: 'provingJobProducer',
         fetch,
         onResponse: getVersioningResponseHandler(versions)
     });
 }
-export function createProvingJobConsumerClient(url, versions, fetch = makeTracedFetch([
-    1,
-    2,
-    3
-], false)) {
+export function createProvingJobConsumerClient(url, versions, fetch = makeTracedFetch(proverBrokerBackoff, false)) {
     return createSafeJsonRpcClient(url, ProvingJobConsumerSchema, {
         namespaceMethods: 'provingJobConsumer',
         fetch,

package/dest/test/mock_prover.d.ts

@@ -19,9 +19,9 @@ export declare class TestBroker implements ProvingJobProducer {
 }
 export declare class MockProver implements ServerCircuitProver {
     constructor();
-    getAvmProof(_inputs: AvmCircuitInputs, _signal?: AbortSignal, _epochNumber?: number): Promise<import("@aztec/stdlib/proofs").RecursiveProof<16200>>;
-    getBaseParityProof(_inputs: ParityBasePrivateInputs, _signal?: AbortSignal, _epochNumber?: number): Promise<PublicInputsAndRecursiveProof<import("@aztec/stdlib/parity").ParityPublicInputs, 457>>;
-    getRootParityProof(_inputs: ParityRootPrivateInputs, _signal?: AbortSignal, _epochNumber?: number): Promise<PublicInputsAndRecursiveProof<import("@aztec/stdlib/parity").ParityPublicInputs, 457>>;
+    getAvmProof(_inputs: AvmCircuitInputs, _signal?: AbortSignal, _epochNumber?: number): Promise<import("@aztec/stdlib/proofs").RecursiveProof<16400>>;
+    getBaseParityProof(_inputs: ParityBasePrivateInputs, _signal?: AbortSignal, _epochNumber?: number): Promise<PublicInputsAndRecursiveProof<import("@aztec/stdlib/parity").ParityPublicInputs, 410>>;
+    getRootParityProof(_inputs: ParityRootPrivateInputs, _signal?: AbortSignal, _epochNumber?: number): Promise<PublicInputsAndRecursiveProof<import("@aztec/stdlib/parity").ParityPublicInputs, 410>>;
     getPublicChonkVerifierProof(_inputs: PublicChonkVerifierPrivateInputs, _signal?: AbortSignal, _epochNumber?: number): Promise<PublicInputsAndRecursiveProof<PublicChonkVerifierPublicInputs, typeof NESTED_RECURSIVE_ROLLUP_HONK_PROOF_LENGTH>>;
     getPrivateTxBaseRollupProof(_baseRollupInput: PrivateTxBaseRollupPrivateInputs, _signal?: AbortSignal, _epochNumber?: number): Promise<PublicInputsAndRecursiveProof<TxRollupPublicInputs, typeof NESTED_RECURSIVE_ROLLUP_HONK_PROOF_LENGTH>>;
     getPublicTxBaseRollupProof(_inputs: PublicTxBaseRollupPrivateInputs, _signal?: AbortSignal, _epochNumber?: number): Promise<PublicInputsAndRecursiveProof<TxRollupPublicInputs, typeof NESTED_RECURSIVE_ROLLUP_HONK_PROOF_LENGTH>>;
@@ -31,7 +31,7 @@ export declare class MockProver implements ServerCircuitProver {
     getBlockRootEmptyTxFirstRollupProof(_input: BlockRootEmptyTxFirstRollupPrivateInputs, _signal?: AbortSignal, _epochNumber?: number): Promise<PublicInputsAndRecursiveProof<BlockRollupPublicInputs, typeof NESTED_RECURSIVE_ROLLUP_HONK_PROOF_LENGTH>>;
     getBlockRootRollupProof(_input: BlockRootRollupPrivateInputs, _signal?: AbortSignal, _epochNumber?: number): Promise<PublicInputsAndRecursiveProof<BlockRollupPublicInputs, typeof NESTED_RECURSIVE_ROLLUP_HONK_PROOF_LENGTH>>;
     getBlockRootSingleTxRollupProof(_input: BlockRootSingleTxRollupPrivateInputs, _signal?: AbortSignal, _epochNumber?: number): Promise<PublicInputsAndRecursiveProof<BlockRollupPublicInputs, typeof NESTED_RECURSIVE_ROLLUP_HONK_PROOF_LENGTH>>;
-    getBlockMergeRollupProof(_input: BlockMergeRollupPrivateInputs, _signal?: AbortSignal, _epochNumber?: number): Promise<PublicInputsAndRecursiveProof<BlockRollupPublicInputs, 531>>;
+    getBlockMergeRollupProof(_input: BlockMergeRollupPrivateInputs, _signal?: AbortSignal, _epochNumber?: number): Promise<PublicInputsAndRecursiveProof<BlockRollupPublicInputs, 480>>;
     getCheckpointRootRollupProof(_input: CheckpointRootRollupPrivateInputs, _signal?: AbortSignal, _epochNumber?: number): Promise<PublicInputsAndRecursiveProof<CheckpointRollupPublicInputs, typeof NESTED_RECURSIVE_ROLLUP_HONK_PROOF_LENGTH>>;
     getCheckpointRootSingleBlockRollupProof(_input: CheckpointRootSingleBlockRollupPrivateInputs, _signal?: AbortSignal, _epochNumber?: number): Promise<PublicInputsAndRecursiveProof<CheckpointRollupPublicInputs, typeof NESTED_RECURSIVE_ROLLUP_HONK_PROOF_LENGTH>>;
     getCheckpointMergeRollupProof(_input: CheckpointMergeRollupPrivateInputs, _signal?: AbortSignal, _epochNumber?: number): Promise<PublicInputsAndRecursiveProof<CheckpointRollupPublicInputs, typeof NESTED_RECURSIVE_ROLLUP_HONK_PROOF_LENGTH>>;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aztec/prover-client",
-  "version": "0.0.1-commit.e6bd8901",
+  "version": "0.0.1-commit.ec7ac5448",
   "type": "module",
   "exports": {
     ".": "./dest/index.js",
@@ -27,8 +27,8 @@
     "build:dev": "../scripts/tsc.sh --watch",
     "clean": "rm -rf ./dest .tsbuildinfo",
     "bb": "node --no-warnings ./dest/bb/index.js",
-    "test": "NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --testTimeout=3500000 --forceExit",
-    "test:debug": "LOG_LEVEL=debug NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --testTimeout=1500000 --forceExit --testNamePattern prover/bb_prover/parity"
+    "test": "NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --testTimeout=3500000",
+    "test:debug": "LOG_LEVEL=debug NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --testTimeout=1500000 --testNamePattern prover/bb_prover/parity"
   },
   "jest": {
     "moduleNameMapper": {
@@ -68,20 +68,19 @@
     ]
   },
   "dependencies": {
-    "@aztec/bb-prover": "0.0.1-commit.e6bd8901",
-    "@aztec/blob-lib": "0.0.1-commit.e6bd8901",
-    "@aztec/constants": "0.0.1-commit.e6bd8901",
-    "@aztec/ethereum": "0.0.1-commit.e6bd8901",
-    "@aztec/foundation": "0.0.1-commit.e6bd8901",
-    "@aztec/kv-store": "0.0.1-commit.e6bd8901",
-    "@aztec/noir-protocol-circuits-types": "0.0.1-commit.e6bd8901",
-    "@aztec/noir-types": "0.0.1-commit.e6bd8901",
-    "@aztec/protocol-contracts": "0.0.1-commit.e6bd8901",
-    "@aztec/simulator": "0.0.1-commit.e6bd8901",
-    "@aztec/stdlib": "0.0.1-commit.e6bd8901",
-    "@aztec/telemetry-client": "0.0.1-commit.e6bd8901",
-    "@aztec/world-state": "0.0.1-commit.e6bd8901",
-    "@google-cloud/storage": "^7.15.0",
+    "@aztec/bb-prover": "0.0.1-commit.ec7ac5448",
+    "@aztec/blob-lib": "0.0.1-commit.ec7ac5448",
+    "@aztec/constants": "0.0.1-commit.ec7ac5448",
+    "@aztec/ethereum": "0.0.1-commit.ec7ac5448",
+    "@aztec/foundation": "0.0.1-commit.ec7ac5448",
+    "@aztec/kv-store": "0.0.1-commit.ec7ac5448",
+    "@aztec/noir-protocol-circuits-types": "0.0.1-commit.ec7ac5448",
+    "@aztec/noir-types": "0.0.1-commit.ec7ac5448",
+    "@aztec/protocol-contracts": "0.0.1-commit.ec7ac5448",
+    "@aztec/simulator": "0.0.1-commit.ec7ac5448",
+    "@aztec/stdlib": "0.0.1-commit.ec7ac5448",
+    "@aztec/telemetry-client": "0.0.1-commit.ec7ac5448",
+    "@aztec/world-state": "0.0.1-commit.ec7ac5448",
     "@iarna/toml": "^2.2.5",
     "commander": "^12.1.0",
     "lodash.chunk": "^4.2.0",
@@ -90,7 +89,7 @@
     "zod": "^3.23.8"
   },
   "devDependencies": {
-    "@aztec/noir-contracts.js": "0.0.1-commit.e6bd8901",
+    "@aztec/noir-contracts.js": "0.0.1-commit.ec7ac5448",
     "@jest/globals": "^30.0.0",
     "@types/jest": "^30.0.0",
     "@types/node": "^22.15.17",

package/src/config.ts

@@ -44,14 +44,25 @@ export const bbConfigMappings: ConfigMappingsType<BBConfig & ACVMConfig> = {
   },
   numConcurrentIVCVerifiers: {
     env: 'BB_NUM_IVC_VERIFIERS',
-    description: 'Max number of chonk verifiers to run concurrently',
+    description: 'Max concurrent verifications for the RPC verifier (QueuedIVCVerifier).',
     ...numberConfigHelper(8),
   },
   bbIVCConcurrency: {
     env: 'BB_IVC_CONCURRENCY',
-    description: 'Number of threads to use for IVC verification',
+    description: 'Thread count for the RPC IVC verifier.',
     ...numberConfigHelper(1),
   },
+  bbChonkVerifyMaxBatch: {
+    env: 'BB_CHONK_VERIFY_MAX_BATCH',
+    description:
+      'Upper bound on proofs per batch for the peer chonk batch verifier. Proofs are verified immediately as they arrive; this only caps how many can accumulate while a batch is already being processed.',
+    ...numberConfigHelper(16),
+  },
+  bbChonkVerifyConcurrency: {
+    env: 'BB_CHONK_VERIFY_BATCH_CONCURRENCY',
+    description: 'Thread count for the peer batch verifier parallel reduce. 0 = auto.',
+    ...numberConfigHelper(6),
+  },
 };
 
 export const proverClientConfigMappings: ConfigMappingsType<ProverClientUserConfig> = {