@aztec/prover-client 0.0.1-commit.7b97ef96e → 0.0.1-commit.7cbc774

package/dest/proving_broker/proving_broker.js

@@ -239,18 +239,27 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
             this.logger.info(`Cancelling job id=${id}`, {
                 provingJobId: id
             });
-            await this.#reportProvingJobError(id, 'Aborted', false);
+            await this.#reportProvingJobError(id, 'Aborted', false, undefined, true);
         }
     }
     cleanUpProvingJobState(ids) {
+        const idsToClean = new Set(ids);
         for (const id of ids){
             this.jobsCache.delete(id);
+            const deferred = this.promises.get(id);
+            if (deferred) {
+                deferred.resolve({
+                    status: 'rejected',
+                    reason: 'Proving job cleaned up'
+                });
+            }
             this.promises.delete(id);
             this.resultsCache.delete(id);
             this.inProgress.delete(id);
             this.retries.delete(id);
             this.enqueuedAt.delete(id);
         }
+        this.completedJobNotifications = this.completedJobNotifications.filter((id)=>!idsToClean.has(id));
     }
     #getProvingJobStatus(id) {
         const result = this.resultsCache.get(id);
@@ -313,7 +322,7 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
         }
         return undefined;
     }
-    async #reportProvingJobError(id, err, retry = false, filter) {
+    async #reportProvingJobError(id, err, retry = false, filter, aborted = false) {
         const info = this.inProgress.get(id);
         const item = this.jobsCache.get(id);
         const retries = this.retries.get(id) ?? 0;
@@ -360,7 +369,11 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
         this.resultsCache.set(id, result);
         this.promises.get(id).resolve(result);
         this.completedJobNotifications.push(id);
-        this.instrumentation.incRejectedJobs(item.type);
+        if (aborted) {
+            this.instrumentation.incAbortedJobs(item.type);
+        } else {
+            this.instrumentation.incRejectedJobs(item.type);
+        }
         if (info) {
             const duration = this.msTimeSource() - info.startedAt;
             this.instrumentation.recordJobDuration(item.type, duration);
@@ -475,20 +488,20 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
         return this.#getProvingJob(filter);
     }
     async cleanupPass() {
-        this.cleanupStaleJobs();
         this.reEnqueueExpiredJobs();
         const oldestEpochToKeep = this.oldestEpochToKeep();
         if (oldestEpochToKeep > 0) {
+            this.cleanupJobsOlderThanEpoch(EpochNumber(oldestEpochToKeep));
             await this.database.deleteAllProvingJobsOlderThanEpoch(EpochNumber(oldestEpochToKeep));
             this.logger.trace(`Deleted all epochs older than ${oldestEpochToKeep}`);
         }
     }
-    cleanupStaleJobs() {
+    cleanupJobsOlderThanEpoch(epochNumber) {
         const jobIds = Array.from(this.jobsCache.keys());
         const jobsToClean = [];
         for (const id of jobIds){
             const job = this.jobsCache.get(id);
-            if (this.isJobStale(job)) {
+            if (job.epochNumber < epochNumber) {
                 jobsToClean.push(id);
             }
         }
@@ -511,12 +524,28 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
             const now = this.msTimeSource();
             const msSinceLastUpdate = now - metadata.lastUpdatedAt;
             if (msSinceLastUpdate >= this.jobTimeoutMs) {
-                this.logger.warn(`Proving job id=${id} timed out. Adding it back to the queue.`, {
-                    provingJobId: id
-                });
                 this.inProgress.delete(id);
-                this.enqueueJobInternal(item);
                 this.instrumentation.incTimedOutJobs(item.type);
+                const retries = this.retries.get(id) ?? 0;
+                if (retries + 1 < this.maxRetries && !this.isJobStale(item)) {
+                    this.logger.warn(`Proving job id=${id} timed out. Re-enqueueing (retry ${retries + 1}/${this.maxRetries}).`, {
+                        provingJobId: id
+                    });
+                    this.retries.set(id, retries + 1);
+                    this.enqueueJobInternal(item);
+                } else {
+                    this.logger.error(`Proving job id=${id} timed out after ${retries + 1} attempts. Marking as failed.`, {
+                        provingJobId: id
+                    });
+                    const result = {
+                        status: 'rejected',
+                        reason: 'Timed out'
+                    };
+                    this.resultsCache.set(id, result);
+                    this.promises.get(id)?.resolve(result);
+                    this.completedJobNotifications.push(id);
+                    this.instrumentation.incRejectedJobs(item.type);
+                }
             }
         }
     }

package/dest/proving_broker/proving_broker_database/persisted.js

@@ -499,7 +499,7 @@ export class KVBrokerDatabase {
                 continue;
             }
             logger.info(`Loading broker database for epoch ${epochNumber} from ${fullDirectory} with map size ${config.dataStoreMapSizeKb}KB`);
-            const db = await openVersionedStoreAt(fullDirectory, SingleEpochDatabase.SCHEMA_VERSION, config.l1Contracts.rollupAddress, config.dataStoreMapSizeKb);
+            const db = await openVersionedStoreAt(fullDirectory, SingleEpochDatabase.SCHEMA_VERSION, config.rollupAddress, config.dataStoreMapSizeKb);
             const epochDb = new SingleEpochDatabase(db);
             epochs.set(epochNumber, epochDb);
         }
@@ -563,7 +563,7 @@ export class KVBrokerDatabase {
                 recursive: true
             });
             this.logger.info(`Creating broker database for epoch ${epochNumber} at ${newEpochDirectory} with map size ${this.config.dataStoreMapSizeKb}`);
-            const db = await openVersionedStoreAt(newEpochDirectory, SingleEpochDatabase.SCHEMA_VERSION, this.config.l1Contracts.rollupAddress, this.config.dataStoreMapSizeKb);
+            const db = await openVersionedStoreAt(newEpochDirectory, SingleEpochDatabase.SCHEMA_VERSION, this.config.rollupAddress, this.config.dataStoreMapSizeKb);
             epochDb = new SingleEpochDatabase(db);
             this.epochs.set(epochNumber, epochDb);
         }

package/dest/proving_broker/proving_broker_instrumentation.d.ts

@@ -7,6 +7,7 @@ export declare class ProvingBrokerInstrumentation {
     private activeJobs;
     private resolvedJobs;
     private rejectedJobs;
+    private abortedJobs;
     private timedOutJobs;
     private cachedJobs;
     private totalJobs;
@@ -18,6 +19,7 @@ export declare class ProvingBrokerInstrumentation {
     monitorActiveJobs(fn: MonitorCallback): void;
     incResolvedJobs(proofType: ProvingRequestType): void;
     incRejectedJobs(proofType: ProvingRequestType): void;
+    incAbortedJobs(proofType: ProvingRequestType): void;
     incRetriedJobs(proofType: ProvingRequestType): void;
     incTimedOutJobs(proofType: ProvingRequestType): void;
     incCachedJobs(proofType: ProvingRequestType): void;
@@ -26,4 +28,4 @@ export declare class ProvingBrokerInstrumentation {
     recordJobDuration(proofType: ProvingRequestType, msOrTimer: Timer | number): void;
     private observe;
 }
-//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoicHJvdmluZ19icm9rZXJfaW5zdHJ1bWVudGF0aW9uLmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi9zcmMvcHJvdmluZ19icm9rZXIvcHJvdmluZ19icm9rZXJfaW5zdHJ1bWVudGF0aW9uLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLE9BQU8sS0FBSyxFQUFFLEtBQUssRUFBRSxNQUFNLHlCQUF5QixDQUFDO0FBQ3JELE9BQU8sRUFBRSxrQkFBa0IsRUFBRSxNQUFNLHNCQUFzQixDQUFDO0FBQzFELE9BQU8sRUFNTCxLQUFLLGVBQWUsRUFHckIsTUFBTSx5QkFBeUIsQ0FBQztBQUVqQyxNQUFNLE1BQU0sZUFBZSxHQUFHLENBQUMsU0FBUyxFQUFFLGtCQUFrQixLQUFLLE1BQU0sQ0FBQztBQUV4RSxxQkFBYSw0QkFBNEI7SUFDdkMsT0FBTyxDQUFDLFNBQVMsQ0FBa0I7SUFDbkMsT0FBTyxDQUFDLFVBQVUsQ0FBa0I7SUFDcEMsT0FBTyxDQUFDLFlBQVksQ0FBZ0I7SUFDcEMsT0FBTyxDQUFDLFlBQVksQ0FBZ0I7SUFDcEMsT0FBTyxDQUFDLFlBQVksQ0FBZ0I7SUFDcEMsT0FBTyxDQUFDLFVBQVUsQ0FBZ0I7SUFDbEMsT0FBTyxDQUFDLFNBQVMsQ0FBZ0I7SUFDakMsT0FBTyxDQUFDLE9BQU8sQ0FBWTtJQUMzQixPQUFPLENBQUMsV0FBVyxDQUFZO0lBQy9CLE9BQU8sQ0FBQyxXQUFXLENBQWdCO0lBRW5DLFlBQVksTUFBTSxFQUFFLGVBQWUsRUFBRSxJQUFJLFNBQWtCLEVBeUIxRDtJQUVELGlCQUFpQixDQUFDLEVBQUUsRUFBRSxlQUFlLFFBRXBDO0lBRUQsaUJBQWlCLENBQUMsRUFBRSxFQUFFLGVBQWUsUUFFcEM7SUFFRCxlQUFlLENBQUMsU0FBUyxFQUFFLGtCQUFrQixRQUk1QztJQUVELGVBQWUsQ0FBQyxTQUFTLEVBQUUsa0JBQWtCLFFBSTVDO0lBRUQsY0FBYyxDQUFDLFNBQVMsRUFBRSxrQkFBa0IsUUFJM0M7SUFFRCxlQUFlLENBQUMsU0FBUyxFQUFFLGtCQUFrQixRQUk1QztJQUVELGFBQWEsQ0FBQyxTQUFTLEVBQUUsa0JBQWtCLFFBSTFDO0lBRUQsWUFBWSxDQUFDLFNBQVMsRUFBRSxrQkFBa0IsUUFJekM7SUFFRCxhQUFhLENBQUMsU0FBUyxFQUFFLGtCQUFrQixFQUFFLFNBQVMsRUFBRSxLQUFLLEdBQUcsTUFBTSxRQUtyRTtJQUVELGlCQUFpQixDQUFDLFNBQVMsRUFBRSxrQkFBa0IsRUFBRSxTQUFTLEVBQUUsS0FBSyxHQUFHLE1BQU0sUUFLekU7SUFFRCxPQUFPLENBQUMsT0FBTztDQVdoQiJ9
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoicHJvdmluZ19icm9rZXJfaW5zdHJ1bWVudGF0aW9uLmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi9zcmMvcHJvdmluZ19icm9rZXIvcHJvdmluZ19icm9rZXJfaW5zdHJ1bWVudGF0aW9uLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLE9BQU8sS0FBSyxFQUFFLEtBQUssRUFBRSxNQUFNLHlCQUF5QixDQUFDO0FBQ3JELE9BQU8sRUFBRSxrQkFBa0IsRUFBRSxNQUFNLHNCQUFzQixDQUFDO0FBQzFELE9BQU8sRUFNTCxLQUFLLGVBQWUsRUFHckIsTUFBTSx5QkFBeUIsQ0FBQztBQUVqQyxNQUFNLE1BQU0sZUFBZSxHQUFHLENBQUMsU0FBUyxFQUFFLGtCQUFrQixLQUFLLE1BQU0sQ0FBQztBQUV4RSxxQkFBYSw0QkFBNEI7SUFDdkMsT0FBTyxDQUFDLFNBQVMsQ0FBa0I7SUFDbkMsT0FBTyxDQUFDLFVBQVUsQ0FBa0I7SUFDcEMsT0FBTyxDQUFDLFlBQVksQ0FBZ0I7SUFDcEMsT0FBTyxDQUFDLFlBQVksQ0FBZ0I7SUFDcEMsT0FBTyxDQUFDLFdBQVcsQ0FBZ0I7SUFDbkMsT0FBTyxDQUFDLFlBQVksQ0FBZ0I7SUFDcEMsT0FBTyxDQUFDLFVBQVUsQ0FBZ0I7SUFDbEMsT0FBTyxDQUFDLFNBQVMsQ0FBZ0I7SUFDakMsT0FBTyxDQUFDLE9BQU8sQ0FBWTtJQUMzQixPQUFPLENBQUMsV0FBVyxDQUFZO0lBQy9CLE9BQU8sQ0FBQyxXQUFXLENBQWdCO0lBRW5DLFlBQVksTUFBTSxFQUFFLGVBQWUsRUFBRSxJQUFJLFNBQWtCLEVBMkIxRDtJQUVELGlCQUFpQixDQUFDLEVBQUUsRUFBRSxlQUFlLFFBRXBDO0lBRUQsaUJBQWlCLENBQUMsRUFBRSxFQUFFLGVBQWUsUUFFcEM7SUFFRCxlQUFlLENBQUMsU0FBUyxFQUFFLGtCQUFrQixRQUk1QztJQUVELGVBQWUsQ0FBQyxTQUFTLEVBQUUsa0JBQWtCLFFBSTVDO0lBRUQsY0FBYyxDQUFDLFNBQVMsRUFBRSxrQkFBa0IsUUFJM0M7SUFFRCxjQUFjLENBQUMsU0FBUyxFQUFFLGtCQUFrQixRQUkzQztJQUVELGVBQWUsQ0FBQyxTQUFTLEVBQUUsa0JBQWtCLFFBSTVDO0lBRUQsYUFBYSxDQUFDLFNBQVMsRUFBRSxrQkFBa0IsUUFJMUM7SUFFRCxZQUFZLENBQUMsU0FBUyxFQUFFLGtCQUFrQixRQUl6QztJQUVELGFBQWEsQ0FBQyxTQUFTLEVBQUUsa0JBQWtCLEVBQUUsU0FBUyxFQUFFLEtBQUssR0FBRyxNQUFNLFFBS3JFO0lBRUQsaUJBQWlCLENBQUMsU0FBUyxFQUFFLGtCQUFrQixFQUFFLFNBQVMsRUFBRSxLQUFLLEdBQUcsTUFBTSxRQUt6RTtJQUVELE9BQU8sQ0FBQyxPQUFPO0NBV2hCIn0=

package/dest/proving_broker/proving_broker_instrumentation.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"proving_broker_instrumentation.d.ts","sourceRoot":"","sources":["../../src/proving_broker/proving_broker_instrumentation.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AACrD,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAC1D,OAAO,EAML,KAAK,eAAe,EAGrB,MAAM,yBAAyB,CAAC;AAEjC,MAAM,MAAM,eAAe,GAAG,CAAC,SAAS,EAAE,kBAAkB,KAAK,MAAM,CAAC;AAExE,qBAAa,4BAA4B;IACvC,OAAO,CAAC,SAAS,CAAkB;IACnC,OAAO,CAAC,UAAU,CAAkB;IACpC,OAAO,CAAC,YAAY,CAAgB;IACpC,OAAO,CAAC,YAAY,CAAgB;IACpC,OAAO,CAAC,YAAY,CAAgB;IACpC,OAAO,CAAC,UAAU,CAAgB;IAClC,OAAO,CAAC,SAAS,CAAgB;IACjC,OAAO,CAAC,OAAO,CAAY;IAC3B,OAAO,CAAC,WAAW,CAAY;IAC/B,OAAO,CAAC,WAAW,CAAgB;IAEnC,YAAY,MAAM,EAAE,eAAe,EAAE,IAAI,SAAkB,EAyB1D;IAED,iBAAiB,CAAC,EAAE,EAAE,eAAe,QAEpC;IAED,iBAAiB,CAAC,EAAE,EAAE,eAAe,QAEpC;IAED,eAAe,CAAC,SAAS,EAAE,kBAAkB,QAI5C;IAED,eAAe,CAAC,SAAS,EAAE,kBAAkB,QAI5C;IAED,cAAc,CAAC,SAAS,EAAE,kBAAkB,QAI3C;IAED,eAAe,CAAC,SAAS,EAAE,kBAAkB,QAI5C;IAED,aAAa,CAAC,SAAS,EAAE,kBAAkB,QAI1C;IAED,YAAY,CAAC,SAAS,EAAE,kBAAkB,QAIzC;IAED,aAAa,CAAC,SAAS,EAAE,kBAAkB,EAAE,SAAS,EAAE,KAAK,GAAG,MAAM,QAKrE;IAED,iBAAiB,CAAC,SAAS,EAAE,kBAAkB,EAAE,SAAS,EAAE,KAAK,GAAG,MAAM,QAKzE;IAED,OAAO,CAAC,OAAO;CAWhB"}
+{"version":3,"file":"proving_broker_instrumentation.d.ts","sourceRoot":"","sources":["../../src/proving_broker/proving_broker_instrumentation.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AACrD,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAC1D,OAAO,EAML,KAAK,eAAe,EAGrB,MAAM,yBAAyB,CAAC;AAEjC,MAAM,MAAM,eAAe,GAAG,CAAC,SAAS,EAAE,kBAAkB,KAAK,MAAM,CAAC;AAExE,qBAAa,4BAA4B;IACvC,OAAO,CAAC,SAAS,CAAkB;IACnC,OAAO,CAAC,UAAU,CAAkB;IACpC,OAAO,CAAC,YAAY,CAAgB;IACpC,OAAO,CAAC,YAAY,CAAgB;IACpC,OAAO,CAAC,WAAW,CAAgB;IACnC,OAAO,CAAC,YAAY,CAAgB;IACpC,OAAO,CAAC,UAAU,CAAgB;IAClC,OAAO,CAAC,SAAS,CAAgB;IACjC,OAAO,CAAC,OAAO,CAAY;IAC3B,OAAO,CAAC,WAAW,CAAY;IAC/B,OAAO,CAAC,WAAW,CAAgB;IAEnC,YAAY,MAAM,EAAE,eAAe,EAAE,IAAI,SAAkB,EA2B1D;IAED,iBAAiB,CAAC,EAAE,EAAE,eAAe,QAEpC;IAED,iBAAiB,CAAC,EAAE,EAAE,eAAe,QAEpC;IAED,eAAe,CAAC,SAAS,EAAE,kBAAkB,QAI5C;IAED,eAAe,CAAC,SAAS,EAAE,kBAAkB,QAI5C;IAED,cAAc,CAAC,SAAS,EAAE,kBAAkB,QAI3C;IAED,cAAc,CAAC,SAAS,EAAE,kBAAkB,QAI3C;IAED,eAAe,CAAC,SAAS,EAAE,kBAAkB,QAI5C;IAED,aAAa,CAAC,SAAS,EAAE,kBAAkB,QAI1C;IAED,YAAY,CAAC,SAAS,EAAE,kBAAkB,QAIzC;IAED,aAAa,CAAC,SAAS,EAAE,kBAAkB,EAAE,SAAS,EAAE,KAAK,GAAG,MAAM,QAKrE;IAED,iBAAiB,CAAC,SAAS,EAAE,kBAAkB,EAAE,SAAS,EAAE,KAAK,GAAG,MAAM,QAKzE;IAED,OAAO,CAAC,OAAO;CAWhB"}

package/dest/proving_broker/proving_broker_instrumentation.js

@@ -5,6 +5,7 @@ export class ProvingBrokerInstrumentation {
     activeJobs;
     resolvedJobs;
     rejectedJobs;
+    abortedJobs;
     timedOutJobs;
     cachedJobs;
     totalJobs;
@@ -21,6 +22,7 @@ export class ProvingBrokerInstrumentation {
         };
         this.resolvedJobs = createUpDownCounterWithDefault(meter, Metrics.PROVING_QUEUE_RESOLVED_JOBS, provingJobAttrs);
         this.rejectedJobs = createUpDownCounterWithDefault(meter, Metrics.PROVING_QUEUE_REJECTED_JOBS, provingJobAttrs);
+        this.abortedJobs = createUpDownCounterWithDefault(meter, Metrics.PROVING_QUEUE_ABORTED_JOBS, provingJobAttrs);
         this.retriedJobs = createUpDownCounterWithDefault(meter, Metrics.PROVING_QUEUE_RETRIED_JOBS, provingJobAttrs);
         this.timedOutJobs = createUpDownCounterWithDefault(meter, Metrics.PROVING_QUEUE_TIMED_OUT_JOBS, provingJobAttrs);
         this.cachedJobs = createUpDownCounterWithDefault(meter, Metrics.PROVING_QUEUE_CACHED_JOBS, provingJobAttrs);
@@ -44,6 +46,11 @@ export class ProvingBrokerInstrumentation {
             [Attributes.PROVING_JOB_TYPE]: ProvingRequestType[proofType]
         });
     }
+    incAbortedJobs(proofType) {
+        this.abortedJobs.add(1, {
+            [Attributes.PROVING_JOB_TYPE]: ProvingRequestType[proofType]
+        });
+    }
     incRetriedJobs(proofType) {
         this.retriedJobs.add(1, {
             [Attributes.PROVING_JOB_TYPE]: ProvingRequestType[proofType]

package/dest/proving_broker/rpc.d.ts

@@ -1,6 +1,8 @@
 import { type ProvingJobBroker, type ProvingJobBrokerDebug, type ProvingJobConsumer, type ProvingJobProducer } from '@aztec/stdlib/interfaces/server';
 import { type ApiSchemaFor } from '@aztec/stdlib/schemas';
 import { type ComponentsVersions } from '@aztec/stdlib/versioning';
+/** Indefinite backoff for broker communication: 1, 1, 1, 2, 4, 4, 4, ... seconds. */
+export declare function proverBrokerBackoff(): Generator<number, void, unknown>;
 export declare const ProvingJobProducerSchema: ApiSchemaFor<ProvingJobProducer>;
 export declare const ProvingJobConsumerSchema: ApiSchemaFor<ProvingJobConsumer>;
 export declare const ProvingJobBrokerSchema: ApiSchemaFor<ProvingJobBroker>;
@@ -24,4 +26,4 @@ export declare function createProvingJobConsumerClient(url: string, versions: Pa
         get: (header: string) => string | null | undefined;
     };
 }>): ProvingJobConsumer;
-//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoicnBjLmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi9zcmMvcHJvdmluZ19icm9rZXIvcnBjLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUVBLE9BQU8sRUFJTCxLQUFLLGdCQUFnQixFQUNyQixLQUFLLHFCQUFxQixFQUMxQixLQUFLLGtCQUFrQixFQUV2QixLQUFLLGtCQUFrQixFQUV4QixNQUFNLGlDQUFpQyxDQUFDO0FBRXpDLE9BQU8sRUFBRSxLQUFLLFlBQVksRUFBWSxNQUFNLHVCQUF1QixDQUFDO0FBQ3BFLE9BQU8sRUFBRSxLQUFLLGtCQUFrQixFQUFnQyxNQUFNLDBCQUEwQixDQUFDO0FBY2pHLGVBQU8sTUFBTSx3QkFBd0IsRUFBRSxZQUFZLENBQUMsa0JBQWtCLENBS3JFLENBQUM7QUFFRixlQUFPLE1BQU0sd0JBQXdCLEVBQUUsWUFBWSxDQUFDLGtCQUFrQixDQWNyRSxDQUFDO0FBRUYsZUFBTyxNQUFNLHNCQUFzQixFQUFFLFlBQVksQ0FBQyxnQkFBZ0IsQ0FHakUsQ0FBQztBQUVGLGVBQU8sTUFBTSwyQkFBMkIsRUFBRSxZQUFZLENBQUMscUJBQXFCLENBSzNFLENBQUM7QUFFRixlQUFPLE1BQU0sK0JBQStCLEVBQUUsWUFBWSxDQUFDLGdCQUFnQixHQUFHLHFCQUFxQixDQUdsRyxDQUFDO0FBRUYsd0JBQWdCLDRCQUE0QixDQUMxQyxHQUFHLEVBQUUsTUFBTSxFQUNYLFFBQVEsRUFBRSxPQUFPLENBQUMsa0JBQWtCLENBQUMsRUFDckMsS0FBSzs7Ozs7RUFBb0MsR0FDeEMsZ0JBQWdCLENBTWxCO0FBRUQsd0JBQWdCLDhCQUE4QixDQUM1QyxHQUFHLEVBQUUsTUFBTSxFQUNYLFFBQVEsRUFBRSxPQUFPLENBQUMsa0JBQWtCLENBQUMsRUFDckMsS0FBSzs7Ozs7RUFBb0MsR0FDeEMsa0JBQWtCLENBTXBCO0FBRUQsd0JBQWdCLDhCQUE4QixDQUM1QyxHQUFHLEVBQUUsTUFBTSxFQUNYLFFBQVEsRUFBRSxPQUFPLENBQUMsa0JBQWtCLENBQUMsRUFDckMsS0FBSzs7Ozs7RUFBb0MsR0FDeEMsa0JBQWtCLENBTXBCIn0=
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoicnBjLmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi9zcmMvcHJvdmluZ19icm9rZXIvcnBjLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUVBLE9BQU8sRUFJTCxLQUFLLGdCQUFnQixFQUNyQixLQUFLLHFCQUFxQixFQUMxQixLQUFLLGtCQUFrQixFQUV2QixLQUFLLGtCQUFrQixFQUV4QixNQUFNLGlDQUFpQyxDQUFDO0FBRXpDLE9BQU8sRUFBRSxLQUFLLFlBQVksRUFBWSxNQUFNLHVCQUF1QixDQUFDO0FBQ3BFLE9BQU8sRUFBRSxLQUFLLGtCQUFrQixFQUFnQyxNQUFNLDBCQUEwQixDQUFDO0FBS2pHLHFGQUFxRjtBQUNyRix3QkFBaUIsbUJBQW1CLHFDQU1uQztBQVdELGVBQU8sTUFBTSx3QkFBd0IsRUFBRSxZQUFZLENBQUMsa0JBQWtCLENBS3JFLENBQUM7QUFFRixlQUFPLE1BQU0sd0JBQXdCLEVBQUUsWUFBWSxDQUFDLGtCQUFrQixDQWlCckUsQ0FBQztBQUVGLGVBQU8sTUFBTSxzQkFBc0IsRUFBRSxZQUFZLENBQUMsZ0JBQWdCLENBR2pFLENBQUM7QUFFRixlQUFPLE1BQU0sMkJBQTJCLEVBQUUsWUFBWSxDQUFDLHFCQUFxQixDQUszRSxDQUFDO0FBRUYsZUFBTyxNQUFNLCtCQUErQixFQUFFLFlBQVksQ0FBQyxnQkFBZ0IsR0FBRyxxQkFBcUIsQ0FHbEcsQ0FBQztBQUVGLHdCQUFnQiw0QkFBNEIsQ0FDMUMsR0FBRyxFQUFFLE1BQU0sRUFDWCxRQUFRLEVBQUUsT0FBTyxDQUFDLGtCQUFrQixDQUFDLEVBQ3JDLEtBQUs7Ozs7O0VBQThDLEdBQ2xELGdCQUFnQixDQU1sQjtBQUVELHdCQUFnQiw4QkFBOEIsQ0FDNUMsR0FBRyxFQUFFLE1BQU0sRUFDWCxRQUFRLEVBQUUsT0FBTyxDQUFDLGtCQUFrQixDQUFDLEVBQ3JDLEtBQUs7Ozs7O0VBQThDLEdBQ2xELGtCQUFrQixDQU1wQjtBQUVELHdCQUFnQiw4QkFBOEIsQ0FDNUMsR0FBRyxFQUFFLE1BQU0sRUFDWCxRQUFRLEVBQUUsT0FBTyxDQUFDLGtCQUFrQixDQUFDLEVBQ3JDLEtBQUs7Ozs7O0VBQThDLEdBQ2xELGtCQUFrQixDQU1wQiJ9

package/dest/proving_broker/rpc.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"rpc.d.ts","sourceRoot":"","sources":["../../src/proving_broker/rpc.ts"],"names":[],"mappings":"AAEA,OAAO,EAIL,KAAK,gBAAgB,EACrB,KAAK,qBAAqB,EAC1B,KAAK,kBAAkB,EAEvB,KAAK,kBAAkB,EAExB,MAAM,iCAAiC,CAAC;AAEzC,OAAO,EAAE,KAAK,YAAY,EAAY,MAAM,uBAAuB,CAAC;AACpE,OAAO,EAAE,KAAK,kBAAkB,EAAgC,MAAM,0BAA0B,CAAC;AAcjG,eAAO,MAAM,wBAAwB,EAAE,YAAY,CAAC,kBAAkB,CAKrE,CAAC;AAEF,eAAO,MAAM,wBAAwB,EAAE,YAAY,CAAC,kBAAkB,CAcrE,CAAC;AAEF,eAAO,MAAM,sBAAsB,EAAE,YAAY,CAAC,gBAAgB,CAGjE,CAAC;AAEF,eAAO,MAAM,2BAA2B,EAAE,YAAY,CAAC,qBAAqB,CAK3E,CAAC;AAEF,eAAO,MAAM,+BAA+B,EAAE,YAAY,CAAC,gBAAgB,GAAG,qBAAqB,CAGlG,CAAC;AAEF,wBAAgB,4BAA4B,CAC1C,GAAG,EAAE,MAAM,EACX,QAAQ,EAAE,OAAO,CAAC,kBAAkB,CAAC,EACrC,KAAK;;;;;EAAoC,GACxC,gBAAgB,CAMlB;AAED,wBAAgB,8BAA8B,CAC5C,GAAG,EAAE,MAAM,EACX,QAAQ,EAAE,OAAO,CAAC,kBAAkB,CAAC,EACrC,KAAK;;;;;EAAoC,GACxC,kBAAkB,CAMpB;AAED,wBAAgB,8BAA8B,CAC5C,GAAG,EAAE,MAAM,EACX,QAAQ,EAAE,OAAO,CAAC,kBAAkB,CAAC,EACrC,KAAK;;;;;EAAoC,GACxC,kBAAkB,CAMpB"}
+{"version":3,"file":"rpc.d.ts","sourceRoot":"","sources":["../../src/proving_broker/rpc.ts"],"names":[],"mappings":"AAEA,OAAO,EAIL,KAAK,gBAAgB,EACrB,KAAK,qBAAqB,EAC1B,KAAK,kBAAkB,EAEvB,KAAK,kBAAkB,EAExB,MAAM,iCAAiC,CAAC;AAEzC,OAAO,EAAE,KAAK,YAAY,EAAY,MAAM,uBAAuB,CAAC;AACpE,OAAO,EAAE,KAAK,kBAAkB,EAAgC,MAAM,0BAA0B,CAAC;AAKjG,qFAAqF;AACrF,wBAAiB,mBAAmB,qCAMnC;AAWD,eAAO,MAAM,wBAAwB,EAAE,YAAY,CAAC,kBAAkB,CAKrE,CAAC;AAEF,eAAO,MAAM,wBAAwB,EAAE,YAAY,CAAC,kBAAkB,CAiBrE,CAAC;AAEF,eAAO,MAAM,sBAAsB,EAAE,YAAY,CAAC,gBAAgB,CAGjE,CAAC;AAEF,eAAO,MAAM,2BAA2B,EAAE,YAAY,CAAC,qBAAqB,CAK3E,CAAC;AAEF,eAAO,MAAM,+BAA+B,EAAE,YAAY,CAAC,gBAAgB,GAAG,qBAAqB,CAGlG,CAAC;AAEF,wBAAgB,4BAA4B,CAC1C,GAAG,EAAE,MAAM,EACX,QAAQ,EAAE,OAAO,CAAC,kBAAkB,CAAC,EACrC,KAAK;;;;;EAA8C,GAClD,gBAAgB,CAMlB;AAED,wBAAgB,8BAA8B,CAC5C,GAAG,EAAE,MAAM,EACX,QAAQ,EAAE,OAAO,CAAC,kBAAkB,CAAC,EACrC,KAAK;;;;;EAA8C,GAClD,kBAAkB,CAMpB;AAED,wBAAgB,8BAA8B,CAC5C,GAAG,EAAE,MAAM,EACX,QAAQ,EAAE,OAAO,CAAC,kBAAkB,CAAC,EACrC,KAAK;;;;;EAA8C,GAClD,kBAAkB,CAMpB"}

package/dest/proving_broker/rpc.js

@@ -6,6 +6,19 @@ import { optional } from '@aztec/stdlib/schemas';
 import { getVersioningResponseHandler } from '@aztec/stdlib/versioning';
 import { makeTracedFetch } from '@aztec/telemetry-client';
 import { z } from 'zod';
+/** Indefinite backoff for broker communication: 1, 1, 1, 2, 4, 4, 4, ... seconds. */ export function* proverBrokerBackoff() {
+    const v = [
+        1,
+        1,
+        1,
+        2,
+        4
+    ];
+    let i = 0;
+    while(true){
+        yield v[Math.min(i++, v.length - 1)];
+    }
+}
 const ProvingJobFilterSchema = z.object({
     allowList: z.array(z.nativeEnum(ProvingRequestType))
 });
@@ -14,55 +27,98 @@ const GetProvingJobResponse = z.object({
     time: z.number()
 });
 export const ProvingJobProducerSchema = {
-    enqueueProvingJob: z.function().args(ProvingJob).returns(ProvingJobStatus),
-    getProvingJobStatus: z.function().args(ProvingJobId).returns(ProvingJobStatus),
-    cancelProvingJob: z.function().args(ProvingJobId).returns(z.void()),
-    getCompletedJobs: z.function().args(z.array(ProvingJobId)).returns(z.array(ProvingJobId))
+    enqueueProvingJob: z.function({
+        input: z.tuple([
+            ProvingJob
+        ]),
+        output: ProvingJobStatus
+    }),
+    getProvingJobStatus: z.function({
+        input: z.tuple([
+            ProvingJobId
+        ]),
+        output: ProvingJobStatus
+    }),
+    cancelProvingJob: z.function({
+        input: z.tuple([
+            ProvingJobId
+        ]),
+        output: z.void()
+    }),
+    getCompletedJobs: z.function({
+        input: z.tuple([
+            z.array(ProvingJobId)
+        ]),
+        output: z.array(ProvingJobId)
+    })
 };
 export const ProvingJobConsumerSchema = {
-    getProvingJob: z.function().args(optional(ProvingJobFilterSchema)).returns(GetProvingJobResponse.optional()),
-    reportProvingJobError: z.function().args(ProvingJobId, z.string(), optional(z.boolean()), optional(ProvingJobFilterSchema)).returns(GetProvingJobResponse.optional()),
-    reportProvingJobProgress: z.function().args(ProvingJobId, z.number(), optional(ProvingJobFilterSchema)).returns(GetProvingJobResponse.optional()),
-    reportProvingJobSuccess: z.function().args(ProvingJobId, ProofUri, optional(ProvingJobFilterSchema)).returns(GetProvingJobResponse.optional())
+    getProvingJob: z.function({
+        input: z.tuple([
+            optional(ProvingJobFilterSchema)
+        ]),
+        output: GetProvingJobResponse.optional()
+    }),
+    reportProvingJobError: z.function({
+        input: z.tuple([
+            ProvingJobId,
+            z.string(),
+            optional(z.boolean()),
+            optional(ProvingJobFilterSchema)
+        ]),
+        output: GetProvingJobResponse.optional()
+    }),
+    reportProvingJobProgress: z.function({
+        input: z.tuple([
+            ProvingJobId,
+            z.number(),
+            optional(ProvingJobFilterSchema)
+        ]),
+        output: GetProvingJobResponse.optional()
+    }),
+    reportProvingJobSuccess: z.function({
+        input: z.tuple([
+            ProvingJobId,
+            ProofUri,
+            optional(ProvingJobFilterSchema)
+        ]),
+        output: GetProvingJobResponse.optional()
+    })
 };
 export const ProvingJobBrokerSchema = {
     ...ProvingJobConsumerSchema,
     ...ProvingJobProducerSchema
 };
 export const ProvingJobBrokerDebugSchema = {
-    replayProvingJob: z.function().args(ProvingJobId, z.nativeEnum(ProvingRequestType), EpochNumberSchema, ProofUri).returns(ProvingJobStatus)
+    replayProvingJob: z.function({
+        input: z.tuple([
+            ProvingJobId,
+            z.nativeEnum(ProvingRequestType),
+            EpochNumberSchema,
+            ProofUri
+        ]),
+        output: ProvingJobStatus
+    })
 };
 export const ProvingJobBrokerSchemaWithDebug = {
     ...ProvingJobBrokerSchema,
     ...ProvingJobBrokerDebugSchema
 };
-export function createProvingJobBrokerClient(url, versions, fetch = makeTracedFetch([
-    1,
-    2,
-    3
-], false)) {
+export function createProvingJobBrokerClient(url, versions, fetch = makeTracedFetch(proverBrokerBackoff, false)) {
     return createSafeJsonRpcClient(url, ProvingJobBrokerSchema, {
         namespaceMethods: 'proverBroker',
         fetch,
         onResponse: getVersioningResponseHandler(versions)
     });
 }
-export function createProvingJobProducerClient(url, versions, fetch = makeTracedFetch([
-    1,
-    2,
-    3
-], false)) {
+export function createProvingJobProducerClient(url, versions, fetch = makeTracedFetch(proverBrokerBackoff, false)) {
     return createSafeJsonRpcClient(url, ProvingJobProducerSchema, {
         namespaceMethods: 'provingJobProducer',
         fetch,
         onResponse: getVersioningResponseHandler(versions)
     });
 }
-export function createProvingJobConsumerClient(url, versions, fetch = makeTracedFetch([
-    1,
-    2,
-    3
-], false)) {
+export function createProvingJobConsumerClient(url, versions, fetch = makeTracedFetch(proverBrokerBackoff, false)) {
     return createSafeJsonRpcClient(url, ProvingJobConsumerSchema, {
         namespaceMethods: 'provingJobConsumer',
         fetch,

package/dest/test/mock_prover.d.ts

@@ -20,8 +20,8 @@ export declare class TestBroker implements ProvingJobProducer {
 export declare class MockProver implements ServerCircuitProver {
     constructor();
     getAvmProof(_inputs: AvmCircuitInputs, _signal?: AbortSignal, _epochNumber?: number): Promise<import("@aztec/stdlib/proofs").RecursiveProof<16400>>;
-    getBaseParityProof(_inputs: ParityBasePrivateInputs, _signal?: AbortSignal, _epochNumber?: number): Promise<PublicInputsAndRecursiveProof<import("@aztec/stdlib/parity").ParityPublicInputs, 449>>;
-    getRootParityProof(_inputs: ParityRootPrivateInputs, _signal?: AbortSignal, _epochNumber?: number): Promise<PublicInputsAndRecursiveProof<import("@aztec/stdlib/parity").ParityPublicInputs, 449>>;
+    getBaseParityProof(_inputs: ParityBasePrivateInputs, _signal?: AbortSignal, _epochNumber?: number): Promise<PublicInputsAndRecursiveProof<import("@aztec/stdlib/parity").ParityPublicInputs, 410>>;
+    getRootParityProof(_inputs: ParityRootPrivateInputs, _signal?: AbortSignal, _epochNumber?: number): Promise<PublicInputsAndRecursiveProof<import("@aztec/stdlib/parity").ParityPublicInputs, 410>>;
     getPublicChonkVerifierProof(_inputs: PublicChonkVerifierPrivateInputs, _signal?: AbortSignal, _epochNumber?: number): Promise<PublicInputsAndRecursiveProof<PublicChonkVerifierPublicInputs, typeof NESTED_RECURSIVE_ROLLUP_HONK_PROOF_LENGTH>>;
     getPrivateTxBaseRollupProof(_baseRollupInput: PrivateTxBaseRollupPrivateInputs, _signal?: AbortSignal, _epochNumber?: number): Promise<PublicInputsAndRecursiveProof<TxRollupPublicInputs, typeof NESTED_RECURSIVE_ROLLUP_HONK_PROOF_LENGTH>>;
     getPublicTxBaseRollupProof(_inputs: PublicTxBaseRollupPrivateInputs, _signal?: AbortSignal, _epochNumber?: number): Promise<PublicInputsAndRecursiveProof<TxRollupPublicInputs, typeof NESTED_RECURSIVE_ROLLUP_HONK_PROOF_LENGTH>>;
@@ -31,7 +31,7 @@ export declare class MockProver implements ServerCircuitProver {
     getBlockRootEmptyTxFirstRollupProof(_input: BlockRootEmptyTxFirstRollupPrivateInputs, _signal?: AbortSignal, _epochNumber?: number): Promise<PublicInputsAndRecursiveProof<BlockRollupPublicInputs, typeof NESTED_RECURSIVE_ROLLUP_HONK_PROOF_LENGTH>>;
     getBlockRootRollupProof(_input: BlockRootRollupPrivateInputs, _signal?: AbortSignal, _epochNumber?: number): Promise<PublicInputsAndRecursiveProof<BlockRollupPublicInputs, typeof NESTED_RECURSIVE_ROLLUP_HONK_PROOF_LENGTH>>;
     getBlockRootSingleTxRollupProof(_input: BlockRootSingleTxRollupPrivateInputs, _signal?: AbortSignal, _epochNumber?: number): Promise<PublicInputsAndRecursiveProof<BlockRollupPublicInputs, typeof NESTED_RECURSIVE_ROLLUP_HONK_PROOF_LENGTH>>;
-    getBlockMergeRollupProof(_input: BlockMergeRollupPrivateInputs, _signal?: AbortSignal, _epochNumber?: number): Promise<PublicInputsAndRecursiveProof<BlockRollupPublicInputs, 519>>;
+    getBlockMergeRollupProof(_input: BlockMergeRollupPrivateInputs, _signal?: AbortSignal, _epochNumber?: number): Promise<PublicInputsAndRecursiveProof<BlockRollupPublicInputs, 480>>;
     getCheckpointRootRollupProof(_input: CheckpointRootRollupPrivateInputs, _signal?: AbortSignal, _epochNumber?: number): Promise<PublicInputsAndRecursiveProof<CheckpointRollupPublicInputs, typeof NESTED_RECURSIVE_ROLLUP_HONK_PROOF_LENGTH>>;
     getCheckpointRootSingleBlockRollupProof(_input: CheckpointRootSingleBlockRollupPrivateInputs, _signal?: AbortSignal, _epochNumber?: number): Promise<PublicInputsAndRecursiveProof<CheckpointRollupPublicInputs, typeof NESTED_RECURSIVE_ROLLUP_HONK_PROOF_LENGTH>>;
     getCheckpointMergeRollupProof(_input: CheckpointMergeRollupPrivateInputs, _signal?: AbortSignal, _epochNumber?: number): Promise<PublicInputsAndRecursiveProof<CheckpointRollupPublicInputs, typeof NESTED_RECURSIVE_ROLLUP_HONK_PROOF_LENGTH>>;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aztec/prover-client",
-  "version": "0.0.1-commit.7b97ef96e",
+  "version": "0.0.1-commit.7cbc774",
   "type": "module",
   "exports": {
     ".": "./dest/index.js",
@@ -27,8 +27,8 @@
     "build:dev": "../scripts/tsc.sh --watch",
     "clean": "rm -rf ./dest .tsbuildinfo",
     "bb": "node --no-warnings ./dest/bb/index.js",
-    "test": "NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --testTimeout=3500000 --forceExit",
-    "test:debug": "LOG_LEVEL=debug NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --testTimeout=1500000 --forceExit --testNamePattern prover/bb_prover/parity"
+    "test": "NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --testTimeout=3500000",
+    "test:debug": "LOG_LEVEL=\"debug; info: json-rpc, simulator\" NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --testTimeout=1500000 --testNamePattern prover/bb_prover/parity"
   },
   "jest": {
     "moduleNameMapper": {
@@ -68,29 +68,28 @@
     ]
   },
   "dependencies": {
-    "@aztec/bb-prover": "0.0.1-commit.7b97ef96e",
-    "@aztec/blob-lib": "0.0.1-commit.7b97ef96e",
-    "@aztec/constants": "0.0.1-commit.7b97ef96e",
-    "@aztec/ethereum": "0.0.1-commit.7b97ef96e",
-    "@aztec/foundation": "0.0.1-commit.7b97ef96e",
-    "@aztec/kv-store": "0.0.1-commit.7b97ef96e",
-    "@aztec/noir-protocol-circuits-types": "0.0.1-commit.7b97ef96e",
-    "@aztec/noir-types": "0.0.1-commit.7b97ef96e",
-    "@aztec/protocol-contracts": "0.0.1-commit.7b97ef96e",
-    "@aztec/simulator": "0.0.1-commit.7b97ef96e",
-    "@aztec/stdlib": "0.0.1-commit.7b97ef96e",
-    "@aztec/telemetry-client": "0.0.1-commit.7b97ef96e",
-    "@aztec/world-state": "0.0.1-commit.7b97ef96e",
-    "@google-cloud/storage": "^7.15.0",
+    "@aztec/bb-prover": "0.0.1-commit.7cbc774",
+    "@aztec/blob-lib": "0.0.1-commit.7cbc774",
+    "@aztec/constants": "0.0.1-commit.7cbc774",
+    "@aztec/ethereum": "0.0.1-commit.7cbc774",
+    "@aztec/foundation": "0.0.1-commit.7cbc774",
+    "@aztec/kv-store": "0.0.1-commit.7cbc774",
+    "@aztec/noir-protocol-circuits-types": "0.0.1-commit.7cbc774",
+    "@aztec/noir-types": "0.0.1-commit.7cbc774",
+    "@aztec/protocol-contracts": "0.0.1-commit.7cbc774",
+    "@aztec/simulator": "0.0.1-commit.7cbc774",
+    "@aztec/stdlib": "0.0.1-commit.7cbc774",
+    "@aztec/telemetry-client": "0.0.1-commit.7cbc774",
+    "@aztec/world-state": "0.0.1-commit.7cbc774",
     "@iarna/toml": "^2.2.5",
     "commander": "^12.1.0",
     "lodash.chunk": "^4.2.0",
     "source-map-support": "^0.5.21",
     "tslib": "^2.4.0",
-    "zod": "^3.23.8"
+    "zod": "^4"
   },
   "devDependencies": {
-    "@aztec/noir-contracts.js": "0.0.1-commit.7b97ef96e",
+    "@aztec/noir-contracts.js": "0.0.1-commit.7cbc774",
     "@jest/globals": "^30.0.0",
     "@types/jest": "^30.0.0",
     "@types/node": "^22.15.17",

package/src/config.ts

@@ -44,14 +44,30 @@ export const bbConfigMappings: ConfigMappingsType<BBConfig & ACVMConfig> = {
   },
   numConcurrentIVCVerifiers: {
     env: 'BB_NUM_IVC_VERIFIERS',
-    description: 'Max number of chonk verifiers to run concurrently',
+    description: 'Max concurrent verifications for the RPC verifier (QueuedIVCVerifier).',
     ...numberConfigHelper(8),
   },
   bbIVCConcurrency: {
     env: 'BB_IVC_CONCURRENCY',
-    description: 'Number of threads to use for IVC verification',
+    description: 'Thread count for the RPC IVC verifier.',
     ...numberConfigHelper(1),
   },
+  bbChonkVerifyMaxBatch: {
+    env: 'BB_CHONK_VERIFY_MAX_BATCH',
+    description:
+      'Upper bound on proofs per batch for the peer chonk batch verifier. Proofs are verified immediately as they arrive; this only caps how many can accumulate while a batch is already being processed.',
+    ...numberConfigHelper(16),
+  },
+  bbChonkVerifyConcurrency: {
+    env: 'BB_CHONK_VERIFY_BATCH_CONCURRENCY',
+    description: 'Thread count for the peer batch verifier parallel reduce. 0 = auto.',
+    ...numberConfigHelper(6),
+  },
+  bbDebugOutputDir: {
+    env: 'BB_DEBUG_OUTPUT_DIR',
+    description:
+      'When set, bb.js operations write input/output files and log equivalent CLI commands to this directory',
+  },
 };
 
 export const proverClientConfigMappings: ConfigMappingsType<ProverClientUserConfig> = {

package/src/light/lightweight_checkpoint_builder.ts

@@ -1,14 +1,14 @@
 import { SpongeBlob, computeBlobsHashFromBlobs, encodeCheckpointEndMarker, getBlobsPerL1Block } from '@aztec/blob-lib';
-import { NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP } from '@aztec/constants';
 import { type CheckpointNumber, IndexWithinCheckpoint } from '@aztec/foundation/branded-types';
-import { padArrayEnd } from '@aztec/foundation/collection';
 import { Fr } from '@aztec/foundation/curves/bn254';
 import { type Logger, type LoggerBindings, createLogger } from '@aztec/foundation/log';
+import { elapsed } from '@aztec/foundation/timer';
 import { L2Block } from '@aztec/stdlib/block';
 import { Checkpoint } from '@aztec/stdlib/checkpoint';
 import type { MerkleTreeWriteOperations } from '@aztec/stdlib/interfaces/server';
 import {
   accumulateCheckpointOutHashes,
+  appendL1ToL2MessagesToTree,
   computeCheckpointOutHash,
   computeInHashFromL1ToL2Messages,
 } from '@aztec/stdlib/messaging';
@@ -68,10 +68,7 @@ export class LightweightCheckpointBuilder {
     feeAssetPriceModifier: bigint = 0n,
   ): Promise<LightweightCheckpointBuilder> {
     // Insert l1-to-l2 messages into the tree.
-    await db.appendLeaves(
-      MerkleTreeId.L1_TO_L2_MESSAGE_TREE,
-      padArrayEnd<Fr, number>(l1ToL2Messages, Fr.ZERO, NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP),
-    );
+    await appendL1ToL2MessagesToTree(db, l1ToL2Messages);
 
     return new LightweightCheckpointBuilder(
       checkpointNumber,
@@ -116,6 +113,10 @@ export class LightweightCheckpointBuilder {
       blockNumbers: existingBlocks.map(b => b.header.getBlockNumber()),
     });
 
+    if (existingBlocks.length === 0) {
+      throw new Error(`Cannot resume checkpoint ${checkpointNumber} with no existing blocks`);
+    }
+
     // Validate block order and consistency
     for (let i = 1; i < existingBlocks.length; i++) {
       const prev = existingBlocks[i - 1];
@@ -153,6 +154,10 @@ export class LightweightCheckpointBuilder {
     return this.blocks.length;
   }
 
+  public getBlocks() {
+    return this.blocks;
+  }
+
   /**
    * Adds a new block to the checkpoint. The tx effects must have already been inserted into the db if
    * this is called after tx processing, if that's not the case, then set `insertTxsEffects` to true.
@@ -161,7 +166,8 @@ export class LightweightCheckpointBuilder {
     globalVariables: GlobalVariables,
     txs: ProcessedTx[],
     opts: { insertTxsEffects?: boolean; expectedEndState?: StateReference } = {},
-  ): Promise<L2Block> {
+  ): Promise<{ block: L2Block; timings: Record<string, number> }> {
+    const timings: Record<string, number> = {};
     const isFirstBlock = this.blocks.length === 0;
 
     // Empty blocks are only allowed as the first block in a checkpoint
@@ -170,7 +176,9 @@ export class LightweightCheckpointBuilder {
     }
 
     if (isFirstBlock) {
-      this.lastArchives.push(await getTreeSnapshot(MerkleTreeId.ARCHIVE, this.db));
+      const [msGetInitialArchive, initialArchive] = await elapsed(() => getTreeSnapshot(MerkleTreeId.ARCHIVE, this.db));
+      this.lastArchives.push(initialArchive);
+      timings.getInitialArchive = msGetInitialArchive;
     }
 
     const lastArchive = this.lastArchives.at(-1)!;
@@ -180,12 +188,17 @@ export class LightweightCheckpointBuilder {
         `Inserting side effects for ${txs.length} txs for block ${globalVariables.blockNumber} into db`,
         { txs: txs.map(tx => tx.hash.toString()) },
       );
+      let msInsertSideEffects = 0;
       for (const tx of txs) {
-        await insertSideEffects(tx, this.db);
+        const [ms] = await elapsed(() => insertSideEffects(tx, this.db));
+        msInsertSideEffects += ms;
       }
+      timings.insertSideEffects = msInsertSideEffects;
     }
 
-    const endState = await this.db.getStateReference();
+    const [msGetEndState, endState] = await elapsed(() => this.db.getStateReference());
+    timings.getEndState = msGetEndState;
+
     if (opts.expectedEndState && !endState.equals(opts.expectedEndState)) {
       this.logger.error('End state after processing txs does not match expected end state', {
         globalVariables: globalVariables.toInspect(),
@@ -195,26 +208,31 @@ export class LightweightCheckpointBuilder {
       throw new Error(`End state does not match expected end state when building block ${globalVariables.blockNumber}`);
     }
 
-    const { header, body, blockBlobFields } = await buildHeaderAndBodyFromTxs(
-      txs,
-      lastArchive,
-      endState,
-      globalVariables,
-      this.spongeBlob,
-      isFirstBlock,
+    const [msBuildHeaderAndBody, { header, body, blockBlobFields }] = await elapsed(() =>
+      buildHeaderAndBodyFromTxs(txs, lastArchive, endState, globalVariables, this.spongeBlob, isFirstBlock),
     );
+    timings.buildHeaderAndBody = msBuildHeaderAndBody;
 
     header.state.validate();
 
     await this.db.updateArchive(header);
-    const newArchive = await getTreeSnapshot(MerkleTreeId.ARCHIVE, this.db);
+    const [msUpdateArchive, newArchive] = await elapsed(() => getTreeSnapshot(MerkleTreeId.ARCHIVE, this.db));
+    timings.updateArchive = msUpdateArchive;
     this.lastArchives.push(newArchive);
 
+    const expectedNextLeafIndex = Number(globalVariables.blockNumber) + 1;
+    if (newArchive.nextAvailableLeafIndex !== expectedNextLeafIndex) {
+      throw new Error(
+        `Archive tree next leaf index mismatch after building block ${globalVariables.blockNumber} (expected ${expectedNextLeafIndex} but got ${newArchive.nextAvailableLeafIndex})`,
+      );
+    }
+
     const indexWithinCheckpoint = IndexWithinCheckpoint(this.blocks.length);
     const block = new L2Block(newArchive, header, body, this.checkpointNumber, indexWithinCheckpoint);
     this.blocks.push(block);
 
-    await this.spongeBlob.absorb(blockBlobFields);
+    const [msSpongeAbsorb] = await elapsed(() => this.spongeBlob.absorb(blockBlobFields));
+    timings.spongeAbsorb = msSpongeAbsorb;
     this.blobFields.push(...blockBlobFields);
 
     this.logger.debug(`Built block ${header.getBlockNumber()}`, {
@@ -225,7 +243,7 @@ export class LightweightCheckpointBuilder {
       txs: block.body.txEffects.map(tx => tx.txHash.toString()),
     });
 
-    return block;
+    return { block, timings };
   }
 
   async completeCheckpoint(): Promise<Checkpoint> {
@@ -272,6 +290,14 @@ export class LightweightCheckpointBuilder {
       totalManaUsed,
     });
 
+    this.logger.debug(`Completed checkpoint ${this.checkpointNumber}`, {
+      checkpointNumber: this.checkpointNumber,
+      headerHash: header.hash().toString(),
+      checkpointOutHash: checkpointOutHash.toString(),
+      numPreviousCheckpointOutHashes: this.previousCheckpointOutHashes.length,
+      ...header.toInspect(),
+    });
+
     return new Checkpoint(newArchive, header, blocks, this.checkpointNumber, this.feeAssetPriceModifier);
   }