@aztec-labs/prover-client 6.0.0-nightly.20260829

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (170) hide show
  1. package/README.md +1 -0
  2. package/dest/config.d.ts +17 -0
  3. package/dest/config.d.ts.map +1 -0
  4. package/dest/config.js +63 -0
  5. package/dest/index.d.ts +4 -0
  6. package/dest/index.d.ts.map +1 -0
  7. package/dest/index.js +2 -0
  8. package/dest/light/index.d.ts +2 -0
  9. package/dest/light/index.d.ts.map +1 -0
  10. package/dest/light/index.js +1 -0
  11. package/dest/light/lightweight_checkpoint_builder.d.ts +58 -0
  12. package/dest/light/lightweight_checkpoint_builder.d.ts.map +1 -0
  13. package/dest/light/lightweight_checkpoint_builder.js +243 -0
  14. package/dest/mocks/fixtures.d.ts +20 -0
  15. package/dest/mocks/fixtures.d.ts.map +1 -0
  16. package/dest/mocks/fixtures.js +96 -0
  17. package/dest/mocks/test_context.d.ts +93 -0
  18. package/dest/mocks/test_context.d.ts.map +1 -0
  19. package/dest/mocks/test_context.js +354 -0
  20. package/dest/orchestrator/block-building-helpers.d.ts +67 -0
  21. package/dest/orchestrator/block-building-helpers.d.ts.map +1 -0
  22. package/dest/orchestrator/block-building-helpers.js +274 -0
  23. package/dest/orchestrator/block-proving-state.d.ts +107 -0
  24. package/dest/orchestrator/block-proving-state.d.ts.map +1 -0
  25. package/dest/orchestrator/block-proving-state.js +339 -0
  26. package/dest/orchestrator/checkpoint-proving-state.d.ts +67 -0
  27. package/dest/orchestrator/checkpoint-proving-state.d.ts.map +1 -0
  28. package/dest/orchestrator/checkpoint-proving-state.js +175 -0
  29. package/dest/orchestrator/checkpoint-sub-tree-orchestrator.d.ts +169 -0
  30. package/dest/orchestrator/checkpoint-sub-tree-orchestrator.d.ts.map +1 -0
  31. package/dest/orchestrator/checkpoint-sub-tree-orchestrator.js +1047 -0
  32. package/dest/orchestrator/chonk-cache.d.ts +39 -0
  33. package/dest/orchestrator/chonk-cache.d.ts.map +1 -0
  34. package/dest/orchestrator/chonk-cache.js +80 -0
  35. package/dest/orchestrator/index.d.ts +4 -0
  36. package/dest/orchestrator/index.d.ts.map +1 -0
  37. package/dest/orchestrator/index.js +3 -0
  38. package/dest/orchestrator/orchestrator_metrics.d.ts +8 -0
  39. package/dest/orchestrator/orchestrator_metrics.d.ts.map +1 -0
  40. package/dest/orchestrator/orchestrator_metrics.js +13 -0
  41. package/dest/orchestrator/proving-scheduler.d.ts +81 -0
  42. package/dest/orchestrator/proving-scheduler.d.ts.map +1 -0
  43. package/dest/orchestrator/proving-scheduler.js +120 -0
  44. package/dest/orchestrator/top-tree-orchestrator.d.ts +89 -0
  45. package/dest/orchestrator/top-tree-orchestrator.d.ts.map +1 -0
  46. package/dest/orchestrator/top-tree-orchestrator.js +233 -0
  47. package/dest/orchestrator/top-tree-proving-state.d.ts +61 -0
  48. package/dest/orchestrator/top-tree-proving-state.d.ts.map +1 -0
  49. package/dest/orchestrator/top-tree-proving-state.js +185 -0
  50. package/dest/orchestrator/tx-proving-state.d.ts +37 -0
  51. package/dest/orchestrator/tx-proving-state.d.ts.map +1 -0
  52. package/dest/orchestrator/tx-proving-state.js +77 -0
  53. package/dest/prover-client/factory.d.ts +6 -0
  54. package/dest/prover-client/factory.d.ts.map +1 -0
  55. package/dest/prover-client/factory.js +5 -0
  56. package/dest/prover-client/index.d.ts +3 -0
  57. package/dest/prover-client/index.d.ts.map +1 -0
  58. package/dest/prover-client/index.js +2 -0
  59. package/dest/prover-client/prover-client.d.ts +99 -0
  60. package/dest/prover-client/prover-client.d.ts.map +1 -0
  61. package/dest/prover-client/prover-client.js +163 -0
  62. package/dest/proving_broker/broker_prover_facade.d.ts +52 -0
  63. package/dest/proving_broker/broker_prover_facade.d.ts.map +1 -0
  64. package/dest/proving_broker/broker_prover_facade.js +360 -0
  65. package/dest/proving_broker/config.d.ts +40 -0
  66. package/dest/proving_broker/config.d.ts.map +1 -0
  67. package/dest/proving_broker/config.js +138 -0
  68. package/dest/proving_broker/factory.d.ts +5 -0
  69. package/dest/proving_broker/factory.d.ts.map +1 -0
  70. package/dest/proving_broker/factory.js +13 -0
  71. package/dest/proving_broker/fixtures.d.ts +6 -0
  72. package/dest/proving_broker/fixtures.d.ts.map +1 -0
  73. package/dest/proving_broker/fixtures.js +13 -0
  74. package/dest/proving_broker/index.d.ts +11 -0
  75. package/dest/proving_broker/index.d.ts.map +1 -0
  76. package/dest/proving_broker/index.js +10 -0
  77. package/dest/proving_broker/proof_store/factory.d.ts +3 -0
  78. package/dest/proving_broker/proof_store/factory.d.ts.map +1 -0
  79. package/dest/proving_broker/proof_store/factory.js +13 -0
  80. package/dest/proving_broker/proof_store/file_store_proof_store.d.ts +18 -0
  81. package/dest/proving_broker/proof_store/file_store_proof_store.d.ts.map +1 -0
  82. package/dest/proving_broker/proof_store/file_store_proof_store.js +60 -0
  83. package/dest/proving_broker/proof_store/index.d.ts +5 -0
  84. package/dest/proving_broker/proof_store/index.d.ts.map +1 -0
  85. package/dest/proving_broker/proof_store/index.js +4 -0
  86. package/dest/proving_broker/proof_store/inline_proof_store.d.ts +15 -0
  87. package/dest/proving_broker/proof_store/inline_proof_store.d.ts.map +1 -0
  88. package/dest/proving_broker/proof_store/inline_proof_store.js +41 -0
  89. package/dest/proving_broker/proof_store/proof_store.d.ts +36 -0
  90. package/dest/proving_broker/proof_store/proof_store.d.ts.map +1 -0
  91. package/dest/proving_broker/proof_store/proof_store.js +3 -0
  92. package/dest/proving_broker/proving_agent.d.ts +42 -0
  93. package/dest/proving_broker/proving_agent.d.ts.map +1 -0
  94. package/dest/proving_broker/proving_agent.js +155 -0
  95. package/dest/proving_broker/proving_broker.d.ts +77 -0
  96. package/dest/proving_broker/proving_broker.d.ts.map +1 -0
  97. package/dest/proving_broker/proving_broker.js +780 -0
  98. package/dest/proving_broker/proving_broker_database/memory.d.ts +20 -0
  99. package/dest/proving_broker/proving_broker_database/memory.d.ts.map +1 -0
  100. package/dest/proving_broker/proving_broker_database/memory.js +67 -0
  101. package/dest/proving_broker/proving_broker_database/persisted.d.ts +40 -0
  102. package/dest/proving_broker/proving_broker_database/persisted.d.ts.map +1 -0
  103. package/dest/proving_broker/proving_broker_database/persisted.js +623 -0
  104. package/dest/proving_broker/proving_broker_database.d.ts +66 -0
  105. package/dest/proving_broker/proving_broker_database.d.ts.map +1 -0
  106. package/dest/proving_broker/proving_broker_database.js +3 -0
  107. package/dest/proving_broker/proving_broker_instrumentation.d.ts +31 -0
  108. package/dest/proving_broker/proving_broker_instrumentation.d.ts.map +1 -0
  109. package/dest/proving_broker/proving_broker_instrumentation.js +97 -0
  110. package/dest/proving_broker/proving_job_controller.d.ts +34 -0
  111. package/dest/proving_broker/proving_job_controller.d.ts.map +1 -0
  112. package/dest/proving_broker/proving_job_controller.js +184 -0
  113. package/dest/proving_broker/rpc.d.ts +38 -0
  114. package/dest/proving_broker/rpc.d.ts.map +1 -0
  115. package/dest/proving_broker/rpc.js +138 -0
  116. package/dest/test/epoch_settlement.d.ts +36 -0
  117. package/dest/test/epoch_settlement.d.ts.map +1 -0
  118. package/dest/test/epoch_settlement.js +52 -0
  119. package/dest/test/index.d.ts +2 -0
  120. package/dest/test/index.d.ts.map +1 -0
  121. package/dest/test/index.js +1 -0
  122. package/dest/test/mock_proof_store.d.ts +9 -0
  123. package/dest/test/mock_proof_store.d.ts.map +1 -0
  124. package/dest/test/mock_proof_store.js +10 -0
  125. package/dest/test/mock_prover.d.ts +38 -0
  126. package/dest/test/mock_prover.d.ts.map +1 -0
  127. package/dest/test/mock_prover.js +91 -0
  128. package/package.json +113 -0
  129. package/src/config.ts +87 -0
  130. package/src/index.ts +4 -0
  131. package/src/light/index.ts +1 -0
  132. package/src/light/lightweight_checkpoint_builder.ts +342 -0
  133. package/src/mocks/fixtures.ts +120 -0
  134. package/src/mocks/test_context.ts +477 -0
  135. package/src/orchestrator/block-building-helpers.ts +486 -0
  136. package/src/orchestrator/block-proving-state.ts +457 -0
  137. package/src/orchestrator/checkpoint-proving-state.ts +247 -0
  138. package/src/orchestrator/checkpoint-sub-tree-orchestrator.ts +1010 -0
  139. package/src/orchestrator/chonk-cache.ts +99 -0
  140. package/src/orchestrator/index.ts +12 -0
  141. package/src/orchestrator/orchestrator_metrics.ts +18 -0
  142. package/src/orchestrator/proving-scheduler.ts +160 -0
  143. package/src/orchestrator/top-tree-orchestrator.ts +394 -0
  144. package/src/orchestrator/top-tree-proving-state.ts +219 -0
  145. package/src/orchestrator/tx-proving-state.ts +121 -0
  146. package/src/prover-client/factory.ts +18 -0
  147. package/src/prover-client/index.ts +2 -0
  148. package/src/prover-client/prover-client.ts +286 -0
  149. package/src/proving_broker/broker_prover_facade.ts +626 -0
  150. package/src/proving_broker/config.ts +178 -0
  151. package/src/proving_broker/factory.ts +19 -0
  152. package/src/proving_broker/fixtures.ts +20 -0
  153. package/src/proving_broker/index.ts +10 -0
  154. package/src/proving_broker/proof_store/factory.ts +20 -0
  155. package/src/proving_broker/proof_store/file_store_proof_store.ts +78 -0
  156. package/src/proving_broker/proof_store/index.ts +4 -0
  157. package/src/proving_broker/proof_store/inline_proof_store.ts +63 -0
  158. package/src/proving_broker/proof_store/proof_store.ts +54 -0
  159. package/src/proving_broker/proving_agent.ts +210 -0
  160. package/src/proving_broker/proving_broker.ts +888 -0
  161. package/src/proving_broker/proving_broker_database/memory.ts +78 -0
  162. package/src/proving_broker/proving_broker_database/persisted.ts +296 -0
  163. package/src/proving_broker/proving_broker_database.ts +75 -0
  164. package/src/proving_broker/proving_broker_instrumentation.ts +133 -0
  165. package/src/proving_broker/proving_job_controller.ts +199 -0
  166. package/src/proving_broker/rpc.ts +131 -0
  167. package/src/test/epoch_settlement.ts +82 -0
  168. package/src/test/index.ts +1 -0
  169. package/src/test/mock_proof_store.ts +14 -0
  170. package/src/test/mock_prover.ts +304 -0
@@ -0,0 +1,626 @@
1
+ import type {
2
+ AVM_V2_PROOF_LENGTH_IN_FIELDS,
3
+ NESTED_RECURSIVE_ROLLUP_HONK_PROOF_LENGTH,
4
+ RECURSIVE_PROOF_LENGTH,
5
+ } from '@aztec-labs/constants';
6
+ import { asyncPool } from '@aztec-labs/foundation/async-pool';
7
+ import { EpochNumber } from '@aztec-labs/foundation/branded-types';
8
+ import { chunk } from '@aztec-labs/foundation/collection';
9
+ import { type Logger, type LoggerBindings, createLogger } from '@aztec-labs/foundation/log';
10
+ import { type PromiseWithResolvers, RunningPromise, promiseWithResolvers } from '@aztec-labs/foundation/promise';
11
+ import { truncate } from '@aztec-labs/foundation/string';
12
+ import type { AvmCircuitInputs } from '@aztec-labs/stdlib/avm';
13
+ import {
14
+ type ProofUri,
15
+ type ProvingJobId,
16
+ type ProvingJobInputsMap,
17
+ type ProvingJobProducer,
18
+ type ProvingJobResultsMap,
19
+ type ProvingJobStatus,
20
+ type PublicInputsAndRecursiveProof,
21
+ type ServerCircuitProver,
22
+ makeProvingJobId,
23
+ } from '@aztec-labs/stdlib/interfaces/server';
24
+ import type { InboxParityPrivateInputs, ParityPublicInputs } from '@aztec-labs/stdlib/parity';
25
+ import { ProvingRequestType, RecursiveProof } from '@aztec-labs/stdlib/proofs';
26
+ import type {
27
+ BlockMergeRollupPrivateInputs,
28
+ BlockRollupPublicInputs,
29
+ BlockRootNoTxsRollupPrivateInputs,
30
+ BlockRootRollupPrivateInputs,
31
+ BlockRootSingleTxRollupPrivateInputs,
32
+ CheckpointMergeRollupPrivateInputs,
33
+ CheckpointPaddingRollupPrivateInputs,
34
+ CheckpointRollupPublicInputs,
35
+ CheckpointRootRollupPrivateInputs,
36
+ CheckpointRootSingleBlockRollupPrivateInputs,
37
+ PrivateTxBaseRollupPrivateInputs,
38
+ PublicChonkVerifierPrivateInputs,
39
+ PublicChonkVerifierPublicInputs,
40
+ PublicTxBaseRollupPrivateInputs,
41
+ RootRollupPrivateInputs,
42
+ RootRollupPublicInputs,
43
+ TxMergeRollupPrivateInputs,
44
+ TxRollupPublicInputs,
45
+ } from '@aztec-labs/stdlib/rollup';
46
+ import { createHash } from 'node:crypto';
47
+
48
+ import { InlineProofStore, type ProofStore } from './proof_store/index.js';
49
+
50
+ // Perform a snapshot sync every 30 seconds
51
+ const SNAPSHOT_SYNC_INTERVAL_MS = 30_000;
52
+
53
+ const MAX_CONCURRENT_JOB_SETTLED_REQUESTS = 10;
54
+ const SNAPSHOT_SYNC_CHECK_MAX_REQUEST_SIZE = 1000;
55
+
56
+ type ProvingJob = {
57
+ id: ProvingJobId;
58
+ inputsUri?: ProofUri;
59
+ type: ProvingRequestType;
60
+ deferred: PromiseWithResolvers<any>;
61
+ abortFn: () => void;
62
+ signal?: AbortSignal;
63
+ };
64
+
65
+ export class BrokerCircuitProverFacade implements ServerCircuitProver {
66
+ private jobs: Map<ProvingJobId, ProvingJob> = new Map();
67
+ private runningPromise?: RunningPromise;
68
+ private timeOfLastSnapshotSync = Date.now();
69
+ private jobsToRetrieve: Set<ProvingJobId> = new Set();
70
+ private log: Logger;
71
+
72
+ constructor(
73
+ private broker: ProvingJobProducer,
74
+ private proofStore: ProofStore = new InlineProofStore(),
75
+ private failedProofStore?: ProofStore,
76
+ private pollIntervalMs = 1000,
77
+ bindings?: LoggerBindings,
78
+ ) {
79
+ this.log = createLogger('prover-client:broker-circuit-prover-facade', bindings);
80
+ }
81
+
82
+ /**
83
+ * This is a critical section. This function can not be async since it writes
84
+ * to the jobs map which acts as a mutex, ensuring a job is only ever created once.
85
+ *
86
+ * This could be called in a SerialQueue if it needs to become async.
87
+ */
88
+ private getOrCreateProvingJob<T extends ProvingRequestType>(
89
+ id: ProvingJobId,
90
+ type: T,
91
+ signal?: AbortSignal,
92
+ ): { job: ProvingJob; isEnqueued: boolean } {
93
+ // Check if there is already a promise for this job
94
+ const existingJob = this.jobs.get(id);
95
+ if (existingJob) {
96
+ this.log.verbose(`Job already found in facade id=${id} type=${ProvingRequestType[type]}`, {
97
+ provingJobId: id,
98
+ provingJobType: ProvingRequestType[type],
99
+ });
100
+ return { job: existingJob, isEnqueued: true };
101
+ }
102
+
103
+ // Create a promise for this job id, regardless of whether it was enqueued at the broker
104
+ // The running promise will monitor for the job to be completed and resolve it either way
105
+ // We install an error handler to prevent unhandled rejections in the process before the
106
+ // job promise is awaited by the caller (see #13166)
107
+ const promise = promiseWithResolvers<ProvingJobResultsMap[T]>();
108
+ promise.promise.catch(err =>
109
+ this.log.error(`Job errored with '${err.message ?? err}' id=${id} type=${ProvingRequestType[type]}`, {
110
+ provingJobId: id,
111
+ provingJobType: ProvingRequestType[type],
112
+ }),
113
+ );
114
+ const abortFn = () => {
115
+ signal?.removeEventListener('abort', abortFn);
116
+ void this.broker.cancelProvingJob(id).catch(err => this.log.warn(`Error cancelling job id=${id}`, err));
117
+ };
118
+ const job: ProvingJob = {
119
+ id,
120
+ type,
121
+ deferred: promise,
122
+ abortFn,
123
+ signal,
124
+ };
125
+
126
+ this.jobs.set(id, job);
127
+ return { job, isEnqueued: false };
128
+ }
129
+
130
+ private async enqueueJob<T extends ProvingRequestType>(
131
+ id: ProvingJobId,
132
+ type: T,
133
+ inputs: ProvingJobInputsMap[T],
134
+ epochNumber = EpochNumber.ZERO,
135
+ signal?: AbortSignal,
136
+ ): Promise<ProvingJobResultsMap[T]> {
137
+ const { job: job, isEnqueued } = this.getOrCreateProvingJob(id, type, signal);
138
+ if (isEnqueued) {
139
+ return job.deferred.promise;
140
+ }
141
+
142
+ try {
143
+ const inputsUri = await this.proofStore.saveProofInput(id, type, inputs);
144
+ // Only retain the inputs URI on the long-lived job when a failed-proof store is configured: it is
145
+ // read solely by `backupFailedProofInputs` on failure. With the default `InlineProofStore` the URI
146
+ // embeds the full circuit inputs, so retaining it for every in-flight job would pin those inputs in
147
+ // memory until the job settles for no purpose when no failed-proof store exists. The broker still
148
+ // receives the URI below regardless; this only governs what the facade holds onto.
149
+ if (this.failedProofStore) {
150
+ job.inputsUri = inputsUri;
151
+ }
152
+
153
+ // Send the job to the broker
154
+ const jobStatus = await this.broker.enqueueProvingJob({ id, type, inputsUri, epochNumber });
155
+
156
+ const jobLogText = `id=${id} type=${ProvingRequestType[type]} epochNumber=${epochNumber}`;
157
+ const jobLogData = {
158
+ provingJobId: id,
159
+ provingJobType: ProvingRequestType[type],
160
+ epochNumber,
161
+ inputsUri: truncate(inputsUri),
162
+ status: jobStatus.status,
163
+ numOutstandingJobs: this.jobs.size,
164
+ };
165
+
166
+ // If we are here then the job was successfully accepted by the broker
167
+ // the returned status is for before any action was performed
168
+ if (jobStatus.status === 'fulfilled' || jobStatus.status === 'rejected') {
169
+ // Job was already completed by the broker
170
+ // No need to notify the broker on aborted job
171
+ this.log.verbose(`Job already completed when sent to broker ${jobLogText}`, jobLogData);
172
+
173
+ // Job was not enqueued. It must be completed already, add to our set of already completed jobs
174
+ this.jobsToRetrieve.add(id);
175
+ } else {
176
+ // notify the broker if job is aborted
177
+ signal?.addEventListener('abort', job.abortFn);
178
+
179
+ // Job added for the first time
180
+ if (jobStatus.status === 'not-found') {
181
+ this.log.verbose(`Job enqueued with broker ${jobLogText}`, jobLogData);
182
+ } else {
183
+ // Job was previously sent to the broker but is not completed
184
+ this.log.verbose(`Job already in queue or in progress when sent to broker ${jobLogText}`, jobLogData);
185
+ }
186
+ }
187
+ } catch (err) {
188
+ this.jobs.delete(job.id);
189
+ job.deferred.reject(err);
190
+ }
191
+
192
+ return job.deferred.promise as Promise<ProvingJobResultsMap[T]>;
193
+ }
194
+
195
+ public start() {
196
+ if (this.runningPromise) {
197
+ throw new Error('BrokerCircuitProverFacade already started');
198
+ }
199
+
200
+ this.log.verbose('Starting BrokerCircuitProverFacade');
201
+
202
+ this.runningPromise = new RunningPromise(() => this.monitorForCompletedJobs(), this.log, this.pollIntervalMs);
203
+ this.runningPromise.start();
204
+ }
205
+
206
+ public async stop(): Promise<void> {
207
+ if (!this.runningPromise) {
208
+ throw new Error('BrokerCircuitProverFacade not started');
209
+ }
210
+ this.log.verbose('Stopping BrokerCircuitProverFacade');
211
+ await this.runningPromise.stop();
212
+
213
+ // Reject any outstanding promises as stopped
214
+ for (const [_, v] of this.jobs) {
215
+ v.deferred.reject(new Error('Broker facade stopped'));
216
+ }
217
+ this.jobs.clear();
218
+ }
219
+
220
+ private async updateCompletedJobs() {
221
+ // Here we check for completed jobs. If everything works well (there are no service restarts etc) then all we need to do
222
+ // to maintain correct job state is to check for incrementally completed jobs. i.e. call getCompletedJobs with an empty array
223
+ // However, if there are any problems then we may lose sync with the broker's actual set of completed jobs.
224
+ // In this case we need to perform a full snapshot sync. This involves sending all of our outstanding job Ids to the broker
225
+ // and have the broker report on whether they are completed or not.
226
+ // We perform an incremental sync on every call of this function with a full snapshot sync periodically.
227
+ // This should keep us in sync without over-burdening the broker with snapshot sync requests
228
+
229
+ const getAllCompletedJobs = async (ids: ProvingJobId[]) => {
230
+ // In this function we take whatever set of snapshot ids and we ask the broker for completed job notifications
231
+ // We collect all returned notifications and return them
232
+ const allCompleted = new Set<ProvingJobId>();
233
+ try {
234
+ const batches = ids.length > 0 ? chunk(ids, SNAPSHOT_SYNC_CHECK_MAX_REQUEST_SIZE) : [[]];
235
+ await asyncPool(1, batches, async batch => {
236
+ const completed = await this.broker.getCompletedJobs(batch);
237
+ completed.forEach(id => allCompleted.add(id));
238
+ });
239
+ } catch (err) {
240
+ this.log.error(`Error thrown when requesting completed job notifications from the broker`, err);
241
+ }
242
+ return allCompleted;
243
+ };
244
+
245
+ const snapshotSyncIds: string[] = [];
246
+ const currentTime = Date.now();
247
+ const secondsSinceLastSnapshotSync = currentTime - this.timeOfLastSnapshotSync;
248
+ if (secondsSinceLastSnapshotSync > SNAPSHOT_SYNC_INTERVAL_MS) {
249
+ this.timeOfLastSnapshotSync = currentTime;
250
+ snapshotSyncIds.push(...this.jobs.keys());
251
+ this.log.trace(`Performing full snapshot sync of completed jobs with ${snapshotSyncIds.length} job(s)`);
252
+ } else {
253
+ this.log.trace(`Performing incremental sync of completed jobs`, { snapshotSyncIds });
254
+ }
255
+
256
+ // Now request the notifications from the broker
257
+ const snapshotIdsLength = snapshotSyncIds.length;
258
+ const completedJobs = await getAllCompletedJobs(snapshotSyncIds);
259
+
260
+ // We now have an additional set of completed job notifications to add to our cached set giving us the full set of jobs that we have been told are ready
261
+ // We filter this list to what we actually need, in case for any reason it is different and store in our cache
262
+ const allJobsReady = [...completedJobs, ...this.jobsToRetrieve];
263
+ this.jobsToRetrieve = new Set(allJobsReady.filter(id => this.jobs.has(id)));
264
+
265
+ if (completedJobs.size > 0) {
266
+ this.log.verbose(
267
+ `Check for job completion notifications returned ${completedJobs.size} job(s), snapshot ids length: ${snapshotIdsLength}, num outstanding jobs: ${this.jobs.size}, total jobs ready: ${this.jobsToRetrieve.size}`,
268
+ );
269
+ } else {
270
+ this.log.trace(
271
+ `Check for job completion notifications returned 0 jobs, snapshot ids length: ${snapshotIdsLength}, num outstanding jobs: ${this.jobs.size}, total jobs ready: ${this.jobsToRetrieve.size}`,
272
+ );
273
+ }
274
+ }
275
+
276
+ private async retrieveJobsThatShouldBeReady() {
277
+ const convertJobResult = async <T extends ProvingRequestType>(
278
+ result: ProvingJobStatus,
279
+ jobType: ProvingRequestType,
280
+ ): Promise<{
281
+ success: boolean;
282
+ reason?: string;
283
+ result?: ProvingJobResultsMap[T];
284
+ }> => {
285
+ if (result.status === 'fulfilled') {
286
+ const output = await this.proofStore.getProofOutput(result.value);
287
+ if (output.type === jobType) {
288
+ return { result: output.result as ProvingJobResultsMap[T], success: true };
289
+ } else {
290
+ return { success: false, reason: `Unexpected proof type: ${output.type}. Expected: ${jobType}` };
291
+ }
292
+ } else if (result.status === 'rejected') {
293
+ return { success: false, reason: result.reason };
294
+ } else if (result.status === 'aborted') {
295
+ return { success: false, reason: 'Aborted' };
296
+ } else {
297
+ throw new Error(`Unexpected proving job status ${result.status}`);
298
+ }
299
+ };
300
+
301
+ const processJob = async (job: ProvingJob) => {
302
+ // First retrieve the settled job from the broker
303
+ this.log.debug(`Received notification of completed job id=${job.id} type=${ProvingRequestType[job.type]}`);
304
+ let settledResult;
305
+ try {
306
+ settledResult = await this.broker.getProvingJobStatus(job.id);
307
+ } catch (err) {
308
+ // If an error occurs retrieving the job result then just log it and move on.
309
+ // We will try again on the next iteration
310
+ this.log.error(
311
+ `Error retrieving job result from broker job id=${job.id} type=${ProvingRequestType[job.type]}`,
312
+ err,
313
+ );
314
+ return false;
315
+ }
316
+
317
+ // Then convert the result and resolve/reject the promise
318
+ let result;
319
+ try {
320
+ result = await convertJobResult(settledResult, job.type);
321
+ } catch (err) {
322
+ // If an error occurs retrieving the job result then just log it and move on.
323
+ // We will try again on the next iteration
324
+ this.log.error(`Error processing job result job id=${job.id} type=${ProvingRequestType[job.type]}`, err);
325
+ return false;
326
+ }
327
+
328
+ if (result.success) {
329
+ this.log.verbose(`Resolved proving job id=${job.id} type=${ProvingRequestType[job.type]}`);
330
+ job.deferred.resolve(result.result);
331
+ } else {
332
+ this.log.error(
333
+ `Resolving proving job with error id=${job.id} type=${ProvingRequestType[job.type]}`,
334
+ result.reason,
335
+ );
336
+ if (result.reason !== 'Aborted') {
337
+ void this.backupFailedProofInputs(job);
338
+ }
339
+ job.deferred.reject(new Error(result.reason));
340
+ }
341
+
342
+ if (job.abortFn && job.signal) {
343
+ job.signal?.removeEventListener('abort', job.abortFn);
344
+ }
345
+
346
+ // Job is now processed removed from our cache
347
+ this.jobs.delete(job.id);
348
+ this.jobsToRetrieve.delete(job.id);
349
+ return true;
350
+ };
351
+
352
+ const toBeRetrieved = Array.from(this.jobsToRetrieve.values())
353
+ .map(id => this.jobs.get(id)!)
354
+ .filter(x => x !== undefined);
355
+ const totalJobsToRetrieve = toBeRetrieved.length;
356
+ const results = await asyncPool(MAX_CONCURRENT_JOB_SETTLED_REQUESTS, toBeRetrieved, job => processJob(job));
357
+ const totalJobsRetrieved = results.filter(x => x).length;
358
+ if (totalJobsToRetrieve > 0) {
359
+ this.log.verbose(
360
+ `Successfully retrieved ${totalJobsRetrieved} of ${totalJobsToRetrieve} jobs that should be ready, total ready jobs is now: ${this.jobsToRetrieve.size}`,
361
+ );
362
+ }
363
+ }
364
+
365
+ private async backupFailedProofInputs(job: ProvingJob) {
366
+ try {
367
+ if (!this.failedProofStore || !job.inputsUri) {
368
+ return;
369
+ }
370
+ const inputs = await this.proofStore.getProofInput(job.inputsUri);
371
+ const uri = await this.failedProofStore.saveProofInput(job.id, inputs.type, inputs.inputs);
372
+ this.log.info(`Stored proof inputs for failed job id=${job.id} type=${ProvingRequestType[job.type]} at ${uri}`, {
373
+ id: job.id,
374
+ type: job.type,
375
+ uri,
376
+ });
377
+ } catch (err) {
378
+ this.log.error(
379
+ `Error backing up proof inputs for failed job id=${job.id} type=${ProvingRequestType[job.type]}`,
380
+ err,
381
+ );
382
+ }
383
+ }
384
+
385
+ private async monitorForCompletedJobs() {
386
+ // Monitoring for completed jobs involves 2 stages.
387
+
388
+ // 1. Update our list of completed jobs.
389
+ // We poll the broker for any new job completion notifications and after filtering/deduplication add them to our cached
390
+ // list of jobs that we have been told are ready.
391
+ await this.updateCompletedJobs();
392
+
393
+ // 2. Retrieve the jobs that should be ready.
394
+ // We have a list of jobs that we have been told are ready, so we go ahead and ask for their results
395
+ await this.retrieveJobsThatShouldBeReady();
396
+ }
397
+
398
+ getAvmProof(
399
+ inputs: AvmCircuitInputs,
400
+ signal?: AbortSignal,
401
+ epochNumber?: EpochNumber,
402
+ ): Promise<RecursiveProof<typeof AVM_V2_PROOF_LENGTH_IN_FIELDS>> {
403
+ return this.enqueueJob(
404
+ this.generateId(ProvingRequestType.PUBLIC_VM, inputs, epochNumber),
405
+ ProvingRequestType.PUBLIC_VM,
406
+ inputs,
407
+ epochNumber,
408
+ signal,
409
+ );
410
+ }
411
+
412
+ getInboxParityProof(
413
+ inputs: InboxParityPrivateInputs,
414
+ signal?: AbortSignal,
415
+ epochNumber?: EpochNumber,
416
+ ): Promise<PublicInputsAndRecursiveProof<ParityPublicInputs, typeof RECURSIVE_PROOF_LENGTH>> {
417
+ return this.enqueueJob(
418
+ this.generateId(ProvingRequestType.INBOX_PARITY, inputs, epochNumber),
419
+ ProvingRequestType.INBOX_PARITY,
420
+ inputs,
421
+ epochNumber,
422
+ signal,
423
+ );
424
+ }
425
+
426
+ getTxMergeRollupProof(
427
+ input: TxMergeRollupPrivateInputs,
428
+ signal?: AbortSignal,
429
+ epochNumber?: EpochNumber,
430
+ ): Promise<PublicInputsAndRecursiveProof<TxRollupPublicInputs, typeof NESTED_RECURSIVE_ROLLUP_HONK_PROOF_LENGTH>> {
431
+ return this.enqueueJob(
432
+ this.generateId(ProvingRequestType.TX_MERGE_ROLLUP, input, epochNumber),
433
+ ProvingRequestType.TX_MERGE_ROLLUP,
434
+ input,
435
+ epochNumber,
436
+ signal,
437
+ );
438
+ }
439
+
440
+ getPublicChonkVerifierProof(
441
+ inputs: PublicChonkVerifierPrivateInputs,
442
+ signal?: AbortSignal,
443
+ epochNumber?: EpochNumber,
444
+ ): Promise<
445
+ PublicInputsAndRecursiveProof<PublicChonkVerifierPublicInputs, typeof NESTED_RECURSIVE_ROLLUP_HONK_PROOF_LENGTH>
446
+ > {
447
+ return this.enqueueJob(
448
+ this.generateId(ProvingRequestType.PUBLIC_CHONK_VERIFIER, inputs, epochNumber),
449
+ ProvingRequestType.PUBLIC_CHONK_VERIFIER,
450
+ inputs,
451
+ epochNumber,
452
+ signal,
453
+ );
454
+ }
455
+
456
+ getPrivateTxBaseRollupProof(
457
+ baseRollupInput: PrivateTxBaseRollupPrivateInputs,
458
+ signal?: AbortSignal,
459
+ epochNumber?: EpochNumber,
460
+ ): Promise<PublicInputsAndRecursiveProof<TxRollupPublicInputs, typeof NESTED_RECURSIVE_ROLLUP_HONK_PROOF_LENGTH>> {
461
+ return this.enqueueJob(
462
+ this.generateId(ProvingRequestType.PRIVATE_TX_BASE_ROLLUP, baseRollupInput, epochNumber),
463
+ ProvingRequestType.PRIVATE_TX_BASE_ROLLUP,
464
+ baseRollupInput,
465
+ epochNumber,
466
+ signal,
467
+ );
468
+ }
469
+
470
+ getPublicTxBaseRollupProof(
471
+ inputs: PublicTxBaseRollupPrivateInputs,
472
+ signal?: AbortSignal,
473
+ epochNumber?: EpochNumber,
474
+ ): Promise<PublicInputsAndRecursiveProof<TxRollupPublicInputs, typeof NESTED_RECURSIVE_ROLLUP_HONK_PROOF_LENGTH>> {
475
+ return this.enqueueJob(
476
+ this.generateId(ProvingRequestType.PUBLIC_TX_BASE_ROLLUP, inputs, epochNumber),
477
+ ProvingRequestType.PUBLIC_TX_BASE_ROLLUP,
478
+ inputs,
479
+ epochNumber,
480
+ signal,
481
+ );
482
+ }
483
+
484
+ getBlockRootNoTxsRollupProof(
485
+ input: BlockRootNoTxsRollupPrivateInputs,
486
+ signal?: AbortSignal,
487
+ epochNumber?: EpochNumber,
488
+ ): Promise<PublicInputsAndRecursiveProof<BlockRollupPublicInputs, typeof NESTED_RECURSIVE_ROLLUP_HONK_PROOF_LENGTH>> {
489
+ return this.enqueueJob(
490
+ this.generateId(ProvingRequestType.BLOCK_ROOT_NO_TXS_ROLLUP, input, epochNumber),
491
+ ProvingRequestType.BLOCK_ROOT_NO_TXS_ROLLUP,
492
+ input,
493
+ epochNumber,
494
+ signal,
495
+ );
496
+ }
497
+
498
+ getBlockRootRollupProof(
499
+ input: BlockRootRollupPrivateInputs,
500
+ signal?: AbortSignal,
501
+ epochNumber?: EpochNumber,
502
+ ): Promise<PublicInputsAndRecursiveProof<BlockRollupPublicInputs, typeof NESTED_RECURSIVE_ROLLUP_HONK_PROOF_LENGTH>> {
503
+ return this.enqueueJob(
504
+ this.generateId(ProvingRequestType.BLOCK_ROOT_ROLLUP, input, epochNumber),
505
+ ProvingRequestType.BLOCK_ROOT_ROLLUP,
506
+ input,
507
+ epochNumber,
508
+ signal,
509
+ );
510
+ }
511
+
512
+ getBlockRootSingleTxRollupProof(
513
+ input: BlockRootSingleTxRollupPrivateInputs,
514
+ signal?: AbortSignal,
515
+ epochNumber?: EpochNumber,
516
+ ): Promise<PublicInputsAndRecursiveProof<BlockRollupPublicInputs, typeof NESTED_RECURSIVE_ROLLUP_HONK_PROOF_LENGTH>> {
517
+ return this.enqueueJob(
518
+ this.generateId(ProvingRequestType.BLOCK_ROOT_SINGLE_TX_ROLLUP, input, epochNumber),
519
+ ProvingRequestType.BLOCK_ROOT_SINGLE_TX_ROLLUP,
520
+ input,
521
+ epochNumber,
522
+ signal,
523
+ );
524
+ }
525
+
526
+ getBlockMergeRollupProof(
527
+ input: BlockMergeRollupPrivateInputs,
528
+ signal?: AbortSignal,
529
+ epochNumber?: EpochNumber,
530
+ ): Promise<PublicInputsAndRecursiveProof<BlockRollupPublicInputs, typeof NESTED_RECURSIVE_ROLLUP_HONK_PROOF_LENGTH>> {
531
+ return this.enqueueJob(
532
+ this.generateId(ProvingRequestType.BLOCK_MERGE_ROLLUP, input, epochNumber),
533
+ ProvingRequestType.BLOCK_MERGE_ROLLUP,
534
+ input,
535
+ epochNumber,
536
+ signal,
537
+ );
538
+ }
539
+
540
+ getCheckpointRootRollupProof(
541
+ input: CheckpointRootRollupPrivateInputs,
542
+ signal?: AbortSignal,
543
+ epochNumber?: EpochNumber,
544
+ ): Promise<
545
+ PublicInputsAndRecursiveProof<CheckpointRollupPublicInputs, typeof NESTED_RECURSIVE_ROLLUP_HONK_PROOF_LENGTH>
546
+ > {
547
+ return this.enqueueJob(
548
+ this.generateId(ProvingRequestType.CHECKPOINT_ROOT_ROLLUP, input, epochNumber),
549
+ ProvingRequestType.CHECKPOINT_ROOT_ROLLUP,
550
+ input,
551
+ epochNumber,
552
+ signal,
553
+ );
554
+ }
555
+
556
+ getCheckpointRootSingleBlockRollupProof(
557
+ input: CheckpointRootSingleBlockRollupPrivateInputs,
558
+ signal?: AbortSignal,
559
+ epochNumber?: EpochNumber,
560
+ ): Promise<
561
+ PublicInputsAndRecursiveProof<CheckpointRollupPublicInputs, typeof NESTED_RECURSIVE_ROLLUP_HONK_PROOF_LENGTH>
562
+ > {
563
+ return this.enqueueJob(
564
+ this.generateId(ProvingRequestType.CHECKPOINT_ROOT_SINGLE_BLOCK_ROLLUP, input, epochNumber),
565
+ ProvingRequestType.CHECKPOINT_ROOT_SINGLE_BLOCK_ROLLUP,
566
+ input,
567
+ epochNumber,
568
+ signal,
569
+ );
570
+ }
571
+
572
+ getCheckpointPaddingRollupProof(
573
+ input: CheckpointPaddingRollupPrivateInputs,
574
+ signal?: AbortSignal,
575
+ epochNumber?: EpochNumber,
576
+ ): Promise<
577
+ PublicInputsAndRecursiveProof<CheckpointRollupPublicInputs, typeof NESTED_RECURSIVE_ROLLUP_HONK_PROOF_LENGTH>
578
+ > {
579
+ return this.enqueueJob(
580
+ this.generateId(ProvingRequestType.CHECKPOINT_PADDING_ROLLUP, input, epochNumber),
581
+ ProvingRequestType.CHECKPOINT_PADDING_ROLLUP,
582
+ input,
583
+ epochNumber,
584
+ signal,
585
+ );
586
+ }
587
+
588
+ getCheckpointMergeRollupProof(
589
+ input: CheckpointMergeRollupPrivateInputs,
590
+ signal?: AbortSignal,
591
+ epochNumber?: EpochNumber,
592
+ ): Promise<
593
+ PublicInputsAndRecursiveProof<CheckpointRollupPublicInputs, typeof NESTED_RECURSIVE_ROLLUP_HONK_PROOF_LENGTH>
594
+ > {
595
+ return this.enqueueJob(
596
+ this.generateId(ProvingRequestType.CHECKPOINT_MERGE_ROLLUP, input, epochNumber),
597
+ ProvingRequestType.CHECKPOINT_MERGE_ROLLUP,
598
+ input,
599
+ epochNumber,
600
+ signal,
601
+ );
602
+ }
603
+
604
+ getRootRollupProof(
605
+ input: RootRollupPrivateInputs,
606
+ signal?: AbortSignal,
607
+ epochNumber?: EpochNumber,
608
+ ): Promise<PublicInputsAndRecursiveProof<RootRollupPublicInputs, typeof RECURSIVE_PROOF_LENGTH>> {
609
+ return this.enqueueJob(
610
+ this.generateId(ProvingRequestType.ROOT_ROLLUP, input, epochNumber),
611
+ ProvingRequestType.ROOT_ROLLUP,
612
+ input,
613
+ epochNumber,
614
+ signal,
615
+ );
616
+ }
617
+
618
+ private generateId(
619
+ type: ProvingRequestType,
620
+ inputs: { toBuffer(): Buffer },
621
+ epochNumber = EpochNumber.ZERO,
622
+ ): ProvingJobId {
623
+ const inputsHash = createHash('sha256').update(inputs.toBuffer()).digest('hex');
624
+ return makeProvingJobId(epochNumber, type, inputsHash);
625
+ }
626
+ }