@aztec/prover-client 0.0.1-commit.c31f2472 → 0.0.1-commit.c52d6e7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (133) hide show
  1. package/dest/config.d.ts +1 -1
  2. package/dest/config.d.ts.map +1 -1
  3. package/dest/config.js +16 -2
  4. package/dest/light/lightweight_checkpoint_builder.d.ts +12 -5
  5. package/dest/light/lightweight_checkpoint_builder.d.ts.map +1 -1
  6. package/dest/light/lightweight_checkpoint_builder.js +59 -23
  7. package/dest/mocks/fixtures.d.ts +1 -1
  8. package/dest/mocks/fixtures.d.ts.map +1 -1
  9. package/dest/mocks/fixtures.js +3 -3
  10. package/dest/mocks/test_context.d.ts +11 -11
  11. package/dest/mocks/test_context.d.ts.map +1 -1
  12. package/dest/mocks/test_context.js +31 -30
  13. package/dest/orchestrator/block-building-helpers.d.ts +4 -4
  14. package/dest/orchestrator/block-building-helpers.d.ts.map +1 -1
  15. package/dest/orchestrator/block-building-helpers.js +4 -4
  16. package/dest/orchestrator/block-proving-state.d.ts +4 -1
  17. package/dest/orchestrator/block-proving-state.d.ts.map +1 -1
  18. package/dest/orchestrator/block-proving-state.js +7 -0
  19. package/dest/orchestrator/checkpoint-proving-state.d.ts +20 -36
  20. package/dest/orchestrator/checkpoint-proving-state.d.ts.map +1 -1
  21. package/dest/orchestrator/checkpoint-proving-state.js +22 -120
  22. package/dest/orchestrator/checkpoint-sub-tree-orchestrator.d.ts +161 -0
  23. package/dest/orchestrator/checkpoint-sub-tree-orchestrator.d.ts.map +1 -0
  24. package/dest/orchestrator/{orchestrator.js → checkpoint-sub-tree-orchestrator.js} +285 -462
  25. package/dest/orchestrator/chonk-cache.d.ts +39 -0
  26. package/dest/orchestrator/chonk-cache.d.ts.map +1 -0
  27. package/dest/orchestrator/chonk-cache.js +80 -0
  28. package/dest/orchestrator/index.d.ts +4 -2
  29. package/dest/orchestrator/index.d.ts.map +1 -1
  30. package/dest/orchestrator/index.js +3 -1
  31. package/dest/orchestrator/proving-scheduler.d.ts +81 -0
  32. package/dest/orchestrator/proving-scheduler.d.ts.map +1 -0
  33. package/dest/orchestrator/proving-scheduler.js +120 -0
  34. package/dest/orchestrator/top-tree-orchestrator.d.ts +88 -0
  35. package/dest/orchestrator/top-tree-orchestrator.d.ts.map +1 -0
  36. package/dest/orchestrator/top-tree-orchestrator.js +232 -0
  37. package/dest/orchestrator/top-tree-proving-state.d.ts +61 -0
  38. package/dest/orchestrator/top-tree-proving-state.d.ts.map +1 -0
  39. package/dest/orchestrator/top-tree-proving-state.js +185 -0
  40. package/dest/orchestrator/tx-proving-state.d.ts +3 -3
  41. package/dest/orchestrator/tx-proving-state.d.ts.map +1 -1
  42. package/dest/prover-client/prover-client.d.ts +64 -7
  43. package/dest/prover-client/prover-client.d.ts.map +1 -1
  44. package/dest/prover-client/prover-client.js +62 -13
  45. package/dest/proving_broker/broker_prover_facade.d.ts +3 -3
  46. package/dest/proving_broker/broker_prover_facade.d.ts.map +1 -1
  47. package/dest/proving_broker/broker_prover_facade.js +26 -20
  48. package/dest/proving_broker/config.d.ts +11 -67
  49. package/dest/proving_broker/config.d.ts.map +1 -1
  50. package/dest/proving_broker/config.js +16 -5
  51. package/dest/proving_broker/index.d.ts +2 -1
  52. package/dest/proving_broker/index.d.ts.map +1 -1
  53. package/dest/proving_broker/index.js +1 -0
  54. package/dest/proving_broker/proof_store/factory.d.ts +2 -5
  55. package/dest/proving_broker/proof_store/factory.d.ts.map +1 -1
  56. package/dest/proving_broker/proof_store/factory.js +7 -30
  57. package/dest/proving_broker/proof_store/file_store_proof_store.d.ts +18 -0
  58. package/dest/proving_broker/proof_store/file_store_proof_store.d.ts.map +1 -0
  59. package/dest/proving_broker/proof_store/file_store_proof_store.js +60 -0
  60. package/dest/proving_broker/proof_store/index.d.ts +2 -2
  61. package/dest/proving_broker/proof_store/index.d.ts.map +1 -1
  62. package/dest/proving_broker/proof_store/index.js +1 -1
  63. package/dest/proving_broker/proving_broker.d.ts +9 -5
  64. package/dest/proving_broker/proving_broker.d.ts.map +1 -1
  65. package/dest/proving_broker/proving_broker.js +245 -70
  66. package/dest/proving_broker/proving_broker_database/memory.d.ts +5 -2
  67. package/dest/proving_broker/proving_broker_database/memory.d.ts.map +1 -1
  68. package/dest/proving_broker/proving_broker_database/memory.js +14 -1
  69. package/dest/proving_broker/proving_broker_database/persisted.d.ts +15 -2
  70. package/dest/proving_broker/proving_broker_database/persisted.d.ts.map +1 -1
  71. package/dest/proving_broker/proving_broker_database/persisted.js +57 -6
  72. package/dest/proving_broker/proving_broker_database.d.ts +27 -1
  73. package/dest/proving_broker/proving_broker_database.d.ts.map +1 -1
  74. package/dest/proving_broker/proving_broker_instrumentation.d.ts +3 -1
  75. package/dest/proving_broker/proving_broker_instrumentation.d.ts.map +1 -1
  76. package/dest/proving_broker/proving_broker_instrumentation.js +7 -0
  77. package/dest/proving_broker/rpc.d.ts +18 -5
  78. package/dest/proving_broker/rpc.d.ts.map +1 -1
  79. package/dest/proving_broker/rpc.js +98 -23
  80. package/dest/test/epoch_settlement.d.ts +36 -0
  81. package/dest/test/epoch_settlement.d.ts.map +1 -0
  82. package/dest/test/epoch_settlement.js +52 -0
  83. package/dest/test/index.d.ts +2 -0
  84. package/dest/test/index.d.ts.map +1 -0
  85. package/dest/test/index.js +1 -0
  86. package/dest/test/mock_prover.d.ts +4 -4
  87. package/dest/test/mock_prover.js +2 -2
  88. package/package.json +20 -20
  89. package/src/config.ts +18 -2
  90. package/src/light/lightweight_checkpoint_builder.ts +63 -25
  91. package/src/mocks/fixtures.ts +2 -4
  92. package/src/mocks/test_context.ts +27 -43
  93. package/src/orchestrator/block-building-helpers.ts +4 -4
  94. package/src/orchestrator/block-proving-state.ts +9 -0
  95. package/src/orchestrator/checkpoint-proving-state.ts +21 -162
  96. package/src/orchestrator/checkpoint-sub-tree-orchestrator.ts +1044 -0
  97. package/src/orchestrator/chonk-cache.ts +99 -0
  98. package/src/orchestrator/index.ts +8 -1
  99. package/src/orchestrator/proving-scheduler.ts +160 -0
  100. package/src/orchestrator/top-tree-orchestrator.ts +384 -0
  101. package/src/orchestrator/top-tree-proving-state.ts +219 -0
  102. package/src/orchestrator/tx-proving-state.ts +3 -3
  103. package/src/prover-client/prover-client.ts +134 -25
  104. package/src/proving_broker/broker_prover_facade.ts +29 -23
  105. package/src/proving_broker/config.ts +16 -2
  106. package/src/proving_broker/index.ts +1 -0
  107. package/src/proving_broker/proof_store/factory.ts +10 -32
  108. package/src/proving_broker/proof_store/file_store_proof_store.ts +78 -0
  109. package/src/proving_broker/proof_store/index.ts +1 -1
  110. package/src/proving_broker/proving_broker.ts +237 -72
  111. package/src/proving_broker/proving_broker_database/memory.ts +16 -2
  112. package/src/proving_broker/proving_broker_database/persisted.ts +71 -8
  113. package/src/proving_broker/proving_broker_database.ts +30 -0
  114. package/src/proving_broker/proving_broker_instrumentation.ts +9 -0
  115. package/src/proving_broker/rpc.ts +62 -20
  116. package/src/test/epoch_settlement.ts +82 -0
  117. package/src/test/index.ts +1 -0
  118. package/src/test/mock_prover.ts +2 -2
  119. package/dest/orchestrator/epoch-proving-state.d.ts +0 -75
  120. package/dest/orchestrator/epoch-proving-state.d.ts.map +0 -1
  121. package/dest/orchestrator/epoch-proving-state.js +0 -267
  122. package/dest/orchestrator/orchestrator.d.ts +0 -128
  123. package/dest/orchestrator/orchestrator.d.ts.map +0 -1
  124. package/dest/prover-client/server-epoch-prover.d.ts +0 -32
  125. package/dest/prover-client/server-epoch-prover.d.ts.map +0 -1
  126. package/dest/prover-client/server-epoch-prover.js +0 -40
  127. package/dest/proving_broker/proof_store/gcs_proof_store.d.ts +0 -14
  128. package/dest/proving_broker/proof_store/gcs_proof_store.d.ts.map +0 -1
  129. package/dest/proving_broker/proof_store/gcs_proof_store.js +0 -52
  130. package/src/orchestrator/epoch-proving-state.ts +0 -378
  131. package/src/orchestrator/orchestrator.ts +0 -1281
  132. package/src/prover-client/server-epoch-prover.ts +0 -69
  133. package/src/proving_broker/proof_store/gcs_proof_store.ts +0 -76
@@ -9,6 +9,18 @@ import { getTelemetryClient } from '@aztec/telemetry-client';
9
9
  import assert from 'assert';
10
10
  import { defaultProverBrokerConfig } from './config.js';
11
11
  import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.js';
12
+ function toJobMetadata(job) {
13
+ return {
14
+ id: job.id,
15
+ type: job.type,
16
+ epochNumber: job.epochNumber
17
+ };
18
+ }
19
+ function toSettledRecord(result) {
20
+ return result.status === 'fulfilled' ? {
21
+ status: 'fulfilled'
22
+ } : result;
23
+ }
12
24
  /**
13
25
  * A broker that manages proof requests and distributes them to workers based on their priority.
14
26
  * It takes a backend that is responsible for storing and retrieving proof requests and results.
@@ -16,11 +28,17 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
16
28
  database;
17
29
  logger;
18
30
  queues;
19
- // holds a copy of the database in memory in order to quickly fulfill requests
20
- // this is fine because this broker is the only one that can modify the database
31
+ // Scheduling metadata for every known job (id, type, epochNumber). The large `inputsUri` is NOT held
32
+ // here it lives in the database and is read on demand when a job is dispatched to an agent.
21
33
  jobsCache;
22
- // as above, but for results
34
+ // Settled status for every settled job. The large fulfilled proof `value` is NOT held here — it lives
35
+ // in the database (read on demand in getProvingJobStatus) and, transiently, in `pendingResults` while
36
+ // its database write is in flight. Small payloads (rejected reason, aborted) are kept inline.
23
37
  resultsCache;
38
+ // Transient hold of a fulfilled result's full value, from the moment it settles until its database
39
+ // write commits, so status reads are read-your-writes without retaining every proof. Evicted once the
40
+ // value is durable in the database; kept only if that write failed (preserving the result in memory).
41
+ pendingResults;
24
42
  // tracks when each job was enqueued
25
43
  enqueuedAt;
26
44
  // keeps track of which jobs are currently being processed
@@ -30,7 +48,8 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
30
48
  inProgress;
31
49
  // keep track of which proving job has been retried
32
50
  retries;
33
- // a map of promises that will be resolved when a job is settled
51
+ // a map of promises that are resolved when a job settles. Carries no payload (settled status/results
52
+ // are read from resultsCache/pendingResults/DB) so it never pins a proof value in memory.
34
53
  promises;
35
54
  cleanupPromise;
36
55
  msTimeSource;
@@ -51,7 +70,8 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
51
70
  */ epochHeight;
52
71
  maxEpochsToKeepResultsFor;
53
72
  started;
54
- constructor(database, { proverBrokerJobTimeoutMs, proverBrokerPollIntervalMs, proverBrokerJobMaxRetries, proverBrokerMaxEpochsToKeepResultsFor } = defaultProverBrokerConfig, client = getTelemetryClient(), logger = createLogger('prover-client:proving-broker')){
73
+ debugReplayEnabled;
74
+ constructor(database, { proverBrokerJobTimeoutMs, proverBrokerPollIntervalMs, proverBrokerJobMaxRetries, proverBrokerMaxEpochsToKeepResultsFor, proverBrokerDebugReplayEnabled } = defaultProverBrokerConfig, client = getTelemetryClient(), logger = createLogger('prover-client:proving-broker')){
55
75
  this.database = database;
56
76
  this.logger = logger;
57
77
  this.queues = {
@@ -76,6 +96,7 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
76
96
  };
77
97
  this.jobsCache = new Map();
78
98
  this.resultsCache = new Map();
99
+ this.pendingResults = new Map();
79
100
  this.enqueuedAt = new Map();
80
101
  this.inProgress = new Map();
81
102
  this.retries = new Map();
@@ -104,6 +125,7 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
104
125
  this.jobTimeoutMs = proverBrokerJobTimeoutMs;
105
126
  this.maxRetries = proverBrokerJobMaxRetries;
106
127
  this.maxEpochsToKeepResultsFor = proverBrokerMaxEpochsToKeepResultsFor;
128
+ this.debugReplayEnabled = proverBrokerDebugReplayEnabled ?? false;
107
129
  }
108
130
  measureQueueDepth;
109
131
  countActiveJobs;
@@ -118,11 +140,11 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
118
140
  provingJobId: item.id,
119
141
  status: result ? result.status : 'pending'
120
142
  });
121
- this.jobsCache.set(item.id, item);
143
+ this.jobsCache.set(item.id, toJobMetadata(item));
122
144
  this.promises.set(item.id, promiseWithResolvers());
123
145
  if (result) {
124
- this.promises.get(item.id).resolve(result);
125
- this.resultsCache.set(item.id, result);
146
+ this.promises.get(item.id).resolve();
147
+ this.resultsCache.set(item.id, toSettledRecord(result));
126
148
  } else {
127
149
  this.enqueueJobInternal(item);
128
150
  }
@@ -146,13 +168,13 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
146
168
  return this.#cancelProvingJob(id);
147
169
  }
148
170
  getProvingJobStatus(id) {
149
- return Promise.resolve(this.#getProvingJobStatus(id));
171
+ return this.#getProvingJobStatus(id);
150
172
  }
151
173
  getCompletedJobs(ids) {
152
174
  return this.#getCompletedJobs(ids);
153
175
  }
154
176
  getProvingJob(filter) {
155
- return Promise.resolve(this.#getProvingJob(filter));
177
+ return this.#getProvingJob(filter);
156
178
  }
157
179
  reportProvingJobSuccess(id, value, filter) {
158
180
  return this.#reportProvingJobSuccess(id, value, filter);
@@ -161,19 +183,86 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
161
183
  return this.#reportProvingJobError(id, err, retry, filter);
162
184
  }
163
185
  reportProvingJobProgress(id, startedAt, filter) {
164
- return Promise.resolve(this.#reportProvingJobProgress(id, startedAt, filter));
186
+ return this.#reportProvingJobProgress(id, startedAt, filter);
187
+ }
188
+ async replayProvingJob(jobId, type, epochNumber, inputsUri) {
189
+ if (!this.debugReplayEnabled) {
190
+ throw new Error('Debug replay not enabled. Set PROVER_BROKER_DEBUG_REPLAY_ENABLED=true');
191
+ }
192
+ this.logger.info(`Replaying proving job`, {
193
+ provingJobId: jobId,
194
+ epochNumber,
195
+ inputsUri
196
+ });
197
+ // Clear existing state and enqueue
198
+ this.cleanUpProvingJobState([
199
+ jobId
200
+ ]);
201
+ const job = {
202
+ id: jobId,
203
+ type,
204
+ epochNumber,
205
+ inputsUri
206
+ };
207
+ this.jobsCache.set(jobId, toJobMetadata(job));
208
+ await this.database.addProvingJob(job);
209
+ this.enqueueJobInternal(job);
210
+ return {
211
+ status: 'in-queue'
212
+ };
165
213
  }
166
214
  async #enqueueProvingJob(job) {
167
- // We return the job status at the start of this call
168
- const jobStatus = this.#getProvingJobStatus(job.id);
215
+ // The status returned to the caller reflects the job's state at the start of this call: `not-found`
216
+ // for a brand-new job (it did not exist yet), or the cached status for one already known. Crucially
217
+ // this must NOT `await` before the synchronous `jobsCache`/`resultsCache` gate below, or the
218
+ // enqueue/revive lock would no longer hold. So the not-found default is set synchronously here and
219
+ // the cached branch computes the real status (which may read the DB) only when it returns.
220
+ let jobStatus = {
221
+ status: 'not-found'
222
+ };
169
223
  if (this.jobsCache.has(job.id)) {
170
224
  const existing = this.jobsCache.get(job.id);
171
- assert.deepStrictEqual(job, existing, 'Duplicate proving job ID');
172
- this.logger.warn(`Cached proving job id=${job.id} epochNumber=${job.epochNumber}. Not enqueuing again`, {
173
- provingJobId: job.id
174
- });
175
- this.instrumentation.incCachedJobs(job.type);
176
- return jobStatus;
225
+ // Identity check is metadata-only: `inputsUri` now lives in the database, not memory, and job ids
226
+ // are content-addressed (same id same inputs), so a mismatch shows up in id/type/epochNumber.
227
+ assert.deepStrictEqual(toJobMetadata(job), existing, 'Duplicate proving job ID');
228
+ if (this.resultsCache.get(job.id)?.status === 'aborted') {
229
+ // The producer is re-requesting a job it previously cancelled: revive it rather than
230
+ // returning the cached abort, clearing the aborted state in memory and in the database so the
231
+ // revival survives a restart.
232
+ //
233
+ // Concurrency model: `jobsCache` is the enqueue lock. Every path that puts a job on the queue
234
+ // populates `jobsCache` *synchronously, before its first await* (see the "New proving job"
235
+ // block below), so a second concurrent enqueue of the same id observes the entry at the top
236
+ // of this method and takes a cached, no-op branch instead of enqueuing a duplicate. The revive
237
+ // must keep holding that lock: we tear down the settled state and re-set `jobsCache` in a
238
+ // single synchronous span (no await in between), and only then await the database. Because a
239
+ // concurrent re-request can only interleave at that await — by which point `jobsCache` is
240
+ // populated again and the aborted result is gone — it falls into the cached branch and no-ops,
241
+ // so the job is enqueued exactly once. (`cleanUpProvingJobState` also drops the promise, so
242
+ // `enqueueJobInternal` below mints a fresh one for the retry.) This holds unchanged with the
243
+ // slimmer caches: `jobsCache` (now metadata) and `resultsCache` (now settled status) are still
244
+ // synchronous in-memory maps, so the tear-down + re-set span still contains no await.
245
+ this.logger.info(`Reviving aborted proving job id=${job.id} epochNumber=${job.epochNumber}`, {
246
+ provingJobId: job.id
247
+ });
248
+ this.cleanUpProvingJobState([
249
+ job.id
250
+ ]);
251
+ this.jobsCache.set(job.id, toJobMetadata(job));
252
+ await this.database.deleteProvingJobResult(job.id);
253
+ // The job is re-set in the cache and about to be re-enqueued below: its start status is in-queue.
254
+ jobStatus = {
255
+ status: 'in-queue'
256
+ };
257
+ } else {
258
+ this.logger.warn(`Cached proving job id=${job.id} epochNumber=${job.epochNumber}. Not enqueuing again`, {
259
+ provingJobId: job.id
260
+ });
261
+ this.instrumentation.incCachedJobs(job.type);
262
+ // Return the job's current status. This reads the fulfilled proof value from the DB when needed;
263
+ // the await is fine here because this branch does not enqueue, so it is outside the lock span.
264
+ return await this.#getProvingJobStatus(job.id);
265
+ }
177
266
  }
178
267
  if (this.isJobStale(job)) {
179
268
  this.logger.warn(`Tried enqueueing stale proving job id=${job.id} epochNumber=${job.epochNumber}`, {
@@ -186,7 +275,7 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
186
275
  });
187
276
  try {
188
277
  // do this first so it acts as a "lock". If this job is enqueued again while we're saving it the if at the top will catch it.
189
- this.jobsCache.set(job.id, job);
278
+ this.jobsCache.set(job.id, toJobMetadata(job));
190
279
  await this.database.addProvingJob(job);
191
280
  this.enqueueJobInternal(job);
192
281
  this.instrumentation.incTotalJobs(job.type);
@@ -200,46 +289,89 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
200
289
  return jobStatus;
201
290
  }
202
291
  async #cancelProvingJob(id) {
203
- if (!this.jobsCache.has(id)) {
292
+ const job = this.jobsCache.get(id);
293
+ if (!job) {
204
294
  this.logger.warn(`Can't cancel a job that doesn't exist id=${id}`, {
205
295
  provingJobId: id
206
296
  });
207
297
  return;
208
298
  }
209
- // notify listeners of the cancellation
210
- if (!this.resultsCache.has(id)) {
211
- this.logger.info(`Cancelling job id=${id}`, {
299
+ // Leave jobs that have already settled (completed or failed) alone: those results are terminal.
300
+ if (this.resultsCache.has(id)) {
301
+ return;
302
+ }
303
+ this.logger.info(`Cancelling job id=${id}`, {
304
+ provingJobId: id
305
+ });
306
+ this.inProgress.delete(id);
307
+ // Record the cancellation as its own settled state and persist it, so it survives a restart and
308
+ // notifies the current waiter. Unlike a completion or failure this is not terminal: re-enqueuing
309
+ // the same job id revives it (see #enqueueProvingJob), so the abort never permanently blocks the
310
+ // proof.
311
+ this.resultsCache.set(id, {
312
+ status: 'aborted'
313
+ });
314
+ this.promises.get(id)?.resolve();
315
+ this.completedJobNotifications.push(id);
316
+ this.instrumentation.incAbortedJobs(job.type);
317
+ try {
318
+ await this.database.setProvingJobAborted(id);
319
+ } catch (saveErr) {
320
+ this.logger.error(`Failed to save proving job aborted status id=${id}`, saveErr, {
212
321
  provingJobId: id
213
322
  });
214
- await this.#reportProvingJobError(id, 'Aborted', false);
323
+ throw saveErr;
215
324
  }
216
325
  }
217
326
  cleanUpProvingJobState(ids) {
327
+ const idsToClean = new Set(ids);
218
328
  for (const id of ids){
219
329
  this.jobsCache.delete(id);
330
+ const deferred = this.promises.get(id);
331
+ if (deferred) {
332
+ deferred.resolve();
333
+ }
220
334
  this.promises.delete(id);
221
335
  this.resultsCache.delete(id);
336
+ this.pendingResults.delete(id);
222
337
  this.inProgress.delete(id);
223
338
  this.retries.delete(id);
224
339
  this.enqueuedAt.delete(id);
225
340
  }
341
+ this.completedJobNotifications = this.completedJobNotifications.filter((id)=>!idsToClean.has(id));
226
342
  }
227
- #getProvingJobStatus(id) {
228
- const result = this.resultsCache.get(id);
229
- if (result) {
230
- return result;
231
- } else {
232
- // no result yet, check if we know the item
233
- const item = this.jobsCache.get(id);
234
- if (!item) {
235
- return {
236
- status: 'not-found'
237
- };
343
+ async #getProvingJobStatus(id) {
344
+ const settled = this.resultsCache.get(id);
345
+ if (settled) {
346
+ if (settled.status !== 'fulfilled') {
347
+ // rejected/aborted carry their (small) payload inline.
348
+ return settled;
349
+ }
350
+ // A fulfilled job's proof value is not held in memory: serve it from the transient hold if its
351
+ // database write is still in flight, otherwise read it back from the database.
352
+ const full = this.pendingResults.get(id) ?? await this.database.getProvingJobResult(id);
353
+ if (full?.status === 'fulfilled') {
354
+ return full;
238
355
  }
356
+ // Settled as fulfilled but the value is nowhere to be found — should not happen (it is held in
357
+ // pendingResults until its DB write commits). Report not-found rather than a torn result.
358
+ this.logger.error(`Fulfilled proving job id=${id} has no retrievable result value`, {
359
+ provingJobId: id
360
+ });
239
361
  return {
240
- status: this.inProgress.has(id) ? 'in-progress' : 'in-queue'
362
+ status: 'not-found'
241
363
  };
242
364
  }
365
+ // no result yet, check if we know the item
366
+ const item = this.jobsCache.get(id);
367
+ if (!item) {
368
+ return {
369
+ status: 'not-found'
370
+ };
371
+ }
372
+ return {
373
+ status: this.inProgress.has(id) ? 'in-progress' : 'in-queue'
374
+ };
243
375
  }
244
376
  #getCompletedJobs(ids) {
245
377
  const completedJobs = ids.filter((id)=>this.resultsCache.has(id));
@@ -247,7 +379,7 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
247
379
  this.completedJobNotifications = [];
248
380
  return Promise.resolve(notifications.concat(completedJobs));
249
381
  }
250
- #getProvingJob(filter = {
382
+ async #getProvingJob(filter = {
251
383
  allowList: []
252
384
  }) {
253
385
  const allowedProofs = Array.isArray(filter.allowList) && filter.allowList.length > 0 ? [
@@ -262,20 +394,43 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
262
394
  // this can happen if the broker crashes and restarts
263
395
  // it's possible agents will report progress or results for jobs that are in the queue (after the restart)
264
396
  while(enqueuedJob = queue.getImmediate()){
265
- const job = this.jobsCache.get(enqueuedJob.id);
266
- if (job && !this.inProgress.has(enqueuedJob.id) && !this.resultsCache.has(enqueuedJob.id)) {
397
+ const meta = this.jobsCache.get(enqueuedJob.id);
398
+ if (meta && !this.inProgress.has(enqueuedJob.id) && !this.resultsCache.has(enqueuedJob.id)) {
267
399
  const time = this.msTimeSource();
268
- this.inProgress.set(job.id, {
269
- id: job.id,
400
+ // Claim the job synchronously (before the await below) so a concurrent dispatch can't re-pick
401
+ // it. The same id can sit in a queue more than once (an abort leaves a stale entry that a
402
+ // later revive re-enqueues alongside), so this in-progress claim — not the queue pop — is what
403
+ // dedups those copies across concurrently-polling agents. It must be set before the await.
404
+ this.inProgress.set(meta.id, {
405
+ id: meta.id,
270
406
  startedAt: time,
271
407
  lastUpdatedAt: time
272
408
  });
273
- const enqueuedAt = this.enqueuedAt.get(job.id);
409
+ // The large inputs are not kept in memory; read them from the database. They were durably
410
+ // persisted (addProvingJob's write commits) before the job became dispatchable, so this hits.
411
+ const inputsUri = await this.database.getProvingJobInputs(meta.id);
412
+ if (!inputsUri) {
413
+ // The job was cleaned up (or its inputs lost) after we claimed it — release the claim and
414
+ // keep draining the queue for another candidate. Nothing else was mutated yet, so there is
415
+ // no wait metric or enqueued-at timer to unwind.
416
+ this.inProgress.delete(meta.id);
417
+ this.logger.warn(`No inputs found for proving job id=${meta.id}; skipping dispatch`, {
418
+ provingJobId: meta.id
419
+ });
420
+ continue;
421
+ }
422
+ // The dispatch is now committed: record the queue wait and clear the enqueued-at timer.
423
+ const enqueuedAt = this.enqueuedAt.get(meta.id);
274
424
  if (enqueuedAt) {
275
- this.instrumentation.recordJobWait(job.type, enqueuedAt);
276
- // we can clear this flag now.
277
- this.enqueuedAt.delete(job.id);
425
+ this.instrumentation.recordJobWait(meta.type, enqueuedAt);
426
+ this.enqueuedAt.delete(meta.id);
278
427
  }
428
+ const job = {
429
+ id: meta.id,
430
+ type: meta.type,
431
+ epochNumber: meta.epochNumber,
432
+ inputsUri
433
+ };
279
434
  return {
280
435
  job,
281
436
  time
@@ -323,14 +478,13 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
323
478
  this.logger.info(`Marking proving job as failed id=${id} type=${ProvingRequestType[item.type]} totalAttempts=${retries + 1} err=${err}`, {
324
479
  provingJobId: id
325
480
  });
326
- // save the result to the cache and notify clients of the job status
327
- // this should work even if our database breaks because the result is cached in memory
328
- const result = {
481
+ // save the (small) rejection reason to the cache and notify clients of the job status. The reason
482
+ // is kept in memory, so status reads work even if the database write below fails.
483
+ this.resultsCache.set(id, {
329
484
  status: 'rejected',
330
485
  reason: String(err)
331
- };
332
- this.resultsCache.set(id, result);
333
- this.promises.get(id).resolve(result);
486
+ });
487
+ this.promises.get(id).resolve();
334
488
  this.completedJobNotifications.push(id);
335
489
  this.instrumentation.incRejectedJobs(item.type);
336
490
  if (info) {
@@ -347,19 +501,19 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
347
501
  }
348
502
  return this.#getProvingJob(filter);
349
503
  }
350
- #reportProvingJobProgress(id, startedAt, filter) {
504
+ async #reportProvingJobProgress(id, startedAt, filter) {
351
505
  const job = this.jobsCache.get(id);
352
506
  if (!job) {
353
507
  this.logger.warn(`Proving job id=${id} does not exist`, {
354
508
  provingJobId: id
355
509
  });
356
- return this.#getProvingJob(filter);
510
+ return await this.#getProvingJob(filter);
357
511
  }
358
512
  if (this.resultsCache.has(id)) {
359
513
  this.logger.warn(`Proving job id=${id} has already been completed`, {
360
514
  provingJobId: id
361
515
  });
362
- return this.#getProvingJob(filter);
516
+ return await this.#getProvingJob(filter);
363
517
  }
364
518
  const metadata = this.inProgress.get(id);
365
519
  const now = this.msTimeSource();
@@ -393,7 +547,7 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
393
547
  this.logger.warn(`Proving job id=${id} type=${ProvingRequestType[job.type]} already being worked on by another agent. Sending new one`, {
394
548
  provingJobId: id
395
549
  });
396
- return this.#getProvingJob(filter);
550
+ return await this.#getProvingJob(filter);
397
551
  }
398
552
  async #reportProvingJobSuccess(id, value, filter) {
399
553
  const info = this.inProgress.get(id);
@@ -421,15 +575,17 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
421
575
  this.logger.info(`Proving job complete id=${id} type=${ProvingRequestType[item.type]} totalAttempts=${retries + 1}`, {
422
576
  provingJobId: id
423
577
  });
424
- // save result to our local cache and notify clients
425
- // if save to database fails, that's ok because we have the result in memory
426
- // if the broker crashes and needs the result again, we're covered because we can just recompute it
427
- const result = {
578
+ // Mark settled synchronously (the gate that makes concurrent settles no-op, unchanged from before),
579
+ // holding only the status in resultsCache. The large proof value is kept transiently in
580
+ // pendingResults so status reads are read-your-writes until the database write below commits.
581
+ this.resultsCache.set(id, {
582
+ status: 'fulfilled'
583
+ });
584
+ this.pendingResults.set(id, {
428
585
  status: 'fulfilled',
429
586
  value
430
- };
431
- this.resultsCache.set(id, result);
432
- this.promises.get(id).resolve(result);
587
+ });
588
+ this.promises.get(id).resolve();
433
589
  this.completedJobNotifications.push(id);
434
590
  this.instrumentation.incResolvedJobs(item.type);
435
591
  if (info) {
@@ -438,7 +594,11 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
438
594
  }
439
595
  try {
440
596
  await this.database.setProvingJobResult(id, value);
597
+ // The value is durable in the database now; drop the transient in-memory copy.
598
+ this.pendingResults.delete(id);
441
599
  } catch (saveErr) {
600
+ // Keep the value in pendingResults so status/result reads still succeed despite the failed write —
601
+ // this preserves the previous "works even if the database breaks" behaviour for this edge.
442
602
  this.logger.error(`Failed to save proving job result id=${id}`, saveErr, {
443
603
  provingJobId: id
444
604
  });
@@ -447,20 +607,20 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
447
607
  return this.#getProvingJob(filter);
448
608
  }
449
609
  async cleanupPass() {
450
- this.cleanupStaleJobs();
451
610
  this.reEnqueueExpiredJobs();
452
611
  const oldestEpochToKeep = this.oldestEpochToKeep();
453
612
  if (oldestEpochToKeep > 0) {
613
+ this.cleanupJobsOlderThanEpoch(EpochNumber(oldestEpochToKeep));
454
614
  await this.database.deleteAllProvingJobsOlderThanEpoch(EpochNumber(oldestEpochToKeep));
455
615
  this.logger.trace(`Deleted all epochs older than ${oldestEpochToKeep}`);
456
616
  }
457
617
  }
458
- cleanupStaleJobs() {
618
+ cleanupJobsOlderThanEpoch(epochNumber) {
459
619
  const jobIds = Array.from(this.jobsCache.keys());
460
620
  const jobsToClean = [];
461
621
  for (const id of jobIds){
462
622
  const job = this.jobsCache.get(id);
463
- if (this.isJobStale(job)) {
623
+ if (job.epochNumber < epochNumber) {
464
624
  jobsToClean.push(id);
465
625
  }
466
626
  }
@@ -483,12 +643,27 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
483
643
  const now = this.msTimeSource();
484
644
  const msSinceLastUpdate = now - metadata.lastUpdatedAt;
485
645
  if (msSinceLastUpdate >= this.jobTimeoutMs) {
486
- this.logger.warn(`Proving job id=${id} timed out. Adding it back to the queue.`, {
487
- provingJobId: id
488
- });
489
646
  this.inProgress.delete(id);
490
- this.enqueueJobInternal(item);
491
647
  this.instrumentation.incTimedOutJobs(item.type);
648
+ const retries = this.retries.get(id) ?? 0;
649
+ if (retries + 1 < this.maxRetries && !this.isJobStale(item)) {
650
+ this.logger.warn(`Proving job id=${id} timed out. Re-enqueueing (retry ${retries + 1}/${this.maxRetries}).`, {
651
+ provingJobId: id
652
+ });
653
+ this.retries.set(id, retries + 1);
654
+ this.enqueueJobInternal(item);
655
+ } else {
656
+ this.logger.error(`Proving job id=${id} timed out after ${retries + 1} attempts. Marking as failed.`, {
657
+ provingJobId: id
658
+ });
659
+ this.resultsCache.set(id, {
660
+ status: 'rejected',
661
+ reason: 'Timed out'
662
+ });
663
+ this.promises.get(id)?.resolve();
664
+ this.completedJobNotifications.push(id);
665
+ this.instrumentation.incRejectedJobs(item.type);
666
+ }
492
667
  }
493
668
  }
494
669
  }
@@ -5,13 +5,16 @@ export declare class InMemoryBrokerDatabase implements ProvingBrokerDatabase {
5
5
  private jobs;
6
6
  private results;
7
7
  getProvingJob(id: ProvingJobId): ProvingJob | undefined;
8
- getProvingJobResult(id: ProvingJobId): ProvingJobSettledResult | undefined;
8
+ getProvingJobInputs(id: ProvingJobId): Promise<ProofUri | undefined>;
9
+ getProvingJobResult(id: ProvingJobId): Promise<ProvingJobSettledResult | undefined>;
9
10
  addProvingJob(job: ProvingJob): Promise<void>;
10
11
  setProvingJobResult(id: ProvingJobId, value: ProofUri): Promise<void>;
11
12
  setProvingJobError(id: ProvingJobId, reason: string): Promise<void>;
13
+ setProvingJobAborted(id: ProvingJobId): Promise<void>;
14
+ deleteProvingJobResult(id: ProvingJobId): Promise<void>;
12
15
  deleteProvingJobs(ids: ProvingJobId[]): Promise<void>;
13
16
  deleteAllProvingJobsOlderThanEpoch(epochNumber: EpochNumber): Promise<void>;
14
17
  allProvingJobs(): AsyncIterableIterator<[ProvingJob, ProvingJobSettledResult | undefined]>;
15
18
  close(): Promise<void>;
16
19
  }
17
- //# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoibWVtb3J5LmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi8uLi9zcmMvcHJvdmluZ19icm9rZXIvcHJvdmluZ19icm9rZXJfZGF0YWJhc2UvbWVtb3J5LnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLE9BQU8sRUFBRSxXQUFXLEVBQUUsTUFBTSxpQ0FBaUMsQ0FBQztBQUM5RCxPQUFPLEVBQ0wsS0FBSyxRQUFRLEVBQ2IsS0FBSyxVQUFVLEVBQ2YsS0FBSyxZQUFZLEVBQ2pCLEtBQUssdUJBQXVCLEVBRTdCLE1BQU0saUNBQWlDLENBQUM7QUFFekMsT0FBTyxLQUFLLEVBQUUscUJBQXFCLEVBQUUsTUFBTSwrQkFBK0IsQ0FBQztBQUUzRSxxQkFBYSxzQkFBdUIsWUFBVyxxQkFBcUI7SUFDbEUsT0FBTyxDQUFDLElBQUksQ0FBdUM7SUFDbkQsT0FBTyxDQUFDLE9BQU8sQ0FBb0Q7SUFFbkUsYUFBYSxDQUFDLEVBQUUsRUFBRSxZQUFZLEdBQUcsVUFBVSxHQUFHLFNBQVMsQ0FFdEQ7SUFFRCxtQkFBbUIsQ0FBQyxFQUFFLEVBQUUsWUFBWSxHQUFHLHVCQUF1QixHQUFHLFNBQVMsQ0FFekU7SUFFRCxhQUFhLENBQUMsR0FBRyxFQUFFLFVBQVUsR0FBRyxPQUFPLENBQUMsSUFBSSxDQUFDLENBRzVDO0lBRUQsbUJBQW1CLENBQUMsRUFBRSxFQUFFLFlBQVksRUFBRSxLQUFLLEVBQUUsUUFBUSxHQUFHLE9BQU8sQ0FBQyxJQUFJLENBQUMsQ0FHcEU7SUFFRCxrQkFBa0IsQ0FBQyxFQUFFLEVBQUUsWUFBWSxFQUFFLE1BQU0sRUFBRSxNQUFNLEdBQUcsT0FBTyxDQUFDLElBQUksQ0FBQyxDQUdsRTtJQUVELGlCQUFpQixDQUFDLEdBQUcsRUFBRSxZQUFZLEVBQUUsR0FBRyxPQUFPLENBQUMsSUFBSSxDQUFDLENBTXBEO0lBRUQsa0NBQWtDLENBQUMsV0FBVyxFQUFFLFdBQVcsR0FBRyxPQUFPLENBQUMsSUFBSSxDQUFDLENBTTFFO0lBRU0sY0FBYyxJQUFJLHFCQUFxQixDQUFDLENBQUMsVUFBVSxFQUFFLHVCQUF1QixHQUFHLFNBQVMsQ0FBQyxDQUFDLENBSWhHO0lBRUQsS0FBSyxJQUFJLE9BQU8sQ0FBQyxJQUFJLENBQUMsQ0FFckI7Q0FDRiJ9
20
+ //# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoibWVtb3J5LmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi8uLi9zcmMvcHJvdmluZ19icm9rZXIvcHJvdmluZ19icm9rZXJfZGF0YWJhc2UvbWVtb3J5LnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLE9BQU8sRUFBRSxXQUFXLEVBQUUsTUFBTSxpQ0FBaUMsQ0FBQztBQUM5RCxPQUFPLEVBQ0wsS0FBSyxRQUFRLEVBQ2IsS0FBSyxVQUFVLEVBQ2YsS0FBSyxZQUFZLEVBQ2pCLEtBQUssdUJBQXVCLEVBRTdCLE1BQU0saUNBQWlDLENBQUM7QUFFekMsT0FBTyxLQUFLLEVBQUUscUJBQXFCLEVBQUUsTUFBTSwrQkFBK0IsQ0FBQztBQUUzRSxxQkFBYSxzQkFBdUIsWUFBVyxxQkFBcUI7SUFDbEUsT0FBTyxDQUFDLElBQUksQ0FBdUM7SUFDbkQsT0FBTyxDQUFDLE9BQU8sQ0FBb0Q7SUFFbkUsYUFBYSxDQUFDLEVBQUUsRUFBRSxZQUFZLEdBQUcsVUFBVSxHQUFHLFNBQVMsQ0FFdEQ7SUFFRCxtQkFBbUIsQ0FBQyxFQUFFLEVBQUUsWUFBWSxHQUFHLE9BQU8sQ0FBQyxRQUFRLEdBQUcsU0FBUyxDQUFDLENBRW5FO0lBRUQsbUJBQW1CLENBQUMsRUFBRSxFQUFFLFlBQVksR0FBRyxPQUFPLENBQUMsdUJBQXVCLEdBQUcsU0FBUyxDQUFDLENBRWxGO0lBRUQsYUFBYSxDQUFDLEdBQUcsRUFBRSxVQUFVLEdBQUcsT0FBTyxDQUFDLElBQUksQ0FBQyxDQUc1QztJQUVELG1CQUFtQixDQUFDLEVBQUUsRUFBRSxZQUFZLEVBQUUsS0FBSyxFQUFFLFFBQVEsR0FBRyxPQUFPLENBQUMsSUFBSSxDQUFDLENBR3BFO0lBRUQsa0JBQWtCLENBQUMsRUFBRSxFQUFFLFlBQVksRUFBRSxNQUFNLEVBQUUsTUFBTSxHQUFHLE9BQU8sQ0FBQyxJQUFJLENBQUMsQ0FHbEU7SUFFRCxvQkFBb0IsQ0FBQyxFQUFFLEVBQUUsWUFBWSxHQUFHLE9BQU8sQ0FBQyxJQUFJLENBQUMsQ0FHcEQ7SUFFRCxzQkFBc0IsQ0FBQyxFQUFFLEVBQUUsWUFBWSxHQUFHLE9BQU8sQ0FBQyxJQUFJLENBQUMsQ0FHdEQ7SUFFRCxpQkFBaUIsQ0FBQyxHQUFHLEVBQUUsWUFBWSxFQUFFLEdBQUcsT0FBTyxDQUFDLElBQUksQ0FBQyxDQU1wRDtJQUVELGtDQUFrQyxDQUFDLFdBQVcsRUFBRSxXQUFXLEdBQUcsT0FBTyxDQUFDLElBQUksQ0FBQyxDQU0xRTtJQUVNLGNBQWMsSUFBSSxxQkFBcUIsQ0FBQyxDQUFDLFVBQVUsRUFBRSx1QkFBdUIsR0FBRyxTQUFTLENBQUMsQ0FBQyxDQUloRztJQUVELEtBQUssSUFBSSxPQUFPLENBQUMsSUFBSSxDQUFDLENBRXJCO0NBQ0YifQ==
@@ -1 +1 @@
1
- {"version":3,"file":"memory.d.ts","sourceRoot":"","sources":["../../../src/proving_broker/proving_broker_database/memory.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iCAAiC,CAAC;AAC9D,OAAO,EACL,KAAK,QAAQ,EACb,KAAK,UAAU,EACf,KAAK,YAAY,EACjB,KAAK,uBAAuB,EAE7B,MAAM,iCAAiC,CAAC;AAEzC,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,+BAA+B,CAAC;AAE3E,qBAAa,sBAAuB,YAAW,qBAAqB;IAClE,OAAO,CAAC,IAAI,CAAuC;IACnD,OAAO,CAAC,OAAO,CAAoD;IAEnE,aAAa,CAAC,EAAE,EAAE,YAAY,GAAG,UAAU,GAAG,SAAS,CAEtD;IAED,mBAAmB,CAAC,EAAE,EAAE,YAAY,GAAG,uBAAuB,GAAG,SAAS,CAEzE;IAED,aAAa,CAAC,GAAG,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAG5C;IAED,mBAAmB,CAAC,EAAE,EAAE,YAAY,EAAE,KAAK,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CAGpE;IAED,kBAAkB,CAAC,EAAE,EAAE,YAAY,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAGlE;IAED,iBAAiB,CAAC,GAAG,EAAE,YAAY,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAMpD;IAED,kCAAkC,CAAC,WAAW,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAM1E;IAEM,cAAc,IAAI,qBAAqB,CAAC,CAAC,UAAU,EAAE,uBAAuB,GAAG,SAAS,CAAC,CAAC,CAIhG;IAED,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAErB;CACF"}
1
+ {"version":3,"file":"memory.d.ts","sourceRoot":"","sources":["../../../src/proving_broker/proving_broker_database/memory.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iCAAiC,CAAC;AAC9D,OAAO,EACL,KAAK,QAAQ,EACb,KAAK,UAAU,EACf,KAAK,YAAY,EACjB,KAAK,uBAAuB,EAE7B,MAAM,iCAAiC,CAAC;AAEzC,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,+BAA+B,CAAC;AAE3E,qBAAa,sBAAuB,YAAW,qBAAqB;IAClE,OAAO,CAAC,IAAI,CAAuC;IACnD,OAAO,CAAC,OAAO,CAAoD;IAEnE,aAAa,CAAC,EAAE,EAAE,YAAY,GAAG,UAAU,GAAG,SAAS,CAEtD;IAED,mBAAmB,CAAC,EAAE,EAAE,YAAY,GAAG,OAAO,CAAC,QAAQ,GAAG,SAAS,CAAC,CAEnE;IAED,mBAAmB,CAAC,EAAE,EAAE,YAAY,GAAG,OAAO,CAAC,uBAAuB,GAAG,SAAS,CAAC,CAElF;IAED,aAAa,CAAC,GAAG,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAG5C;IAED,mBAAmB,CAAC,EAAE,EAAE,YAAY,EAAE,KAAK,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CAGpE;IAED,kBAAkB,CAAC,EAAE,EAAE,YAAY,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAGlE;IAED,oBAAoB,CAAC,EAAE,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAGpD;IAED,sBAAsB,CAAC,EAAE,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAGtD;IAED,iBAAiB,CAAC,GAAG,EAAE,YAAY,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAMpD;IAED,kCAAkC,CAAC,WAAW,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAM1E;IAEM,cAAc,IAAI,qBAAqB,CAAC,CAAC,UAAU,EAAE,uBAAuB,GAAG,SAAS,CAAC,CAAC,CAIhG;IAED,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAErB;CACF"}
@@ -5,8 +5,11 @@ export class InMemoryBrokerDatabase {
5
5
  getProvingJob(id) {
6
6
  return this.jobs.get(id);
7
7
  }
8
+ getProvingJobInputs(id) {
9
+ return Promise.resolve(this.jobs.get(id)?.inputsUri);
10
+ }
8
11
  getProvingJobResult(id) {
9
- return this.results.get(id);
12
+ return Promise.resolve(this.results.get(id));
10
13
  }
11
14
  addProvingJob(job) {
12
15
  this.jobs.set(job.id, job);
@@ -26,6 +29,16 @@ export class InMemoryBrokerDatabase {
26
29
  });
27
30
  return Promise.resolve();
28
31
  }
32
+ setProvingJobAborted(id) {
33
+ this.results.set(id, {
34
+ status: 'aborted'
35
+ });
36
+ return Promise.resolve();
37
+ }
38
+ deleteProvingJobResult(id) {
39
+ this.results.delete(id);
40
+ return Promise.resolve();
41
+ }
29
42
  deleteProvingJobs(ids) {
30
43
  for (const id of ids){
31
44
  this.jobs.delete(id);
@@ -4,6 +4,14 @@ import { type ProofUri, ProvingJob, type ProvingJobId, ProvingJobSettledResult }
4
4
  import { type TelemetryClient, type Tracer } from '@aztec/telemetry-client';
5
5
  import type { ProverBrokerConfig } from '../config.js';
6
6
  import type { ProvingBrokerDatabase } from '../proving_broker_database.js';
7
+ /**
8
+ * An item queued for a batched write. A {@link ProvingJob} adds/updates a job; a `[id, result]`
9
+ * tuple sets a job's settled result; a `[id, undefined]` tuple deletes a job's result. Deletes ride
10
+ * the same queue as writes so they stay FIFO-ordered against them: a revive's result-delete must
11
+ * land after the abort that preceded it, otherwise a still-queued abort could flush later and
12
+ * resurrect the cancelled state on disk.
13
+ */
14
+ type BrokerWrite = ProvingJob | [ProvingJobId, ProvingJobSettledResult | undefined];
7
15
  export declare class KVBrokerDatabase implements ProvingBrokerDatabase {
8
16
  private epochs;
9
17
  private config;
@@ -12,7 +20,7 @@ export declare class KVBrokerDatabase implements ProvingBrokerDatabase {
12
20
  private batchQueue;
13
21
  readonly tracer: Tracer;
14
22
  private constructor();
15
- commitWrites(items: Array<ProvingJob | [ProvingJobId, ProvingJobSettledResult]>, epochNumber: number): Promise<void>;
23
+ commitWrites(items: Array<BrokerWrite>, epochNumber: number): Promise<void>;
16
24
  private estimateSize;
17
25
  static new(config: ProverBrokerConfig, client?: TelemetryClient, logger?: Logger): Promise<KVBrokerDatabase>;
18
26
  private start;
@@ -20,8 +28,13 @@ export declare class KVBrokerDatabase implements ProvingBrokerDatabase {
20
28
  deleteAllProvingJobsOlderThanEpoch(epochNumber: EpochNumber): Promise<void>;
21
29
  addProvingJob(job: ProvingJob): Promise<void>;
22
30
  allProvingJobs(): AsyncIterableIterator<[ProvingJob, ProvingJobSettledResult | undefined]>;
31
+ getProvingJobInputs(id: ProvingJobId): Promise<ProofUri | undefined>;
32
+ getProvingJobResult(id: ProvingJobId): Promise<ProvingJobSettledResult | undefined>;
23
33
  setProvingJobError(id: ProvingJobId, reason: string): Promise<void>;
34
+ setProvingJobAborted(id: ProvingJobId): Promise<void>;
35
+ deleteProvingJobResult(id: ProvingJobId): Promise<void>;
24
36
  setProvingJobResult(id: ProvingJobId, value: ProofUri): Promise<void>;
25
37
  private getEpochDatabase;
26
38
  }
27
- //# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoicGVyc2lzdGVkLmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi8uLi9zcmMvcHJvdmluZ19icm9rZXIvcHJvdmluZ19icm9rZXJfZGF0YWJhc2UvcGVyc2lzdGVkLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLE9BQU8sRUFBRSxXQUFXLEVBQUUsTUFBTSxpQ0FBaUMsQ0FBQztBQUU5RCxPQUFPLEVBQUUsS0FBSyxNQUFNLEVBQWdCLE1BQU0sdUJBQXVCLENBQUM7QUFJbEUsT0FBTyxFQUNMLEtBQUssUUFBUSxFQUNiLFVBQVUsRUFDVixLQUFLLFlBQVksRUFDakIsdUJBQXVCLEVBRXhCLE1BQU0saUNBQWlDLENBQUM7QUFDekMsT0FBTyxFQUdMLEtBQUssZUFBZSxFQUNwQixLQUFLLE1BQU0sRUFHWixNQUFNLHlCQUF5QixDQUFDO0FBS2pDLE9BQU8sS0FBSyxFQUFFLGtCQUFrQixFQUFFLE1BQU0sY0FBYyxDQUFDO0FBQ3ZELE9BQU8sS0FBSyxFQUFFLHFCQUFxQixFQUFFLE1BQU0sK0JBQStCLENBQUM7QUF3RDNFLHFCQUFhLGdCQUFpQixZQUFXLHFCQUFxQjtJQVExRCxPQUFPLENBQUMsTUFBTTtJQUNkLE9BQU8sQ0FBQyxNQUFNO0lBRWQsT0FBTyxDQUFDLE1BQU07SUFWaEIsT0FBTyxDQUFDLE9BQU8sQ0FBYztJQUU3QixPQUFPLENBQUMsVUFBVSxDQUEyRTtJQUU3RixTQUFnQixNQUFNLEVBQUUsTUFBTSxDQUFDO0lBRS9CLE9BQU8sZUFzQk47SUFHWSxZQUFZLENBQUMsS0FBSyxFQUFFLEtBQUssQ0FBQyxVQUFVLEdBQUcsQ0FBQyxZQUFZLEVBQUUsdUJBQXVCLENBQUMsQ0FBQyxFQUFFLFdBQVcsRUFBRSxNQUFNLGlCQU1oSDtZQUVhLFlBQVk7SUFVMUIsT0FBb0IsR0FBRyxDQUNyQixNQUFNLEVBQUUsa0JBQWtCLEVBQzFCLE1BQU0sR0FBRSxlQUFzQyxFQUM5QyxNQUFNLFNBQXdELDZCQThCL0Q7SUFFRCxPQUFPLENBQUMsS0FBSztJQUlQLEtBQUssSUFBSSxPQUFPLENBQUMsSUFBSSxDQUFDLENBSzNCO0lBS0ssa0NBQWtDLENBQUMsV0FBVyxFQUFFLFdBQVcsR0FBRyxPQUFPLENBQUMsSUFBSSxDQUFDLENBV2hGO0lBRUQsYUFBYSxDQUFDLEdBQUcsRUFBRSxVQUFVLEdBQUcsT0FBTyxDQUFDLElBQUksQ0FBQyxDQUU1QztJQUVNLGNBQWMsSUFBSSxxQkFBcUIsQ0FBQyxDQUFDLFVBQVUsRUFBRSx1QkFBdUIsR0FBRyxTQUFTLENBQUMsQ0FBQyxDQUtoRztJQUVELGtCQUFrQixDQUFDLEVBQUUsRUFBRSxZQUFZLEVBQUUsTUFBTSxFQUFFLE1BQU0sR0FBRyxPQUFPLENBQUMsSUFBSSxDQUFDLENBRWxFO0lBRUQsbUJBQW1CLENBQUMsRUFBRSxFQUFFLFlBQVksRUFBRSxLQUFLLEVBQUUsUUFBUSxHQUFHLE9BQU8sQ0FBQyxJQUFJLENBQUMsQ0FFcEU7WUFFYSxnQkFBZ0I7Q0FvQi9CIn0=
39
+ export {};
40
+ //# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoicGVyc2lzdGVkLmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi8uLi9zcmMvcHJvdmluZ19icm9rZXIvcHJvdmluZ19icm9rZXJfZGF0YWJhc2UvcGVyc2lzdGVkLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLE9BQU8sRUFBRSxXQUFXLEVBQUUsTUFBTSxpQ0FBaUMsQ0FBQztBQUU5RCxPQUFPLEVBQUUsS0FBSyxNQUFNLEVBQWdCLE1BQU0sdUJBQXVCLENBQUM7QUFJbEUsT0FBTyxFQUNMLEtBQUssUUFBUSxFQUNiLFVBQVUsRUFDVixLQUFLLFlBQVksRUFDakIsdUJBQXVCLEVBRXhCLE1BQU0saUNBQWlDLENBQUM7QUFDekMsT0FBTyxFQUdMLEtBQUssZUFBZSxFQUNwQixLQUFLLE1BQU0sRUFHWixNQUFNLHlCQUF5QixDQUFDO0FBS2pDLE9BQU8sS0FBSyxFQUFFLGtCQUFrQixFQUFFLE1BQU0sY0FBYyxDQUFDO0FBQ3ZELE9BQU8sS0FBSyxFQUFFLHFCQUFxQixFQUFFLE1BQU0sK0JBQStCLENBQUM7QUFnRjNFOzs7Ozs7R0FNRztBQUNILEtBQUssV0FBVyxHQUFHLFVBQVUsR0FBRyxDQUFDLFlBQVksRUFBRSx1QkFBdUIsR0FBRyxTQUFTLENBQUMsQ0FBQztBQUVwRixxQkFBYSxnQkFBaUIsWUFBVyxxQkFBcUI7SUFRMUQsT0FBTyxDQUFDLE1BQU07SUFDZCxPQUFPLENBQUMsTUFBTTtJQUVkLE9BQU8sQ0FBQyxNQUFNO0lBVmhCLE9BQU8sQ0FBQyxPQUFPLENBQWM7SUFFN0IsT0FBTyxDQUFDLFVBQVUsQ0FBa0M7SUFFcEQsU0FBZ0IsTUFBTSxFQUFFLE1BQU0sQ0FBQztJQUUvQixPQUFPLGVBc0JOO0lBR1ksWUFBWSxDQUFDLEtBQUssRUFBRSxLQUFLLENBQUMsV0FBVyxDQUFDLEVBQUUsV0FBVyxFQUFFLE1BQU0saUJBVXZFO1lBRWEsWUFBWTtJQVUxQixPQUFvQixHQUFHLENBQ3JCLE1BQU0sRUFBRSxrQkFBa0IsRUFDMUIsTUFBTSxHQUFFLGVBQXNDLEVBQzlDLE1BQU0sU0FBd0QsNkJBOEIvRDtJQUVELE9BQU8sQ0FBQyxLQUFLO0lBSVAsS0FBSyxJQUFJLE9BQU8sQ0FBQyxJQUFJLENBQUMsQ0FLM0I7SUFLSyxrQ0FBa0MsQ0FBQyxXQUFXLEVBQUUsV0FBVyxHQUFHLE9BQU8sQ0FBQyxJQUFJLENBQUMsQ0FXaEY7SUFPRCxhQUFhLENBQUMsR0FBRyxFQUFFLFVBQVUsR0FBRyxPQUFPLENBQUMsSUFBSSxDQUFDLENBRTVDO0lBRU0sY0FBYyxJQUFJLHFCQUFxQixDQUFDLENBQUMsVUFBVSxFQUFFLHVCQUF1QixHQUFHLFNBQVMsQ0FBQyxDQUFDLENBS2hHO0lBT0QsbUJBQW1CLENBQUMsRUFBRSxFQUFFLFlBQVksR0FBRyxPQUFPLENBQUMsUUFBUSxHQUFHLFNBQVMsQ0FBQyxDQUVuRTtJQUVELG1CQUFtQixDQUFDLEVBQUUsRUFBRSxZQUFZLEdBQUcsT0FBTyxDQUFDLHVCQUF1QixHQUFHLFNBQVMsQ0FBQyxDQUVsRjtJQUVELGtCQUFrQixDQUFDLEVBQUUsRUFBRSxZQUFZLEVBQUUsTUFBTSxFQUFFLE1BQU0sR0FBRyxPQUFPLENBQUMsSUFBSSxDQUFDLENBRWxFO0lBRUQsb0JBQW9CLENBQUMsRUFBRSxFQUFFLFlBQVksR0FBRyxPQUFPLENBQUMsSUFBSSxDQUFDLENBRXBEO0lBRUQsc0JBQXNCLENBQUMsRUFBRSxFQUFFLFlBQVksR0FBRyxPQUFPLENBQUMsSUFBSSxDQUFDLENBRXREO0lBRUQsbUJBQW1CLENBQUMsRUFBRSxFQUFFLFlBQVksRUFBRSxLQUFLLEVBQUUsUUFBUSxHQUFHLE9BQU8sQ0FBQyxJQUFJLENBQUMsQ0FFcEU7WUFFYSxnQkFBZ0I7Q0FvQi9CIn0=
@@ -1 +1 @@
1
- {"version":3,"file":"persisted.d.ts","sourceRoot":"","sources":["../../../src/proving_broker/proving_broker_database/persisted.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iCAAiC,CAAC;AAE9D,OAAO,EAAE,KAAK,MAAM,EAAgB,MAAM,uBAAuB,CAAC;AAIlE,OAAO,EACL,KAAK,QAAQ,EACb,UAAU,EACV,KAAK,YAAY,EACjB,uBAAuB,EAExB,MAAM,iCAAiC,CAAC;AACzC,OAAO,EAGL,KAAK,eAAe,EACpB,KAAK,MAAM,EAGZ,MAAM,yBAAyB,CAAC;AAKjC,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAC;AACvD,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,+BAA+B,CAAC;AAwD3E,qBAAa,gBAAiB,YAAW,qBAAqB;IAQ1D,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,MAAM;IAEd,OAAO,CAAC,MAAM;IAVhB,OAAO,CAAC,OAAO,CAAc;IAE7B,OAAO,CAAC,UAAU,CAA2E;IAE7F,SAAgB,MAAM,EAAE,MAAM,CAAC;IAE/B,OAAO,eAsBN;IAGY,YAAY,CAAC,KAAK,EAAE,KAAK,CAAC,UAAU,GAAG,CAAC,YAAY,EAAE,uBAAuB,CAAC,CAAC,EAAE,WAAW,EAAE,MAAM,iBAMhH;YAEa,YAAY;IAU1B,OAAoB,GAAG,CACrB,MAAM,EAAE,kBAAkB,EAC1B,MAAM,GAAE,eAAsC,EAC9C,MAAM,SAAwD,6BA8B/D;IAED,OAAO,CAAC,KAAK;IAIP,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAK3B;IAKK,kCAAkC,CAAC,WAAW,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAWhF;IAED,aAAa,CAAC,GAAG,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAE5C;IAEM,cAAc,IAAI,qBAAqB,CAAC,CAAC,UAAU,EAAE,uBAAuB,GAAG,SAAS,CAAC,CAAC,CAKhG;IAED,kBAAkB,CAAC,EAAE,EAAE,YAAY,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAElE;IAED,mBAAmB,CAAC,EAAE,EAAE,YAAY,EAAE,KAAK,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CAEpE;YAEa,gBAAgB;CAoB/B"}
1
+ {"version":3,"file":"persisted.d.ts","sourceRoot":"","sources":["../../../src/proving_broker/proving_broker_database/persisted.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iCAAiC,CAAC;AAE9D,OAAO,EAAE,KAAK,MAAM,EAAgB,MAAM,uBAAuB,CAAC;AAIlE,OAAO,EACL,KAAK,QAAQ,EACb,UAAU,EACV,KAAK,YAAY,EACjB,uBAAuB,EAExB,MAAM,iCAAiC,CAAC;AACzC,OAAO,EAGL,KAAK,eAAe,EACpB,KAAK,MAAM,EAGZ,MAAM,yBAAyB,CAAC;AAKjC,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAC;AACvD,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,+BAA+B,CAAC;AAgF3E;;;;;;GAMG;AACH,KAAK,WAAW,GAAG,UAAU,GAAG,CAAC,YAAY,EAAE,uBAAuB,GAAG,SAAS,CAAC,CAAC;AAEpF,qBAAa,gBAAiB,YAAW,qBAAqB;IAQ1D,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,MAAM;IAEd,OAAO,CAAC,MAAM;IAVhB,OAAO,CAAC,OAAO,CAAc;IAE7B,OAAO,CAAC,UAAU,CAAkC;IAEpD,SAAgB,MAAM,EAAE,MAAM,CAAC;IAE/B,OAAO,eAsBN;IAGY,YAAY,CAAC,KAAK,EAAE,KAAK,CAAC,WAAW,CAAC,EAAE,WAAW,EAAE,MAAM,iBAUvE;YAEa,YAAY;IAU1B,OAAoB,GAAG,CACrB,MAAM,EAAE,kBAAkB,EAC1B,MAAM,GAAE,eAAsC,EAC9C,MAAM,SAAwD,6BA8B/D;IAED,OAAO,CAAC,KAAK;IAIP,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAK3B;IAKK,kCAAkC,CAAC,WAAW,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAWhF;IAOD,aAAa,CAAC,GAAG,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAE5C;IAEM,cAAc,IAAI,qBAAqB,CAAC,CAAC,UAAU,EAAE,uBAAuB,GAAG,SAAS,CAAC,CAAC,CAKhG;IAOD,mBAAmB,CAAC,EAAE,EAAE,YAAY,GAAG,OAAO,CAAC,QAAQ,GAAG,SAAS,CAAC,CAEnE;IAED,mBAAmB,CAAC,EAAE,EAAE,YAAY,GAAG,OAAO,CAAC,uBAAuB,GAAG,SAAS,CAAC,CAElF;IAED,kBAAkB,CAAC,EAAE,EAAE,YAAY,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAElE;IAED,oBAAoB,CAAC,EAAE,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAEpD;IAED,sBAAsB,CAAC,EAAE,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAEtD;IAED,mBAAmB,CAAC,EAAE,EAAE,YAAY,EAAE,KAAK,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CAEpE;YAEa,gBAAgB;CAoB/B"}