@aztec/prover-client 0.0.1-commit.3100065 → 0.0.1-commit.330febf
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dest/mocks/fixtures.d.ts +1 -1
- package/dest/mocks/fixtures.d.ts.map +1 -1
- package/dest/mocks/fixtures.js +3 -3
- package/dest/mocks/test_context.d.ts +1 -1
- package/dest/mocks/test_context.d.ts.map +1 -1
- package/dest/mocks/test_context.js +2 -1
- package/dest/orchestrator/block-building-helpers.d.ts +1 -1
- package/dest/proving_broker/broker_prover_facade.d.ts +1 -1
- package/dest/proving_broker/broker_prover_facade.d.ts.map +1 -1
- package/dest/proving_broker/broker_prover_facade.js +8 -1
- package/dest/proving_broker/proving_broker.d.ts +2 -1
- package/dest/proving_broker/proving_broker.d.ts.map +1 -1
- package/dest/proving_broker/proving_broker.js +145 -70
- package/dest/proving_broker/proving_broker_database/memory.d.ts +3 -2
- package/dest/proving_broker/proving_broker_database/memory.d.ts.map +1 -1
- package/dest/proving_broker/proving_broker_database/memory.js +4 -1
- package/dest/proving_broker/proving_broker_database/persisted.d.ts +3 -1
- package/dest/proving_broker/proving_broker_database/persisted.d.ts.map +1 -1
- package/dest/proving_broker/proving_broker_database/persisted.js +25 -1
- package/dest/proving_broker/proving_broker_database.d.ts +13 -1
- package/dest/proving_broker/proving_broker_database.d.ts.map +1 -1
- package/dest/proving_broker/rpc.d.ts +13 -4
- package/dest/proving_broker/rpc.d.ts.map +1 -1
- package/dest/proving_broker/rpc.js +14 -3
- package/dest/test/mock_prover.d.ts +1 -1
- package/package.json +15 -15
- package/src/mocks/fixtures.ts +2 -4
- package/src/mocks/test_context.ts +2 -1
- package/src/proving_broker/broker_prover_facade.ts +8 -1
- package/src/proving_broker/proving_broker.ts +138 -69
- package/src/proving_broker/proving_broker_database/memory.ts +6 -2
- package/src/proving_broker/proving_broker_database/persisted.ts +29 -1
- package/src/proving_broker/proving_broker_database.ts +14 -0
- package/src/proving_broker/rpc.ts +16 -0
|
@@ -9,6 +9,18 @@ import { getTelemetryClient } from '@aztec/telemetry-client';
|
|
|
9
9
|
import assert from 'assert';
|
|
10
10
|
import { defaultProverBrokerConfig } from './config.js';
|
|
11
11
|
import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.js';
|
|
12
|
+
function toJobMetadata(job) {
|
|
13
|
+
return {
|
|
14
|
+
id: job.id,
|
|
15
|
+
type: job.type,
|
|
16
|
+
epochNumber: job.epochNumber
|
|
17
|
+
};
|
|
18
|
+
}
|
|
19
|
+
function toSettledRecord(result) {
|
|
20
|
+
return result.status === 'fulfilled' ? {
|
|
21
|
+
status: 'fulfilled'
|
|
22
|
+
} : result;
|
|
23
|
+
}
|
|
12
24
|
/**
|
|
13
25
|
* A broker that manages proof requests and distributes them to workers based on their priority.
|
|
14
26
|
* It takes a backend that is responsible for storing and retrieving proof requests and results.
|
|
@@ -16,11 +28,17 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
|
|
|
16
28
|
database;
|
|
17
29
|
logger;
|
|
18
30
|
queues;
|
|
19
|
-
//
|
|
20
|
-
//
|
|
31
|
+
// Scheduling metadata for every known job (id, type, epochNumber). The large `inputsUri` is NOT held
|
|
32
|
+
// here — it lives in the database and is read on demand when a job is dispatched to an agent.
|
|
21
33
|
jobsCache;
|
|
22
|
-
//
|
|
34
|
+
// Settled status for every settled job. The large fulfilled proof `value` is NOT held here — it lives
|
|
35
|
+
// in the database (read on demand in getProvingJobStatus) and, transiently, in `pendingResults` while
|
|
36
|
+
// its database write is in flight. Small payloads (rejected reason, aborted) are kept inline.
|
|
23
37
|
resultsCache;
|
|
38
|
+
// Transient hold of a fulfilled result's full value, from the moment it settles until its database
|
|
39
|
+
// write commits, so status reads are read-your-writes without retaining every proof. Evicted once the
|
|
40
|
+
// value is durable in the database; kept only if that write failed (preserving the result in memory).
|
|
41
|
+
pendingResults;
|
|
24
42
|
// tracks when each job was enqueued
|
|
25
43
|
enqueuedAt;
|
|
26
44
|
// keeps track of which jobs are currently being processed
|
|
@@ -30,7 +48,8 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
|
|
|
30
48
|
inProgress;
|
|
31
49
|
// keep track of which proving job has been retried
|
|
32
50
|
retries;
|
|
33
|
-
// a map of promises that
|
|
51
|
+
// a map of promises that are resolved when a job settles. Carries no payload (settled status/results
|
|
52
|
+
// are read from resultsCache/pendingResults/DB) so it never pins a proof value in memory.
|
|
34
53
|
promises;
|
|
35
54
|
cleanupPromise;
|
|
36
55
|
msTimeSource;
|
|
@@ -77,6 +96,7 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
|
|
|
77
96
|
};
|
|
78
97
|
this.jobsCache = new Map();
|
|
79
98
|
this.resultsCache = new Map();
|
|
99
|
+
this.pendingResults = new Map();
|
|
80
100
|
this.enqueuedAt = new Map();
|
|
81
101
|
this.inProgress = new Map();
|
|
82
102
|
this.retries = new Map();
|
|
@@ -120,11 +140,11 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
|
|
|
120
140
|
provingJobId: item.id,
|
|
121
141
|
status: result ? result.status : 'pending'
|
|
122
142
|
});
|
|
123
|
-
this.jobsCache.set(item.id, item);
|
|
143
|
+
this.jobsCache.set(item.id, toJobMetadata(item));
|
|
124
144
|
this.promises.set(item.id, promiseWithResolvers());
|
|
125
145
|
if (result) {
|
|
126
|
-
this.promises.get(item.id).resolve(
|
|
127
|
-
this.resultsCache.set(item.id, result);
|
|
146
|
+
this.promises.get(item.id).resolve();
|
|
147
|
+
this.resultsCache.set(item.id, toSettledRecord(result));
|
|
128
148
|
} else {
|
|
129
149
|
this.enqueueJobInternal(item);
|
|
130
150
|
}
|
|
@@ -148,13 +168,13 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
|
|
|
148
168
|
return this.#cancelProvingJob(id);
|
|
149
169
|
}
|
|
150
170
|
getProvingJobStatus(id) {
|
|
151
|
-
return
|
|
171
|
+
return this.#getProvingJobStatus(id);
|
|
152
172
|
}
|
|
153
173
|
getCompletedJobs(ids) {
|
|
154
174
|
return this.#getCompletedJobs(ids);
|
|
155
175
|
}
|
|
156
176
|
getProvingJob(filter) {
|
|
157
|
-
return
|
|
177
|
+
return this.#getProvingJob(filter);
|
|
158
178
|
}
|
|
159
179
|
reportProvingJobSuccess(id, value, filter) {
|
|
160
180
|
return this.#reportProvingJobSuccess(id, value, filter);
|
|
@@ -163,7 +183,7 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
|
|
|
163
183
|
return this.#reportProvingJobError(id, err, retry, filter);
|
|
164
184
|
}
|
|
165
185
|
reportProvingJobProgress(id, startedAt, filter) {
|
|
166
|
-
return
|
|
186
|
+
return this.#reportProvingJobProgress(id, startedAt, filter);
|
|
167
187
|
}
|
|
168
188
|
async replayProvingJob(jobId, type, epochNumber, inputsUri) {
|
|
169
189
|
if (!this.debugReplayEnabled) {
|
|
@@ -184,7 +204,7 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
|
|
|
184
204
|
epochNumber,
|
|
185
205
|
inputsUri
|
|
186
206
|
};
|
|
187
|
-
this.jobsCache.set(jobId, job);
|
|
207
|
+
this.jobsCache.set(jobId, toJobMetadata(job));
|
|
188
208
|
await this.database.addProvingJob(job);
|
|
189
209
|
this.enqueueJobInternal(job);
|
|
190
210
|
return {
|
|
@@ -192,11 +212,19 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
|
|
|
192
212
|
};
|
|
193
213
|
}
|
|
194
214
|
async #enqueueProvingJob(job) {
|
|
195
|
-
//
|
|
196
|
-
|
|
215
|
+
// The status returned to the caller reflects the job's state at the start of this call: `not-found`
|
|
216
|
+
// for a brand-new job (it did not exist yet), or the cached status for one already known. Crucially
|
|
217
|
+
// this must NOT `await` before the synchronous `jobsCache`/`resultsCache` gate below, or the
|
|
218
|
+
// enqueue/revive lock would no longer hold. So the not-found default is set synchronously here and
|
|
219
|
+
// the cached branch computes the real status (which may read the DB) only when it returns.
|
|
220
|
+
let jobStatus = {
|
|
221
|
+
status: 'not-found'
|
|
222
|
+
};
|
|
197
223
|
if (this.jobsCache.has(job.id)) {
|
|
198
224
|
const existing = this.jobsCache.get(job.id);
|
|
199
|
-
|
|
225
|
+
// Identity check is metadata-only: `inputsUri` now lives in the database, not memory, and job ids
|
|
226
|
+
// are content-addressed (same id ⇒ same inputs), so a mismatch shows up in id/type/epochNumber.
|
|
227
|
+
assert.deepStrictEqual(toJobMetadata(job), existing, 'Duplicate proving job ID');
|
|
200
228
|
if (this.resultsCache.get(job.id)?.status === 'aborted') {
|
|
201
229
|
// The producer is re-requesting a job it previously cancelled: revive it rather than
|
|
202
230
|
// returning the cached abort, clearing the aborted state in memory and in the database so the
|
|
@@ -210,23 +238,30 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
|
|
|
210
238
|
// single synchronous span (no await in between), and only then await the database. Because a
|
|
211
239
|
// concurrent re-request can only interleave at that await — by which point `jobsCache` is
|
|
212
240
|
// populated again and the aborted result is gone — it falls into the cached branch and no-ops,
|
|
213
|
-
// so the job is enqueued exactly once. (`cleanUpProvingJobState` also drops the promise,
|
|
214
|
-
//
|
|
241
|
+
// so the job is enqueued exactly once. (`cleanUpProvingJobState` also drops the promise, so
|
|
242
|
+
// `enqueueJobInternal` below mints a fresh one for the retry.) This holds unchanged with the
|
|
243
|
+
// slimmer caches: `jobsCache` (now metadata) and `resultsCache` (now settled status) are still
|
|
244
|
+
// synchronous in-memory maps, so the tear-down + re-set span still contains no await.
|
|
215
245
|
this.logger.info(`Reviving aborted proving job id=${job.id} epochNumber=${job.epochNumber}`, {
|
|
216
246
|
provingJobId: job.id
|
|
217
247
|
});
|
|
218
248
|
this.cleanUpProvingJobState([
|
|
219
249
|
job.id
|
|
220
250
|
]);
|
|
221
|
-
this.jobsCache.set(job.id, job);
|
|
251
|
+
this.jobsCache.set(job.id, toJobMetadata(job));
|
|
222
252
|
await this.database.deleteProvingJobResult(job.id);
|
|
223
|
-
|
|
253
|
+
// The job is re-set in the cache and about to be re-enqueued below: its start status is in-queue.
|
|
254
|
+
jobStatus = {
|
|
255
|
+
status: 'in-queue'
|
|
256
|
+
};
|
|
224
257
|
} else {
|
|
225
258
|
this.logger.warn(`Cached proving job id=${job.id} epochNumber=${job.epochNumber}. Not enqueuing again`, {
|
|
226
259
|
provingJobId: job.id
|
|
227
260
|
});
|
|
228
261
|
this.instrumentation.incCachedJobs(job.type);
|
|
229
|
-
|
|
262
|
+
// Return the job's current status. This reads the fulfilled proof value from the DB when needed;
|
|
263
|
+
// the await is fine here because this branch does not enqueue, so it is outside the lock span.
|
|
264
|
+
return await this.#getProvingJobStatus(job.id);
|
|
230
265
|
}
|
|
231
266
|
}
|
|
232
267
|
if (this.isJobStale(job)) {
|
|
@@ -240,7 +275,7 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
|
|
|
240
275
|
});
|
|
241
276
|
try {
|
|
242
277
|
// do this first so it acts as a "lock". If this job is enqueued again while we're saving it the if at the top will catch it.
|
|
243
|
-
this.jobsCache.set(job.id, job);
|
|
278
|
+
this.jobsCache.set(job.id, toJobMetadata(job));
|
|
244
279
|
await this.database.addProvingJob(job);
|
|
245
280
|
this.enqueueJobInternal(job);
|
|
246
281
|
this.instrumentation.incTotalJobs(job.type);
|
|
@@ -273,11 +308,10 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
|
|
|
273
308
|
// notifies the current waiter. Unlike a completion or failure this is not terminal: re-enqueuing
|
|
274
309
|
// the same job id revives it (see #enqueueProvingJob), so the abort never permanently blocks the
|
|
275
310
|
// proof.
|
|
276
|
-
|
|
311
|
+
this.resultsCache.set(id, {
|
|
277
312
|
status: 'aborted'
|
|
278
|
-
};
|
|
279
|
-
this.
|
|
280
|
-
this.promises.get(id)?.resolve(result);
|
|
313
|
+
});
|
|
314
|
+
this.promises.get(id)?.resolve();
|
|
281
315
|
this.completedJobNotifications.push(id);
|
|
282
316
|
this.instrumentation.incAbortedJobs(job.type);
|
|
283
317
|
try {
|
|
@@ -295,35 +329,49 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
|
|
|
295
329
|
this.jobsCache.delete(id);
|
|
296
330
|
const deferred = this.promises.get(id);
|
|
297
331
|
if (deferred) {
|
|
298
|
-
deferred.resolve(
|
|
299
|
-
status: 'rejected',
|
|
300
|
-
reason: 'Proving job cleaned up'
|
|
301
|
-
});
|
|
332
|
+
deferred.resolve();
|
|
302
333
|
}
|
|
303
334
|
this.promises.delete(id);
|
|
304
335
|
this.resultsCache.delete(id);
|
|
336
|
+
this.pendingResults.delete(id);
|
|
305
337
|
this.inProgress.delete(id);
|
|
306
338
|
this.retries.delete(id);
|
|
307
339
|
this.enqueuedAt.delete(id);
|
|
308
340
|
}
|
|
309
341
|
this.completedJobNotifications = this.completedJobNotifications.filter((id)=>!idsToClean.has(id));
|
|
310
342
|
}
|
|
311
|
-
#getProvingJobStatus(id) {
|
|
312
|
-
const
|
|
313
|
-
if (
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
if
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
343
|
+
async #getProvingJobStatus(id) {
|
|
344
|
+
const settled = this.resultsCache.get(id);
|
|
345
|
+
if (settled) {
|
|
346
|
+
if (settled.status !== 'fulfilled') {
|
|
347
|
+
// rejected/aborted carry their (small) payload inline.
|
|
348
|
+
return settled;
|
|
349
|
+
}
|
|
350
|
+
// A fulfilled job's proof value is not held in memory: serve it from the transient hold if its
|
|
351
|
+
// database write is still in flight, otherwise read it back from the database.
|
|
352
|
+
const full = this.pendingResults.get(id) ?? await this.database.getProvingJobResult(id);
|
|
353
|
+
if (full?.status === 'fulfilled') {
|
|
354
|
+
return full;
|
|
322
355
|
}
|
|
356
|
+
// Settled as fulfilled but the value is nowhere to be found — should not happen (it is held in
|
|
357
|
+
// pendingResults until its DB write commits). Report not-found rather than a torn result.
|
|
358
|
+
this.logger.error(`Fulfilled proving job id=${id} has no retrievable result value`, {
|
|
359
|
+
provingJobId: id
|
|
360
|
+
});
|
|
361
|
+
return {
|
|
362
|
+
status: 'not-found'
|
|
363
|
+
};
|
|
364
|
+
}
|
|
365
|
+
// no result yet, check if we know the item
|
|
366
|
+
const item = this.jobsCache.get(id);
|
|
367
|
+
if (!item) {
|
|
323
368
|
return {
|
|
324
|
-
status:
|
|
369
|
+
status: 'not-found'
|
|
325
370
|
};
|
|
326
371
|
}
|
|
372
|
+
return {
|
|
373
|
+
status: this.inProgress.has(id) ? 'in-progress' : 'in-queue'
|
|
374
|
+
};
|
|
327
375
|
}
|
|
328
376
|
#getCompletedJobs(ids) {
|
|
329
377
|
const completedJobs = ids.filter((id)=>this.resultsCache.has(id));
|
|
@@ -331,7 +379,7 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
|
|
|
331
379
|
this.completedJobNotifications = [];
|
|
332
380
|
return Promise.resolve(notifications.concat(completedJobs));
|
|
333
381
|
}
|
|
334
|
-
#getProvingJob(filter = {
|
|
382
|
+
async #getProvingJob(filter = {
|
|
335
383
|
allowList: []
|
|
336
384
|
}) {
|
|
337
385
|
const allowedProofs = Array.isArray(filter.allowList) && filter.allowList.length > 0 ? [
|
|
@@ -346,20 +394,43 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
|
|
|
346
394
|
// this can happen if the broker crashes and restarts
|
|
347
395
|
// it's possible agents will report progress or results for jobs that are in the queue (after the restart)
|
|
348
396
|
while(enqueuedJob = queue.getImmediate()){
|
|
349
|
-
const
|
|
350
|
-
if (
|
|
397
|
+
const meta = this.jobsCache.get(enqueuedJob.id);
|
|
398
|
+
if (meta && !this.inProgress.has(enqueuedJob.id) && !this.resultsCache.has(enqueuedJob.id)) {
|
|
351
399
|
const time = this.msTimeSource();
|
|
352
|
-
|
|
353
|
-
|
|
400
|
+
// Claim the job synchronously (before the await below) so a concurrent dispatch can't re-pick
|
|
401
|
+
// it. The same id can sit in a queue more than once (an abort leaves a stale entry that a
|
|
402
|
+
// later revive re-enqueues alongside), so this in-progress claim — not the queue pop — is what
|
|
403
|
+
// dedups those copies across concurrently-polling agents. It must be set before the await.
|
|
404
|
+
this.inProgress.set(meta.id, {
|
|
405
|
+
id: meta.id,
|
|
354
406
|
startedAt: time,
|
|
355
407
|
lastUpdatedAt: time
|
|
356
408
|
});
|
|
357
|
-
|
|
409
|
+
// The large inputs are not kept in memory; read them from the database. They were durably
|
|
410
|
+
// persisted (addProvingJob's write commits) before the job became dispatchable, so this hits.
|
|
411
|
+
const inputsUri = await this.database.getProvingJobInputs(meta.id);
|
|
412
|
+
if (!inputsUri) {
|
|
413
|
+
// The job was cleaned up (or its inputs lost) after we claimed it — release the claim and
|
|
414
|
+
// keep draining the queue for another candidate. Nothing else was mutated yet, so there is
|
|
415
|
+
// no wait metric or enqueued-at timer to unwind.
|
|
416
|
+
this.inProgress.delete(meta.id);
|
|
417
|
+
this.logger.warn(`No inputs found for proving job id=${meta.id}; skipping dispatch`, {
|
|
418
|
+
provingJobId: meta.id
|
|
419
|
+
});
|
|
420
|
+
continue;
|
|
421
|
+
}
|
|
422
|
+
// The dispatch is now committed: record the queue wait and clear the enqueued-at timer.
|
|
423
|
+
const enqueuedAt = this.enqueuedAt.get(meta.id);
|
|
358
424
|
if (enqueuedAt) {
|
|
359
|
-
this.instrumentation.recordJobWait(
|
|
360
|
-
|
|
361
|
-
this.enqueuedAt.delete(job.id);
|
|
425
|
+
this.instrumentation.recordJobWait(meta.type, enqueuedAt);
|
|
426
|
+
this.enqueuedAt.delete(meta.id);
|
|
362
427
|
}
|
|
428
|
+
const job = {
|
|
429
|
+
id: meta.id,
|
|
430
|
+
type: meta.type,
|
|
431
|
+
epochNumber: meta.epochNumber,
|
|
432
|
+
inputsUri
|
|
433
|
+
};
|
|
363
434
|
return {
|
|
364
435
|
job,
|
|
365
436
|
time
|
|
@@ -407,14 +478,13 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
|
|
|
407
478
|
this.logger.info(`Marking proving job as failed id=${id} type=${ProvingRequestType[item.type]} totalAttempts=${retries + 1} err=${err}`, {
|
|
408
479
|
provingJobId: id
|
|
409
480
|
});
|
|
410
|
-
// save the
|
|
411
|
-
//
|
|
412
|
-
|
|
481
|
+
// save the (small) rejection reason to the cache and notify clients of the job status. The reason
|
|
482
|
+
// is kept in memory, so status reads work even if the database write below fails.
|
|
483
|
+
this.resultsCache.set(id, {
|
|
413
484
|
status: 'rejected',
|
|
414
485
|
reason: String(err)
|
|
415
|
-
};
|
|
416
|
-
this.
|
|
417
|
-
this.promises.get(id).resolve(result);
|
|
486
|
+
});
|
|
487
|
+
this.promises.get(id).resolve();
|
|
418
488
|
this.completedJobNotifications.push(id);
|
|
419
489
|
this.instrumentation.incRejectedJobs(item.type);
|
|
420
490
|
if (info) {
|
|
@@ -431,19 +501,19 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
|
|
|
431
501
|
}
|
|
432
502
|
return this.#getProvingJob(filter);
|
|
433
503
|
}
|
|
434
|
-
#reportProvingJobProgress(id, startedAt, filter) {
|
|
504
|
+
async #reportProvingJobProgress(id, startedAt, filter) {
|
|
435
505
|
const job = this.jobsCache.get(id);
|
|
436
506
|
if (!job) {
|
|
437
507
|
this.logger.warn(`Proving job id=${id} does not exist`, {
|
|
438
508
|
provingJobId: id
|
|
439
509
|
});
|
|
440
|
-
return this.#getProvingJob(filter);
|
|
510
|
+
return await this.#getProvingJob(filter);
|
|
441
511
|
}
|
|
442
512
|
if (this.resultsCache.has(id)) {
|
|
443
513
|
this.logger.warn(`Proving job id=${id} has already been completed`, {
|
|
444
514
|
provingJobId: id
|
|
445
515
|
});
|
|
446
|
-
return this.#getProvingJob(filter);
|
|
516
|
+
return await this.#getProvingJob(filter);
|
|
447
517
|
}
|
|
448
518
|
const metadata = this.inProgress.get(id);
|
|
449
519
|
const now = this.msTimeSource();
|
|
@@ -477,7 +547,7 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
|
|
|
477
547
|
this.logger.warn(`Proving job id=${id} type=${ProvingRequestType[job.type]} already being worked on by another agent. Sending new one`, {
|
|
478
548
|
provingJobId: id
|
|
479
549
|
});
|
|
480
|
-
return this.#getProvingJob(filter);
|
|
550
|
+
return await this.#getProvingJob(filter);
|
|
481
551
|
}
|
|
482
552
|
async #reportProvingJobSuccess(id, value, filter) {
|
|
483
553
|
const info = this.inProgress.get(id);
|
|
@@ -505,15 +575,17 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
|
|
|
505
575
|
this.logger.info(`Proving job complete id=${id} type=${ProvingRequestType[item.type]} totalAttempts=${retries + 1}`, {
|
|
506
576
|
provingJobId: id
|
|
507
577
|
});
|
|
508
|
-
//
|
|
509
|
-
//
|
|
510
|
-
//
|
|
511
|
-
|
|
578
|
+
// Mark settled synchronously (the gate that makes concurrent settles no-op, unchanged from before),
|
|
579
|
+
// holding only the status in resultsCache. The large proof value is kept transiently in
|
|
580
|
+
// pendingResults so status reads are read-your-writes until the database write below commits.
|
|
581
|
+
this.resultsCache.set(id, {
|
|
582
|
+
status: 'fulfilled'
|
|
583
|
+
});
|
|
584
|
+
this.pendingResults.set(id, {
|
|
512
585
|
status: 'fulfilled',
|
|
513
586
|
value
|
|
514
|
-
};
|
|
515
|
-
this.
|
|
516
|
-
this.promises.get(id).resolve(result);
|
|
587
|
+
});
|
|
588
|
+
this.promises.get(id).resolve();
|
|
517
589
|
this.completedJobNotifications.push(id);
|
|
518
590
|
this.instrumentation.incResolvedJobs(item.type);
|
|
519
591
|
if (info) {
|
|
@@ -522,7 +594,11 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
|
|
|
522
594
|
}
|
|
523
595
|
try {
|
|
524
596
|
await this.database.setProvingJobResult(id, value);
|
|
597
|
+
// The value is durable in the database now; drop the transient in-memory copy.
|
|
598
|
+
this.pendingResults.delete(id);
|
|
525
599
|
} catch (saveErr) {
|
|
600
|
+
// Keep the value in pendingResults so status/result reads still succeed despite the failed write —
|
|
601
|
+
// this preserves the previous "works even if the database breaks" behaviour for this edge.
|
|
526
602
|
this.logger.error(`Failed to save proving job result id=${id}`, saveErr, {
|
|
527
603
|
provingJobId: id
|
|
528
604
|
});
|
|
@@ -580,12 +656,11 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
|
|
|
580
656
|
this.logger.error(`Proving job id=${id} timed out after ${retries + 1} attempts. Marking as failed.`, {
|
|
581
657
|
provingJobId: id
|
|
582
658
|
});
|
|
583
|
-
|
|
659
|
+
this.resultsCache.set(id, {
|
|
584
660
|
status: 'rejected',
|
|
585
661
|
reason: 'Timed out'
|
|
586
|
-
};
|
|
587
|
-
this.
|
|
588
|
-
this.promises.get(id)?.resolve(result);
|
|
662
|
+
});
|
|
663
|
+
this.promises.get(id)?.resolve();
|
|
589
664
|
this.completedJobNotifications.push(id);
|
|
590
665
|
this.instrumentation.incRejectedJobs(item.type);
|
|
591
666
|
}
|
|
@@ -5,7 +5,8 @@ export declare class InMemoryBrokerDatabase implements ProvingBrokerDatabase {
|
|
|
5
5
|
private jobs;
|
|
6
6
|
private results;
|
|
7
7
|
getProvingJob(id: ProvingJobId): ProvingJob | undefined;
|
|
8
|
-
|
|
8
|
+
getProvingJobInputs(id: ProvingJobId): Promise<ProofUri | undefined>;
|
|
9
|
+
getProvingJobResult(id: ProvingJobId): Promise<ProvingJobSettledResult | undefined>;
|
|
9
10
|
addProvingJob(job: ProvingJob): Promise<void>;
|
|
10
11
|
setProvingJobResult(id: ProvingJobId, value: ProofUri): Promise<void>;
|
|
11
12
|
setProvingJobError(id: ProvingJobId, reason: string): Promise<void>;
|
|
@@ -16,4 +17,4 @@ export declare class InMemoryBrokerDatabase implements ProvingBrokerDatabase {
|
|
|
16
17
|
allProvingJobs(): AsyncIterableIterator<[ProvingJob, ProvingJobSettledResult | undefined]>;
|
|
17
18
|
close(): Promise<void>;
|
|
18
19
|
}
|
|
19
|
-
//# sourceMappingURL=data:application/json;base64,
|
|
20
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoibWVtb3J5LmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi8uLi9zcmMvcHJvdmluZ19icm9rZXIvcHJvdmluZ19icm9rZXJfZGF0YWJhc2UvbWVtb3J5LnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLE9BQU8sRUFBRSxXQUFXLEVBQUUsTUFBTSxpQ0FBaUMsQ0FBQztBQUM5RCxPQUFPLEVBQ0wsS0FBSyxRQUFRLEVBQ2IsS0FBSyxVQUFVLEVBQ2YsS0FBSyxZQUFZLEVBQ2pCLEtBQUssdUJBQXVCLEVBRTdCLE1BQU0saUNBQWlDLENBQUM7QUFFekMsT0FBTyxLQUFLLEVBQUUscUJBQXFCLEVBQUUsTUFBTSwrQkFBK0IsQ0FBQztBQUUzRSxxQkFBYSxzQkFBdUIsWUFBVyxxQkFBcUI7SUFDbEUsT0FBTyxDQUFDLElBQUksQ0FBdUM7SUFDbkQsT0FBTyxDQUFDLE9BQU8sQ0FBb0Q7SUFFbkUsYUFBYSxDQUFDLEVBQUUsRUFBRSxZQUFZLEdBQUcsVUFBVSxHQUFHLFNBQVMsQ0FFdEQ7SUFFRCxtQkFBbUIsQ0FBQyxFQUFFLEVBQUUsWUFBWSxHQUFHLE9BQU8sQ0FBQyxRQUFRLEdBQUcsU0FBUyxDQUFDLENBRW5FO0lBRUQsbUJBQW1CLENBQUMsRUFBRSxFQUFFLFlBQVksR0FBRyxPQUFPLENBQUMsdUJBQXVCLEdBQUcsU0FBUyxDQUFDLENBRWxGO0lBRUQsYUFBYSxDQUFDLEdBQUcsRUFBRSxVQUFVLEdBQUcsT0FBTyxDQUFDLElBQUksQ0FBQyxDQUc1QztJQUVELG1CQUFtQixDQUFDLEVBQUUsRUFBRSxZQUFZLEVBQUUsS0FBSyxFQUFFLFFBQVEsR0FBRyxPQUFPLENBQUMsSUFBSSxDQUFDLENBR3BFO0lBRUQsa0JBQWtCLENBQUMsRUFBRSxFQUFFLFlBQVksRUFBRSxNQUFNLEVBQUUsTUFBTSxHQUFHLE9BQU8sQ0FBQyxJQUFJLENBQUMsQ0FHbEU7SUFFRCxvQkFBb0IsQ0FBQyxFQUFFLEVBQUUsWUFBWSxHQUFHLE9BQU8sQ0FBQyxJQUFJLENBQUMsQ0FHcEQ7SUFFRCxzQkFBc0IsQ0FBQyxFQUFFLEVBQUUsWUFBWSxHQUFHLE9BQU8sQ0FBQyxJQUFJLENBQUMsQ0FHdEQ7SUFFRCxpQkFBaUIsQ0FBQyxHQUFHLEVBQUUsWUFBWSxFQUFFLEdBQUcsT0FBTyxDQUFDLElBQUksQ0FBQyxDQU1wRDtJQUVELGtDQUFrQyxDQUFDLFdBQVcsRUFBRSxXQUFXLEdBQUcsT0FBTyxDQUFDLElBQUksQ0FBQyxDQU0xRTtJQUVNLGNBQWMsSUFBSSxxQkFBcUIsQ0FBQyxDQUFDLFVBQVUsRUFBRSx1QkFBdUIsR0FBRyxTQUFTLENBQUMsQ0FBQyxDQUloRztJQUVELEtBQUssSUFBSSxPQUFPLENBQUMsSUFBSSxDQUFDLENBRXJCO0NBQ0YifQ==
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"memory.d.ts","sourceRoot":"","sources":["../../../src/proving_broker/proving_broker_database/memory.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iCAAiC,CAAC;AAC9D,OAAO,EACL,KAAK,QAAQ,EACb,KAAK,UAAU,EACf,KAAK,YAAY,EACjB,KAAK,uBAAuB,EAE7B,MAAM,iCAAiC,CAAC;AAEzC,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,+BAA+B,CAAC;AAE3E,qBAAa,sBAAuB,YAAW,qBAAqB;IAClE,OAAO,CAAC,IAAI,CAAuC;IACnD,OAAO,CAAC,OAAO,CAAoD;IAEnE,aAAa,CAAC,EAAE,EAAE,YAAY,GAAG,UAAU,GAAG,SAAS,CAEtD;IAED,mBAAmB,CAAC,EAAE,EAAE,YAAY,GAAG,uBAAuB,GAAG,SAAS,
|
|
1
|
+
{"version":3,"file":"memory.d.ts","sourceRoot":"","sources":["../../../src/proving_broker/proving_broker_database/memory.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iCAAiC,CAAC;AAC9D,OAAO,EACL,KAAK,QAAQ,EACb,KAAK,UAAU,EACf,KAAK,YAAY,EACjB,KAAK,uBAAuB,EAE7B,MAAM,iCAAiC,CAAC;AAEzC,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,+BAA+B,CAAC;AAE3E,qBAAa,sBAAuB,YAAW,qBAAqB;IAClE,OAAO,CAAC,IAAI,CAAuC;IACnD,OAAO,CAAC,OAAO,CAAoD;IAEnE,aAAa,CAAC,EAAE,EAAE,YAAY,GAAG,UAAU,GAAG,SAAS,CAEtD;IAED,mBAAmB,CAAC,EAAE,EAAE,YAAY,GAAG,OAAO,CAAC,QAAQ,GAAG,SAAS,CAAC,CAEnE;IAED,mBAAmB,CAAC,EAAE,EAAE,YAAY,GAAG,OAAO,CAAC,uBAAuB,GAAG,SAAS,CAAC,CAElF;IAED,aAAa,CAAC,GAAG,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAG5C;IAED,mBAAmB,CAAC,EAAE,EAAE,YAAY,EAAE,KAAK,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CAGpE;IAED,kBAAkB,CAAC,EAAE,EAAE,YAAY,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAGlE;IAED,oBAAoB,CAAC,EAAE,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAGpD;IAED,sBAAsB,CAAC,EAAE,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAGtD;IAED,iBAAiB,CAAC,GAAG,EAAE,YAAY,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAMpD;IAED,kCAAkC,CAAC,WAAW,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAM1E;IAEM,cAAc,IAAI,qBAAqB,CAAC,CAAC,UAAU,EAAE,uBAAuB,GAAG,SAAS,CAAC,CAAC,CAIhG;IAED,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAErB;CACF"}
|
|
@@ -5,8 +5,11 @@ export class InMemoryBrokerDatabase {
|
|
|
5
5
|
getProvingJob(id) {
|
|
6
6
|
return this.jobs.get(id);
|
|
7
7
|
}
|
|
8
|
+
getProvingJobInputs(id) {
|
|
9
|
+
return Promise.resolve(this.jobs.get(id)?.inputsUri);
|
|
10
|
+
}
|
|
8
11
|
getProvingJobResult(id) {
|
|
9
|
-
return this.results.get(id);
|
|
12
|
+
return Promise.resolve(this.results.get(id));
|
|
10
13
|
}
|
|
11
14
|
addProvingJob(job) {
|
|
12
15
|
this.jobs.set(job.id, job);
|
|
@@ -28,6 +28,8 @@ export declare class KVBrokerDatabase implements ProvingBrokerDatabase {
|
|
|
28
28
|
deleteAllProvingJobsOlderThanEpoch(epochNumber: EpochNumber): Promise<void>;
|
|
29
29
|
addProvingJob(job: ProvingJob): Promise<void>;
|
|
30
30
|
allProvingJobs(): AsyncIterableIterator<[ProvingJob, ProvingJobSettledResult | undefined]>;
|
|
31
|
+
getProvingJobInputs(id: ProvingJobId): Promise<ProofUri | undefined>;
|
|
32
|
+
getProvingJobResult(id: ProvingJobId): Promise<ProvingJobSettledResult | undefined>;
|
|
31
33
|
setProvingJobError(id: ProvingJobId, reason: string): Promise<void>;
|
|
32
34
|
setProvingJobAborted(id: ProvingJobId): Promise<void>;
|
|
33
35
|
deleteProvingJobResult(id: ProvingJobId): Promise<void>;
|
|
@@ -35,4 +37,4 @@ export declare class KVBrokerDatabase implements ProvingBrokerDatabase {
|
|
|
35
37
|
private getEpochDatabase;
|
|
36
38
|
}
|
|
37
39
|
export {};
|
|
38
|
-
//# sourceMappingURL=data:application/json;base64,
|
|
40
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoicGVyc2lzdGVkLmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi8uLi9zcmMvcHJvdmluZ19icm9rZXIvcHJvdmluZ19icm9rZXJfZGF0YWJhc2UvcGVyc2lzdGVkLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLE9BQU8sRUFBRSxXQUFXLEVBQUUsTUFBTSxpQ0FBaUMsQ0FBQztBQUU5RCxPQUFPLEVBQUUsS0FBSyxNQUFNLEVBQWdCLE1BQU0sdUJBQXVCLENBQUM7QUFJbEUsT0FBTyxFQUNMLEtBQUssUUFBUSxFQUNiLFVBQVUsRUFDVixLQUFLLFlBQVksRUFDakIsdUJBQXVCLEVBRXhCLE1BQU0saUNBQWlDLENBQUM7QUFDekMsT0FBTyxFQUdMLEtBQUssZUFBZSxFQUNwQixLQUFLLE1BQU0sRUFHWixNQUFNLHlCQUF5QixDQUFDO0FBS2pDLE9BQU8sS0FBSyxFQUFFLGtCQUFrQixFQUFFLE1BQU0sY0FBYyxDQUFDO0FBQ3ZELE9BQU8sS0FBSyxFQUFFLHFCQUFxQixFQUFFLE1BQU0sK0JBQStCLENBQUM7QUFnRjNFOzs7Ozs7R0FNRztBQUNILEtBQUssV0FBVyxHQUFHLFVBQVUsR0FBRyxDQUFDLFlBQVksRUFBRSx1QkFBdUIsR0FBRyxTQUFTLENBQUMsQ0FBQztBQUVwRixxQkFBYSxnQkFBaUIsWUFBVyxxQkFBcUI7SUFRMUQsT0FBTyxDQUFDLE1BQU07SUFDZCxPQUFPLENBQUMsTUFBTTtJQUVkLE9BQU8sQ0FBQyxNQUFNO0lBVmhCLE9BQU8sQ0FBQyxPQUFPLENBQWM7SUFFN0IsT0FBTyxDQUFDLFVBQVUsQ0FBa0M7SUFFcEQsU0FBZ0IsTUFBTSxFQUFFLE1BQU0sQ0FBQztJQUUvQixPQUFPLGVBc0JOO0lBR1ksWUFBWSxDQUFDLEtBQUssRUFBRSxLQUFLLENBQUMsV0FBVyxDQUFDLEVBQUUsV0FBVyxFQUFFLE1BQU0saUJBVXZFO1lBRWEsWUFBWTtJQVUxQixPQUFvQixHQUFHLENBQ3JCLE1BQU0sRUFBRSxrQkFBa0IsRUFDMUIsTUFBTSxHQUFFLGVBQXNDLEVBQzlDLE1BQU0sU0FBd0QsNkJBOEIvRDtJQUVELE9BQU8sQ0FBQyxLQUFLO0lBSVAsS0FBSyxJQUFJLE9BQU8sQ0FBQyxJQUFJLENBQUMsQ0FLM0I7SUFLSyxrQ0FBa0MsQ0FBQyxXQUFXLEVBQUUsV0FBVyxHQUFHLE9BQU8sQ0FBQyxJQUFJLENBQUMsQ0FXaEY7SUFPRCxhQUFhLENBQUMsR0FBRyxFQUFFLFVBQVUsR0FBRyxPQUFPLENBQUMsSUFBSSxDQUFDLENBRTVDO0lBRU0sY0FBYyxJQUFJLHFCQUFxQixDQUFDLENBQUMsVUFBVSxFQUFFLHVCQUF1QixHQUFHLFNBQVMsQ0FBQyxDQUFDLENBS2hHO0lBT0QsbUJBQW1CLENBQUMsRUFBRSxFQUFFLFlBQVksR0FBRyxPQUFPLENBQUMsUUFBUSxHQUFHLFNBQVMsQ0FBQyxDQUVuRTtJQUVELG1CQUFtQixDQUFDLEVBQUUsRUFBRSxZQUFZLEdBQUcsT0FBTyxDQUFDLHVCQUF1QixHQUFHLFNBQVMsQ0FBQyxDQUVsRjtJQUVELGtCQUFrQixDQUFDLEVBQUUsRUFBRSxZQUFZLEVBQUUsTUFBTSxFQUFFLE1BQU0sR0FBRyxPQUFPLENBQUMsSUFBSSxDQUFDLENBRWxFO0lBRUQsb0JBQW9CLENBQUMsRUFBRSxFQUFFLFlBQVksR0FBRyxPQUFPLENBQUMsSUFBSSxDQUFDLENBRXBEO0lBRUQsc0JBQXNCLENBQUMsRUFBRSxFQUFFLFlBQVksR0FBRyxPQUFPLENBQUMsSUFBSSxDQUFDLENBRXREO0lBRUQsbUJBQW1CLENBQUMsRUFBRSxFQUFFLFlBQVksRUFBRSxLQUFLLEVBQUUsUUFBUSxHQUFHLE9BQU8sQ0FBQyxJQUFJLENBQUMsQ0FFcEU7WUFFYSxnQkFBZ0I7Q0FvQi9CIn0=
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"persisted.d.ts","sourceRoot":"","sources":["../../../src/proving_broker/proving_broker_database/persisted.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iCAAiC,CAAC;AAE9D,OAAO,EAAE,KAAK,MAAM,EAAgB,MAAM,uBAAuB,CAAC;AAIlE,OAAO,EACL,KAAK,QAAQ,EACb,UAAU,EACV,KAAK,YAAY,EACjB,uBAAuB,EAExB,MAAM,iCAAiC,CAAC;AACzC,OAAO,EAGL,KAAK,eAAe,EACpB,KAAK,MAAM,EAGZ,MAAM,yBAAyB,CAAC;AAKjC,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAC;AACvD,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,+BAA+B,CAAC;
|
|
1
|
+
{"version":3,"file":"persisted.d.ts","sourceRoot":"","sources":["../../../src/proving_broker/proving_broker_database/persisted.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iCAAiC,CAAC;AAE9D,OAAO,EAAE,KAAK,MAAM,EAAgB,MAAM,uBAAuB,CAAC;AAIlE,OAAO,EACL,KAAK,QAAQ,EACb,UAAU,EACV,KAAK,YAAY,EACjB,uBAAuB,EAExB,MAAM,iCAAiC,CAAC;AACzC,OAAO,EAGL,KAAK,eAAe,EACpB,KAAK,MAAM,EAGZ,MAAM,yBAAyB,CAAC;AAKjC,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAC;AACvD,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,+BAA+B,CAAC;AAgF3E;;;;;;GAMG;AACH,KAAK,WAAW,GAAG,UAAU,GAAG,CAAC,YAAY,EAAE,uBAAuB,GAAG,SAAS,CAAC,CAAC;AAEpF,qBAAa,gBAAiB,YAAW,qBAAqB;IAQ1D,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,MAAM;IAEd,OAAO,CAAC,MAAM;IAVhB,OAAO,CAAC,OAAO,CAAc;IAE7B,OAAO,CAAC,UAAU,CAAkC;IAEpD,SAAgB,MAAM,EAAE,MAAM,CAAC;IAE/B,OAAO,eAsBN;IAGY,YAAY,CAAC,KAAK,EAAE,KAAK,CAAC,WAAW,CAAC,EAAE,WAAW,EAAE,MAAM,iBAUvE;YAEa,YAAY;IAU1B,OAAoB,GAAG,CACrB,MAAM,EAAE,kBAAkB,EAC1B,MAAM,GAAE,eAAsC,EAC9C,MAAM,SAAwD,6BA8B/D;IAED,OAAO,CAAC,KAAK;IAIP,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAK3B;IAKK,kCAAkC,CAAC,WAAW,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAWhF;IAOD,aAAa,CAAC,GAAG,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAE5C;IAEM,cAAc,IAAI,qBAAqB,CAAC,CAAC,UAAU,EAAE,uBAAuB,GAAG,SAAS,CAAC,CAAC,CAKhG;IAOD,mBAAmB,CAAC,EAAE,EAAE,YAAY,GAAG,OAAO,CAAC,QAAQ,GAAG,SAAS,CAAC,CAEnE;IAED,mBAAmB,CAAC,EAAE,EAAE,YAAY,GAAG,OAAO,CAAC,uBAAuB,GAAG,SAAS,CAAC,CAElF;IAED,kBAAkB,CAAC,EAAE,EAAE,YAAY,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAElE;IAED,oBAAoB,CAAC,EAAE,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAEpD;IAED,sBAAsB,CAAC,EAAE,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAEtD;IAED,mBAAmB,CAAC,EAAE,EAAE,YAAY,EAAE,KAAK,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CAEpE;YAEa,gBAAgB;CAoB/B"}
|
|
@@ -419,6 +419,14 @@ class SingleEpochDatabase {
|
|
|
419
419
|
];
|
|
420
420
|
}
|
|
421
421
|
}
|
|
422
|
+
async getProvingJobInputs(id) {
|
|
423
|
+
const jobStr = await this.jobs.getAsync(id);
|
|
424
|
+
return jobStr ? jsonParseWithSchema(jobStr, ProvingJob).inputsUri : undefined;
|
|
425
|
+
}
|
|
426
|
+
async getProvingJobResult(id) {
|
|
427
|
+
const resultStr = await this.jobResults.getAsync(id);
|
|
428
|
+
return resultStr ? jsonParseWithSchema(resultStr, ProvingJobSettledResult) : undefined;
|
|
429
|
+
}
|
|
422
430
|
async setProvingJobError(id, reason) {
|
|
423
431
|
const result = {
|
|
424
432
|
status: 'rejected',
|
|
@@ -541,8 +549,13 @@ export class KVBrokerDatabase {
|
|
|
541
549
|
this.epochs.delete(old);
|
|
542
550
|
}
|
|
543
551
|
}
|
|
552
|
+
// Key jobs by the id-derived epoch, the same epoch results are written under (setProvingJobResult et
|
|
553
|
+
// al.). This keeps a job's inputs and its result co-located in one epoch database so both can be read
|
|
554
|
+
// back with a single keyed lookup (getProvingJobInputs/getProvingJobResult) rather than a scan. The id
|
|
555
|
+
// embeds the epoch and equals `job.epochNumber` at every construction site, so this matches the prior
|
|
556
|
+
// storage epoch while making the id the single source of truth for placement.
|
|
544
557
|
addProvingJob(job) {
|
|
545
|
-
return this.batchQueue.put(job, job.
|
|
558
|
+
return this.batchQueue.put(job, getEpochFromProvingJobId(job.id));
|
|
546
559
|
}
|
|
547
560
|
async *allProvingJobs() {
|
|
548
561
|
const iterators = Array.from(this.epochs.values()).map((x)=>x.allProvingJobs());
|
|
@@ -550,6 +563,17 @@ export class KVBrokerDatabase {
|
|
|
550
563
|
yield* it;
|
|
551
564
|
}
|
|
552
565
|
}
|
|
566
|
+
// A job id is `${epochNumber}:${type}:${inputsHash}`, so its epoch is recoverable directly and maps to
|
|
567
|
+
// exactly one epoch database. Results are already written under this same id-derived epoch (see
|
|
568
|
+
// setProvingJobResult et al.), and a job's `epochNumber` equals its id-epoch at every construction site,
|
|
569
|
+
// so inputs live there too. Use `epochs.get` (not `getEpochDatabase`, which would create the store on a
|
|
570
|
+
// miss) so a read for an unknown id has no side effects.
|
|
571
|
+
getProvingJobInputs(id) {
|
|
572
|
+
return this.epochs.get(getEpochFromProvingJobId(id))?.getProvingJobInputs(id) ?? Promise.resolve(undefined);
|
|
573
|
+
}
|
|
574
|
+
getProvingJobResult(id) {
|
|
575
|
+
return this.epochs.get(getEpochFromProvingJobId(id))?.getProvingJobResult(id) ?? Promise.resolve(undefined);
|
|
576
|
+
}
|
|
553
577
|
setProvingJobError(id, reason) {
|
|
554
578
|
return this.batchQueue.put([
|
|
555
579
|
id,
|
|
@@ -18,6 +18,18 @@ export interface ProvingBrokerDatabase {
|
|
|
18
18
|
* Returns an iterator over all saved proving jobs
|
|
19
19
|
*/
|
|
20
20
|
allProvingJobs(): AsyncIterableIterator<[ProvingJob, ProvingJobSettledResult | undefined]>;
|
|
21
|
+
/**
|
|
22
|
+
* Reads a single job's inputs URI by id, or undefined if the job is unknown. The broker keeps only
|
|
23
|
+
* job metadata in memory and reads the (large) inputs from here on demand when dispatching to an agent.
|
|
24
|
+
* @param id - The ID of the proof request
|
|
25
|
+
*/
|
|
26
|
+
getProvingJobInputs(id: ProvingJobId): Promise<ProofUri | undefined>;
|
|
27
|
+
/**
|
|
28
|
+
* Reads a single job's settled result by id, or undefined if not settled. The broker keeps only the
|
|
29
|
+
* settled status in memory and reads the (large) fulfilled proof from here on demand.
|
|
30
|
+
* @param id - The ID of the proof request
|
|
31
|
+
*/
|
|
32
|
+
getProvingJobResult(id: ProvingJobId): Promise<ProvingJobSettledResult | undefined>;
|
|
21
33
|
/**
|
|
22
34
|
* Saves the result of a proof request
|
|
23
35
|
* @param id - The ID of the proof request to save the result for
|
|
@@ -51,4 +63,4 @@ export interface ProvingBrokerDatabase {
|
|
|
51
63
|
*/
|
|
52
64
|
close(): Promise<void>;
|
|
53
65
|
}
|
|
54
|
-
//# sourceMappingURL=data:application/json;base64,
|
|
66
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoicHJvdmluZ19icm9rZXJfZGF0YWJhc2UuZC50cyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uL3NyYy9wcm92aW5nX2Jyb2tlci9wcm92aW5nX2Jyb2tlcl9kYXRhYmFzZS50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSxPQUFPLEVBQUUsV0FBVyxFQUFFLE1BQU0saUNBQWlDLENBQUM7QUFDOUQsT0FBTyxLQUFLLEVBQUUsUUFBUSxFQUFFLFVBQVUsRUFBRSxZQUFZLEVBQUUsdUJBQXVCLEVBQUUsTUFBTSxpQ0FBaUMsQ0FBQztBQUVuSDs7R0FFRztBQUNILE1BQU0sV0FBVyxxQkFBcUI7SUFDcEM7OztPQUdHO0lBQ0gsYUFBYSxDQUFDLEdBQUcsRUFBRSxVQUFVLEdBQUcsT0FBTyxDQUFDLElBQUksQ0FBQyxDQUFDO0lBRTlDOzs7T0FHRztJQUNILGtDQUFrQyxDQUFDLFdBQVcsRUFBRSxXQUFXLEdBQUcsT0FBTyxDQUFDLElBQUksQ0FBQyxDQUFDO0lBRTVFOztPQUVHO0lBQ0gsY0FBYyxJQUFJLHFCQUFxQixDQUFDLENBQUMsVUFBVSxFQUFFLHVCQUF1QixHQUFHLFNBQVMsQ0FBQyxDQUFDLENBQUM7SUFFM0Y7Ozs7T0FJRztJQUNILG1CQUFtQixDQUFDLEVBQUUsRUFBRSxZQUFZLEdBQUcsT0FBTyxDQUFDLFFBQVEsR0FBRyxTQUFTLENBQUMsQ0FBQztJQUVyRTs7OztPQUlHO0lBQ0gsbUJBQW1CLENBQUMsRUFBRSxFQUFFLFlBQVksR0FBRyxPQUFPLENBQUMsdUJBQXVCLEdBQUcsU0FBUyxDQUFDLENBQUM7SUFFcEY7Ozs7O09BS0c7SUFDSCxtQkFBbUIsQ0FBQyxFQUFFLEVBQUUsWUFBWSxFQUFFLEtBQUssRUFBRSxRQUFRLEdBQUcsT0FBTyxDQUFDLElBQUksQ0FBQyxDQUFDO0lBRXRFOzs7OztPQUtHO0lBQ0gsa0JBQWtCLENBQUMsRUFBRSxFQUFFLFlBQVksRUFBRSxHQUFHLEVBQUUsTUFBTSxHQUFHLE9BQU8sQ0FBQyxJQUFJLENBQUMsQ0FBQztJQUVqRTs7Ozs7T0FLRztJQUNILG9CQUFvQixDQUFDLEVBQUUsRUFBRSxZQUFZLEdBQUcsT0FBTyxDQUFDLElBQUksQ0FBQyxDQUFDO0lBRXREOzs7OztPQUtHO0lBQ0gsc0JBQXNCLENBQUMsRUFBRSxFQUFFLFlBQVksR0FBRyxPQUFPLENBQUMsSUFBSSxDQUFDLENBQUM7SUFFeEQ7O09BRUc7SUFDSCxLQUFLLElBQUksT0FBTyxDQUFDLElBQUksQ0FBQyxDQUFDO0NBQ3hCIn0=
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"proving_broker_database.d.ts","sourceRoot":"","sources":["../../src/proving_broker/proving_broker_database.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iCAAiC,CAAC;AAC9D,OAAO,KAAK,EAAE,QAAQ,EAAE,UAAU,EAAE,YAAY,EAAE,uBAAuB,EAAE,MAAM,iCAAiC,CAAC;AAEnH;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC;;;OAGG;IACH,aAAa,CAAC,GAAG,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAE9C;;;OAGG;IACH,kCAAkC,CAAC,WAAW,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAE5E;;OAEG;IACH,cAAc,IAAI,qBAAqB,CAAC,CAAC,UAAU,EAAE,uBAAuB,GAAG,SAAS,CAAC,CAAC,CAAC;IAE3F;;;;;OAKG;IACH,mBAAmB,CAAC,EAAE,EAAE,YAAY,EAAE,KAAK,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAEtE;;;;;OAKG;IACH,kBAAkB,CAAC,EAAE,EAAE,YAAY,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAEjE;;;;;OAKG;IACH,oBAAoB,CAAC,EAAE,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAEtD;;;;;OAKG;IACH,sBAAsB,CAAC,EAAE,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAExD;;OAEG;IACH,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACxB"}
|
|
1
|
+
{"version":3,"file":"proving_broker_database.d.ts","sourceRoot":"","sources":["../../src/proving_broker/proving_broker_database.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iCAAiC,CAAC;AAC9D,OAAO,KAAK,EAAE,QAAQ,EAAE,UAAU,EAAE,YAAY,EAAE,uBAAuB,EAAE,MAAM,iCAAiC,CAAC;AAEnH;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC;;;OAGG;IACH,aAAa,CAAC,GAAG,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAE9C;;;OAGG;IACH,kCAAkC,CAAC,WAAW,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAE5E;;OAEG;IACH,cAAc,IAAI,qBAAqB,CAAC,CAAC,UAAU,EAAE,uBAAuB,GAAG,SAAS,CAAC,CAAC,CAAC;IAE3F;;;;OAIG;IACH,mBAAmB,CAAC,EAAE,EAAE,YAAY,GAAG,OAAO,CAAC,QAAQ,GAAG,SAAS,CAAC,CAAC;IAErE;;;;OAIG;IACH,mBAAmB,CAAC,EAAE,EAAE,YAAY,GAAG,OAAO,CAAC,uBAAuB,GAAG,SAAS,CAAC,CAAC;IAEpF;;;;;OAKG;IACH,mBAAmB,CAAC,EAAE,EAAE,YAAY,EAAE,KAAK,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAEtE;;;;;OAKG;IACH,kBAAkB,CAAC,EAAE,EAAE,YAAY,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAEjE;;;;;OAKG;IACH,oBAAoB,CAAC,EAAE,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAEtD;;;;;OAKG;IACH,sBAAsB,CAAC,EAAE,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAExD;;OAEG;IACH,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACxB"}
|
|
@@ -1,6 +1,15 @@
|
|
|
1
1
|
import { type ProvingJobBroker, type ProvingJobBrokerDebug, type ProvingJobConsumer, type ProvingJobProducer } from '@aztec/stdlib/interfaces/server';
|
|
2
2
|
import { type ApiSchemaFor } from '@aztec/stdlib/schemas';
|
|
3
3
|
import { type ComponentsVersions } from '@aztec/stdlib/versioning';
|
|
4
|
+
/**
|
|
5
|
+
* Max JSON-RPC request body the broker clients will send.
|
|
6
|
+
*
|
|
7
|
+
* Job inputs and proof results travel inline as `data:` URIs on this path, so a single call is routinely larger than
|
|
8
|
+
* the 1 mb the generic JSON-RPC client allows — that default sizes batches of small node requests against the node
|
|
9
|
+
* server's own default. An AVM (`PUBLIC_VM`) result is around 1.3 mb today. This value must stay at or below the
|
|
10
|
+
* broker server's `RPC_MAX_BODY_SIZE`, which deployments set to 50mb.
|
|
11
|
+
*/
|
|
12
|
+
export declare const PROVER_BROKER_MAX_REQUEST_BODY_SIZE: number;
|
|
4
13
|
/** Indefinite backoff for broker communication: 1, 1, 1, 2, 4, 4, 4, ... seconds. */
|
|
5
14
|
export declare function proverBrokerBackoff(): Generator<number, void, unknown>;
|
|
6
15
|
export declare const ProvingJobProducerSchema: ApiSchemaFor<ProvingJobProducer>;
|
|
@@ -13,17 +22,17 @@ export declare function createProvingJobBrokerClient(url: string, versions: Part
|
|
|
13
22
|
headers: {
|
|
14
23
|
get: (header: string) => string | null | undefined;
|
|
15
24
|
};
|
|
16
|
-
}
|
|
25
|
+
}>, maxRequestBodySize?: number): ProvingJobBroker;
|
|
17
26
|
export declare function createProvingJobProducerClient(url: string, versions: Partial<ComponentsVersions>, fetch?: (host: string, body: unknown, extraHeaders?: Record<string, string> | undefined, noRetry?: boolean | undefined) => Promise<{
|
|
18
27
|
response: any;
|
|
19
28
|
headers: {
|
|
20
29
|
get: (header: string) => string | null | undefined;
|
|
21
30
|
};
|
|
22
|
-
}
|
|
31
|
+
}>, maxRequestBodySize?: number): ProvingJobProducer;
|
|
23
32
|
export declare function createProvingJobConsumerClient(url: string, versions: Partial<ComponentsVersions>, fetch?: (host: string, body: unknown, extraHeaders?: Record<string, string> | undefined, noRetry?: boolean | undefined) => Promise<{
|
|
24
33
|
response: any;
|
|
25
34
|
headers: {
|
|
26
35
|
get: (header: string) => string | null | undefined;
|
|
27
36
|
};
|
|
28
|
-
}
|
|
29
|
-
//# sourceMappingURL=data:application/json;base64,
|
|
37
|
+
}>, maxRequestBodySize?: number): ProvingJobConsumer;
|
|
38
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoicnBjLmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi9zcmMvcHJvdmluZ19icm9rZXIvcnBjLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUVBLE9BQU8sRUFJTCxLQUFLLGdCQUFnQixFQUNyQixLQUFLLHFCQUFxQixFQUMxQixLQUFLLGtCQUFrQixFQUV2QixLQUFLLGtCQUFrQixFQUV4QixNQUFNLGlDQUFpQyxDQUFDO0FBRXpDLE9BQU8sRUFBRSxLQUFLLFlBQVksRUFBWSxNQUFNLHVCQUF1QixDQUFDO0FBQ3BFLE9BQU8sRUFBRSxLQUFLLGtCQUFrQixFQUFnQyxNQUFNLDBCQUEwQixDQUFDO0FBS2pHOzs7Ozs7O0dBT0c7QUFDSCxlQUFPLE1BQU0sbUNBQW1DLFFBQW1CLENBQUM7QUFFcEUscUZBQXFGO0FBQ3JGLHdCQUFpQixtQkFBbUIscUNBTW5DO0FBV0QsZUFBTyxNQUFNLHdCQUF3QixFQUFFLFlBQVksQ0FBQyxrQkFBa0IsQ0FLckUsQ0FBQztBQUVGLGVBQU8sTUFBTSx3QkFBd0IsRUFBRSxZQUFZLENBQUMsa0JBQWtCLENBaUJyRSxDQUFDO0FBRUYsZUFBTyxNQUFNLHNCQUFzQixFQUFFLFlBQVksQ0FBQyxnQkFBZ0IsQ0FHakUsQ0FBQztBQUVGLGVBQU8sTUFBTSwyQkFBMkIsRUFBRSxZQUFZLENBQUMscUJBQXFCLENBSzNFLENBQUM7QUFFRixlQUFPLE1BQU0sK0JBQStCLEVBQUUsWUFBWSxDQUFDLGdCQUFnQixHQUFHLHFCQUFxQixDQUdsRyxDQUFDO0FBRUYsd0JBQWdCLDRCQUE0QixDQUMxQyxHQUFHLEVBQUUsTUFBTSxFQUNYLFFBQVEsRUFBRSxPQUFPLENBQUMsa0JBQWtCLENBQUMsRUFDckMsS0FBSzs7Ozs7RUFBOEMsRUFDbkQsa0JBQWtCLFNBQXNDLEdBQ3ZELGdCQUFnQixDQU9sQjtBQUVELHdCQUFnQiw4QkFBOEIsQ0FDNUMsR0FBRyxFQUFFLE1BQU0sRUFDWCxRQUFRLEVBQUUsT0FBTyxDQUFDLGtCQUFrQixDQUFDLEVBQ3JDLEtBQUs7Ozs7O0VBQThDLEVBQ25ELGtCQUFrQixTQUFzQyxHQUN2RCxrQkFBa0IsQ0FPcEI7QUFFRCx3QkFBZ0IsOEJBQThCLENBQzVDLEdBQUcsRUFBRSxNQUFNLEVBQ1gsUUFBUSxFQUFFLE9BQU8sQ0FBQyxrQkFBa0IsQ0FBQyxFQUNyQyxLQUFLOzs7OztFQUE4QyxFQUNuRCxrQkFBa0IsU0FBc0MsR0FDdkQsa0JBQWtCLENBT3BCIn0=
|