@aztec/prover-client 0.0.1-commit.f2ce05ee → 0.0.1-commit.f5a9928
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dest/config.d.ts +1 -1
- package/dest/config.d.ts.map +1 -1
- package/dest/config.js +16 -2
- package/dest/light/lightweight_checkpoint_builder.d.ts +10 -5
- package/dest/light/lightweight_checkpoint_builder.d.ts.map +1 -1
- package/dest/light/lightweight_checkpoint_builder.js +56 -23
- package/dest/mocks/test_context.d.ts +11 -11
- package/dest/mocks/test_context.d.ts.map +1 -1
- package/dest/mocks/test_context.js +30 -30
- package/dest/orchestrator/block-building-helpers.d.ts +4 -4
- package/dest/orchestrator/block-building-helpers.d.ts.map +1 -1
- package/dest/orchestrator/block-building-helpers.js +2 -2
- package/dest/orchestrator/block-proving-state.d.ts +4 -1
- package/dest/orchestrator/block-proving-state.d.ts.map +1 -1
- package/dest/orchestrator/block-proving-state.js +7 -0
- package/dest/orchestrator/checkpoint-proving-state.d.ts +20 -36
- package/dest/orchestrator/checkpoint-proving-state.d.ts.map +1 -1
- package/dest/orchestrator/checkpoint-proving-state.js +22 -120
- package/dest/orchestrator/checkpoint-sub-tree-orchestrator.d.ts +161 -0
- package/dest/orchestrator/checkpoint-sub-tree-orchestrator.d.ts.map +1 -0
- package/dest/orchestrator/{orchestrator.js → checkpoint-sub-tree-orchestrator.js} +285 -462
- package/dest/orchestrator/chonk-cache.d.ts +39 -0
- package/dest/orchestrator/chonk-cache.d.ts.map +1 -0
- package/dest/orchestrator/chonk-cache.js +80 -0
- package/dest/orchestrator/index.d.ts +4 -2
- package/dest/orchestrator/index.d.ts.map +1 -1
- package/dest/orchestrator/index.js +3 -1
- package/dest/orchestrator/proving-scheduler.d.ts +81 -0
- package/dest/orchestrator/proving-scheduler.d.ts.map +1 -0
- package/dest/orchestrator/proving-scheduler.js +120 -0
- package/dest/orchestrator/top-tree-orchestrator.d.ts +88 -0
- package/dest/orchestrator/top-tree-orchestrator.d.ts.map +1 -0
- package/dest/orchestrator/top-tree-orchestrator.js +232 -0
- package/dest/orchestrator/top-tree-proving-state.d.ts +61 -0
- package/dest/orchestrator/top-tree-proving-state.d.ts.map +1 -0
- package/dest/orchestrator/top-tree-proving-state.js +185 -0
- package/dest/orchestrator/tx-proving-state.d.ts +3 -3
- package/dest/orchestrator/tx-proving-state.d.ts.map +1 -1
- package/dest/prover-client/prover-client.d.ts +62 -5
- package/dest/prover-client/prover-client.d.ts.map +1 -1
- package/dest/prover-client/prover-client.js +54 -7
- package/dest/proving_broker/broker_prover_facade.d.ts +3 -3
- package/dest/proving_broker/broker_prover_facade.d.ts.map +1 -1
- package/dest/proving_broker/broker_prover_facade.js +26 -20
- package/dest/proving_broker/config.d.ts +9 -73
- package/dest/proving_broker/config.d.ts.map +1 -1
- package/dest/proving_broker/config.js +3 -3
- package/dest/proving_broker/index.d.ts +2 -1
- package/dest/proving_broker/index.d.ts.map +1 -1
- package/dest/proving_broker/index.js +1 -0
- package/dest/proving_broker/proving_broker.d.ts +3 -2
- package/dest/proving_broker/proving_broker.d.ts.map +1 -1
- package/dest/proving_broker/proving_broker.js +217 -70
- package/dest/proving_broker/proving_broker_database/memory.d.ts +5 -2
- package/dest/proving_broker/proving_broker_database/memory.d.ts.map +1 -1
- package/dest/proving_broker/proving_broker_database/memory.js +14 -1
- package/dest/proving_broker/proving_broker_database/persisted.d.ts +15 -2
- package/dest/proving_broker/proving_broker_database/persisted.d.ts.map +1 -1
- package/dest/proving_broker/proving_broker_database/persisted.js +57 -6
- package/dest/proving_broker/proving_broker_database.d.ts +27 -1
- package/dest/proving_broker/proving_broker_database.d.ts.map +1 -1
- package/dest/proving_broker/proving_broker_instrumentation.d.ts +3 -1
- package/dest/proving_broker/proving_broker_instrumentation.d.ts.map +1 -1
- package/dest/proving_broker/proving_broker_instrumentation.js +7 -0
- package/dest/proving_broker/rpc.d.ts +3 -1
- package/dest/proving_broker/rpc.d.ts.map +1 -1
- package/dest/proving_broker/rpc.js +80 -24
- package/dest/test/epoch_settlement.d.ts +36 -0
- package/dest/test/epoch_settlement.d.ts.map +1 -0
- package/dest/test/epoch_settlement.js +52 -0
- package/dest/test/index.d.ts +2 -0
- package/dest/test/index.d.ts.map +1 -0
- package/dest/test/index.js +1 -0
- package/dest/test/mock_prover.d.ts +4 -4
- package/dest/test/mock_prover.js +2 -2
- package/package.json +20 -20
- package/src/config.ts +18 -2
- package/src/light/lightweight_checkpoint_builder.ts +58 -25
- package/src/mocks/test_context.ts +26 -43
- package/src/orchestrator/block-building-helpers.ts +2 -2
- package/src/orchestrator/block-proving-state.ts +9 -0
- package/src/orchestrator/checkpoint-proving-state.ts +21 -162
- package/src/orchestrator/checkpoint-sub-tree-orchestrator.ts +1044 -0
- package/src/orchestrator/chonk-cache.ts +99 -0
- package/src/orchestrator/index.ts +8 -1
- package/src/orchestrator/proving-scheduler.ts +160 -0
- package/src/orchestrator/top-tree-orchestrator.ts +384 -0
- package/src/orchestrator/top-tree-proving-state.ts +219 -0
- package/src/orchestrator/tx-proving-state.ts +3 -3
- package/src/prover-client/prover-client.ts +129 -18
- package/src/proving_broker/broker_prover_facade.ts +29 -23
- package/src/proving_broker/config.ts +3 -2
- package/src/proving_broker/index.ts +1 -0
- package/src/proving_broker/proving_broker.ts +208 -72
- package/src/proving_broker/proving_broker_database/memory.ts +16 -2
- package/src/proving_broker/proving_broker_database/persisted.ts +71 -8
- package/src/proving_broker/proving_broker_database.ts +30 -0
- package/src/proving_broker/proving_broker_instrumentation.ts +9 -0
- package/src/proving_broker/rpc.ts +36 -24
- package/src/test/epoch_settlement.ts +82 -0
- package/src/test/index.ts +1 -0
- package/src/test/mock_prover.ts +2 -2
- package/dest/orchestrator/epoch-proving-state.d.ts +0 -75
- package/dest/orchestrator/epoch-proving-state.d.ts.map +0 -1
- package/dest/orchestrator/epoch-proving-state.js +0 -267
- package/dest/orchestrator/orchestrator.d.ts +0 -128
- package/dest/orchestrator/orchestrator.d.ts.map +0 -1
- package/dest/prover-client/server-epoch-prover.d.ts +0 -32
- package/dest/prover-client/server-epoch-prover.d.ts.map +0 -1
- package/dest/prover-client/server-epoch-prover.js +0 -40
- package/src/orchestrator/epoch-proving-state.ts +0 -378
- package/src/orchestrator/orchestrator.ts +0 -1281
- package/src/prover-client/server-epoch-prover.ts +0 -69
|
@@ -9,6 +9,18 @@ import { getTelemetryClient } from '@aztec/telemetry-client';
|
|
|
9
9
|
import assert from 'assert';
|
|
10
10
|
import { defaultProverBrokerConfig } from './config.js';
|
|
11
11
|
import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.js';
|
|
12
|
+
function toJobMetadata(job) {
|
|
13
|
+
return {
|
|
14
|
+
id: job.id,
|
|
15
|
+
type: job.type,
|
|
16
|
+
epochNumber: job.epochNumber
|
|
17
|
+
};
|
|
18
|
+
}
|
|
19
|
+
function toSettledRecord(result) {
|
|
20
|
+
return result.status === 'fulfilled' ? {
|
|
21
|
+
status: 'fulfilled'
|
|
22
|
+
} : result;
|
|
23
|
+
}
|
|
12
24
|
/**
|
|
13
25
|
* A broker that manages proof requests and distributes them to workers based on their priority.
|
|
14
26
|
* It takes a backend that is responsible for storing and retrieving proof requests and results.
|
|
@@ -16,11 +28,17 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
|
|
|
16
28
|
database;
|
|
17
29
|
logger;
|
|
18
30
|
queues;
|
|
19
|
-
//
|
|
20
|
-
//
|
|
31
|
+
// Scheduling metadata for every known job (id, type, epochNumber). The large `inputsUri` is NOT held
|
|
32
|
+
// here — it lives in the database and is read on demand when a job is dispatched to an agent.
|
|
21
33
|
jobsCache;
|
|
22
|
-
//
|
|
34
|
+
// Settled status for every settled job. The large fulfilled proof `value` is NOT held here — it lives
|
|
35
|
+
// in the database (read on demand in getProvingJobStatus) and, transiently, in `pendingResults` while
|
|
36
|
+
// its database write is in flight. Small payloads (rejected reason, aborted) are kept inline.
|
|
23
37
|
resultsCache;
|
|
38
|
+
// Transient hold of a fulfilled result's full value, from the moment it settles until its database
|
|
39
|
+
// write commits, so status reads are read-your-writes without retaining every proof. Evicted once the
|
|
40
|
+
// value is durable in the database; kept only if that write failed (preserving the result in memory).
|
|
41
|
+
pendingResults;
|
|
24
42
|
// tracks when each job was enqueued
|
|
25
43
|
enqueuedAt;
|
|
26
44
|
// keeps track of which jobs are currently being processed
|
|
@@ -30,7 +48,8 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
|
|
|
30
48
|
inProgress;
|
|
31
49
|
// keep track of which proving job has been retried
|
|
32
50
|
retries;
|
|
33
|
-
// a map of promises that
|
|
51
|
+
// a map of promises that are resolved when a job settles. Carries no payload (settled status/results
|
|
52
|
+
// are read from resultsCache/pendingResults/DB) so it never pins a proof value in memory.
|
|
34
53
|
promises;
|
|
35
54
|
cleanupPromise;
|
|
36
55
|
msTimeSource;
|
|
@@ -77,6 +96,7 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
|
|
|
77
96
|
};
|
|
78
97
|
this.jobsCache = new Map();
|
|
79
98
|
this.resultsCache = new Map();
|
|
99
|
+
this.pendingResults = new Map();
|
|
80
100
|
this.enqueuedAt = new Map();
|
|
81
101
|
this.inProgress = new Map();
|
|
82
102
|
this.retries = new Map();
|
|
@@ -120,11 +140,11 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
|
|
|
120
140
|
provingJobId: item.id,
|
|
121
141
|
status: result ? result.status : 'pending'
|
|
122
142
|
});
|
|
123
|
-
this.jobsCache.set(item.id, item);
|
|
143
|
+
this.jobsCache.set(item.id, toJobMetadata(item));
|
|
124
144
|
this.promises.set(item.id, promiseWithResolvers());
|
|
125
145
|
if (result) {
|
|
126
|
-
this.promises.get(item.id).resolve(
|
|
127
|
-
this.resultsCache.set(item.id, result);
|
|
146
|
+
this.promises.get(item.id).resolve();
|
|
147
|
+
this.resultsCache.set(item.id, toSettledRecord(result));
|
|
128
148
|
} else {
|
|
129
149
|
this.enqueueJobInternal(item);
|
|
130
150
|
}
|
|
@@ -148,13 +168,13 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
|
|
|
148
168
|
return this.#cancelProvingJob(id);
|
|
149
169
|
}
|
|
150
170
|
getProvingJobStatus(id) {
|
|
151
|
-
return
|
|
171
|
+
return this.#getProvingJobStatus(id);
|
|
152
172
|
}
|
|
153
173
|
getCompletedJobs(ids) {
|
|
154
174
|
return this.#getCompletedJobs(ids);
|
|
155
175
|
}
|
|
156
176
|
getProvingJob(filter) {
|
|
157
|
-
return
|
|
177
|
+
return this.#getProvingJob(filter);
|
|
158
178
|
}
|
|
159
179
|
reportProvingJobSuccess(id, value, filter) {
|
|
160
180
|
return this.#reportProvingJobSuccess(id, value, filter);
|
|
@@ -163,7 +183,7 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
|
|
|
163
183
|
return this.#reportProvingJobError(id, err, retry, filter);
|
|
164
184
|
}
|
|
165
185
|
reportProvingJobProgress(id, startedAt, filter) {
|
|
166
|
-
return
|
|
186
|
+
return this.#reportProvingJobProgress(id, startedAt, filter);
|
|
167
187
|
}
|
|
168
188
|
async replayProvingJob(jobId, type, epochNumber, inputsUri) {
|
|
169
189
|
if (!this.debugReplayEnabled) {
|
|
@@ -184,7 +204,7 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
|
|
|
184
204
|
epochNumber,
|
|
185
205
|
inputsUri
|
|
186
206
|
};
|
|
187
|
-
this.jobsCache.set(jobId, job);
|
|
207
|
+
this.jobsCache.set(jobId, toJobMetadata(job));
|
|
188
208
|
await this.database.addProvingJob(job);
|
|
189
209
|
this.enqueueJobInternal(job);
|
|
190
210
|
return {
|
|
@@ -192,16 +212,57 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
|
|
|
192
212
|
};
|
|
193
213
|
}
|
|
194
214
|
async #enqueueProvingJob(job) {
|
|
195
|
-
//
|
|
196
|
-
|
|
215
|
+
// The status returned to the caller reflects the job's state at the start of this call: `not-found`
|
|
216
|
+
// for a brand-new job (it did not exist yet), or the cached status for one already known. Crucially
|
|
217
|
+
// this must NOT `await` before the synchronous `jobsCache`/`resultsCache` gate below, or the
|
|
218
|
+
// enqueue/revive lock would no longer hold. So the not-found default is set synchronously here and
|
|
219
|
+
// the cached branch computes the real status (which may read the DB) only when it returns.
|
|
220
|
+
let jobStatus = {
|
|
221
|
+
status: 'not-found'
|
|
222
|
+
};
|
|
197
223
|
if (this.jobsCache.has(job.id)) {
|
|
198
224
|
const existing = this.jobsCache.get(job.id);
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
225
|
+
// Identity check is metadata-only: `inputsUri` now lives in the database, not memory, and job ids
|
|
226
|
+
// are content-addressed (same id ⇒ same inputs), so a mismatch shows up in id/type/epochNumber.
|
|
227
|
+
assert.deepStrictEqual(toJobMetadata(job), existing, 'Duplicate proving job ID');
|
|
228
|
+
if (this.resultsCache.get(job.id)?.status === 'aborted') {
|
|
229
|
+
// The producer is re-requesting a job it previously cancelled: revive it rather than
|
|
230
|
+
// returning the cached abort, clearing the aborted state in memory and in the database so the
|
|
231
|
+
// revival survives a restart.
|
|
232
|
+
//
|
|
233
|
+
// Concurrency model: `jobsCache` is the enqueue lock. Every path that puts a job on the queue
|
|
234
|
+
// populates `jobsCache` *synchronously, before its first await* (see the "New proving job"
|
|
235
|
+
// block below), so a second concurrent enqueue of the same id observes the entry at the top
|
|
236
|
+
// of this method and takes a cached, no-op branch instead of enqueuing a duplicate. The revive
|
|
237
|
+
// must keep holding that lock: we tear down the settled state and re-set `jobsCache` in a
|
|
238
|
+
// single synchronous span (no await in between), and only then await the database. Because a
|
|
239
|
+
// concurrent re-request can only interleave at that await — by which point `jobsCache` is
|
|
240
|
+
// populated again and the aborted result is gone — it falls into the cached branch and no-ops,
|
|
241
|
+
// so the job is enqueued exactly once. (`cleanUpProvingJobState` also drops the promise, so
|
|
242
|
+
// `enqueueJobInternal` below mints a fresh one for the retry.) This holds unchanged with the
|
|
243
|
+
// slimmer caches: `jobsCache` (now metadata) and `resultsCache` (now settled status) are still
|
|
244
|
+
// synchronous in-memory maps, so the tear-down + re-set span still contains no await.
|
|
245
|
+
this.logger.info(`Reviving aborted proving job id=${job.id} epochNumber=${job.epochNumber}`, {
|
|
246
|
+
provingJobId: job.id
|
|
247
|
+
});
|
|
248
|
+
this.cleanUpProvingJobState([
|
|
249
|
+
job.id
|
|
250
|
+
]);
|
|
251
|
+
this.jobsCache.set(job.id, toJobMetadata(job));
|
|
252
|
+
await this.database.deleteProvingJobResult(job.id);
|
|
253
|
+
// The job is re-set in the cache and about to be re-enqueued below: its start status is in-queue.
|
|
254
|
+
jobStatus = {
|
|
255
|
+
status: 'in-queue'
|
|
256
|
+
};
|
|
257
|
+
} else {
|
|
258
|
+
this.logger.warn(`Cached proving job id=${job.id} epochNumber=${job.epochNumber}. Not enqueuing again`, {
|
|
259
|
+
provingJobId: job.id
|
|
260
|
+
});
|
|
261
|
+
this.instrumentation.incCachedJobs(job.type);
|
|
262
|
+
// Return the job's current status. This reads the fulfilled proof value from the DB when needed;
|
|
263
|
+
// the await is fine here because this branch does not enqueue, so it is outside the lock span.
|
|
264
|
+
return await this.#getProvingJobStatus(job.id);
|
|
265
|
+
}
|
|
205
266
|
}
|
|
206
267
|
if (this.isJobStale(job)) {
|
|
207
268
|
this.logger.warn(`Tried enqueueing stale proving job id=${job.id} epochNumber=${job.epochNumber}`, {
|
|
@@ -214,7 +275,7 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
|
|
|
214
275
|
});
|
|
215
276
|
try {
|
|
216
277
|
// do this first so it acts as a "lock". If this job is enqueued again while we're saving it the if at the top will catch it.
|
|
217
|
-
this.jobsCache.set(job.id, job);
|
|
278
|
+
this.jobsCache.set(job.id, toJobMetadata(job));
|
|
218
279
|
await this.database.addProvingJob(job);
|
|
219
280
|
this.enqueueJobInternal(job);
|
|
220
281
|
this.instrumentation.incTotalJobs(job.type);
|
|
@@ -228,46 +289,89 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
|
|
|
228
289
|
return jobStatus;
|
|
229
290
|
}
|
|
230
291
|
async #cancelProvingJob(id) {
|
|
231
|
-
|
|
292
|
+
const job = this.jobsCache.get(id);
|
|
293
|
+
if (!job) {
|
|
232
294
|
this.logger.warn(`Can't cancel a job that doesn't exist id=${id}`, {
|
|
233
295
|
provingJobId: id
|
|
234
296
|
});
|
|
235
297
|
return;
|
|
236
298
|
}
|
|
237
|
-
//
|
|
238
|
-
if (
|
|
239
|
-
|
|
299
|
+
// Leave jobs that have already settled (completed or failed) alone: those results are terminal.
|
|
300
|
+
if (this.resultsCache.has(id)) {
|
|
301
|
+
return;
|
|
302
|
+
}
|
|
303
|
+
this.logger.info(`Cancelling job id=${id}`, {
|
|
304
|
+
provingJobId: id
|
|
305
|
+
});
|
|
306
|
+
this.inProgress.delete(id);
|
|
307
|
+
// Record the cancellation as its own settled state and persist it, so it survives a restart and
|
|
308
|
+
// notifies the current waiter. Unlike a completion or failure this is not terminal: re-enqueuing
|
|
309
|
+
// the same job id revives it (see #enqueueProvingJob), so the abort never permanently blocks the
|
|
310
|
+
// proof.
|
|
311
|
+
this.resultsCache.set(id, {
|
|
312
|
+
status: 'aborted'
|
|
313
|
+
});
|
|
314
|
+
this.promises.get(id)?.resolve();
|
|
315
|
+
this.completedJobNotifications.push(id);
|
|
316
|
+
this.instrumentation.incAbortedJobs(job.type);
|
|
317
|
+
try {
|
|
318
|
+
await this.database.setProvingJobAborted(id);
|
|
319
|
+
} catch (saveErr) {
|
|
320
|
+
this.logger.error(`Failed to save proving job aborted status id=${id}`, saveErr, {
|
|
240
321
|
provingJobId: id
|
|
241
322
|
});
|
|
242
|
-
|
|
323
|
+
throw saveErr;
|
|
243
324
|
}
|
|
244
325
|
}
|
|
245
326
|
cleanUpProvingJobState(ids) {
|
|
327
|
+
const idsToClean = new Set(ids);
|
|
246
328
|
for (const id of ids){
|
|
247
329
|
this.jobsCache.delete(id);
|
|
330
|
+
const deferred = this.promises.get(id);
|
|
331
|
+
if (deferred) {
|
|
332
|
+
deferred.resolve();
|
|
333
|
+
}
|
|
248
334
|
this.promises.delete(id);
|
|
249
335
|
this.resultsCache.delete(id);
|
|
336
|
+
this.pendingResults.delete(id);
|
|
250
337
|
this.inProgress.delete(id);
|
|
251
338
|
this.retries.delete(id);
|
|
252
339
|
this.enqueuedAt.delete(id);
|
|
253
340
|
}
|
|
341
|
+
this.completedJobNotifications = this.completedJobNotifications.filter((id)=>!idsToClean.has(id));
|
|
254
342
|
}
|
|
255
|
-
#getProvingJobStatus(id) {
|
|
256
|
-
const
|
|
257
|
-
if (
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
const item = this.jobsCache.get(id);
|
|
262
|
-
if (!item) {
|
|
263
|
-
return {
|
|
264
|
-
status: 'not-found'
|
|
265
|
-
};
|
|
343
|
+
async #getProvingJobStatus(id) {
|
|
344
|
+
const settled = this.resultsCache.get(id);
|
|
345
|
+
if (settled) {
|
|
346
|
+
if (settled.status !== 'fulfilled') {
|
|
347
|
+
// rejected/aborted carry their (small) payload inline.
|
|
348
|
+
return settled;
|
|
266
349
|
}
|
|
350
|
+
// A fulfilled job's proof value is not held in memory: serve it from the transient hold if its
|
|
351
|
+
// database write is still in flight, otherwise read it back from the database.
|
|
352
|
+
const full = this.pendingResults.get(id) ?? await this.database.getProvingJobResult(id);
|
|
353
|
+
if (full?.status === 'fulfilled') {
|
|
354
|
+
return full;
|
|
355
|
+
}
|
|
356
|
+
// Settled as fulfilled but the value is nowhere to be found — should not happen (it is held in
|
|
357
|
+
// pendingResults until its DB write commits). Report not-found rather than a torn result.
|
|
358
|
+
this.logger.error(`Fulfilled proving job id=${id} has no retrievable result value`, {
|
|
359
|
+
provingJobId: id
|
|
360
|
+
});
|
|
361
|
+
return {
|
|
362
|
+
status: 'not-found'
|
|
363
|
+
};
|
|
364
|
+
}
|
|
365
|
+
// no result yet, check if we know the item
|
|
366
|
+
const item = this.jobsCache.get(id);
|
|
367
|
+
if (!item) {
|
|
267
368
|
return {
|
|
268
|
-
status:
|
|
369
|
+
status: 'not-found'
|
|
269
370
|
};
|
|
270
371
|
}
|
|
372
|
+
return {
|
|
373
|
+
status: this.inProgress.has(id) ? 'in-progress' : 'in-queue'
|
|
374
|
+
};
|
|
271
375
|
}
|
|
272
376
|
#getCompletedJobs(ids) {
|
|
273
377
|
const completedJobs = ids.filter((id)=>this.resultsCache.has(id));
|
|
@@ -275,7 +379,7 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
|
|
|
275
379
|
this.completedJobNotifications = [];
|
|
276
380
|
return Promise.resolve(notifications.concat(completedJobs));
|
|
277
381
|
}
|
|
278
|
-
#getProvingJob(filter = {
|
|
382
|
+
async #getProvingJob(filter = {
|
|
279
383
|
allowList: []
|
|
280
384
|
}) {
|
|
281
385
|
const allowedProofs = Array.isArray(filter.allowList) && filter.allowList.length > 0 ? [
|
|
@@ -290,20 +394,43 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
|
|
|
290
394
|
// this can happen if the broker crashes and restarts
|
|
291
395
|
// it's possible agents will report progress or results for jobs that are in the queue (after the restart)
|
|
292
396
|
while(enqueuedJob = queue.getImmediate()){
|
|
293
|
-
const
|
|
294
|
-
if (
|
|
397
|
+
const meta = this.jobsCache.get(enqueuedJob.id);
|
|
398
|
+
if (meta && !this.inProgress.has(enqueuedJob.id) && !this.resultsCache.has(enqueuedJob.id)) {
|
|
295
399
|
const time = this.msTimeSource();
|
|
296
|
-
|
|
297
|
-
|
|
400
|
+
// Claim the job synchronously (before the await below) so a concurrent dispatch can't re-pick
|
|
401
|
+
// it. The same id can sit in a queue more than once (an abort leaves a stale entry that a
|
|
402
|
+
// later revive re-enqueues alongside), so this in-progress claim — not the queue pop — is what
|
|
403
|
+
// dedups those copies across concurrently-polling agents. It must be set before the await.
|
|
404
|
+
this.inProgress.set(meta.id, {
|
|
405
|
+
id: meta.id,
|
|
298
406
|
startedAt: time,
|
|
299
407
|
lastUpdatedAt: time
|
|
300
408
|
});
|
|
301
|
-
|
|
409
|
+
// The large inputs are not kept in memory; read them from the database. They were durably
|
|
410
|
+
// persisted (addProvingJob's write commits) before the job became dispatchable, so this hits.
|
|
411
|
+
const inputsUri = await this.database.getProvingJobInputs(meta.id);
|
|
412
|
+
if (!inputsUri) {
|
|
413
|
+
// The job was cleaned up (or its inputs lost) after we claimed it — release the claim and
|
|
414
|
+
// keep draining the queue for another candidate. Nothing else was mutated yet, so there is
|
|
415
|
+
// no wait metric or enqueued-at timer to unwind.
|
|
416
|
+
this.inProgress.delete(meta.id);
|
|
417
|
+
this.logger.warn(`No inputs found for proving job id=${meta.id}; skipping dispatch`, {
|
|
418
|
+
provingJobId: meta.id
|
|
419
|
+
});
|
|
420
|
+
continue;
|
|
421
|
+
}
|
|
422
|
+
// The dispatch is now committed: record the queue wait and clear the enqueued-at timer.
|
|
423
|
+
const enqueuedAt = this.enqueuedAt.get(meta.id);
|
|
302
424
|
if (enqueuedAt) {
|
|
303
|
-
this.instrumentation.recordJobWait(
|
|
304
|
-
|
|
305
|
-
this.enqueuedAt.delete(job.id);
|
|
425
|
+
this.instrumentation.recordJobWait(meta.type, enqueuedAt);
|
|
426
|
+
this.enqueuedAt.delete(meta.id);
|
|
306
427
|
}
|
|
428
|
+
const job = {
|
|
429
|
+
id: meta.id,
|
|
430
|
+
type: meta.type,
|
|
431
|
+
epochNumber: meta.epochNumber,
|
|
432
|
+
inputsUri
|
|
433
|
+
};
|
|
307
434
|
return {
|
|
308
435
|
job,
|
|
309
436
|
time
|
|
@@ -351,14 +478,13 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
|
|
|
351
478
|
this.logger.info(`Marking proving job as failed id=${id} type=${ProvingRequestType[item.type]} totalAttempts=${retries + 1} err=${err}`, {
|
|
352
479
|
provingJobId: id
|
|
353
480
|
});
|
|
354
|
-
// save the
|
|
355
|
-
//
|
|
356
|
-
|
|
481
|
+
// save the (small) rejection reason to the cache and notify clients of the job status. The reason
|
|
482
|
+
// is kept in memory, so status reads work even if the database write below fails.
|
|
483
|
+
this.resultsCache.set(id, {
|
|
357
484
|
status: 'rejected',
|
|
358
485
|
reason: String(err)
|
|
359
|
-
};
|
|
360
|
-
this.
|
|
361
|
-
this.promises.get(id).resolve(result);
|
|
486
|
+
});
|
|
487
|
+
this.promises.get(id).resolve();
|
|
362
488
|
this.completedJobNotifications.push(id);
|
|
363
489
|
this.instrumentation.incRejectedJobs(item.type);
|
|
364
490
|
if (info) {
|
|
@@ -375,19 +501,19 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
|
|
|
375
501
|
}
|
|
376
502
|
return this.#getProvingJob(filter);
|
|
377
503
|
}
|
|
378
|
-
#reportProvingJobProgress(id, startedAt, filter) {
|
|
504
|
+
async #reportProvingJobProgress(id, startedAt, filter) {
|
|
379
505
|
const job = this.jobsCache.get(id);
|
|
380
506
|
if (!job) {
|
|
381
507
|
this.logger.warn(`Proving job id=${id} does not exist`, {
|
|
382
508
|
provingJobId: id
|
|
383
509
|
});
|
|
384
|
-
return this.#getProvingJob(filter);
|
|
510
|
+
return await this.#getProvingJob(filter);
|
|
385
511
|
}
|
|
386
512
|
if (this.resultsCache.has(id)) {
|
|
387
513
|
this.logger.warn(`Proving job id=${id} has already been completed`, {
|
|
388
514
|
provingJobId: id
|
|
389
515
|
});
|
|
390
|
-
return this.#getProvingJob(filter);
|
|
516
|
+
return await this.#getProvingJob(filter);
|
|
391
517
|
}
|
|
392
518
|
const metadata = this.inProgress.get(id);
|
|
393
519
|
const now = this.msTimeSource();
|
|
@@ -421,7 +547,7 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
|
|
|
421
547
|
this.logger.warn(`Proving job id=${id} type=${ProvingRequestType[job.type]} already being worked on by another agent. Sending new one`, {
|
|
422
548
|
provingJobId: id
|
|
423
549
|
});
|
|
424
|
-
return this.#getProvingJob(filter);
|
|
550
|
+
return await this.#getProvingJob(filter);
|
|
425
551
|
}
|
|
426
552
|
async #reportProvingJobSuccess(id, value, filter) {
|
|
427
553
|
const info = this.inProgress.get(id);
|
|
@@ -449,15 +575,17 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
|
|
|
449
575
|
this.logger.info(`Proving job complete id=${id} type=${ProvingRequestType[item.type]} totalAttempts=${retries + 1}`, {
|
|
450
576
|
provingJobId: id
|
|
451
577
|
});
|
|
452
|
-
//
|
|
453
|
-
//
|
|
454
|
-
//
|
|
455
|
-
|
|
578
|
+
// Mark settled synchronously (the gate that makes concurrent settles no-op, unchanged from before),
|
|
579
|
+
// holding only the status in resultsCache. The large proof value is kept transiently in
|
|
580
|
+
// pendingResults so status reads are read-your-writes until the database write below commits.
|
|
581
|
+
this.resultsCache.set(id, {
|
|
582
|
+
status: 'fulfilled'
|
|
583
|
+
});
|
|
584
|
+
this.pendingResults.set(id, {
|
|
456
585
|
status: 'fulfilled',
|
|
457
586
|
value
|
|
458
|
-
};
|
|
459
|
-
this.
|
|
460
|
-
this.promises.get(id).resolve(result);
|
|
587
|
+
});
|
|
588
|
+
this.promises.get(id).resolve();
|
|
461
589
|
this.completedJobNotifications.push(id);
|
|
462
590
|
this.instrumentation.incResolvedJobs(item.type);
|
|
463
591
|
if (info) {
|
|
@@ -466,7 +594,11 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
|
|
|
466
594
|
}
|
|
467
595
|
try {
|
|
468
596
|
await this.database.setProvingJobResult(id, value);
|
|
597
|
+
// The value is durable in the database now; drop the transient in-memory copy.
|
|
598
|
+
this.pendingResults.delete(id);
|
|
469
599
|
} catch (saveErr) {
|
|
600
|
+
// Keep the value in pendingResults so status/result reads still succeed despite the failed write —
|
|
601
|
+
// this preserves the previous "works even if the database breaks" behaviour for this edge.
|
|
470
602
|
this.logger.error(`Failed to save proving job result id=${id}`, saveErr, {
|
|
471
603
|
provingJobId: id
|
|
472
604
|
});
|
|
@@ -475,20 +607,20 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
|
|
|
475
607
|
return this.#getProvingJob(filter);
|
|
476
608
|
}
|
|
477
609
|
async cleanupPass() {
|
|
478
|
-
this.cleanupStaleJobs();
|
|
479
610
|
this.reEnqueueExpiredJobs();
|
|
480
611
|
const oldestEpochToKeep = this.oldestEpochToKeep();
|
|
481
612
|
if (oldestEpochToKeep > 0) {
|
|
613
|
+
this.cleanupJobsOlderThanEpoch(EpochNumber(oldestEpochToKeep));
|
|
482
614
|
await this.database.deleteAllProvingJobsOlderThanEpoch(EpochNumber(oldestEpochToKeep));
|
|
483
615
|
this.logger.trace(`Deleted all epochs older than ${oldestEpochToKeep}`);
|
|
484
616
|
}
|
|
485
617
|
}
|
|
486
|
-
|
|
618
|
+
cleanupJobsOlderThanEpoch(epochNumber) {
|
|
487
619
|
const jobIds = Array.from(this.jobsCache.keys());
|
|
488
620
|
const jobsToClean = [];
|
|
489
621
|
for (const id of jobIds){
|
|
490
622
|
const job = this.jobsCache.get(id);
|
|
491
|
-
if (
|
|
623
|
+
if (job.epochNumber < epochNumber) {
|
|
492
624
|
jobsToClean.push(id);
|
|
493
625
|
}
|
|
494
626
|
}
|
|
@@ -511,12 +643,27 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
|
|
|
511
643
|
const now = this.msTimeSource();
|
|
512
644
|
const msSinceLastUpdate = now - metadata.lastUpdatedAt;
|
|
513
645
|
if (msSinceLastUpdate >= this.jobTimeoutMs) {
|
|
514
|
-
this.logger.warn(`Proving job id=${id} timed out. Adding it back to the queue.`, {
|
|
515
|
-
provingJobId: id
|
|
516
|
-
});
|
|
517
646
|
this.inProgress.delete(id);
|
|
518
|
-
this.enqueueJobInternal(item);
|
|
519
647
|
this.instrumentation.incTimedOutJobs(item.type);
|
|
648
|
+
const retries = this.retries.get(id) ?? 0;
|
|
649
|
+
if (retries + 1 < this.maxRetries && !this.isJobStale(item)) {
|
|
650
|
+
this.logger.warn(`Proving job id=${id} timed out. Re-enqueueing (retry ${retries + 1}/${this.maxRetries}).`, {
|
|
651
|
+
provingJobId: id
|
|
652
|
+
});
|
|
653
|
+
this.retries.set(id, retries + 1);
|
|
654
|
+
this.enqueueJobInternal(item);
|
|
655
|
+
} else {
|
|
656
|
+
this.logger.error(`Proving job id=${id} timed out after ${retries + 1} attempts. Marking as failed.`, {
|
|
657
|
+
provingJobId: id
|
|
658
|
+
});
|
|
659
|
+
this.resultsCache.set(id, {
|
|
660
|
+
status: 'rejected',
|
|
661
|
+
reason: 'Timed out'
|
|
662
|
+
});
|
|
663
|
+
this.promises.get(id)?.resolve();
|
|
664
|
+
this.completedJobNotifications.push(id);
|
|
665
|
+
this.instrumentation.incRejectedJobs(item.type);
|
|
666
|
+
}
|
|
520
667
|
}
|
|
521
668
|
}
|
|
522
669
|
}
|
|
@@ -5,13 +5,16 @@ export declare class InMemoryBrokerDatabase implements ProvingBrokerDatabase {
|
|
|
5
5
|
private jobs;
|
|
6
6
|
private results;
|
|
7
7
|
getProvingJob(id: ProvingJobId): ProvingJob | undefined;
|
|
8
|
-
|
|
8
|
+
getProvingJobInputs(id: ProvingJobId): Promise<ProofUri | undefined>;
|
|
9
|
+
getProvingJobResult(id: ProvingJobId): Promise<ProvingJobSettledResult | undefined>;
|
|
9
10
|
addProvingJob(job: ProvingJob): Promise<void>;
|
|
10
11
|
setProvingJobResult(id: ProvingJobId, value: ProofUri): Promise<void>;
|
|
11
12
|
setProvingJobError(id: ProvingJobId, reason: string): Promise<void>;
|
|
13
|
+
setProvingJobAborted(id: ProvingJobId): Promise<void>;
|
|
14
|
+
deleteProvingJobResult(id: ProvingJobId): Promise<void>;
|
|
12
15
|
deleteProvingJobs(ids: ProvingJobId[]): Promise<void>;
|
|
13
16
|
deleteAllProvingJobsOlderThanEpoch(epochNumber: EpochNumber): Promise<void>;
|
|
14
17
|
allProvingJobs(): AsyncIterableIterator<[ProvingJob, ProvingJobSettledResult | undefined]>;
|
|
15
18
|
close(): Promise<void>;
|
|
16
19
|
}
|
|
17
|
-
//# sourceMappingURL=data:application/json;base64,
|
|
20
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoibWVtb3J5LmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi8uLi9zcmMvcHJvdmluZ19icm9rZXIvcHJvdmluZ19icm9rZXJfZGF0YWJhc2UvbWVtb3J5LnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLE9BQU8sRUFBRSxXQUFXLEVBQUUsTUFBTSxpQ0FBaUMsQ0FBQztBQUM5RCxPQUFPLEVBQ0wsS0FBSyxRQUFRLEVBQ2IsS0FBSyxVQUFVLEVBQ2YsS0FBSyxZQUFZLEVBQ2pCLEtBQUssdUJBQXVCLEVBRTdCLE1BQU0saUNBQWlDLENBQUM7QUFFekMsT0FBTyxLQUFLLEVBQUUscUJBQXFCLEVBQUUsTUFBTSwrQkFBK0IsQ0FBQztBQUUzRSxxQkFBYSxzQkFBdUIsWUFBVyxxQkFBcUI7SUFDbEUsT0FBTyxDQUFDLElBQUksQ0FBdUM7SUFDbkQsT0FBTyxDQUFDLE9BQU8sQ0FBb0Q7SUFFbkUsYUFBYSxDQUFDLEVBQUUsRUFBRSxZQUFZLEdBQUcsVUFBVSxHQUFHLFNBQVMsQ0FFdEQ7SUFFRCxtQkFBbUIsQ0FBQyxFQUFFLEVBQUUsWUFBWSxHQUFHLE9BQU8sQ0FBQyxRQUFRLEdBQUcsU0FBUyxDQUFDLENBRW5FO0lBRUQsbUJBQW1CLENBQUMsRUFBRSxFQUFFLFlBQVksR0FBRyxPQUFPLENBQUMsdUJBQXVCLEdBQUcsU0FBUyxDQUFDLENBRWxGO0lBRUQsYUFBYSxDQUFDLEdBQUcsRUFBRSxVQUFVLEdBQUcsT0FBTyxDQUFDLElBQUksQ0FBQyxDQUc1QztJQUVELG1CQUFtQixDQUFDLEVBQUUsRUFBRSxZQUFZLEVBQUUsS0FBSyxFQUFFLFFBQVEsR0FBRyxPQUFPLENBQUMsSUFBSSxDQUFDLENBR3BFO0lBRUQsa0JBQWtCLENBQUMsRUFBRSxFQUFFLFlBQVksRUFBRSxNQUFNLEVBQUUsTUFBTSxHQUFHLE9BQU8sQ0FBQyxJQUFJLENBQUMsQ0FHbEU7SUFFRCxvQkFBb0IsQ0FBQyxFQUFFLEVBQUUsWUFBWSxHQUFHLE9BQU8sQ0FBQyxJQUFJLENBQUMsQ0FHcEQ7SUFFRCxzQkFBc0IsQ0FBQyxFQUFFLEVBQUUsWUFBWSxHQUFHLE9BQU8sQ0FBQyxJQUFJLENBQUMsQ0FHdEQ7SUFFRCxpQkFBaUIsQ0FBQyxHQUFHLEVBQUUsWUFBWSxFQUFFLEdBQUcsT0FBTyxDQUFDLElBQUksQ0FBQyxDQU1wRDtJQUVELGtDQUFrQyxDQUFDLFdBQVcsRUFBRSxXQUFXLEdBQUcsT0FBTyxDQUFDLElBQUksQ0FBQyxDQU0xRTtJQUVNLGNBQWMsSUFBSSxxQkFBcUIsQ0FBQyxDQUFDLFVBQVUsRUFBRSx1QkFBdUIsR0FBRyxTQUFTLENBQUMsQ0FBQyxDQUloRztJQUVELEtBQUssSUFBSSxPQUFPLENBQUMsSUFBSSxDQUFDLENBRXJCO0NBQ0YifQ==
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"memory.d.ts","sourceRoot":"","sources":["../../../src/proving_broker/proving_broker_database/memory.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iCAAiC,CAAC;AAC9D,OAAO,EACL,KAAK,QAAQ,EACb,KAAK,UAAU,EACf,KAAK,YAAY,EACjB,KAAK,uBAAuB,EAE7B,MAAM,iCAAiC,CAAC;AAEzC,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,+BAA+B,CAAC;AAE3E,qBAAa,sBAAuB,YAAW,qBAAqB;IAClE,OAAO,CAAC,IAAI,CAAuC;IACnD,OAAO,CAAC,OAAO,CAAoD;IAEnE,aAAa,CAAC,EAAE,EAAE,YAAY,GAAG,UAAU,GAAG,SAAS,CAEtD;IAED,mBAAmB,CAAC,EAAE,EAAE,YAAY,GAAG,uBAAuB,GAAG,SAAS,
|
|
1
|
+
{"version":3,"file":"memory.d.ts","sourceRoot":"","sources":["../../../src/proving_broker/proving_broker_database/memory.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iCAAiC,CAAC;AAC9D,OAAO,EACL,KAAK,QAAQ,EACb,KAAK,UAAU,EACf,KAAK,YAAY,EACjB,KAAK,uBAAuB,EAE7B,MAAM,iCAAiC,CAAC;AAEzC,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,+BAA+B,CAAC;AAE3E,qBAAa,sBAAuB,YAAW,qBAAqB;IAClE,OAAO,CAAC,IAAI,CAAuC;IACnD,OAAO,CAAC,OAAO,CAAoD;IAEnE,aAAa,CAAC,EAAE,EAAE,YAAY,GAAG,UAAU,GAAG,SAAS,CAEtD;IAED,mBAAmB,CAAC,EAAE,EAAE,YAAY,GAAG,OAAO,CAAC,QAAQ,GAAG,SAAS,CAAC,CAEnE;IAED,mBAAmB,CAAC,EAAE,EAAE,YAAY,GAAG,OAAO,CAAC,uBAAuB,GAAG,SAAS,CAAC,CAElF;IAED,aAAa,CAAC,GAAG,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAG5C;IAED,mBAAmB,CAAC,EAAE,EAAE,YAAY,EAAE,KAAK,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CAGpE;IAED,kBAAkB,CAAC,EAAE,EAAE,YAAY,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAGlE;IAED,oBAAoB,CAAC,EAAE,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAGpD;IAED,sBAAsB,CAAC,EAAE,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAGtD;IAED,iBAAiB,CAAC,GAAG,EAAE,YAAY,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAMpD;IAED,kCAAkC,CAAC,WAAW,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAM1E;IAEM,cAAc,IAAI,qBAAqB,CAAC,CAAC,UAAU,EAAE,uBAAuB,GAAG,SAAS,CAAC,CAAC,CAIhG;IAED,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAErB;CACF"}
|
|
@@ -5,8 +5,11 @@ export class InMemoryBrokerDatabase {
|
|
|
5
5
|
getProvingJob(id) {
|
|
6
6
|
return this.jobs.get(id);
|
|
7
7
|
}
|
|
8
|
+
getProvingJobInputs(id) {
|
|
9
|
+
return Promise.resolve(this.jobs.get(id)?.inputsUri);
|
|
10
|
+
}
|
|
8
11
|
getProvingJobResult(id) {
|
|
9
|
-
return this.results.get(id);
|
|
12
|
+
return Promise.resolve(this.results.get(id));
|
|
10
13
|
}
|
|
11
14
|
addProvingJob(job) {
|
|
12
15
|
this.jobs.set(job.id, job);
|
|
@@ -26,6 +29,16 @@ export class InMemoryBrokerDatabase {
|
|
|
26
29
|
});
|
|
27
30
|
return Promise.resolve();
|
|
28
31
|
}
|
|
32
|
+
setProvingJobAborted(id) {
|
|
33
|
+
this.results.set(id, {
|
|
34
|
+
status: 'aborted'
|
|
35
|
+
});
|
|
36
|
+
return Promise.resolve();
|
|
37
|
+
}
|
|
38
|
+
deleteProvingJobResult(id) {
|
|
39
|
+
this.results.delete(id);
|
|
40
|
+
return Promise.resolve();
|
|
41
|
+
}
|
|
29
42
|
deleteProvingJobs(ids) {
|
|
30
43
|
for (const id of ids){
|
|
31
44
|
this.jobs.delete(id);
|
|
@@ -4,6 +4,14 @@ import { type ProofUri, ProvingJob, type ProvingJobId, ProvingJobSettledResult }
|
|
|
4
4
|
import { type TelemetryClient, type Tracer } from '@aztec/telemetry-client';
|
|
5
5
|
import type { ProverBrokerConfig } from '../config.js';
|
|
6
6
|
import type { ProvingBrokerDatabase } from '../proving_broker_database.js';
|
|
7
|
+
/**
|
|
8
|
+
* An item queued for a batched write. A {@link ProvingJob} adds/updates a job; a `[id, result]`
|
|
9
|
+
* tuple sets a job's settled result; a `[id, undefined]` tuple deletes a job's result. Deletes ride
|
|
10
|
+
* the same queue as writes so they stay FIFO-ordered against them: a revive's result-delete must
|
|
11
|
+
* land after the abort that preceded it, otherwise a still-queued abort could flush later and
|
|
12
|
+
* resurrect the cancelled state on disk.
|
|
13
|
+
*/
|
|
14
|
+
type BrokerWrite = ProvingJob | [ProvingJobId, ProvingJobSettledResult | undefined];
|
|
7
15
|
export declare class KVBrokerDatabase implements ProvingBrokerDatabase {
|
|
8
16
|
private epochs;
|
|
9
17
|
private config;
|
|
@@ -12,7 +20,7 @@ export declare class KVBrokerDatabase implements ProvingBrokerDatabase {
|
|
|
12
20
|
private batchQueue;
|
|
13
21
|
readonly tracer: Tracer;
|
|
14
22
|
private constructor();
|
|
15
|
-
commitWrites(items: Array<
|
|
23
|
+
commitWrites(items: Array<BrokerWrite>, epochNumber: number): Promise<void>;
|
|
16
24
|
private estimateSize;
|
|
17
25
|
static new(config: ProverBrokerConfig, client?: TelemetryClient, logger?: Logger): Promise<KVBrokerDatabase>;
|
|
18
26
|
private start;
|
|
@@ -20,8 +28,13 @@ export declare class KVBrokerDatabase implements ProvingBrokerDatabase {
|
|
|
20
28
|
deleteAllProvingJobsOlderThanEpoch(epochNumber: EpochNumber): Promise<void>;
|
|
21
29
|
addProvingJob(job: ProvingJob): Promise<void>;
|
|
22
30
|
allProvingJobs(): AsyncIterableIterator<[ProvingJob, ProvingJobSettledResult | undefined]>;
|
|
31
|
+
getProvingJobInputs(id: ProvingJobId): Promise<ProofUri | undefined>;
|
|
32
|
+
getProvingJobResult(id: ProvingJobId): Promise<ProvingJobSettledResult | undefined>;
|
|
23
33
|
setProvingJobError(id: ProvingJobId, reason: string): Promise<void>;
|
|
34
|
+
setProvingJobAborted(id: ProvingJobId): Promise<void>;
|
|
35
|
+
deleteProvingJobResult(id: ProvingJobId): Promise<void>;
|
|
24
36
|
setProvingJobResult(id: ProvingJobId, value: ProofUri): Promise<void>;
|
|
25
37
|
private getEpochDatabase;
|
|
26
38
|
}
|
|
27
|
-
|
|
39
|
+
export {};
|
|
40
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoicGVyc2lzdGVkLmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi8uLi9zcmMvcHJvdmluZ19icm9rZXIvcHJvdmluZ19icm9rZXJfZGF0YWJhc2UvcGVyc2lzdGVkLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLE9BQU8sRUFBRSxXQUFXLEVBQUUsTUFBTSxpQ0FBaUMsQ0FBQztBQUU5RCxPQUFPLEVBQUUsS0FBSyxNQUFNLEVBQWdCLE1BQU0sdUJBQXVCLENBQUM7QUFJbEUsT0FBTyxFQUNMLEtBQUssUUFBUSxFQUNiLFVBQVUsRUFDVixLQUFLLFlBQVksRUFDakIsdUJBQXVCLEVBRXhCLE1BQU0saUNBQWlDLENBQUM7QUFDekMsT0FBTyxFQUdMLEtBQUssZUFBZSxFQUNwQixLQUFLLE1BQU0sRUFHWixNQUFNLHlCQUF5QixDQUFDO0FBS2pDLE9BQU8sS0FBSyxFQUFFLGtCQUFrQixFQUFFLE1BQU0sY0FBYyxDQUFDO0FBQ3ZELE9BQU8sS0FBSyxFQUFFLHFCQUFxQixFQUFFLE1BQU0sK0JBQStCLENBQUM7QUFnRjNFOzs7Ozs7R0FNRztBQUNILEtBQUssV0FBVyxHQUFHLFVBQVUsR0FBRyxDQUFDLFlBQVksRUFBRSx1QkFBdUIsR0FBRyxTQUFTLENBQUMsQ0FBQztBQUVwRixxQkFBYSxnQkFBaUIsWUFBVyxxQkFBcUI7SUFRMUQsT0FBTyxDQUFDLE1BQU07SUFDZCxPQUFPLENBQUMsTUFBTTtJQUVkLE9BQU8sQ0FBQyxNQUFNO0lBVmhCLE9BQU8sQ0FBQyxPQUFPLENBQWM7SUFFN0IsT0FBTyxDQUFDLFVBQVUsQ0FBa0M7SUFFcEQsU0FBZ0IsTUFBTSxFQUFFLE1BQU0sQ0FBQztJQUUvQixPQUFPLGVBc0JOO0lBR1ksWUFBWSxDQUFDLEtBQUssRUFBRSxLQUFLLENBQUMsV0FBVyxDQUFDLEVBQUUsV0FBVyxFQUFFLE1BQU0saUJBVXZFO1lBRWEsWUFBWTtJQVUxQixPQUFvQixHQUFHLENBQ3JCLE1BQU0sRUFBRSxrQkFBa0IsRUFDMUIsTUFBTSxHQUFFLGVBQXNDLEVBQzlDLE1BQU0sU0FBd0QsNkJBOEIvRDtJQUVELE9BQU8sQ0FBQyxLQUFLO0lBSVAsS0FBSyxJQUFJLE9BQU8sQ0FBQyxJQUFJLENBQUMsQ0FLM0I7SUFLSyxrQ0FBa0MsQ0FBQyxXQUFXLEVBQUUsV0FBVyxHQUFHLE9BQU8sQ0FBQyxJQUFJLENBQUMsQ0FXaEY7SUFPRCxhQUFhLENBQUMsR0FBRyxFQUFFLFVBQVUsR0FBRyxPQUFPLENBQUMsSUFBSSxDQUFDLENBRTVDO0lBRU0sY0FBYyxJQUFJLHFCQUFxQixDQUFDLENBQUMsVUFBVSxFQUFFLHVCQUF1QixHQUFHLFNBQVMsQ0FBQyxDQUFDLENBS2hHO0lBT0QsbUJBQW1CLENBQUMsRUFBRSxFQUFFLFlBQVksR0FBRyxPQUFPLENBQUMsUUFBUSxHQUFHLFNBQVMsQ0FBQyxDQUVuRTtJQUVELG1CQUFtQixDQUFDLEVBQUUsRUFBRSxZQUFZLEdBQUcsT0FBTyxDQUFDLHVCQUF1QixHQUFHLFNBQVMsQ0FBQyxDQUVsRjtJQUVELGtCQUFrQixDQUFDLEVBQUUsRUFBRSxZQUFZLEVBQUUsTUFBTSxFQUFFLE1BQU0sR0FBRyxPQUFPLENBQUMsSUFBSSxDQUFDLENBRWxFO0lBRUQsb0JBQW9CLENBQUMsRUFBRSxFQUFFLFlBQVksR0FBRyxPQUFPLENBQUMsSUFBSSxDQUFDLENBRXBEO0lBRUQsc0JBQXNCLENBQUMsRUFBRSxFQUFFLFlBQVksR0FBRyxPQUFPLENBQUMsSUFBSSxDQUFDLENBRXREO0lBRUQsbUJBQW1CLENBQUMsRUFBRSxFQUFFLFlBQVksRUFBRSxLQUFLLEVBQUUsUUFBUSxHQUFHLE9BQU8sQ0FBQyxJQUFJLENBQUMsQ0FFcEU7WUFFYSxnQkFBZ0I7Q0FvQi9CIn0=
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"persisted.d.ts","sourceRoot":"","sources":["../../../src/proving_broker/proving_broker_database/persisted.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iCAAiC,CAAC;AAE9D,OAAO,EAAE,KAAK,MAAM,EAAgB,MAAM,uBAAuB,CAAC;AAIlE,OAAO,EACL,KAAK,QAAQ,EACb,UAAU,EACV,KAAK,YAAY,EACjB,uBAAuB,EAExB,MAAM,iCAAiC,CAAC;AACzC,OAAO,EAGL,KAAK,eAAe,EACpB,KAAK,MAAM,EAGZ,MAAM,yBAAyB,CAAC;AAKjC,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAC;AACvD,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,+BAA+B,CAAC;
|
|
1
|
+
{"version":3,"file":"persisted.d.ts","sourceRoot":"","sources":["../../../src/proving_broker/proving_broker_database/persisted.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iCAAiC,CAAC;AAE9D,OAAO,EAAE,KAAK,MAAM,EAAgB,MAAM,uBAAuB,CAAC;AAIlE,OAAO,EACL,KAAK,QAAQ,EACb,UAAU,EACV,KAAK,YAAY,EACjB,uBAAuB,EAExB,MAAM,iCAAiC,CAAC;AACzC,OAAO,EAGL,KAAK,eAAe,EACpB,KAAK,MAAM,EAGZ,MAAM,yBAAyB,CAAC;AAKjC,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAC;AACvD,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,+BAA+B,CAAC;AAgF3E;;;;;;GAMG;AACH,KAAK,WAAW,GAAG,UAAU,GAAG,CAAC,YAAY,EAAE,uBAAuB,GAAG,SAAS,CAAC,CAAC;AAEpF,qBAAa,gBAAiB,YAAW,qBAAqB;IAQ1D,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,MAAM;IAEd,OAAO,CAAC,MAAM;IAVhB,OAAO,CAAC,OAAO,CAAc;IAE7B,OAAO,CAAC,UAAU,CAAkC;IAEpD,SAAgB,MAAM,EAAE,MAAM,CAAC;IAE/B,OAAO,eAsBN;IAGY,YAAY,CAAC,KAAK,EAAE,KAAK,CAAC,WAAW,CAAC,EAAE,WAAW,EAAE,MAAM,iBAUvE;YAEa,YAAY;IAU1B,OAAoB,GAAG,CACrB,MAAM,EAAE,kBAAkB,EAC1B,MAAM,GAAE,eAAsC,EAC9C,MAAM,SAAwD,6BA8B/D;IAED,OAAO,CAAC,KAAK;IAIP,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAK3B;IAKK,kCAAkC,CAAC,WAAW,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAWhF;IAOD,aAAa,CAAC,GAAG,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAE5C;IAEM,cAAc,IAAI,qBAAqB,CAAC,CAAC,UAAU,EAAE,uBAAuB,GAAG,SAAS,CAAC,CAAC,CAKhG;IAOD,mBAAmB,CAAC,EAAE,EAAE,YAAY,GAAG,OAAO,CAAC,QAAQ,GAAG,SAAS,CAAC,CAEnE;IAED,mBAAmB,CAAC,EAAE,EAAE,YAAY,GAAG,OAAO,CAAC,uBAAuB,GAAG,SAAS,CAAC,CAElF;IAED,kBAAkB,CAAC,EAAE,EAAE,YAAY,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAElE;IAED,oBAAoB,CAAC,EAAE,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAEpD;IAED,sBAAsB,CAAC,EAAE,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAEtD;IAED,mBAAmB,CAAC,EAAE,EAAE,YAAY,EAAE,KAAK,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CAEpE;YAEa,gBAAgB;CAoB/B"}
|