@aztec/prover-client 0.0.0-test.1 → 0.0.1-commit.017a351
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dest/config.d.ts +8 -8
- package/dest/config.d.ts.map +1 -1
- package/dest/config.js +26 -2
- package/dest/index.d.ts +1 -1
- package/dest/light/index.d.ts +2 -0
- package/dest/light/index.d.ts.map +1 -0
- package/dest/light/index.js +1 -0
- package/dest/light/lightweight_checkpoint_builder.d.ts +52 -0
- package/dest/light/lightweight_checkpoint_builder.d.ts.map +1 -0
- package/dest/light/lightweight_checkpoint_builder.js +231 -0
- package/dest/mocks/fixtures.d.ts +8 -8
- package/dest/mocks/fixtures.d.ts.map +1 -1
- package/dest/mocks/fixtures.js +36 -17
- package/dest/mocks/test_context.d.ts +45 -32
- package/dest/mocks/test_context.d.ts.map +1 -1
- package/dest/mocks/test_context.js +162 -90
- package/dest/orchestrator/block-building-helpers.d.ts +36 -29
- package/dest/orchestrator/block-building-helpers.d.ts.map +1 -1
- package/dest/orchestrator/block-building-helpers.js +170 -189
- package/dest/orchestrator/block-proving-state.d.ts +73 -48
- package/dest/orchestrator/block-proving-state.d.ts.map +1 -1
- package/dest/orchestrator/block-proving-state.js +289 -177
- package/dest/orchestrator/checkpoint-proving-state.d.ts +83 -0
- package/dest/orchestrator/checkpoint-proving-state.d.ts.map +1 -0
- package/dest/orchestrator/checkpoint-proving-state.js +252 -0
- package/dest/orchestrator/checkpoint-sub-tree-orchestrator.d.ts +107 -0
- package/dest/orchestrator/checkpoint-sub-tree-orchestrator.d.ts.map +1 -0
- package/dest/orchestrator/checkpoint-sub-tree-orchestrator.js +151 -0
- package/dest/orchestrator/epoch-proving-context.d.ts +51 -0
- package/dest/orchestrator/epoch-proving-context.d.ts.map +1 -0
- package/dest/orchestrator/epoch-proving-context.js +81 -0
- package/dest/orchestrator/epoch-proving-state.d.ts +43 -28
- package/dest/orchestrator/epoch-proving-state.d.ts.map +1 -1
- package/dest/orchestrator/epoch-proving-state.js +179 -73
- package/dest/orchestrator/index.d.ts +4 -1
- package/dest/orchestrator/index.d.ts.map +1 -1
- package/dest/orchestrator/index.js +3 -0
- package/dest/orchestrator/orchestrator.d.ts +59 -51
- package/dest/orchestrator/orchestrator.d.ts.map +1 -1
- package/dest/orchestrator/orchestrator.js +808 -385
- package/dest/orchestrator/orchestrator_metrics.d.ts +1 -1
- package/dest/orchestrator/orchestrator_metrics.d.ts.map +1 -1
- package/dest/orchestrator/orchestrator_metrics.js +2 -6
- package/dest/orchestrator/proving-scheduler.d.ts +72 -0
- package/dest/orchestrator/proving-scheduler.d.ts.map +1 -0
- package/dest/orchestrator/proving-scheduler.js +117 -0
- package/dest/orchestrator/top-tree-orchestrator.d.ts +83 -0
- package/dest/orchestrator/top-tree-orchestrator.d.ts.map +1 -0
- package/dest/orchestrator/top-tree-orchestrator.js +182 -0
- package/dest/orchestrator/top-tree-proving-scheduler.d.ts +62 -0
- package/dest/orchestrator/top-tree-proving-scheduler.d.ts.map +1 -0
- package/dest/orchestrator/top-tree-proving-scheduler.js +73 -0
- package/dest/orchestrator/top-tree-proving-state.d.ts +61 -0
- package/dest/orchestrator/top-tree-proving-state.d.ts.map +1 -0
- package/dest/orchestrator/top-tree-proving-state.js +185 -0
- package/dest/orchestrator/tx-proving-state.d.ts +15 -12
- package/dest/orchestrator/tx-proving-state.d.ts.map +1 -1
- package/dest/orchestrator/tx-proving-state.js +27 -44
- package/dest/prover-client/factory.d.ts +3 -3
- package/dest/prover-client/factory.d.ts.map +1 -1
- package/dest/prover-client/index.d.ts +1 -1
- package/dest/prover-client/prover-client.d.ts +68 -9
- package/dest/prover-client/prover-client.d.ts.map +1 -1
- package/dest/prover-client/prover-client.js +68 -14
- package/dest/prover-client/server-epoch-prover.d.ts +16 -12
- package/dest/prover-client/server-epoch-prover.d.ts.map +1 -1
- package/dest/prover-client/server-epoch-prover.js +11 -11
- package/dest/proving_broker/broker_prover_facade.d.ts +28 -19
- package/dest/proving_broker/broker_prover_facade.d.ts.map +1 -1
- package/dest/proving_broker/broker_prover_facade.js +74 -61
- package/dest/proving_broker/config.d.ts +18 -61
- package/dest/proving_broker/config.d.ts.map +1 -1
- package/dest/proving_broker/config.js +43 -9
- package/dest/proving_broker/factory.d.ts +2 -2
- package/dest/proving_broker/factory.d.ts.map +1 -1
- package/dest/proving_broker/factory.js +5 -1
- package/dest/proving_broker/fixtures.d.ts +3 -2
- package/dest/proving_broker/fixtures.d.ts.map +1 -1
- package/dest/proving_broker/fixtures.js +3 -2
- package/dest/proving_broker/index.d.ts +2 -1
- package/dest/proving_broker/index.d.ts.map +1 -1
- package/dest/proving_broker/index.js +1 -0
- package/dest/proving_broker/proof_store/factory.d.ts +2 -5
- package/dest/proving_broker/proof_store/factory.d.ts.map +1 -1
- package/dest/proving_broker/proof_store/factory.js +7 -30
- package/dest/proving_broker/proof_store/file_store_proof_store.d.ts +18 -0
- package/dest/proving_broker/proof_store/file_store_proof_store.d.ts.map +1 -0
- package/dest/proving_broker/proof_store/file_store_proof_store.js +60 -0
- package/dest/proving_broker/proof_store/index.d.ts +2 -1
- package/dest/proving_broker/proof_store/index.d.ts.map +1 -1
- package/dest/proving_broker/proof_store/index.js +1 -0
- package/dest/proving_broker/proof_store/inline_proof_store.d.ts +1 -1
- package/dest/proving_broker/proof_store/inline_proof_store.d.ts.map +1 -1
- package/dest/proving_broker/proof_store/proof_store.d.ts +1 -1
- package/dest/proving_broker/proving_agent.d.ts +8 -12
- package/dest/proving_broker/proving_agent.d.ts.map +1 -1
- package/dest/proving_broker/proving_agent.js +86 -65
- package/dest/proving_broker/proving_broker.d.ts +18 -6
- package/dest/proving_broker/proving_broker.d.ts.map +1 -1
- package/dest/proving_broker/proving_broker.js +108 -44
- package/dest/proving_broker/proving_broker_database/memory.d.ts +3 -2
- package/dest/proving_broker/proving_broker_database/memory.d.ts.map +1 -1
- package/dest/proving_broker/proving_broker_database/memory.js +1 -1
- package/dest/proving_broker/proving_broker_database/persisted.d.ts +5 -3
- package/dest/proving_broker/proving_broker_database/persisted.d.ts.map +1 -1
- package/dest/proving_broker/proving_broker_database/persisted.js +401 -11
- package/dest/proving_broker/proving_broker_database.d.ts +3 -2
- package/dest/proving_broker/proving_broker_database.d.ts.map +1 -1
- package/dest/proving_broker/proving_broker_instrumentation.d.ts +3 -1
- package/dest/proving_broker/proving_broker_instrumentation.d.ts.map +1 -1
- package/dest/proving_broker/proving_broker_instrumentation.js +22 -35
- package/dest/proving_broker/proving_job_controller.d.ts +11 -10
- package/dest/proving_broker/proving_job_controller.d.ts.map +1 -1
- package/dest/proving_broker/proving_job_controller.js +92 -62
- package/dest/proving_broker/rpc.d.ts +9 -7
- package/dest/proving_broker/rpc.d.ts.map +1 -1
- package/dest/proving_broker/rpc.js +88 -27
- package/dest/test/mock_proof_store.d.ts +9 -0
- package/dest/test/mock_proof_store.d.ts.map +1 -0
- package/dest/test/mock_proof_store.js +10 -0
- package/dest/test/mock_prover.d.ts +23 -17
- package/dest/test/mock_prover.d.ts.map +1 -1
- package/dest/test/mock_prover.js +38 -20
- package/package.json +34 -33
- package/src/config.ts +41 -9
- package/src/light/index.ts +1 -0
- package/src/light/lightweight_checkpoint_builder.ts +320 -0
- package/src/mocks/fixtures.ts +46 -40
- package/src/mocks/test_context.ts +231 -120
- package/src/orchestrator/block-building-helpers.ts +258 -334
- package/src/orchestrator/block-proving-state.ts +334 -231
- package/src/orchestrator/checkpoint-proving-state.ts +362 -0
- package/src/orchestrator/checkpoint-sub-tree-orchestrator.ts +271 -0
- package/src/orchestrator/epoch-proving-context.ts +101 -0
- package/src/orchestrator/epoch-proving-state.ts +239 -111
- package/src/orchestrator/index.ts +8 -0
- package/src/orchestrator/orchestrator.ts +591 -451
- package/src/orchestrator/orchestrator_metrics.ts +2 -6
- package/src/orchestrator/proving-scheduler.ts +156 -0
- package/src/orchestrator/top-tree-orchestrator.ts +314 -0
- package/src/orchestrator/top-tree-proving-scheduler.ts +154 -0
- package/src/orchestrator/top-tree-proving-state.ts +220 -0
- package/src/orchestrator/tx-proving-state.ts +48 -66
- package/src/prover-client/factory.ts +6 -2
- package/src/prover-client/prover-client.ts +173 -38
- package/src/prover-client/server-epoch-prover.ts +40 -22
- package/src/proving_broker/broker_prover_facade.ts +227 -149
- package/src/proving_broker/config.ts +49 -8
- package/src/proving_broker/factory.ts +2 -1
- package/src/proving_broker/fixtures.ts +8 -3
- package/src/proving_broker/index.ts +1 -0
- package/src/proving_broker/proof_store/factory.ts +10 -32
- package/src/proving_broker/proof_store/file_store_proof_store.ts +78 -0
- package/src/proving_broker/proof_store/index.ts +1 -0
- package/src/proving_broker/proof_store/inline_proof_store.ts +1 -1
- package/src/proving_broker/proving_agent.ts +95 -66
- package/src/proving_broker/proving_broker.ts +121 -49
- package/src/proving_broker/proving_broker_database/memory.ts +3 -2
- package/src/proving_broker/proving_broker_database/persisted.ts +31 -15
- package/src/proving_broker/proving_broker_database.ts +2 -1
- package/src/proving_broker/proving_broker_instrumentation.ts +23 -35
- package/src/proving_broker/proving_job_controller.ts +100 -83
- package/src/proving_broker/rpc.ts +46 -25
- package/src/test/mock_proof_store.ts +14 -0
- package/src/test/mock_prover.ts +156 -64
- package/dest/bin/get-proof-inputs.d.ts +0 -2
- package/dest/bin/get-proof-inputs.d.ts.map +0 -1
- package/dest/bin/get-proof-inputs.js +0 -51
- package/dest/block_builder/index.d.ts +0 -6
- package/dest/block_builder/index.d.ts.map +0 -1
- package/dest/block_builder/index.js +0 -1
- package/dest/block_builder/light.d.ts +0 -33
- package/dest/block_builder/light.d.ts.map +0 -1
- package/dest/block_builder/light.js +0 -82
- package/dest/proving_broker/proof_store/gcs_proof_store.d.ts +0 -14
- package/dest/proving_broker/proof_store/gcs_proof_store.d.ts.map +0 -1
- package/dest/proving_broker/proof_store/gcs_proof_store.js +0 -51
- package/dest/proving_broker/proving_agent_instrumentation.d.ts +0 -8
- package/dest/proving_broker/proving_agent_instrumentation.d.ts.map +0 -1
- package/dest/proving_broker/proving_agent_instrumentation.js +0 -16
- package/src/bin/get-proof-inputs.ts +0 -59
- package/src/block_builder/index.ts +0 -6
- package/src/block_builder/light.ts +0 -101
- package/src/proving_broker/proof_store/gcs_proof_store.ts +0 -72
- package/src/proving_broker/proving_agent_instrumentation.ts +0 -21
|
@@ -1,15 +1,11 @@
|
|
|
1
|
-
|
|
2
|
-
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
3
|
-
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
4
|
-
else for(var i = decorators.length - 1; i >= 0; i--)if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
5
|
-
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
6
|
-
}
|
|
1
|
+
import { EpochNumber } from '@aztec/foundation/branded-types';
|
|
7
2
|
import { createLogger } from '@aztec/foundation/log';
|
|
8
3
|
import { RunningPromise, promiseWithResolvers } from '@aztec/foundation/promise';
|
|
9
4
|
import { PriorityMemoryQueue } from '@aztec/foundation/queue';
|
|
10
5
|
import { Timer } from '@aztec/foundation/timer';
|
|
6
|
+
import { tryStop } from '@aztec/stdlib/interfaces/server';
|
|
11
7
|
import { ProvingRequestType } from '@aztec/stdlib/proofs';
|
|
12
|
-
import { getTelemetryClient
|
|
8
|
+
import { getTelemetryClient } from '@aztec/telemetry-client';
|
|
13
9
|
import assert from 'assert';
|
|
14
10
|
import { defaultProverBrokerConfig } from './config.js';
|
|
15
11
|
import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.js';
|
|
@@ -46,7 +42,7 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
|
|
|
46
42
|
/**
|
|
47
43
|
* The broker keeps track of the highest epoch its seen.
|
|
48
44
|
* This information is used for garbage collection: once it reaches the next epoch, it can start pruning the database of old state.
|
|
49
|
-
* It is important that this value is
|
|
45
|
+
* It is important that this value is initialized to zero. This ensures that we don't delete any old jobs until the current
|
|
50
46
|
* process instance receives a job request informing it of the actual current highest epoch
|
|
51
47
|
* Example:
|
|
52
48
|
* proving epoch 11 - the broker will wipe all jobs for epochs 9 and lower
|
|
@@ -55,22 +51,29 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
|
|
|
55
51
|
*/ epochHeight;
|
|
56
52
|
maxEpochsToKeepResultsFor;
|
|
57
53
|
started;
|
|
58
|
-
|
|
54
|
+
debugReplayEnabled;
|
|
55
|
+
constructor(database, { proverBrokerJobTimeoutMs, proverBrokerPollIntervalMs, proverBrokerJobMaxRetries, proverBrokerMaxEpochsToKeepResultsFor, proverBrokerDebugReplayEnabled } = defaultProverBrokerConfig, client = getTelemetryClient(), logger = createLogger('prover-client:proving-broker')){
|
|
59
56
|
this.database = database;
|
|
60
57
|
this.logger = logger;
|
|
61
58
|
this.queues = {
|
|
62
59
|
[ProvingRequestType.PUBLIC_VM]: new PriorityMemoryQueue(provingJobComparator),
|
|
63
|
-
[ProvingRequestType.
|
|
64
|
-
[ProvingRequestType.
|
|
65
|
-
[ProvingRequestType.
|
|
66
|
-
[ProvingRequestType.
|
|
60
|
+
[ProvingRequestType.PUBLIC_CHONK_VERIFIER]: new PriorityMemoryQueue(provingJobComparator),
|
|
61
|
+
[ProvingRequestType.PRIVATE_TX_BASE_ROLLUP]: new PriorityMemoryQueue(provingJobComparator),
|
|
62
|
+
[ProvingRequestType.PUBLIC_TX_BASE_ROLLUP]: new PriorityMemoryQueue(provingJobComparator),
|
|
63
|
+
[ProvingRequestType.TX_MERGE_ROLLUP]: new PriorityMemoryQueue(provingJobComparator),
|
|
67
64
|
[ProvingRequestType.ROOT_ROLLUP]: new PriorityMemoryQueue(provingJobComparator),
|
|
68
65
|
[ProvingRequestType.BLOCK_MERGE_ROLLUP]: new PriorityMemoryQueue(provingJobComparator),
|
|
66
|
+
[ProvingRequestType.BLOCK_ROOT_FIRST_ROLLUP]: new PriorityMemoryQueue(provingJobComparator),
|
|
67
|
+
[ProvingRequestType.BLOCK_ROOT_SINGLE_TX_FIRST_ROLLUP]: new PriorityMemoryQueue(provingJobComparator),
|
|
68
|
+
[ProvingRequestType.BLOCK_ROOT_EMPTY_TX_FIRST_ROLLUP]: new PriorityMemoryQueue(provingJobComparator),
|
|
69
69
|
[ProvingRequestType.BLOCK_ROOT_ROLLUP]: new PriorityMemoryQueue(provingJobComparator),
|
|
70
|
-
[ProvingRequestType.
|
|
71
|
-
[ProvingRequestType.
|
|
72
|
-
[ProvingRequestType.
|
|
73
|
-
[ProvingRequestType.
|
|
70
|
+
[ProvingRequestType.BLOCK_ROOT_SINGLE_TX_ROLLUP]: new PriorityMemoryQueue(provingJobComparator),
|
|
71
|
+
[ProvingRequestType.CHECKPOINT_ROOT_ROLLUP]: new PriorityMemoryQueue(provingJobComparator),
|
|
72
|
+
[ProvingRequestType.CHECKPOINT_ROOT_SINGLE_BLOCK_ROLLUP]: new PriorityMemoryQueue(provingJobComparator),
|
|
73
|
+
[ProvingRequestType.CHECKPOINT_MERGE_ROLLUP]: new PriorityMemoryQueue(provingJobComparator),
|
|
74
|
+
[ProvingRequestType.CHECKPOINT_PADDING_ROLLUP]: new PriorityMemoryQueue(provingJobComparator),
|
|
75
|
+
[ProvingRequestType.PARITY_BASE]: new PriorityMemoryQueue(provingJobComparator),
|
|
76
|
+
[ProvingRequestType.PARITY_ROOT]: new PriorityMemoryQueue(provingJobComparator)
|
|
74
77
|
};
|
|
75
78
|
this.jobsCache = new Map();
|
|
76
79
|
this.resultsCache = new Map();
|
|
@@ -102,6 +105,7 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
|
|
|
102
105
|
this.jobTimeoutMs = proverBrokerJobTimeoutMs;
|
|
103
106
|
this.maxRetries = proverBrokerJobMaxRetries;
|
|
104
107
|
this.maxEpochsToKeepResultsFor = proverBrokerMaxEpochsToKeepResultsFor;
|
|
108
|
+
this.debugReplayEnabled = proverBrokerDebugReplayEnabled ?? false;
|
|
105
109
|
}
|
|
106
110
|
measureQueueDepth;
|
|
107
111
|
countActiveJobs;
|
|
@@ -135,7 +139,7 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
|
|
|
135
139
|
this.logger.warn('ProvingBroker not started');
|
|
136
140
|
return Promise.resolve();
|
|
137
141
|
}
|
|
138
|
-
await this.cleanupPromise
|
|
142
|
+
await tryStop(this.cleanupPromise);
|
|
139
143
|
}
|
|
140
144
|
enqueueProvingJob(job) {
|
|
141
145
|
return this.#enqueueProvingJob(job);
|
|
@@ -161,6 +165,32 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
|
|
|
161
165
|
reportProvingJobProgress(id, startedAt, filter) {
|
|
162
166
|
return Promise.resolve(this.#reportProvingJobProgress(id, startedAt, filter));
|
|
163
167
|
}
|
|
168
|
+
async replayProvingJob(jobId, type, epochNumber, inputsUri) {
|
|
169
|
+
if (!this.debugReplayEnabled) {
|
|
170
|
+
throw new Error('Debug replay not enabled. Set PROVER_BROKER_DEBUG_REPLAY_ENABLED=true');
|
|
171
|
+
}
|
|
172
|
+
this.logger.info(`Replaying proving job`, {
|
|
173
|
+
provingJobId: jobId,
|
|
174
|
+
epochNumber,
|
|
175
|
+
inputsUri
|
|
176
|
+
});
|
|
177
|
+
// Clear existing state and enqueue
|
|
178
|
+
this.cleanUpProvingJobState([
|
|
179
|
+
jobId
|
|
180
|
+
]);
|
|
181
|
+
const job = {
|
|
182
|
+
id: jobId,
|
|
183
|
+
type,
|
|
184
|
+
epochNumber,
|
|
185
|
+
inputsUri
|
|
186
|
+
};
|
|
187
|
+
this.jobsCache.set(jobId, job);
|
|
188
|
+
await this.database.addProvingJob(job);
|
|
189
|
+
this.enqueueJobInternal(job);
|
|
190
|
+
return {
|
|
191
|
+
status: 'in-queue'
|
|
192
|
+
};
|
|
193
|
+
}
|
|
164
194
|
async #enqueueProvingJob(job) {
|
|
165
195
|
// We return the job status at the start of this call
|
|
166
196
|
const jobStatus = this.#getProvingJobStatus(job.id);
|
|
@@ -209,17 +239,27 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
|
|
|
209
239
|
this.logger.info(`Cancelling job id=${id}`, {
|
|
210
240
|
provingJobId: id
|
|
211
241
|
});
|
|
212
|
-
await this.#reportProvingJobError(id, 'Aborted', false);
|
|
242
|
+
await this.#reportProvingJobError(id, 'Aborted', false, undefined, true);
|
|
213
243
|
}
|
|
214
244
|
}
|
|
215
245
|
cleanUpProvingJobState(ids) {
|
|
246
|
+
const idsToClean = new Set(ids);
|
|
216
247
|
for (const id of ids){
|
|
217
248
|
this.jobsCache.delete(id);
|
|
249
|
+
const deferred = this.promises.get(id);
|
|
250
|
+
if (deferred) {
|
|
251
|
+
deferred.resolve({
|
|
252
|
+
status: 'rejected',
|
|
253
|
+
reason: 'Proving job cleaned up'
|
|
254
|
+
});
|
|
255
|
+
}
|
|
218
256
|
this.promises.delete(id);
|
|
219
257
|
this.resultsCache.delete(id);
|
|
220
258
|
this.inProgress.delete(id);
|
|
221
259
|
this.retries.delete(id);
|
|
260
|
+
this.enqueuedAt.delete(id);
|
|
222
261
|
}
|
|
262
|
+
this.completedJobNotifications = this.completedJobNotifications.filter((id)=>!idsToClean.has(id));
|
|
223
263
|
}
|
|
224
264
|
#getProvingJobStatus(id) {
|
|
225
265
|
const result = this.resultsCache.get(id);
|
|
@@ -244,7 +284,6 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
|
|
|
244
284
|
this.completedJobNotifications = [];
|
|
245
285
|
return Promise.resolve(notifications.concat(completedJobs));
|
|
246
286
|
}
|
|
247
|
-
// eslint-disable-next-line require-await
|
|
248
287
|
#getProvingJob(filter = {
|
|
249
288
|
allowList: []
|
|
250
289
|
}) {
|
|
@@ -271,6 +310,8 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
|
|
|
271
310
|
const enqueuedAt = this.enqueuedAt.get(job.id);
|
|
272
311
|
if (enqueuedAt) {
|
|
273
312
|
this.instrumentation.recordJobWait(job.type, enqueuedAt);
|
|
313
|
+
// we can clear this flag now.
|
|
314
|
+
this.enqueuedAt.delete(job.id);
|
|
274
315
|
}
|
|
275
316
|
return {
|
|
276
317
|
job,
|
|
@@ -281,7 +322,7 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
|
|
|
281
322
|
}
|
|
282
323
|
return undefined;
|
|
283
324
|
}
|
|
284
|
-
async #reportProvingJobError(id, err, retry = false, filter) {
|
|
325
|
+
async #reportProvingJobError(id, err, retry = false, filter, aborted = false) {
|
|
285
326
|
const info = this.inProgress.get(id);
|
|
286
327
|
const item = this.jobsCache.get(id);
|
|
287
328
|
const retries = this.retries.get(id) ?? 0;
|
|
@@ -328,7 +369,11 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
|
|
|
328
369
|
this.resultsCache.set(id, result);
|
|
329
370
|
this.promises.get(id).resolve(result);
|
|
330
371
|
this.completedJobNotifications.push(id);
|
|
331
|
-
|
|
372
|
+
if (aborted) {
|
|
373
|
+
this.instrumentation.incAbortedJobs(item.type);
|
|
374
|
+
} else {
|
|
375
|
+
this.instrumentation.incRejectedJobs(item.type);
|
|
376
|
+
}
|
|
332
377
|
if (info) {
|
|
333
378
|
const duration = this.msTimeSource() - info.startedAt;
|
|
334
379
|
this.instrumentation.recordJobDuration(item.type, duration);
|
|
@@ -443,26 +488,26 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
|
|
|
443
488
|
return this.#getProvingJob(filter);
|
|
444
489
|
}
|
|
445
490
|
async cleanupPass() {
|
|
446
|
-
this.cleanupStaleJobs();
|
|
447
491
|
this.reEnqueueExpiredJobs();
|
|
448
492
|
const oldestEpochToKeep = this.oldestEpochToKeep();
|
|
449
493
|
if (oldestEpochToKeep > 0) {
|
|
450
|
-
|
|
494
|
+
this.cleanupJobsOlderThanEpoch(EpochNumber(oldestEpochToKeep));
|
|
495
|
+
await this.database.deleteAllProvingJobsOlderThanEpoch(EpochNumber(oldestEpochToKeep));
|
|
451
496
|
this.logger.trace(`Deleted all epochs older than ${oldestEpochToKeep}`);
|
|
452
497
|
}
|
|
453
498
|
}
|
|
454
|
-
|
|
499
|
+
cleanupJobsOlderThanEpoch(epochNumber) {
|
|
455
500
|
const jobIds = Array.from(this.jobsCache.keys());
|
|
456
501
|
const jobsToClean = [];
|
|
457
502
|
for (const id of jobIds){
|
|
458
503
|
const job = this.jobsCache.get(id);
|
|
459
|
-
if (
|
|
504
|
+
if (job.epochNumber < epochNumber) {
|
|
460
505
|
jobsToClean.push(id);
|
|
461
506
|
}
|
|
462
507
|
}
|
|
463
508
|
if (jobsToClean.length > 0) {
|
|
464
509
|
this.cleanUpProvingJobState(jobsToClean);
|
|
465
|
-
this.logger.
|
|
510
|
+
this.logger.verbose(`Cleaned up proving jobs=${jobsToClean.length}`);
|
|
466
511
|
}
|
|
467
512
|
}
|
|
468
513
|
reEnqueueExpiredJobs() {
|
|
@@ -479,12 +524,28 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
|
|
|
479
524
|
const now = this.msTimeSource();
|
|
480
525
|
const msSinceLastUpdate = now - metadata.lastUpdatedAt;
|
|
481
526
|
if (msSinceLastUpdate >= this.jobTimeoutMs) {
|
|
482
|
-
this.logger.warn(`Proving job id=${id} timed out. Adding it back to the queue.`, {
|
|
483
|
-
provingJobId: id
|
|
484
|
-
});
|
|
485
527
|
this.inProgress.delete(id);
|
|
486
|
-
this.enqueueJobInternal(item);
|
|
487
528
|
this.instrumentation.incTimedOutJobs(item.type);
|
|
529
|
+
const retries = this.retries.get(id) ?? 0;
|
|
530
|
+
if (retries + 1 < this.maxRetries && !this.isJobStale(item)) {
|
|
531
|
+
this.logger.warn(`Proving job id=${id} timed out. Re-enqueueing (retry ${retries + 1}/${this.maxRetries}).`, {
|
|
532
|
+
provingJobId: id
|
|
533
|
+
});
|
|
534
|
+
this.retries.set(id, retries + 1);
|
|
535
|
+
this.enqueueJobInternal(item);
|
|
536
|
+
} else {
|
|
537
|
+
this.logger.error(`Proving job id=${id} timed out after ${retries + 1} attempts. Marking as failed.`, {
|
|
538
|
+
provingJobId: id
|
|
539
|
+
});
|
|
540
|
+
const result = {
|
|
541
|
+
status: 'rejected',
|
|
542
|
+
reason: 'Timed out'
|
|
543
|
+
};
|
|
544
|
+
this.resultsCache.set(id, result);
|
|
545
|
+
this.promises.get(id)?.resolve(result);
|
|
546
|
+
this.completedJobNotifications.push(id);
|
|
547
|
+
this.instrumentation.incRejectedJobs(item.type);
|
|
548
|
+
}
|
|
488
549
|
}
|
|
489
550
|
}
|
|
490
551
|
}
|
|
@@ -506,9 +567,6 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
|
|
|
506
567
|
return this.epochHeight - this.maxEpochsToKeepResultsFor;
|
|
507
568
|
}
|
|
508
569
|
}
|
|
509
|
-
_ts_decorate([
|
|
510
|
-
trackSpan('ProvingBroker.cleanupPass')
|
|
511
|
-
], ProvingBroker.prototype, "cleanupPass", null);
|
|
512
570
|
/**
|
|
513
571
|
* Compares two proving jobs and selects which one's more important
|
|
514
572
|
* @param a - A proving job
|
|
@@ -554,17 +612,23 @@ _ts_decorate([
|
|
|
554
612
|
*
|
|
555
613
|
* The aim is that this will speed up block proving as the closer we get to a block's root proof the more likely it
|
|
556
614
|
* is to get picked up by agents
|
|
557
|
-
*/ const PROOF_TYPES_IN_PRIORITY_ORDER = [
|
|
615
|
+
*/ export const PROOF_TYPES_IN_PRIORITY_ORDER = [
|
|
616
|
+
ProvingRequestType.ROOT_ROLLUP,
|
|
617
|
+
ProvingRequestType.BLOCK_ROOT_FIRST_ROLLUP,
|
|
618
|
+
ProvingRequestType.BLOCK_ROOT_SINGLE_TX_FIRST_ROLLUP,
|
|
619
|
+
ProvingRequestType.BLOCK_ROOT_EMPTY_TX_FIRST_ROLLUP,
|
|
558
620
|
ProvingRequestType.BLOCK_ROOT_ROLLUP,
|
|
559
|
-
ProvingRequestType.
|
|
621
|
+
ProvingRequestType.BLOCK_ROOT_SINGLE_TX_ROLLUP,
|
|
560
622
|
ProvingRequestType.BLOCK_MERGE_ROLLUP,
|
|
561
|
-
ProvingRequestType.
|
|
562
|
-
ProvingRequestType.
|
|
563
|
-
ProvingRequestType.
|
|
564
|
-
ProvingRequestType.
|
|
623
|
+
ProvingRequestType.CHECKPOINT_ROOT_ROLLUP,
|
|
624
|
+
ProvingRequestType.CHECKPOINT_ROOT_SINGLE_BLOCK_ROLLUP,
|
|
625
|
+
ProvingRequestType.CHECKPOINT_MERGE_ROLLUP,
|
|
626
|
+
ProvingRequestType.CHECKPOINT_PADDING_ROLLUP,
|
|
627
|
+
ProvingRequestType.TX_MERGE_ROLLUP,
|
|
628
|
+
ProvingRequestType.PUBLIC_TX_BASE_ROLLUP,
|
|
629
|
+
ProvingRequestType.PRIVATE_TX_BASE_ROLLUP,
|
|
565
630
|
ProvingRequestType.PUBLIC_VM,
|
|
566
|
-
ProvingRequestType.
|
|
567
|
-
ProvingRequestType.
|
|
568
|
-
ProvingRequestType.
|
|
569
|
-
ProvingRequestType.EMPTY_BLOCK_ROOT_ROLLUP
|
|
631
|
+
ProvingRequestType.PUBLIC_CHONK_VERIFIER,
|
|
632
|
+
ProvingRequestType.PARITY_ROOT,
|
|
633
|
+
ProvingRequestType.PARITY_BASE
|
|
570
634
|
];
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { EpochNumber } from '@aztec/foundation/branded-types';
|
|
1
2
|
import { type ProofUri, type ProvingJob, type ProvingJobId, type ProvingJobSettledResult } from '@aztec/stdlib/interfaces/server';
|
|
2
3
|
import type { ProvingBrokerDatabase } from '../proving_broker_database.js';
|
|
3
4
|
export declare class InMemoryBrokerDatabase implements ProvingBrokerDatabase {
|
|
@@ -9,8 +10,8 @@ export declare class InMemoryBrokerDatabase implements ProvingBrokerDatabase {
|
|
|
9
10
|
setProvingJobResult(id: ProvingJobId, value: ProofUri): Promise<void>;
|
|
10
11
|
setProvingJobError(id: ProvingJobId, reason: string): Promise<void>;
|
|
11
12
|
deleteProvingJobs(ids: ProvingJobId[]): Promise<void>;
|
|
12
|
-
deleteAllProvingJobsOlderThanEpoch(epochNumber:
|
|
13
|
+
deleteAllProvingJobsOlderThanEpoch(epochNumber: EpochNumber): Promise<void>;
|
|
13
14
|
allProvingJobs(): AsyncIterableIterator<[ProvingJob, ProvingJobSettledResult | undefined]>;
|
|
14
15
|
close(): Promise<void>;
|
|
15
16
|
}
|
|
16
|
-
//# sourceMappingURL=
|
|
17
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoibWVtb3J5LmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi8uLi9zcmMvcHJvdmluZ19icm9rZXIvcHJvdmluZ19icm9rZXJfZGF0YWJhc2UvbWVtb3J5LnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLE9BQU8sRUFBRSxXQUFXLEVBQUUsTUFBTSxpQ0FBaUMsQ0FBQztBQUM5RCxPQUFPLEVBQ0wsS0FBSyxRQUFRLEVBQ2IsS0FBSyxVQUFVLEVBQ2YsS0FBSyxZQUFZLEVBQ2pCLEtBQUssdUJBQXVCLEVBRTdCLE1BQU0saUNBQWlDLENBQUM7QUFFekMsT0FBTyxLQUFLLEVBQUUscUJBQXFCLEVBQUUsTUFBTSwrQkFBK0IsQ0FBQztBQUUzRSxxQkFBYSxzQkFBdUIsWUFBVyxxQkFBcUI7SUFDbEUsT0FBTyxDQUFDLElBQUksQ0FBdUM7SUFDbkQsT0FBTyxDQUFDLE9BQU8sQ0FBb0Q7SUFFbkUsYUFBYSxDQUFDLEVBQUUsRUFBRSxZQUFZLEdBQUcsVUFBVSxHQUFHLFNBQVMsQ0FFdEQ7SUFFRCxtQkFBbUIsQ0FBQyxFQUFFLEVBQUUsWUFBWSxHQUFHLHVCQUF1QixHQUFHLFNBQVMsQ0FFekU7SUFFRCxhQUFhLENBQUMsR0FBRyxFQUFFLFVBQVUsR0FBRyxPQUFPLENBQUMsSUFBSSxDQUFDLENBRzVDO0lBRUQsbUJBQW1CLENBQUMsRUFBRSxFQUFFLFlBQVksRUFBRSxLQUFLLEVBQUUsUUFBUSxHQUFHLE9BQU8sQ0FBQyxJQUFJLENBQUMsQ0FHcEU7SUFFRCxrQkFBa0IsQ0FBQyxFQUFFLEVBQUUsWUFBWSxFQUFFLE1BQU0sRUFBRSxNQUFNLEdBQUcsT0FBTyxDQUFDLElBQUksQ0FBQyxDQUdsRTtJQUVELGlCQUFpQixDQUFDLEdBQUcsRUFBRSxZQUFZLEVBQUUsR0FBRyxPQUFPLENBQUMsSUFBSSxDQUFDLENBTXBEO0lBRUQsa0NBQWtDLENBQUMsV0FBVyxFQUFFLFdBQVcsR0FBRyxPQUFPLENBQUMsSUFBSSxDQUFDLENBTTFFO0lBRU0sY0FBYyxJQUFJLHFCQUFxQixDQUFDLENBQUMsVUFBVSxFQUFFLHVCQUF1QixHQUFHLFNBQVMsQ0FBQyxDQUFDLENBSWhHO0lBRUQsS0FBSyxJQUFJLE9BQU8sQ0FBQyxJQUFJLENBQUMsQ0FFckI7Q0FDRiJ9
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"memory.d.ts","sourceRoot":"","sources":["../../../src/proving_broker/proving_broker_database/memory.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,QAAQ,EACb,KAAK,UAAU,EACf,KAAK,YAAY,EACjB,KAAK,uBAAuB,EAE7B,MAAM,iCAAiC,CAAC;AAEzC,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,+BAA+B,CAAC;AAE3E,qBAAa,sBAAuB,YAAW,qBAAqB;IAClE,OAAO,CAAC,IAAI,CAAuC;IACnD,OAAO,CAAC,OAAO,CAAoD;IAEnE,aAAa,CAAC,EAAE,EAAE,YAAY,GAAG,UAAU,GAAG,SAAS;
|
|
1
|
+
{"version":3,"file":"memory.d.ts","sourceRoot":"","sources":["../../../src/proving_broker/proving_broker_database/memory.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iCAAiC,CAAC;AAC9D,OAAO,EACL,KAAK,QAAQ,EACb,KAAK,UAAU,EACf,KAAK,YAAY,EACjB,KAAK,uBAAuB,EAE7B,MAAM,iCAAiC,CAAC;AAEzC,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,+BAA+B,CAAC;AAE3E,qBAAa,sBAAuB,YAAW,qBAAqB;IAClE,OAAO,CAAC,IAAI,CAAuC;IACnD,OAAO,CAAC,OAAO,CAAoD;IAEnE,aAAa,CAAC,EAAE,EAAE,YAAY,GAAG,UAAU,GAAG,SAAS,CAEtD;IAED,mBAAmB,CAAC,EAAE,EAAE,YAAY,GAAG,uBAAuB,GAAG,SAAS,CAEzE;IAED,aAAa,CAAC,GAAG,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAG5C;IAED,mBAAmB,CAAC,EAAE,EAAE,YAAY,EAAE,KAAK,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CAGpE;IAED,kBAAkB,CAAC,EAAE,EAAE,YAAY,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAGlE;IAED,iBAAiB,CAAC,GAAG,EAAE,YAAY,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAMpD;IAED,kCAAkC,CAAC,WAAW,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAM1E;IAEM,cAAc,IAAI,qBAAqB,CAAC,CAAC,UAAU,EAAE,uBAAuB,GAAG,SAAS,CAAC,CAAC,CAIhG;IAED,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAErB;CACF"}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
|
+
import { EpochNumber } from '@aztec/foundation/branded-types';
|
|
1
2
|
import { type Logger } from '@aztec/foundation/log';
|
|
2
3
|
import { type ProofUri, ProvingJob, type ProvingJobId, ProvingJobSettledResult } from '@aztec/stdlib/interfaces/server';
|
|
3
|
-
import { type TelemetryClient } from '@aztec/telemetry-client';
|
|
4
|
+
import { type TelemetryClient, type Tracer } from '@aztec/telemetry-client';
|
|
4
5
|
import type { ProverBrokerConfig } from '../config.js';
|
|
5
6
|
import type { ProvingBrokerDatabase } from '../proving_broker_database.js';
|
|
6
7
|
export declare class KVBrokerDatabase implements ProvingBrokerDatabase {
|
|
@@ -9,17 +10,18 @@ export declare class KVBrokerDatabase implements ProvingBrokerDatabase {
|
|
|
9
10
|
private logger;
|
|
10
11
|
private metrics;
|
|
11
12
|
private batchQueue;
|
|
13
|
+
readonly tracer: Tracer;
|
|
12
14
|
private constructor();
|
|
13
15
|
commitWrites(items: Array<ProvingJob | [ProvingJobId, ProvingJobSettledResult]>, epochNumber: number): Promise<void>;
|
|
14
16
|
private estimateSize;
|
|
15
17
|
static new(config: ProverBrokerConfig, client?: TelemetryClient, logger?: Logger): Promise<KVBrokerDatabase>;
|
|
16
18
|
private start;
|
|
17
19
|
close(): Promise<void>;
|
|
18
|
-
deleteAllProvingJobsOlderThanEpoch(epochNumber:
|
|
20
|
+
deleteAllProvingJobsOlderThanEpoch(epochNumber: EpochNumber): Promise<void>;
|
|
19
21
|
addProvingJob(job: ProvingJob): Promise<void>;
|
|
20
22
|
allProvingJobs(): AsyncIterableIterator<[ProvingJob, ProvingJobSettledResult | undefined]>;
|
|
21
23
|
setProvingJobError(id: ProvingJobId, reason: string): Promise<void>;
|
|
22
24
|
setProvingJobResult(id: ProvingJobId, value: ProofUri): Promise<void>;
|
|
23
25
|
private getEpochDatabase;
|
|
24
26
|
}
|
|
25
|
-
//# sourceMappingURL=
|
|
27
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoicGVyc2lzdGVkLmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi8uLi9zcmMvcHJvdmluZ19icm9rZXIvcHJvdmluZ19icm9rZXJfZGF0YWJhc2UvcGVyc2lzdGVkLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLE9BQU8sRUFBRSxXQUFXLEVBQUUsTUFBTSxpQ0FBaUMsQ0FBQztBQUU5RCxPQUFPLEVBQUUsS0FBSyxNQUFNLEVBQWdCLE1BQU0sdUJBQXVCLENBQUM7QUFJbEUsT0FBTyxFQUNMLEtBQUssUUFBUSxFQUNiLFVBQVUsRUFDVixLQUFLLFlBQVksRUFDakIsdUJBQXVCLEVBRXhCLE1BQU0saUNBQWlDLENBQUM7QUFDekMsT0FBTyxFQUdMLEtBQUssZUFBZSxFQUNwQixLQUFLLE1BQU0sRUFHWixNQUFNLHlCQUF5QixDQUFDO0FBS2pDLE9BQU8sS0FBSyxFQUFFLGtCQUFrQixFQUFFLE1BQU0sY0FBYyxDQUFDO0FBQ3ZELE9BQU8sS0FBSyxFQUFFLHFCQUFxQixFQUFFLE1BQU0sK0JBQStCLENBQUM7QUF3RDNFLHFCQUFhLGdCQUFpQixZQUFXLHFCQUFxQjtJQVExRCxPQUFPLENBQUMsTUFBTTtJQUNkLE9BQU8sQ0FBQyxNQUFNO0lBRWQsT0FBTyxDQUFDLE1BQU07SUFWaEIsT0FBTyxDQUFDLE9BQU8sQ0FBYztJQUU3QixPQUFPLENBQUMsVUFBVSxDQUEyRTtJQUU3RixTQUFnQixNQUFNLEVBQUUsTUFBTSxDQUFDO0lBRS9CLE9BQU8sZUFzQk47SUFHWSxZQUFZLENBQUMsS0FBSyxFQUFFLEtBQUssQ0FBQyxVQUFVLEdBQUcsQ0FBQyxZQUFZLEVBQUUsdUJBQXVCLENBQUMsQ0FBQyxFQUFFLFdBQVcsRUFBRSxNQUFNLGlCQU1oSDtZQUVhLFlBQVk7SUFVMUIsT0FBb0IsR0FBRyxDQUNyQixNQUFNLEVBQUUsa0JBQWtCLEVBQzFCLE1BQU0sR0FBRSxlQUFzQyxFQUM5QyxNQUFNLFNBQXdELDZCQThCL0Q7SUFFRCxPQUFPLENBQUMsS0FBSztJQUlQLEtBQUssSUFBSSxPQUFPLENBQUMsSUFBSSxDQUFDLENBSzNCO0lBS0ssa0NBQWtDLENBQUMsV0FBVyxFQUFFLFdBQVcsR0FBRyxPQUFPLENBQUMsSUFBSSxDQUFDLENBV2hGO0lBRUQsYUFBYSxDQUFDLEdBQUcsRUFBRSxVQUFVLEdBQUcsT0FBTyxDQUFDLElBQUksQ0FBQyxDQUU1QztJQUVNLGNBQWMsSUFBSSxxQkFBcUIsQ0FBQyxDQUFDLFVBQVUsRUFBRSx1QkFBdUIsR0FBRyxTQUFTLENBQUMsQ0FBQyxDQUtoRztJQUVELGtCQUFrQixDQUFDLEVBQUUsRUFBRSxZQUFZLEVBQUUsTUFBTSxFQUFFLE1BQU0sR0FBRyxPQUFPLENBQUMsSUFBSSxDQUFDLENBRWxFO0lBRUQsbUJBQW1CLENBQUMsRUFBRSxFQUFFLFlBQVksRUFBRSxLQUFLLEVBQUUsUUFBUSxHQUFHLE9BQU8sQ0FBQyxJQUFJLENBQUMsQ0FFcEU7WUFFYSxnQkFBZ0I7Q0FvQi9CIn0=
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"persisted.d.ts","sourceRoot":"","sources":["../../../src/proving_broker/proving_broker_database/persisted.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"persisted.d.ts","sourceRoot":"","sources":["../../../src/proving_broker/proving_broker_database/persisted.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iCAAiC,CAAC;AAE9D,OAAO,EAAE,KAAK,MAAM,EAAgB,MAAM,uBAAuB,CAAC;AAIlE,OAAO,EACL,KAAK,QAAQ,EACb,UAAU,EACV,KAAK,YAAY,EACjB,uBAAuB,EAExB,MAAM,iCAAiC,CAAC;AACzC,OAAO,EAGL,KAAK,eAAe,EACpB,KAAK,MAAM,EAGZ,MAAM,yBAAyB,CAAC;AAKjC,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAC;AACvD,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,+BAA+B,CAAC;AAwD3E,qBAAa,gBAAiB,YAAW,qBAAqB;IAQ1D,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,MAAM;IAEd,OAAO,CAAC,MAAM;IAVhB,OAAO,CAAC,OAAO,CAAc;IAE7B,OAAO,CAAC,UAAU,CAA2E;IAE7F,SAAgB,MAAM,EAAE,MAAM,CAAC;IAE/B,OAAO,eAsBN;IAGY,YAAY,CAAC,KAAK,EAAE,KAAK,CAAC,UAAU,GAAG,CAAC,YAAY,EAAE,uBAAuB,CAAC,CAAC,EAAE,WAAW,EAAE,MAAM,iBAMhH;YAEa,YAAY;IAU1B,OAAoB,GAAG,CACrB,MAAM,EAAE,kBAAkB,EAC1B,MAAM,GAAE,eAAsC,EAC9C,MAAM,SAAwD,6BA8B/D;IAED,OAAO,CAAC,KAAK;IAIP,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAK3B;IAKK,kCAAkC,CAAC,WAAW,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAWhF;IAED,aAAa,CAAC,GAAG,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAE5C;IAEM,cAAc,IAAI,qBAAqB,CAAC,CAAC,UAAU,EAAE,uBAAuB,GAAG,SAAS,CAAC,CAAC,CAKhG;IAED,kBAAkB,CAAC,EAAE,EAAE,YAAY,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAElE;IAED,mBAAmB,CAAC,EAAE,EAAE,YAAY,EAAE,KAAK,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CAEpE;YAEa,gBAAgB;CAoB/B"}
|