@aztec/prover-client 0.76.4-devnet-test-rc3 → 0.77.0-testnet-ignition.17

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (136) hide show
  1. package/dest/bin/get-proof-inputs.js +20 -17
  2. package/dest/block_builder/index.d.ts +1 -1
  3. package/dest/block_builder/index.d.ts.map +1 -1
  4. package/dest/block_builder/index.js +0 -1
  5. package/dest/block_builder/light.d.ts +4 -2
  6. package/dest/block_builder/light.d.ts.map +1 -1
  7. package/dest/block_builder/light.js +28 -15
  8. package/dest/config.d.ts +2 -2
  9. package/dest/config.d.ts.map +1 -1
  10. package/dest/config.js +10 -12
  11. package/dest/index.d.ts +1 -1
  12. package/dest/index.d.ts.map +1 -1
  13. package/dest/index.js +0 -1
  14. package/dest/mocks/fixtures.d.ts +4 -3
  15. package/dest/mocks/fixtures.d.ts.map +1 -1
  16. package/dest/mocks/fixtures.js +37 -30
  17. package/dest/mocks/test_context.d.ts +16 -12
  18. package/dest/mocks/test_context.d.ts.map +1 -1
  19. package/dest/mocks/test_context.js +85 -62
  20. package/dest/orchestrator/block-building-helpers.d.ts +12 -7
  21. package/dest/orchestrator/block-building-helpers.d.ts.map +1 -1
  22. package/dest/orchestrator/block-building-helpers.js +98 -87
  23. package/dest/orchestrator/block-proving-state.d.ts +13 -8
  24. package/dest/orchestrator/block-proving-state.d.ts.map +1 -1
  25. package/dest/orchestrator/block-proving-state.js +101 -73
  26. package/dest/orchestrator/epoch-proving-state.d.ts +10 -6
  27. package/dest/orchestrator/epoch-proving-state.d.ts.map +1 -1
  28. package/dest/orchestrator/epoch-proving-state.js +57 -41
  29. package/dest/orchestrator/index.js +0 -1
  30. package/dest/orchestrator/orchestrator.d.ts +7 -5
  31. package/dest/orchestrator/orchestrator.d.ts.map +1 -1
  32. package/dest/orchestrator/orchestrator.js +656 -654
  33. package/dest/orchestrator/orchestrator_metrics.js +4 -3
  34. package/dest/orchestrator/tx-proving-state.d.ts +7 -4
  35. package/dest/orchestrator/tx-proving-state.d.ts.map +1 -1
  36. package/dest/orchestrator/tx-proving-state.js +54 -52
  37. package/dest/prover-client/factory.d.ts +2 -2
  38. package/dest/prover-client/factory.d.ts.map +1 -1
  39. package/dest/prover-client/factory.js +0 -1
  40. package/dest/prover-client/index.js +0 -1
  41. package/dest/prover-client/prover-client.d.ts +3 -3
  42. package/dest/prover-client/prover-client.d.ts.map +1 -1
  43. package/dest/prover-client/prover-client.js +31 -26
  44. package/dest/prover-client/server-epoch-prover.d.ts +8 -5
  45. package/dest/prover-client/server-epoch-prover.d.ts.map +1 -1
  46. package/dest/prover-client/server-epoch-prover.js +4 -4
  47. package/dest/proving_broker/broker_prover_facade.d.ts +5 -3
  48. package/dest/proving_broker/broker_prover_facade.d.ts.map +1 -1
  49. package/dest/proving_broker/broker_prover_facade.js +80 -67
  50. package/dest/proving_broker/config.d.ts +15 -2
  51. package/dest/proving_broker/config.d.ts.map +1 -1
  52. package/dest/proving_broker/config.js +49 -44
  53. package/dest/proving_broker/factory.d.ts +2 -2
  54. package/dest/proving_broker/factory.d.ts.map +1 -1
  55. package/dest/proving_broker/factory.js +1 -6
  56. package/dest/proving_broker/fixtures.d.ts +1 -1
  57. package/dest/proving_broker/fixtures.d.ts.map +1 -1
  58. package/dest/proving_broker/fixtures.js +2 -2
  59. package/dest/proving_broker/index.js +0 -1
  60. package/dest/proving_broker/proof_store/factory.d.ts +1 -1
  61. package/dest/proving_broker/proof_store/factory.d.ts.map +1 -1
  62. package/dest/proving_broker/proof_store/factory.js +9 -12
  63. package/dest/proving_broker/proof_store/gcs_proof_store.d.ts +3 -2
  64. package/dest/proving_broker/proof_store/gcs_proof_store.d.ts.map +1 -1
  65. package/dest/proving_broker/proof_store/gcs_proof_store.js +12 -7
  66. package/dest/proving_broker/proof_store/index.js +0 -1
  67. package/dest/proving_broker/proof_store/inline_proof_store.d.ts +3 -2
  68. package/dest/proving_broker/proof_store/inline_proof_store.d.ts.map +1 -1
  69. package/dest/proving_broker/proof_store/inline_proof_store.js +11 -7
  70. package/dest/proving_broker/proof_store/proof_store.d.ts +2 -1
  71. package/dest/proving_broker/proof_store/proof_store.d.ts.map +1 -1
  72. package/dest/proving_broker/proof_store/proof_store.js +3 -2
  73. package/dest/proving_broker/proving_agent.d.ts +3 -2
  74. package/dest/proving_broker/proving_agent.d.ts.map +1 -1
  75. package/dest/proving_broker/proving_agent.js +121 -124
  76. package/dest/proving_broker/proving_agent_instrumentation.d.ts +1 -1
  77. package/dest/proving_broker/proving_agent_instrumentation.d.ts.map +1 -1
  78. package/dest/proving_broker/proving_agent_instrumentation.js +6 -6
  79. package/dest/proving_broker/proving_broker.d.ts +4 -11
  80. package/dest/proving_broker/proving_broker.d.ts.map +1 -1
  81. package/dest/proving_broker/proving_broker.js +490 -446
  82. package/dest/proving_broker/proving_broker_database/memory.d.ts +2 -2
  83. package/dest/proving_broker/proving_broker_database/memory.d.ts.map +1 -1
  84. package/dest/proving_broker/proving_broker_database/memory.js +19 -13
  85. package/dest/proving_broker/proving_broker_database/persisted.d.ts +3 -3
  86. package/dest/proving_broker/proving_broker_database/persisted.d.ts.map +1 -1
  87. package/dest/proving_broker/proving_broker_database/persisted.js +62 -29
  88. package/dest/proving_broker/proving_broker_database.d.ts +1 -1
  89. package/dest/proving_broker/proving_broker_database.d.ts.map +1 -1
  90. package/dest/proving_broker/proving_broker_database.js +3 -2
  91. package/dest/proving_broker/proving_broker_instrumentation.d.ts +6 -2
  92. package/dest/proving_broker/proving_broker_instrumentation.d.ts.map +1 -1
  93. package/dest/proving_broker/proving_broker_instrumentation.js +47 -22
  94. package/dest/proving_broker/proving_job_controller.d.ts +2 -1
  95. package/dest/proving_broker/proving_job_controller.d.ts.map +1 -1
  96. package/dest/proving_broker/proving_job_controller.js +81 -62
  97. package/dest/proving_broker/rpc.d.ts +4 -3
  98. package/dest/proving_broker/rpc.d.ts.map +1 -1
  99. package/dest/proving_broker/rpc.js +29 -25
  100. package/dest/test/mock_prover.d.ts +7 -5
  101. package/dest/test/mock_prover.d.ts.map +1 -1
  102. package/dest/test/mock_prover.js +13 -9
  103. package/package.json +16 -15
  104. package/src/bin/get-proof-inputs.ts +2 -1
  105. package/src/block_builder/index.ts +1 -1
  106. package/src/block_builder/light.ts +7 -10
  107. package/src/config.ts +2 -2
  108. package/src/index.ts +1 -1
  109. package/src/mocks/fixtures.ts +13 -15
  110. package/src/mocks/test_context.ts +60 -102
  111. package/src/orchestrator/block-building-helpers.ts +50 -45
  112. package/src/orchestrator/block-proving-state.ts +18 -22
  113. package/src/orchestrator/epoch-proving-state.ts +11 -16
  114. package/src/orchestrator/orchestrator.ts +26 -33
  115. package/src/orchestrator/tx-proving-state.ts +10 -13
  116. package/src/prover-client/factory.ts +2 -2
  117. package/src/prover-client/prover-client.ts +12 -12
  118. package/src/prover-client/server-epoch-prover.ts +8 -5
  119. package/src/proving_broker/broker_prover_facade.ts +29 -31
  120. package/src/proving_broker/config.ts +31 -3
  121. package/src/proving_broker/factory.ts +3 -11
  122. package/src/proving_broker/fixtures.ts +2 -1
  123. package/src/proving_broker/proof_store/factory.ts +1 -1
  124. package/src/proving_broker/proof_store/gcs_proof_store.ts +3 -3
  125. package/src/proving_broker/proof_store/inline_proof_store.ts +5 -5
  126. package/src/proving_broker/proof_store/proof_store.ts +9 -9
  127. package/src/proving_broker/proving_agent.ts +11 -11
  128. package/src/proving_broker/proving_agent_instrumentation.ts +5 -5
  129. package/src/proving_broker/proving_broker.ts +35 -32
  130. package/src/proving_broker/proving_broker_database/memory.ts +2 -2
  131. package/src/proving_broker/proving_broker_database/persisted.ts +8 -8
  132. package/src/proving_broker/proving_broker_database.ts +1 -1
  133. package/src/proving_broker/proving_broker_instrumentation.ts +24 -2
  134. package/src/proving_broker/proving_job_controller.ts +7 -7
  135. package/src/proving_broker/rpc.ts +6 -7
  136. package/src/test/mock_prover.ts +28 -30
@@ -1,474 +1,525 @@
1
- import { __classPrivateFieldGet, __esDecorate, __runInitializers } from "tslib";
2
- import { ProvingRequestType, } from '@aztec/circuit-types';
1
+ function _ts_decorate(decorators, target, key, desc) {
2
+ var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
3
+ if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
4
+ else for(var i = decorators.length - 1; i >= 0; i--)if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
5
+ return c > 3 && r && Object.defineProperty(target, key, r), r;
6
+ }
3
7
  import { createLogger } from '@aztec/foundation/log';
4
8
  import { RunningPromise, promiseWithResolvers } from '@aztec/foundation/promise';
5
9
  import { PriorityMemoryQueue } from '@aztec/foundation/queue';
6
10
  import { Timer } from '@aztec/foundation/timer';
7
- import { getTelemetryClient, trackSpan, } from '@aztec/telemetry-client';
11
+ import { ProvingRequestType } from '@aztec/stdlib/proofs';
12
+ import { getTelemetryClient, trackSpan } from '@aztec/telemetry-client';
8
13
  import assert from 'assert';
14
+ import { defaultProverBrokerConfig } from './config.js';
9
15
  import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.js';
10
16
  /**
11
17
  * A broker that manages proof requests and distributes them to workers based on their priority.
12
18
  * It takes a backend that is responsible for storing and retrieving proof requests and results.
13
- */
14
- let ProvingBroker = (() => {
15
- var _ProvingBroker_instances, _a, _ProvingBroker_enqueueProvingJob, _ProvingBroker_cancelProvingJob, _ProvingBroker_getProvingJobStatus, _ProvingBroker_getCompletedJobs, _ProvingBroker_getProvingJob, _ProvingBroker_reportProvingJobError, _ProvingBroker_reportProvingJobProgress, _ProvingBroker_reportProvingJobSuccess;
16
- let _instanceExtraInitializers = [];
17
- let _cleanupPass_decorators;
18
- return _a = class ProvingBroker {
19
- constructor(database, { jobTimeoutMs = 30000, timeoutIntervalMs = 10000, maxRetries = 3, maxEpochsToKeepResultsFor = 1, } = {}, client = getTelemetryClient(), logger = createLogger('prover-client:proving-broker')) {
20
- _ProvingBroker_instances.add(this);
21
- this.database = (__runInitializers(this, _instanceExtraInitializers), database);
22
- this.logger = logger;
23
- this.queues = {
24
- [ProvingRequestType.PUBLIC_VM]: new PriorityMemoryQueue(provingJobComparator),
25
- [ProvingRequestType.TUBE_PROOF]: new PriorityMemoryQueue(provingJobComparator),
26
- [ProvingRequestType.PRIVATE_BASE_ROLLUP]: new PriorityMemoryQueue(provingJobComparator),
27
- [ProvingRequestType.PUBLIC_BASE_ROLLUP]: new PriorityMemoryQueue(provingJobComparator),
28
- [ProvingRequestType.MERGE_ROLLUP]: new PriorityMemoryQueue(provingJobComparator),
29
- [ProvingRequestType.ROOT_ROLLUP]: new PriorityMemoryQueue(provingJobComparator),
30
- [ProvingRequestType.BLOCK_MERGE_ROLLUP]: new PriorityMemoryQueue(provingJobComparator),
31
- [ProvingRequestType.BLOCK_ROOT_ROLLUP]: new PriorityMemoryQueue(provingJobComparator),
32
- [ProvingRequestType.SINGLE_TX_BLOCK_ROOT_ROLLUP]: new PriorityMemoryQueue(provingJobComparator),
33
- [ProvingRequestType.EMPTY_BLOCK_ROOT_ROLLUP]: new PriorityMemoryQueue(provingJobComparator),
34
- [ProvingRequestType.BASE_PARITY]: new PriorityMemoryQueue(provingJobComparator),
35
- [ProvingRequestType.ROOT_PARITY]: new PriorityMemoryQueue(provingJobComparator),
36
- };
37
- // holds a copy of the database in memory in order to quickly fulfill requests
38
- // this is fine because this broker is the only one that can modify the database
39
- this.jobsCache = new Map();
40
- // as above, but for results
41
- this.resultsCache = new Map();
42
- // tracks when each job was enqueued
43
- this.enqueuedAt = new Map();
44
- // keeps track of which jobs are currently being processed
45
- // in the event of a crash this information is lost, but that's ok
46
- // the next time the broker starts it will recreate jobsCache and still
47
- // accept results from the workers
48
- this.inProgress = new Map();
49
- // keep track of which proving job has been retried
50
- this.retries = new Map();
51
- // a map of promises that will be resolved when a job is settled
52
- this.promises = new Map();
53
- this.msTimeSource = () => Date.now();
54
- this.completedJobNotifications = [];
55
- /**
56
- * The broker keeps track of the highest epoch its seen.
57
- * This information is used for garbage collection: once it reaches the next epoch, it can start pruning the database of old state.
58
- * It is important that this value is initialised to zero. This ensures that we don't delete any old jobs until the current
59
- * process instance receives a job request informing it of the actual current highest epoch
60
- * Example:
61
- * proving epoch 11 - the broker will wipe all jobs for epochs 9 and lower
62
- * finished proving epoch 11 and got first job for epoch 12 -> the broker will wipe all settled jobs for epochs 10 and lower
63
- * reorged back to end of epoch 10 -> epoch 11 is skipped and epoch 12 starts -> the broker will wipe all settled jobs for epochs 10 and lower
64
- */
65
- this.epochHeight = 0;
66
- this.maxEpochsToKeepResultsFor = 1;
67
- this.started = false;
68
- this.measureQueueDepth = (type) => {
69
- return this.queues[type].length();
70
- };
71
- this.countActiveJobs = (type) => {
72
- let count = 0;
73
- for (const { id } of this.inProgress.values()) {
74
- const job = this.jobsCache.get(id);
75
- if (job?.type === type) {
76
- count++;
77
- }
78
- }
79
- return count;
80
- };
81
- this.tracer = client.getTracer('ProvingBroker');
82
- this.instrumentation = new ProvingBrokerInstrumentation(client);
83
- this.cleanupPromise = new RunningPromise(this.cleanupPass.bind(this), this.logger, timeoutIntervalMs);
84
- this.jobTimeoutMs = jobTimeoutMs;
85
- this.maxRetries = maxRetries;
86
- this.maxEpochsToKeepResultsFor = maxEpochsToKeepResultsFor;
87
- }
88
- async start() {
89
- if (this.started) {
90
- this.logger.info('Proving Broker already started');
91
- return Promise.resolve();
92
- }
93
- this.logger.info('Proving Broker started');
94
- for await (const [item, result] of this.database.allProvingJobs()) {
95
- this.logger.info(`Restoring proving job id=${item.id} settled=${!!result}`, {
96
- provingJobId: item.id,
97
- status: result ? result.status : 'pending',
98
- });
99
- this.jobsCache.set(item.id, item);
100
- this.promises.set(item.id, promiseWithResolvers());
101
- if (result) {
102
- this.promises.get(item.id).resolve(result);
103
- this.resultsCache.set(item.id, result);
104
- }
105
- else {
106
- this.enqueueJobInternal(item);
107
- }
108
- }
109
- this.cleanupPromise.start();
110
- this.instrumentation.monitorQueueDepth(this.measureQueueDepth);
111
- this.instrumentation.monitorActiveJobs(this.countActiveJobs);
112
- this.started = true;
113
- }
114
- async stop() {
115
- if (!this.started) {
116
- this.logger.warn('ProvingBroker not started');
117
- return Promise.resolve();
118
- }
119
- await this.cleanupPromise.stop();
120
- }
121
- enqueueProvingJob(job) {
122
- return __classPrivateFieldGet(this, _ProvingBroker_instances, "m", _ProvingBroker_enqueueProvingJob).call(this, job);
123
- }
124
- cancelProvingJob(id) {
125
- return __classPrivateFieldGet(this, _ProvingBroker_instances, "m", _ProvingBroker_cancelProvingJob).call(this, id);
126
- }
127
- getProvingJobStatus(id) {
128
- return Promise.resolve(__classPrivateFieldGet(this, _ProvingBroker_instances, "m", _ProvingBroker_getProvingJobStatus).call(this, id));
129
- }
130
- getCompletedJobs(ids) {
131
- return __classPrivateFieldGet(this, _ProvingBroker_instances, "m", _ProvingBroker_getCompletedJobs).call(this, ids);
132
- }
133
- getProvingJob(filter) {
134
- return Promise.resolve(__classPrivateFieldGet(this, _ProvingBroker_instances, "m", _ProvingBroker_getProvingJob).call(this, filter));
135
- }
136
- reportProvingJobSuccess(id, value, filter) {
137
- return __classPrivateFieldGet(this, _ProvingBroker_instances, "m", _ProvingBroker_reportProvingJobSuccess).call(this, id, value, filter);
138
- }
139
- reportProvingJobError(id, err, retry = false, filter) {
140
- return __classPrivateFieldGet(this, _ProvingBroker_instances, "m", _ProvingBroker_reportProvingJobError).call(this, id, err, retry, filter);
141
- }
142
- reportProvingJobProgress(id, startedAt, filter) {
143
- return Promise.resolve(__classPrivateFieldGet(this, _ProvingBroker_instances, "m", _ProvingBroker_reportProvingJobProgress).call(this, id, startedAt, filter));
144
- }
145
- cleanUpProvingJobState(ids) {
146
- for (const id of ids) {
147
- this.jobsCache.delete(id);
148
- this.promises.delete(id);
149
- this.resultsCache.delete(id);
150
- this.inProgress.delete(id);
151
- this.retries.delete(id);
152
- }
153
- }
154
- async cleanupPass() {
155
- this.cleanupStaleJobs();
156
- this.reEnqueueExpiredJobs();
157
- const oldestEpochToKeep = this.oldestEpochToKeep();
158
- if (oldestEpochToKeep > 0) {
159
- await this.database.deleteAllProvingJobsOlderThanEpoch(oldestEpochToKeep);
160
- this.logger.trace(`Deleted all epochs older than ${oldestEpochToKeep}`);
161
- }
162
- }
163
- cleanupStaleJobs() {
164
- const jobIds = Array.from(this.jobsCache.keys());
165
- const jobsToClean = [];
166
- for (const id of jobIds) {
167
- const job = this.jobsCache.get(id);
168
- if (this.isJobStale(job)) {
169
- jobsToClean.push(id);
170
- }
171
- }
172
- if (jobsToClean.length > 0) {
173
- this.cleanUpProvingJobState(jobsToClean);
174
- this.logger.info(`Cleaned up jobs=${jobsToClean.length}`);
175
- }
176
- }
177
- reEnqueueExpiredJobs() {
178
- const inProgressEntries = Array.from(this.inProgress.entries());
179
- for (const [id, metadata] of inProgressEntries) {
180
- const item = this.jobsCache.get(id);
181
- if (!item) {
182
- this.logger.warn(`Proving job id=${id} not found. Removing it from the queue.`, { provingJobId: id });
183
- this.inProgress.delete(id);
184
- continue;
185
- }
186
- const now = this.msTimeSource();
187
- const msSinceLastUpdate = now - metadata.lastUpdatedAt;
188
- if (msSinceLastUpdate >= this.jobTimeoutMs) {
189
- this.logger.warn(`Proving job id=${id} timed out. Adding it back to the queue.`, { provingJobId: id });
190
- this.inProgress.delete(id);
191
- this.enqueueJobInternal(item);
192
- this.instrumentation.incTimedOutJobs(item.type);
193
- }
194
- }
195
- }
196
- enqueueJobInternal(job) {
197
- if (!this.promises.has(job.id)) {
198
- this.promises.set(job.id, promiseWithResolvers());
19
+ */ export class ProvingBroker {
20
+ database;
21
+ logger;
22
+ queues;
23
+ // holds a copy of the database in memory in order to quickly fulfill requests
24
+ // this is fine because this broker is the only one that can modify the database
25
+ jobsCache;
26
+ // as above, but for results
27
+ resultsCache;
28
+ // tracks when each job was enqueued
29
+ enqueuedAt;
30
+ // keeps track of which jobs are currently being processed
31
+ // in the event of a crash this information is lost, but that's ok
32
+ // the next time the broker starts it will recreate jobsCache and still
33
+ // accept results from the workers
34
+ inProgress;
35
+ // keep track of which proving job has been retried
36
+ retries;
37
+ // a map of promises that will be resolved when a job is settled
38
+ promises;
39
+ cleanupPromise;
40
+ msTimeSource;
41
+ jobTimeoutMs;
42
+ maxRetries;
43
+ instrumentation;
44
+ tracer;
45
+ completedJobNotifications;
46
+ /**
47
+ * The broker keeps track of the highest epoch its seen.
48
+ * This information is used for garbage collection: once it reaches the next epoch, it can start pruning the database of old state.
49
+ * It is important that this value is initialised to zero. This ensures that we don't delete any old jobs until the current
50
+ * process instance receives a job request informing it of the actual current highest epoch
51
+ * Example:
52
+ * proving epoch 11 - the broker will wipe all jobs for epochs 9 and lower
53
+ * finished proving epoch 11 and got first job for epoch 12 -> the broker will wipe all settled jobs for epochs 10 and lower
54
+ * reorged back to end of epoch 10 -> epoch 11 is skipped and epoch 12 starts -> the broker will wipe all settled jobs for epochs 10 and lower
55
+ */ epochHeight;
56
+ maxEpochsToKeepResultsFor;
57
+ started;
58
+ constructor(database, { proverBrokerJobTimeoutMs, proverBrokerPollIntervalMs, proverBrokerJobMaxRetries, proverBrokerMaxEpochsToKeepResultsFor } = defaultProverBrokerConfig, client = getTelemetryClient(), logger = createLogger('prover-client:proving-broker')){
59
+ this.database = database;
60
+ this.logger = logger;
61
+ this.queues = {
62
+ [ProvingRequestType.PUBLIC_VM]: new PriorityMemoryQueue(provingJobComparator),
63
+ [ProvingRequestType.TUBE_PROOF]: new PriorityMemoryQueue(provingJobComparator),
64
+ [ProvingRequestType.PRIVATE_BASE_ROLLUP]: new PriorityMemoryQueue(provingJobComparator),
65
+ [ProvingRequestType.PUBLIC_BASE_ROLLUP]: new PriorityMemoryQueue(provingJobComparator),
66
+ [ProvingRequestType.MERGE_ROLLUP]: new PriorityMemoryQueue(provingJobComparator),
67
+ [ProvingRequestType.ROOT_ROLLUP]: new PriorityMemoryQueue(provingJobComparator),
68
+ [ProvingRequestType.BLOCK_MERGE_ROLLUP]: new PriorityMemoryQueue(provingJobComparator),
69
+ [ProvingRequestType.BLOCK_ROOT_ROLLUP]: new PriorityMemoryQueue(provingJobComparator),
70
+ [ProvingRequestType.SINGLE_TX_BLOCK_ROOT_ROLLUP]: new PriorityMemoryQueue(provingJobComparator),
71
+ [ProvingRequestType.EMPTY_BLOCK_ROOT_ROLLUP]: new PriorityMemoryQueue(provingJobComparator),
72
+ [ProvingRequestType.BASE_PARITY]: new PriorityMemoryQueue(provingJobComparator),
73
+ [ProvingRequestType.ROOT_PARITY]: new PriorityMemoryQueue(provingJobComparator)
74
+ };
75
+ this.jobsCache = new Map();
76
+ this.resultsCache = new Map();
77
+ this.enqueuedAt = new Map();
78
+ this.inProgress = new Map();
79
+ this.retries = new Map();
80
+ this.promises = new Map();
81
+ this.msTimeSource = ()=>Date.now();
82
+ this.completedJobNotifications = [];
83
+ this.epochHeight = 0;
84
+ this.maxEpochsToKeepResultsFor = 1;
85
+ this.started = false;
86
+ this.measureQueueDepth = (type)=>{
87
+ return this.queues[type].length();
88
+ };
89
+ this.countActiveJobs = (type)=>{
90
+ let count = 0;
91
+ for (const { id } of this.inProgress.values()){
92
+ const job = this.jobsCache.get(id);
93
+ if (job?.type === type) {
94
+ count++;
199
95
  }
200
- this.queues[job.type].put({
201
- epochNumber: job.epochNumber,
202
- id: job.id,
203
- });
204
- this.enqueuedAt.set(job.id, new Timer());
205
- this.epochHeight = Math.max(this.epochHeight, job.epochNumber);
206
- }
207
- isJobStale(job) {
208
- return job.epochNumber < this.oldestEpochToKeep();
209
- }
210
- oldestEpochToKeep() {
211
- return this.epochHeight - this.maxEpochsToKeepResultsFor;
212
- }
213
- },
214
- _ProvingBroker_instances = new WeakSet(),
215
- _ProvingBroker_enqueueProvingJob = async function _ProvingBroker_enqueueProvingJob(job) {
216
- // We return the job status at the start of this call
217
- const jobStatus = __classPrivateFieldGet(this, _ProvingBroker_instances, "m", _ProvingBroker_getProvingJobStatus).call(this, job.id);
218
- if (this.jobsCache.has(job.id)) {
219
- const existing = this.jobsCache.get(job.id);
220
- assert.deepStrictEqual(job, existing, 'Duplicate proving job ID');
221
- this.logger.debug(`Duplicate proving job id=${job.id} epochNumber=${job.epochNumber}. Ignoring`, {
222
- provingJobId: job.id,
223
- });
224
- return Promise.resolve(jobStatus);
225
- }
226
- if (this.isJobStale(job)) {
227
- this.logger.warn(`Tried enqueueing stale proving job id=${job.id} epochNumber=${job.epochNumber}`, {
228
- provingJobId: job.id,
229
- });
230
- throw new Error(`Epoch too old: job epoch ${job.epochNumber}, current epoch: ${this.epochHeight}`);
231
- }
232
- this.logger.info(`New proving job id=${job.id} epochNumber=${job.epochNumber}`, { provingJobId: job.id });
233
- try {
234
- // do this first so it acts as a "lock". If this job is enqueued again while we're saving it the if at the top will catch it.
235
- this.jobsCache.set(job.id, job);
236
- await this.database.addProvingJob(job);
237
- this.enqueueJobInternal(job);
238
96
  }
239
- catch (err) {
240
- this.logger.error(`Failed to save proving job id=${job.id}: ${err}`, err, { provingJobId: job.id });
241
- this.jobsCache.delete(job.id);
242
- throw err;
243
- }
244
- return jobStatus;
245
- },
246
- _ProvingBroker_cancelProvingJob = async function _ProvingBroker_cancelProvingJob(id) {
247
- if (!this.jobsCache.has(id)) {
248
- this.logger.warn(`Can't cancel a job that doesn't exist id=${id}`, { provingJobId: id });
249
- return;
250
- }
251
- // notify listeners of the cancellation
252
- if (!this.resultsCache.has(id)) {
253
- this.logger.info(`Cancelling job id=${id}`, { provingJobId: id });
254
- await __classPrivateFieldGet(this, _ProvingBroker_instances, "m", _ProvingBroker_reportProvingJobError).call(this, id, 'Aborted', false);
255
- }
256
- },
257
- _ProvingBroker_getProvingJobStatus = function _ProvingBroker_getProvingJobStatus(id) {
258
- const result = this.resultsCache.get(id);
97
+ return count;
98
+ };
99
+ this.tracer = client.getTracer('ProvingBroker');
100
+ this.instrumentation = new ProvingBrokerInstrumentation(client);
101
+ this.cleanupPromise = new RunningPromise(this.cleanupPass.bind(this), this.logger, proverBrokerPollIntervalMs);
102
+ this.jobTimeoutMs = proverBrokerJobTimeoutMs;
103
+ this.maxRetries = proverBrokerJobMaxRetries;
104
+ this.maxEpochsToKeepResultsFor = proverBrokerMaxEpochsToKeepResultsFor;
105
+ }
106
+ measureQueueDepth;
107
+ countActiveJobs;
108
+ async start() {
109
+ if (this.started) {
110
+ this.logger.info('Proving Broker already started');
111
+ return Promise.resolve();
112
+ }
113
+ this.logger.info('Proving Broker started');
114
+ for await (const [item, result] of this.database.allProvingJobs()){
115
+ this.logger.info(`Restoring proving job id=${item.id} settled=${!!result}`, {
116
+ provingJobId: item.id,
117
+ status: result ? result.status : 'pending'
118
+ });
119
+ this.jobsCache.set(item.id, item);
120
+ this.promises.set(item.id, promiseWithResolvers());
259
121
  if (result) {
260
- return result;
122
+ this.promises.get(item.id).resolve(result);
123
+ this.resultsCache.set(item.id, result);
124
+ } else {
125
+ this.enqueueJobInternal(item);
261
126
  }
262
- else {
263
- // no result yet, check if we know the item
264
- const item = this.jobsCache.get(id);
265
- if (!item) {
266
- return { status: 'not-found' };
267
- }
268
- return { status: this.inProgress.has(id) ? 'in-progress' : 'in-queue' };
127
+ }
128
+ this.cleanupPromise.start();
129
+ this.instrumentation.monitorQueueDepth(this.measureQueueDepth);
130
+ this.instrumentation.monitorActiveJobs(this.countActiveJobs);
131
+ this.started = true;
132
+ }
133
+ async stop() {
134
+ if (!this.started) {
135
+ this.logger.warn('ProvingBroker not started');
136
+ return Promise.resolve();
137
+ }
138
+ await this.cleanupPromise.stop();
139
+ }
140
+ enqueueProvingJob(job) {
141
+ return this.#enqueueProvingJob(job);
142
+ }
143
+ cancelProvingJob(id) {
144
+ return this.#cancelProvingJob(id);
145
+ }
146
+ getProvingJobStatus(id) {
147
+ return Promise.resolve(this.#getProvingJobStatus(id));
148
+ }
149
+ getCompletedJobs(ids) {
150
+ return this.#getCompletedJobs(ids);
151
+ }
152
+ getProvingJob(filter) {
153
+ return Promise.resolve(this.#getProvingJob(filter));
154
+ }
155
+ reportProvingJobSuccess(id, value, filter) {
156
+ return this.#reportProvingJobSuccess(id, value, filter);
157
+ }
158
+ reportProvingJobError(id, err, retry = false, filter) {
159
+ return this.#reportProvingJobError(id, err, retry, filter);
160
+ }
161
+ reportProvingJobProgress(id, startedAt, filter) {
162
+ return Promise.resolve(this.#reportProvingJobProgress(id, startedAt, filter));
163
+ }
164
+ async #enqueueProvingJob(job) {
165
+ // We return the job status at the start of this call
166
+ const jobStatus = this.#getProvingJobStatus(job.id);
167
+ if (this.jobsCache.has(job.id)) {
168
+ const existing = this.jobsCache.get(job.id);
169
+ assert.deepStrictEqual(job, existing, 'Duplicate proving job ID');
170
+ this.logger.warn(`Cached proving job id=${job.id} epochNumber=${job.epochNumber}. Not enqueuing again`, {
171
+ provingJobId: job.id
172
+ });
173
+ this.instrumentation.incCachedJobs(job.type);
174
+ return jobStatus;
175
+ }
176
+ if (this.isJobStale(job)) {
177
+ this.logger.warn(`Tried enqueueing stale proving job id=${job.id} epochNumber=${job.epochNumber}`, {
178
+ provingJobId: job.id
179
+ });
180
+ throw new Error(`Epoch too old: job epoch ${job.epochNumber}, current epoch: ${this.epochHeight}`);
181
+ }
182
+ this.logger.info(`New proving job id=${job.id} epochNumber=${job.epochNumber}`, {
183
+ provingJobId: job.id
184
+ });
185
+ try {
186
+ // do this first so it acts as a "lock". If this job is enqueued again while we're saving it the if at the top will catch it.
187
+ this.jobsCache.set(job.id, job);
188
+ await this.database.addProvingJob(job);
189
+ this.enqueueJobInternal(job);
190
+ this.instrumentation.incTotalJobs(job.type);
191
+ } catch (err) {
192
+ this.logger.error(`Failed to save proving job id=${job.id}: ${err}`, err, {
193
+ provingJobId: job.id
194
+ });
195
+ this.jobsCache.delete(job.id);
196
+ throw err;
197
+ }
198
+ return jobStatus;
199
+ }
200
+ async #cancelProvingJob(id) {
201
+ if (!this.jobsCache.has(id)) {
202
+ this.logger.warn(`Can't cancel a job that doesn't exist id=${id}`, {
203
+ provingJobId: id
204
+ });
205
+ return;
206
+ }
207
+ // notify listeners of the cancellation
208
+ if (!this.resultsCache.has(id)) {
209
+ this.logger.info(`Cancelling job id=${id}`, {
210
+ provingJobId: id
211
+ });
212
+ await this.#reportProvingJobError(id, 'Aborted', false);
213
+ }
214
+ }
215
+ cleanUpProvingJobState(ids) {
216
+ for (const id of ids){
217
+ this.jobsCache.delete(id);
218
+ this.promises.delete(id);
219
+ this.resultsCache.delete(id);
220
+ this.inProgress.delete(id);
221
+ this.retries.delete(id);
222
+ }
223
+ }
224
+ #getProvingJobStatus(id) {
225
+ const result = this.resultsCache.get(id);
226
+ if (result) {
227
+ return result;
228
+ } else {
229
+ // no result yet, check if we know the item
230
+ const item = this.jobsCache.get(id);
231
+ if (!item) {
232
+ return {
233
+ status: 'not-found'
234
+ };
269
235
  }
270
- },
271
- _ProvingBroker_getCompletedJobs = function _ProvingBroker_getCompletedJobs(ids) {
272
- const completedJobs = ids.filter(id => this.resultsCache.has(id));
273
- const notifications = this.completedJobNotifications;
274
- this.completedJobNotifications = [];
275
- return Promise.resolve(notifications.concat(completedJobs));
276
- },
277
- _ProvingBroker_getProvingJob = function _ProvingBroker_getProvingJob(filter = { allowList: [] }) {
278
- const allowedProofs = Array.isArray(filter.allowList) && filter.allowList.length > 0
279
- ? [...filter.allowList]
280
- : Object.values(ProvingRequestType).filter((x) => typeof x === 'number');
281
- allowedProofs.sort(proofTypeComparator);
282
- for (const proofType of allowedProofs) {
283
- const queue = this.queues[proofType];
284
- let enqueuedJob;
285
- // exhaust the queue and make sure we're not sending a job that's already in progress
286
- // or has already been completed
287
- // this can happen if the broker crashes and restarts
288
- // it's possible agents will report progress or results for jobs that are in the queue (after the restart)
289
- while ((enqueuedJob = queue.getImmediate())) {
290
- const job = this.jobsCache.get(enqueuedJob.id);
291
- if (job && !this.inProgress.has(enqueuedJob.id) && !this.resultsCache.has(enqueuedJob.id)) {
292
- const time = this.msTimeSource();
293
- this.inProgress.set(job.id, {
294
- id: job.id,
295
- startedAt: time,
296
- lastUpdatedAt: time,
297
- });
298
- const enqueuedAt = this.enqueuedAt.get(job.id);
299
- if (enqueuedAt) {
300
- this.instrumentation.recordJobWait(job.type, enqueuedAt);
301
- }
302
- return { job, time };
236
+ return {
237
+ status: this.inProgress.has(id) ? 'in-progress' : 'in-queue'
238
+ };
239
+ }
240
+ }
241
+ #getCompletedJobs(ids) {
242
+ const completedJobs = ids.filter((id)=>this.resultsCache.has(id));
243
+ const notifications = this.completedJobNotifications;
244
+ this.completedJobNotifications = [];
245
+ return Promise.resolve(notifications.concat(completedJobs));
246
+ }
247
+ // eslint-disable-next-line require-await
248
+ #getProvingJob(filter = {
249
+ allowList: []
250
+ }) {
251
+ const allowedProofs = Array.isArray(filter.allowList) && filter.allowList.length > 0 ? [
252
+ ...filter.allowList
253
+ ] : Object.values(ProvingRequestType).filter((x)=>typeof x === 'number');
254
+ allowedProofs.sort(proofTypeComparator);
255
+ for (const proofType of allowedProofs){
256
+ const queue = this.queues[proofType];
257
+ let enqueuedJob;
258
+ // exhaust the queue and make sure we're not sending a job that's already in progress
259
+ // or has already been completed
260
+ // this can happen if the broker crashes and restarts
261
+ // it's possible agents will report progress or results for jobs that are in the queue (after the restart)
262
+ while(enqueuedJob = queue.getImmediate()){
263
+ const job = this.jobsCache.get(enqueuedJob.id);
264
+ if (job && !this.inProgress.has(enqueuedJob.id) && !this.resultsCache.has(enqueuedJob.id)) {
265
+ const time = this.msTimeSource();
266
+ this.inProgress.set(job.id, {
267
+ id: job.id,
268
+ startedAt: time,
269
+ lastUpdatedAt: time
270
+ });
271
+ const enqueuedAt = this.enqueuedAt.get(job.id);
272
+ if (enqueuedAt) {
273
+ this.instrumentation.recordJobWait(job.type, enqueuedAt);
303
274
  }
275
+ return {
276
+ job,
277
+ time
278
+ };
304
279
  }
305
280
  }
281
+ }
282
+ return undefined;
283
+ }
284
+ async #reportProvingJobError(id, err, retry = false, filter) {
285
+ const info = this.inProgress.get(id);
286
+ const item = this.jobsCache.get(id);
287
+ const retries = this.retries.get(id) ?? 0;
288
+ if (!item) {
289
+ this.logger.warn(`Can't set error on unknown proving job id=${id} err=${err}`, {
290
+ provingJoId: id
291
+ });
292
+ return;
293
+ }
294
+ if (!info) {
295
+ this.logger.warn(`Proving job id=${id} type=${ProvingRequestType[item.type]} not in the in-progress set`, {
296
+ provingJobId: id
297
+ });
298
+ } else {
299
+ this.inProgress.delete(id);
300
+ }
301
+ if (this.resultsCache.has(id)) {
302
+ this.logger.warn(`Proving job id=${id} is already settled, ignoring err=${err}`, {
303
+ provingJobId: id
304
+ });
305
+ return this.#getProvingJob(filter);
306
+ }
307
+ if (retry && retries + 1 < this.maxRetries && !this.isJobStale(item)) {
308
+ this.logger.info(`Retrying proving job id=${id} type=${ProvingRequestType[item.type]} retry=${retries + 1} err=${err}`, {
309
+ provingJobId: id
310
+ });
311
+ // assign another job to this agent
312
+ // do this first, before we put the failed job back in the queue
313
+ const maybeAnotherJob = this.#getProvingJob(filter);
314
+ this.retries.set(id, retries + 1);
315
+ this.enqueueJobInternal(item);
316
+ this.instrumentation.incRetriedJobs(item.type);
317
+ return maybeAnotherJob;
318
+ }
319
+ this.logger.info(`Marking proving job as failed id=${id} type=${ProvingRequestType[item.type]} totalAttempts=${retries + 1} err=${err}`, {
320
+ provingJobId: id
321
+ });
322
+ // save the result to the cache and notify clients of the job status
323
+ // this should work even if our database breaks because the result is cached in memory
324
+ const result = {
325
+ status: 'rejected',
326
+ reason: String(err)
327
+ };
328
+ this.resultsCache.set(id, result);
329
+ this.promises.get(id).resolve(result);
330
+ this.completedJobNotifications.push(id);
331
+ this.instrumentation.incRejectedJobs(item.type);
332
+ if (info) {
333
+ const duration = this.msTimeSource() - info.startedAt;
334
+ this.instrumentation.recordJobDuration(item.type, duration);
335
+ }
336
+ try {
337
+ await this.database.setProvingJobError(id, err);
338
+ } catch (saveErr) {
339
+ this.logger.error(`Failed to save proving job error status id=${id} jobErr=${err}`, saveErr, {
340
+ provingJobId: id
341
+ });
342
+ throw saveErr;
343
+ }
344
+ return this.#getProvingJob(filter);
345
+ }
346
+ #reportProvingJobProgress(id, startedAt, filter) {
347
+ const job = this.jobsCache.get(id);
348
+ if (!job) {
349
+ this.logger.warn(`Proving job id=${id} does not exist`, {
350
+ provingJobId: id
351
+ });
352
+ return this.#getProvingJob(filter);
353
+ }
354
+ if (this.resultsCache.has(id)) {
355
+ this.logger.warn(`Proving job id=${id} has already been completed`, {
356
+ provingJobId: id
357
+ });
358
+ return this.#getProvingJob(filter);
359
+ }
360
+ const metadata = this.inProgress.get(id);
361
+ const now = this.msTimeSource();
362
+ if (!metadata) {
363
+ this.logger.warn(`Proving job id=${id} type=${ProvingRequestType[job.type]} not found in the in-progress cache, adding it`, {
364
+ provingJobId: id
365
+ });
366
+ // the queue will still contain the item at this point!
367
+ // we need to be careful when popping off the queue to make sure we're not sending
368
+ // a job that's already in progress
369
+ this.inProgress.set(id, {
370
+ id,
371
+ startedAt,
372
+ lastUpdatedAt: this.msTimeSource()
373
+ });
306
374
  return undefined;
307
- },
308
- _ProvingBroker_reportProvingJobError = async function _ProvingBroker_reportProvingJobError(id, err, retry = false, filter) {
309
- const info = this.inProgress.get(id);
310
- const item = this.jobsCache.get(id);
311
- const retries = this.retries.get(id) ?? 0;
312
- if (!item) {
313
- this.logger.warn(`Can't set error on unknown proving job id=${id} err=${err}`, { provingJoId: id });
314
- return;
315
- }
316
- if (!info) {
317
- this.logger.warn(`Proving job id=${id} type=${ProvingRequestType[item.type]} not in the in-progress set`, {
318
- provingJobId: id,
375
+ } else if (startedAt <= metadata.startedAt) {
376
+ if (startedAt < metadata.startedAt) {
377
+ this.logger.info(`Proving job id=${id} type=${ProvingRequestType[job.type]} startedAt=${startedAt} older agent has taken job`, {
378
+ provingJobId: id
319
379
  });
320
- }
321
- else {
322
- this.inProgress.delete(id);
323
- }
324
- if (this.resultsCache.has(id)) {
325
- this.logger.warn(`Proving job id=${id} is already settled, ignoring err=${err}`, {
326
- provingJobId: id,
380
+ } else {
381
+ this.logger.debug(`Proving job id=${id} type=${ProvingRequestType[job.type]} heartbeat`, {
382
+ provingJobId: id
327
383
  });
328
- return __classPrivateFieldGet(this, _ProvingBroker_instances, "m", _ProvingBroker_getProvingJob).call(this, filter);
329
384
  }
330
- if (retry && retries + 1 < this.maxRetries && !this.isJobStale(item)) {
331
- this.logger.info(`Retrying proving job id=${id} type=${ProvingRequestType[item.type]} retry=${retries + 1} err=${err}`, {
332
- provingJobId: id,
333
- });
334
- // assign another job to this agent
335
- // do this first, before we put the failed job back in the queue
336
- const maybeAnotherJob = __classPrivateFieldGet(this, _ProvingBroker_instances, "m", _ProvingBroker_getProvingJob).call(this, filter);
337
- this.retries.set(id, retries + 1);
338
- this.enqueueJobInternal(item);
339
- this.instrumentation.incRetriedJobs(item.type);
340
- return maybeAnotherJob;
341
- }
342
- this.logger.info(`Marking proving job as failed id=${id} type=${ProvingRequestType[item.type]} totalAttempts=${retries + 1} err=${err}`, {
343
- provingJobId: id,
385
+ metadata.startedAt = startedAt;
386
+ metadata.lastUpdatedAt = now;
387
+ return undefined;
388
+ }
389
+ this.logger.warn(`Proving job id=${id} type=${ProvingRequestType[job.type]} already being worked on by another agent. Sending new one`, {
390
+ provingJobId: id
391
+ });
392
+ return this.#getProvingJob(filter);
393
+ }
394
+ async #reportProvingJobSuccess(id, value, filter) {
395
+ const info = this.inProgress.get(id);
396
+ const item = this.jobsCache.get(id);
397
+ const retries = this.retries.get(id) ?? 0;
398
+ if (!item) {
399
+ this.logger.warn(`Proving job id=${id} not found`, {
400
+ provingJobId: id
344
401
  });
345
- // save the result to the cache and notify clients of the job status
346
- // this should work even if our database breaks because the result is cached in memory
347
- const result = { status: 'rejected', reason: String(err) };
348
- this.resultsCache.set(id, result);
349
- this.promises.get(id).resolve(result);
350
- this.completedJobNotifications.push(id);
351
- this.instrumentation.incRejectedJobs(item.type);
352
- if (info) {
353
- const duration = this.msTimeSource() - info.startedAt;
354
- this.instrumentation.recordJobDuration(item.type, duration);
355
- }
356
- try {
357
- await this.database.setProvingJobError(id, err);
358
- }
359
- catch (saveErr) {
360
- this.logger.error(`Failed to save proving job error status id=${id} jobErr=${err}`, saveErr, {
361
- provingJobId: id,
362
- });
363
- throw saveErr;
364
- }
365
- return __classPrivateFieldGet(this, _ProvingBroker_instances, "m", _ProvingBroker_getProvingJob).call(this, filter);
366
- },
367
- _ProvingBroker_reportProvingJobProgress = function _ProvingBroker_reportProvingJobProgress(id, startedAt, filter) {
402
+ return;
403
+ }
404
+ if (!info) {
405
+ this.logger.warn(`Proving job id=${id} type=${ProvingRequestType[item.type]} not in the in-progress set`, {
406
+ provingJobId: id
407
+ });
408
+ } else {
409
+ this.inProgress.delete(id);
410
+ }
411
+ if (this.resultsCache.has(id)) {
412
+ this.logger.warn(`Proving job id=${id} already settled, ignoring result`, {
413
+ provingJobId: id
414
+ });
415
+ return;
416
+ }
417
+ this.logger.info(`Proving job complete id=${id} type=${ProvingRequestType[item.type]} totalAttempts=${retries + 1}`, {
418
+ provingJobId: id
419
+ });
420
+ // save result to our local cache and notify clients
421
+ // if save to database fails, that's ok because we have the result in memory
422
+ // if the broker crashes and needs the result again, we're covered because we can just recompute it
423
+ const result = {
424
+ status: 'fulfilled',
425
+ value
426
+ };
427
+ this.resultsCache.set(id, result);
428
+ this.promises.get(id).resolve(result);
429
+ this.completedJobNotifications.push(id);
430
+ this.instrumentation.incResolvedJobs(item.type);
431
+ if (info) {
432
+ const duration = this.msTimeSource() - info.startedAt;
433
+ this.instrumentation.recordJobDuration(item.type, duration);
434
+ }
435
+ try {
436
+ await this.database.setProvingJobResult(id, value);
437
+ } catch (saveErr) {
438
+ this.logger.error(`Failed to save proving job result id=${id}`, saveErr, {
439
+ provingJobId: id
440
+ });
441
+ throw saveErr;
442
+ }
443
+ return this.#getProvingJob(filter);
444
+ }
445
+ async cleanupPass() {
446
+ this.cleanupStaleJobs();
447
+ this.reEnqueueExpiredJobs();
448
+ const oldestEpochToKeep = this.oldestEpochToKeep();
449
+ if (oldestEpochToKeep > 0) {
450
+ await this.database.deleteAllProvingJobsOlderThanEpoch(oldestEpochToKeep);
451
+ this.logger.trace(`Deleted all epochs older than ${oldestEpochToKeep}`);
452
+ }
453
+ }
454
+ cleanupStaleJobs() {
455
+ const jobIds = Array.from(this.jobsCache.keys());
456
+ const jobsToClean = [];
457
+ for (const id of jobIds){
368
458
  const job = this.jobsCache.get(id);
369
- if (!job) {
370
- this.logger.warn(`Proving job id=${id} does not exist`, { provingJobId: id });
371
- return __classPrivateFieldGet(this, _ProvingBroker_instances, "m", _ProvingBroker_getProvingJob).call(this, filter);
372
- }
373
- if (this.resultsCache.has(id)) {
374
- this.logger.warn(`Proving job id=${id} has already been completed`, { provingJobId: id });
375
- return __classPrivateFieldGet(this, _ProvingBroker_instances, "m", _ProvingBroker_getProvingJob).call(this, filter);
376
- }
377
- const metadata = this.inProgress.get(id);
378
- const now = this.msTimeSource();
379
- if (!metadata) {
380
- this.logger.warn(`Proving job id=${id} type=${ProvingRequestType[job.type]} not found in the in-progress cache, adding it`, { provingJobId: id });
381
- // the queue will still contain the item at this point!
382
- // we need to be careful when popping off the queue to make sure we're not sending
383
- // a job that's already in progress
384
- this.inProgress.set(id, {
385
- id,
386
- startedAt,
387
- lastUpdatedAt: this.msTimeSource(),
388
- });
389
- return undefined;
390
- }
391
- else if (startedAt <= metadata.startedAt) {
392
- if (startedAt < metadata.startedAt) {
393
- this.logger.info(`Proving job id=${id} type=${ProvingRequestType[job.type]} startedAt=${startedAt} older agent has taken job`, { provingJobId: id });
394
- }
395
- else {
396
- this.logger.debug(`Proving job id=${id} type=${ProvingRequestType[job.type]} heartbeat`, { provingJobId: id });
397
- }
398
- metadata.startedAt = startedAt;
399
- metadata.lastUpdatedAt = now;
400
- return undefined;
459
+ if (this.isJobStale(job)) {
460
+ jobsToClean.push(id);
401
461
  }
402
- this.logger.warn(`Proving job id=${id} type=${ProvingRequestType[job.type]} already being worked on by another agent. Sending new one`, { provingJobId: id });
403
- return __classPrivateFieldGet(this, _ProvingBroker_instances, "m", _ProvingBroker_getProvingJob).call(this, filter);
404
- },
405
- _ProvingBroker_reportProvingJobSuccess = async function _ProvingBroker_reportProvingJobSuccess(id, value, filter) {
406
- const info = this.inProgress.get(id);
462
+ }
463
+ if (jobsToClean.length > 0) {
464
+ this.cleanUpProvingJobState(jobsToClean);
465
+ this.logger.info(`Cleaned up jobs=${jobsToClean.length}`);
466
+ }
467
+ }
468
+ reEnqueueExpiredJobs() {
469
+ const inProgressEntries = Array.from(this.inProgress.entries());
470
+ for (const [id, metadata] of inProgressEntries){
407
471
  const item = this.jobsCache.get(id);
408
- const retries = this.retries.get(id) ?? 0;
409
472
  if (!item) {
410
- this.logger.warn(`Proving job id=${id} not found`, { provingJobId: id });
411
- return;
412
- }
413
- if (!info) {
414
- this.logger.warn(`Proving job id=${id} type=${ProvingRequestType[item.type]} not in the in-progress set`, {
415
- provingJobId: id,
473
+ this.logger.warn(`Proving job id=${id} not found. Removing it from the queue.`, {
474
+ provingJobId: id
416
475
  });
417
- }
418
- else {
419
476
  this.inProgress.delete(id);
477
+ continue;
420
478
  }
421
- if (this.resultsCache.has(id)) {
422
- this.logger.warn(`Proving job id=${id} already settled, ignoring result`, { provingJobId: id });
423
- return;
424
- }
425
- this.logger.info(`Proving job complete id=${id} type=${ProvingRequestType[item.type]} totalAttempts=${retries + 1}`, { provingJobId: id });
426
- // save result to our local cache and notify clients
427
- // if save to database fails, that's ok because we have the result in memory
428
- // if the broker crashes and needs the result again, we're covered because we can just recompute it
429
- const result = { status: 'fulfilled', value };
430
- this.resultsCache.set(id, result);
431
- this.promises.get(id).resolve(result);
432
- this.completedJobNotifications.push(id);
433
- this.instrumentation.incResolvedJobs(item.type);
434
- if (info) {
435
- const duration = this.msTimeSource() - info.startedAt;
436
- this.instrumentation.recordJobDuration(item.type, duration);
437
- }
438
- try {
439
- await this.database.setProvingJobResult(id, value);
440
- }
441
- catch (saveErr) {
442
- this.logger.error(`Failed to save proving job result id=${id}`, saveErr, {
443
- provingJobId: id,
479
+ const now = this.msTimeSource();
480
+ const msSinceLastUpdate = now - metadata.lastUpdatedAt;
481
+ if (msSinceLastUpdate >= this.jobTimeoutMs) {
482
+ this.logger.warn(`Proving job id=${id} timed out. Adding it back to the queue.`, {
483
+ provingJobId: id
444
484
  });
445
- throw saveErr;
485
+ this.inProgress.delete(id);
486
+ this.enqueueJobInternal(item);
487
+ this.instrumentation.incTimedOutJobs(item.type);
446
488
  }
447
- return __classPrivateFieldGet(this, _ProvingBroker_instances, "m", _ProvingBroker_getProvingJob).call(this, filter);
448
- },
449
- (() => {
450
- const _metadata = typeof Symbol === "function" && Symbol.metadata ? Object.create(null) : void 0;
451
- _cleanupPass_decorators = [trackSpan('ProvingBroker.cleanupPass')];
452
- __esDecorate(_a, null, _cleanupPass_decorators, { kind: "method", name: "cleanupPass", static: false, private: false, access: { has: obj => "cleanupPass" in obj, get: obj => obj.cleanupPass }, metadata: _metadata }, null, _instanceExtraInitializers);
453
- if (_metadata) Object.defineProperty(_a, Symbol.metadata, { enumerable: true, configurable: true, writable: true, value: _metadata });
454
- })(),
455
- _a;
456
- })();
457
- export { ProvingBroker };
489
+ }
490
+ }
491
+ enqueueJobInternal(job) {
492
+ if (!this.promises.has(job.id)) {
493
+ this.promises.set(job.id, promiseWithResolvers());
494
+ }
495
+ this.queues[job.type].put({
496
+ epochNumber: job.epochNumber,
497
+ id: job.id
498
+ });
499
+ this.enqueuedAt.set(job.id, new Timer());
500
+ this.epochHeight = Math.max(this.epochHeight, job.epochNumber);
501
+ }
502
+ isJobStale(job) {
503
+ return job.epochNumber < this.oldestEpochToKeep();
504
+ }
505
+ oldestEpochToKeep() {
506
+ return this.epochHeight - this.maxEpochsToKeepResultsFor;
507
+ }
508
+ }
509
+ _ts_decorate([
510
+ trackSpan('ProvingBroker.cleanupPass')
511
+ ], ProvingBroker.prototype, "cleanupPass", null);
458
512
  /**
459
513
  * Compares two proving jobs and selects which one's more important
460
514
  * @param a - A proving job
461
515
  * @param b - Another proving job
462
516
  * @returns A number indicating the relative priority of the two proving jobs
463
- */
464
- function provingJobComparator(a, b) {
517
+ */ function provingJobComparator(a, b) {
465
518
  if (a.epochNumber < b.epochNumber) {
466
519
  return -1;
467
- }
468
- else if (a.epochNumber > b.epochNumber) {
520
+ } else if (a.epochNumber > b.epochNumber) {
469
521
  return 1;
470
- }
471
- else {
522
+ } else {
472
523
  return 0;
473
524
  }
474
525
  }
@@ -479,26 +530,21 @@ function provingJobComparator(a, b) {
479
530
  * @param a - A proof type
480
531
  * @param b - Another proof type
481
532
  * @returns A number indicating the relative priority of the two proof types
482
- */
483
- function proofTypeComparator(a, b) {
533
+ */ function proofTypeComparator(a, b) {
484
534
  const indexOfA = PROOF_TYPES_IN_PRIORITY_ORDER.indexOf(a);
485
535
  const indexOfB = PROOF_TYPES_IN_PRIORITY_ORDER.indexOf(b);
486
536
  if (indexOfA === indexOfB) {
487
537
  return 0;
488
- }
489
- else if (indexOfA === -1) {
538
+ } else if (indexOfA === -1) {
490
539
  // a is some new proof that didn't get added to the array
491
540
  // b is more important because we know about it
492
541
  return 1;
493
- }
494
- else if (indexOfB === -1) {
542
+ } else if (indexOfB === -1) {
495
543
  // the opposite of the previous if branch
496
544
  return -1;
497
- }
498
- else if (indexOfA < indexOfB) {
545
+ } else if (indexOfA < indexOfB) {
499
546
  return -1;
500
- }
501
- else {
547
+ } else {
502
548
  return 1;
503
549
  }
504
550
  }
@@ -508,8 +554,7 @@ function proofTypeComparator(a, b) {
508
554
  *
509
555
  * The aim is that this will speed up block proving as the closer we get to a block's root proof the more likely it
510
556
  * is to get picked up by agents
511
- */
512
- const PROOF_TYPES_IN_PRIORITY_ORDER = [
557
+ */ const PROOF_TYPES_IN_PRIORITY_ORDER = [
513
558
  ProvingRequestType.BLOCK_ROOT_ROLLUP,
514
559
  ProvingRequestType.SINGLE_TX_BLOCK_ROOT_ROLLUP,
515
560
  ProvingRequestType.BLOCK_MERGE_ROLLUP,
@@ -521,6 +566,5 @@ const PROOF_TYPES_IN_PRIORITY_ORDER = [
521
566
  ProvingRequestType.TUBE_PROOF,
522
567
  ProvingRequestType.ROOT_PARITY,
523
568
  ProvingRequestType.BASE_PARITY,
524
- ProvingRequestType.EMPTY_BLOCK_ROOT_ROLLUP,
569
+ ProvingRequestType.EMPTY_BLOCK_ROOT_ROLLUP
525
570
  ];
526
- //# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoicHJvdmluZ19icm9rZXIuanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi9zcmMvcHJvdmluZ19icm9rZXIvcHJvdmluZ19icm9rZXIudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IjtBQUFBLE9BQU8sRUFVTCxrQkFBa0IsR0FDbkIsTUFBTSxzQkFBc0IsQ0FBQztBQUM5QixPQUFPLEVBQUUsWUFBWSxFQUFFLE1BQU0sdUJBQXVCLENBQUM7QUFDckQsT0FBTyxFQUE2QixjQUFjLEVBQUUsb0JBQW9CLEVBQUUsTUFBTSwyQkFBMkIsQ0FBQztBQUM1RyxPQUFPLEVBQUUsbUJBQW1CLEVBQUUsTUFBTSx5QkFBeUIsQ0FBQztBQUM5RCxPQUFPLEVBQUUsS0FBSyxFQUFFLE1BQU0seUJBQXlCLENBQUM7QUFDaEQsT0FBTyxFQUlMLGtCQUFrQixFQUNsQixTQUFTLEdBQ1YsTUFBTSx5QkFBeUIsQ0FBQztBQUVqQyxPQUFPLE1BQU0sTUFBTSxRQUFRLENBQUM7QUFHNUIsT0FBTyxFQUF3Qiw0QkFBNEIsRUFBRSxNQUFNLHFDQUFxQyxDQUFDO0FBa0J6Rzs7O0dBR0c7SUFDVSxhQUFhOzs7O3NCQUFiLGFBQWE7WUFpRXhCLFlBQ1UsUUFBK0IsRUFDdkMsRUFDRSxZQUFZLEdBQUcsS0FBTSxFQUNyQixpQkFBaUIsR0FBRyxLQUFNLEVBQzFCLFVBQVUsR0FBRyxDQUFDLEVBQ2QseUJBQXlCLEdBQUcsQ0FBQyxNQUNELEVBQUUsRUFDaEMsU0FBMEIsa0JBQWtCLEVBQUUsRUFDdEMsU0FBUyxZQUFZLENBQUMsOEJBQThCLENBQUM7O2dCQVJyRCxhQUFRLElBbEVQLG1EQUFhLEVBa0VkLFFBQVEsRUFBdUI7Z0JBUS9CLFdBQU0sR0FBTixNQUFNLENBQStDO2dCQXpFdkQsV0FBTSxHQUFrQjtvQkFDOUIsQ0FBQyxrQkFBa0IsQ0FBQyxTQUFTLENBQUMsRUFBRSxJQUFJLG1CQUFtQixDQUFxQixvQkFBb0IsQ0FBQztvQkFDakcsQ0FBQyxrQkFBa0IsQ0FBQyxVQUFVLENBQUMsRUFBRSxJQUFJLG1CQUFtQixDQUFxQixvQkFBb0IsQ0FBQztvQkFFbEcsQ0FBQyxrQkFBa0IsQ0FBQyxtQkFBbUIsQ0FBQyxFQUFFLElBQUksbUJBQW1CLENBQXFCLG9CQUFvQixDQUFDO29CQUMzRyxDQUFDLGtCQUFrQixDQUFDLGtCQUFrQixDQUFDLEVBQUUsSUFBSSxtQkFBbUIsQ0FBcUIsb0JBQW9CLENBQUM7b0JBQzFHLENBQUMsa0JBQWtCLENBQUMsWUFBWSxDQUFDLEVBQUUsSUFBSSxtQkFBbUIsQ0FBcUIsb0JBQW9CLENBQUM7b0JBQ3BHLENBQUMsa0JBQWtCLENBQUMsV0FBVyxDQUFDLEVBQUUsSUFBSSxtQkFBbUIsQ0FBcUIsb0JBQW9CLENBQUM7b0JBRW5HLENBQUMsa0JBQWtCLENBQUMsa0JBQWtCLENBQUMsRUFBRSxJQUFJLG1CQUFtQixDQUFxQixvQkFBb0IsQ0FBQztvQkFDMUcsQ0FBQyxrQkFBa0IsQ0FBQyxpQkFBaUIsQ0FBQyxFQUFFLElBQUksbUJBQW1CLENBQXFCLG9CQUFvQixDQUFDO29CQUN6RyxDQUFDLGtCQUFrQixDQUFDLDJCQUEyQixDQUFDLEVBQUUsSUFBSSxtQkFBbUIsQ0FBcUIsb0JBQW9CLENBQUM7b0JBQ25ILENBQUMsa0JBQWtCLENBQUMsdUJBQXVCLENBQUMsRUFBRSxJQUFJLG1CQUFtQixDQUFxQixvQkFBb0IsQ0FBQztvQkFFL0csQ0FBQyxrQkFBa0IsQ0FBQyxXQUFXLENBQUMsRUFBRSxJQUFJLG1CQUFtQixDQUFxQixvQkFBb0IsQ0FBQztvQkFDbkcsQ0FBQyxrQkFBa0IsQ0FBQyxXQUFXLENBQUMsRUFBRSxJQUFJLG1CQUFtQixDQUFxQixvQkFBb0IsQ0FBQztpQkFDcEcsQ0FBQztnQkFFRiw4RUFBOEU7Z0JBQzlFLGdGQUFnRjtnQkFDeEUsY0FBUyxHQUFHLElBQUksR0FBRyxFQUE0QixDQUFDO2dCQUN4RCw0QkFBNEI7Z0JBQ3BCLGlCQUFZLEdBQUcsSUFBSSxHQUFHLEVBQXlDLENBQUM7Z0JBRXhFLG9DQUFvQztnQkFDNUIsZUFBVSxHQUFHLElBQUksR0FBRyxFQUF1QixDQUFDO2dCQUVwRCwwREFBMEQ7Z0JBQzFELGtFQUFrRTtnQkFDbEUsdUVBQXVFO2dCQUN2RSxrQ0FBa0M7Z0JBQzFCLGVBQVUsR0FBRyxJQUFJLEdBQUcsRUFBb0MsQ0FBQztnQkFFakUsbURBQW1EO2dCQUMzQyxZQUFPLEdBQUcsSUFBSSxHQUFHLEVBQXdCLENBQUM7Z0JBRWxELGdFQUFnRTtnQkFDeEQsYUFBUSxHQUFHLElBQUksR0FBRyxFQUErRCxDQUFDO2dCQUdsRixpQkFBWSxHQUFHLEdBQUcsRUFBRSxDQUFDLElBQUksQ0FBQyxHQUFHLEVBQUUsQ0FBQztnQkFPaEMsOEJBQXlCLEdBQW1CLEVBQUUsQ0FBQztnQkFFdkQ7Ozs7Ozs7OzttQkFTRztnQkFDSyxnQkFBVyxHQUFHLENBQUMsQ0FBQztnQkFDaEIsOEJBQXlCLEdBQUcsQ0FBQyxDQUFDO2dCQUU5QixZQUFPLEdBQUcsS0FBSyxDQUFDO2dCQXFCaEIsc0JBQWlCLEdBQW9CLENBQUMsSUFBd0IsRUFBRSxFQUFFO29CQUN4RSxPQUFPLElBQUksQ0FBQyxNQUFNLENBQUMsSUFBSSxDQUFDLENBQUMsTUFBTSxFQUFFLENBQUM7Z0JBQ3BDLENBQUMsQ0FBQztnQkFFTSxvQkFBZSxHQUFvQixDQUFDLElBQXdCLEVBQUUsRUFBRTtvQkFDdEUsSUFBSSxLQUFLLEdBQUcsQ0FBQyxDQUFDO29CQUNkLEtBQUssTUFBTSxFQUFFLEVBQUUsRUFBRSxJQUFJLElBQUksQ0FBQyxVQUFVLENBQUMsTUFBTSxFQUFFLEVBQUUsQ0FBQzt3QkFDOUMsTUFBTSxHQUFHLEdBQUcsSUFBSSxDQUFDLFNBQVMsQ0FBQyxHQUFHLENBQUMsRUFBRSxDQUFDLENBQUM7d0JBQ25DLElBQUksR0FBRyxFQUFFLElBQUksS0FBSyxJQUFJLEVBQUUsQ0FBQzs0QkFDdkIsS0FBSyxFQUFFLENBQUM7d0JBQ1YsQ0FBQztvQkFDSCxDQUFDO29CQUVELE9BQU8sS0FBSyxDQUFDO2dCQUNmLENBQUMsQ0FBQztnQkF0QkEsSUFBSSxDQUFDLE1BQU0sR0FBRyxNQUFNLENBQUMsU0FBUyxDQUFDLGVBQWUsQ0FBQyxDQUFDO2dCQUNoRCxJQUFJLENBQUMsZUFBZSxHQUFHLElBQUksNEJBQTRCLENBQUMsTUFBTSxDQUFDLENBQUM7Z0JBQ2hFLElBQUksQ0FBQyxjQUFjLEdBQUcsSUFBSSxjQUFjLENBQUMsSUFBSSxDQUFDLFdBQVcsQ0FBQyxJQUFJLENBQUMsSUFBSSxDQUFDLEVBQUUsSUFBSSxDQUFDLE1BQU0sRUFBRSxpQkFBaUIsQ0FBQyxDQUFDO2dCQUN0RyxJQUFJLENBQUMsWUFBWSxHQUFHLFlBQVksQ0FBQztnQkFDakMsSUFBSSxDQUFDLFVBQVUsR0FBRyxVQUFVLENBQUM7Z0JBQzdCLElBQUksQ0FBQyx5QkFBeUIsR0FBRyx5QkFBeUIsQ0FBQztZQUM3RCxDQUFDO1lBa0JNLEtBQUssQ0FBQyxLQUFLO2dCQUNoQixJQUFJLElBQUksQ0FBQyxPQUFPLEVBQUUsQ0FBQztvQkFDakIsSUFBSSxDQUFDLE1BQU0sQ0FBQyxJQUFJLENBQUMsZ0NBQWdDLENBQUMsQ0FBQztvQkFDbkQsT0FBTyxPQUFPLENBQUMsT0FBTyxFQUFFLENBQUM7Z0JBQzNCLENBQUM7Z0JBQ0QsSUFBSSxDQUFDLE1BQU0sQ0FBQyxJQUFJLENBQUMsd0JBQXdCLENBQUMsQ0FBQztnQkFDM0MsSUFBSSxLQUFLLEVBQUUsTUFBTSxDQUFDLElBQUksRUFBRSxNQUFNLENBQUMsSUFBSSxJQUFJLENBQUMsUUFBUSxDQUFDLGNBQWMsRUFBRSxFQUFFLENBQUM7b0JBQ2xFLElBQUksQ0FBQyxNQUFNLENBQUMsSUFBSSxDQUFDLDRCQUE0QixJQUFJLENBQUMsRUFBRSxZQUFZLENBQUMsQ0FBQyxNQUFNLEVBQUUsRUFBRTt3QkFDMUUsWUFBWSxFQUFFLElBQUksQ0FBQyxFQUFFO3dCQUNyQixNQUFNLEVBQUUsTUFBTSxDQUFDLENBQUMsQ0FBQyxNQUFNLENBQUMsTUFBTSxDQUFDLENBQUMsQ0FBQyxTQUFTO3FCQUMzQyxDQUFDLENBQUM7b0JBRUgsSUFBSSxDQUFDLFNBQVMsQ0FBQyxHQUFHLENBQUMsSUFBSSxDQUFDLEVBQUUsRUFBRSxJQUFJLENBQUMsQ0FBQztvQkFDbEMsSUFBSSxDQUFDLFFBQVEsQ0FBQyxHQUFHLENBQUMsSUFBSSxDQUFDLEVBQUUsRUFBRSxvQkFBb0IsRUFBRSxDQUFDLENBQUM7b0JBRW5ELElBQUksTUFBTSxFQUFFLENBQUM7d0JBQ1gsSUFBSSxDQUFDLFFBQVEsQ0FBQyxHQUFHLENBQUMsSUFBSSxDQUFDLEVBQUUsQ0FBRSxDQUFDLE9BQU8sQ0FBQyxNQUFNLENBQUMsQ0FBQzt3QkFDNUMsSUFBSSxDQUFDLFlBQVksQ0FBQyxHQUFHLENBQUMsSUFBSSxDQUFDLEVBQUUsRUFBRSxNQUFNLENBQUMsQ0FBQztvQkFDekMsQ0FBQzt5QkFBTSxDQUFDO3dCQUNOLElBQUksQ0FBQyxrQkFBa0IsQ0FBQyxJQUFJLENBQUMsQ0FBQztvQkFDaEMsQ0FBQztnQkFDSCxDQUFDO2dCQUVELElBQUksQ0FBQyxjQUFjLENBQUMsS0FBSyxFQUFFLENBQUM7Z0JBRTVCLElBQUksQ0FBQyxlQUFlLENBQUMsaUJBQWlCLENBQUMsSUFBSSxDQUFDLGlCQUFpQixDQUFDLENBQUM7Z0JBQy9ELElBQUksQ0FBQyxlQUFlLENBQUMsaUJBQWlCLENBQUMsSUFBSSxDQUFDLGVBQWUsQ0FBQyxDQUFDO2dCQUU3RCxJQUFJLENBQUMsT0FBTyxHQUFHLElBQUksQ0FBQztZQUN0QixDQUFDO1lBRU0sS0FBSyxDQUFDLElBQUk7Z0JBQ2YsSUFBSSxDQUFDLElBQUksQ0FBQyxPQUFPLEVBQUUsQ0FBQztvQkFDbEIsSUFBSSxDQUFDLE1BQU0sQ0FBQyxJQUFJLENBQUMsMkJBQTJCLENBQUMsQ0FBQztvQkFDOUMsT0FBTyxPQUFPLENBQUMsT0FBTyxFQUFFLENBQUM7Z0JBQzNCLENBQUM7Z0JBQ0QsTUFBTSxJQUFJLENBQUMsY0FBYyxDQUFDLElBQUksRUFBRSxDQUFDO1lBQ25DLENBQUM7WUFFTSxpQkFBaUIsQ0FBQyxHQUFlO2dCQUN0QyxPQUFPLHVCQUFBLElBQUksa0VBQW1CLE1BQXZCLElBQUksRUFBb0IsR0FBRyxDQUFDLENBQUM7WUFDdEMsQ0FBQztZQUVNLGdCQUFnQixDQUFDLEVBQWdCO2dCQUN0QyxPQUFPLHVCQUFBLElBQUksaUVBQWtCLE1BQXRCLElBQUksRUFBbUIsRUFBRSxDQUFDLENBQUM7WUFDcEMsQ0FBQztZQUVNLG1CQUFtQixDQUFDLEVBQWdCO2dCQUN6QyxPQUFPLE9BQU8sQ0FBQyxPQUFPLENBQUMsdUJBQUEsSUFBSSxvRUFBcUIsTUFBekIsSUFBSSxFQUFzQixFQUFFLENBQUMsQ0FBQyxDQUFDO1lBQ3hELENBQUM7WUFFTSxnQkFBZ0IsQ0FBQyxHQUFtQjtnQkFDekMsT0FBTyx1QkFBQSxJQUFJLGlFQUFrQixNQUF0QixJQUFJLEVBQW1CLEdBQUcsQ0FBQyxDQUFDO1lBQ3JDLENBQUM7WUFFTSxhQUFhLENBQUMsTUFBeUI7Z0JBQzVDLE9BQU8sT0FBTyxDQUFDLE9BQU8sQ0FBQyx1QkFBQSxJQUFJLDhEQUFlLE1BQW5CLElBQUksRUFBZ0IsTUFBTSxDQUFDLENBQUMsQ0FBQztZQUN0RCxDQUFDO1lBRU0sdUJBQXVCLENBQzVCLEVBQWdCLEVBQ2hCLEtBQWUsRUFDZixNQUF5QjtnQkFFekIsT0FBTyx1QkFBQSxJQUFJLHdFQUF5QixNQUE3QixJQUFJLEVBQTBCLEVBQUUsRUFBRSxLQUFLLEVBQUUsTUFBTSxDQUFDLENBQUM7WUFDMUQsQ0FBQztZQUVNLHFCQUFxQixDQUMxQixFQUFnQixFQUNoQixHQUFXLEVBQ1gsS0FBSyxHQUFHLEtBQUssRUFDYixNQUF5QjtnQkFFekIsT0FBTyx1QkFBQSxJQUFJLHNFQUF1QixNQUEzQixJQUFJLEVBQXdCLEVBQUUsRUFBRSxHQUFHLEVBQUUsS0FBSyxFQUFFLE1BQU0sQ0FBQyxDQUFDO1lBQzdELENBQUM7WUFFTSx3QkFBd0IsQ0FDN0IsRUFBZ0IsRUFDaEIsU0FBaUIsRUFDakIsTUFBeUI7Z0JBRXpCLE9BQU8sT0FBTyxDQUFDLE9BQU8sQ0FBQyx1QkFBQSxJQUFJLHlFQUEwQixNQUE5QixJQUFJLEVBQTJCLEVBQUUsRUFBRSxTQUFTLEVBQUUsTUFBTSxDQUFDLENBQUMsQ0FBQztZQUNoRixDQUFDO1lBZ0RPLHNCQUFzQixDQUFDLEdBQW1CO2dCQUNoRCxLQUFLLE1BQU0sRUFBRSxJQUFJLEdBQUcsRUFBRSxDQUFDO29CQUNyQixJQUFJLENBQUMsU0FBUyxDQUFDLE1BQU0sQ0FBQyxFQUFFLENBQUMsQ0FBQztvQkFDMUIsSUFBSSxDQUFDLFFBQVEsQ0FBQyxNQUFNLENBQUMsRUFBRSxDQUFDLENBQUM7b0JBQ3pCLElBQUksQ0FBQyxZQUFZLENBQUMsTUFBTSxDQUFDLEVBQUUsQ0FBQyxDQUFDO29CQUM3QixJQUFJLENBQUMsVUFBVSxDQUFDLE1BQU0sQ0FBQyxFQUFFLENBQUMsQ0FBQztvQkFDM0IsSUFBSSxDQUFDLE9BQU8sQ0FBQyxNQUFNLENBQUMsRUFBRSxDQUFDLENBQUM7Z0JBQzFCLENBQUM7WUFDSCxDQUFDO1lBcVFPLEtBQUssQ0FBQyxXQUFXO2dCQUN2QixJQUFJLENBQUMsZ0JBQWdCLEVBQUUsQ0FBQztnQkFDeEIsSUFBSSxDQUFDLG9CQUFvQixFQUFFLENBQUM7Z0JBQzVCLE1BQU0saUJBQWlCLEdBQUcsSUFBSSxDQUFDLGlCQUFpQixFQUFFLENBQUM7Z0JBQ25ELElBQUksaUJBQWlCLEdBQUcsQ0FBQyxFQUFFLENBQUM7b0JBQzFCLE1BQU0sSUFBSSxDQUFDLFFBQVEsQ0FBQyxrQ0FBa0MsQ0FBQyxpQkFBaUIsQ0FBQyxDQUFDO29CQUMxRSxJQUFJLENBQUMsTUFBTSxDQUFDLEtBQUssQ0FBQyxpQ0FBaUMsaUJBQWlCLEVBQUUsQ0FBQyxDQUFDO2dCQUMxRSxDQUFDO1lBQ0gsQ0FBQztZQUVPLGdCQUFnQjtnQkFDdEIsTUFBTSxNQUFNLEdBQUcsS0FBSyxDQUFDLElBQUksQ0FBQyxJQUFJLENBQUMsU0FBUyxDQUFDLElBQUksRUFBRSxDQUFDLENBQUM7Z0JBQ2pELE1BQU0sV0FBVyxHQUFtQixFQUFFLENBQUM7Z0JBQ3ZDLEtBQUssTUFBTSxFQUFFLElBQUksTUFBTSxFQUFFLENBQUM7b0JBQ3hCLE1BQU0sR0FBRyxHQUFHLElBQUksQ0FBQyxTQUFTLENBQUMsR0FBRyxDQUFDLEVBQUUsQ0FBRSxDQUFDO29CQUNwQyxJQUFJLElBQUksQ0FBQyxVQUFVLENBQUMsR0FBRyxDQUFDLEVBQUUsQ0FBQzt3QkFDekIsV0FBVyxDQUFDLElBQUksQ0FBQyxFQUFFLENBQUMsQ0FBQztvQkFDdkIsQ0FBQztnQkFDSCxDQUFDO2dCQUVELElBQUksV0FBVyxDQUFDLE1BQU0sR0FBRyxDQUFDLEVBQUUsQ0FBQztvQkFDM0IsSUFBSSxDQUFDLHNCQUFzQixDQUFDLFdBQVcsQ0FBQyxDQUFDO29CQUN6QyxJQUFJLENBQUMsTUFBTSxDQUFDLElBQUksQ0FBQyxtQkFBbUIsV0FBVyxDQUFDLE1BQU0sRUFBRSxDQUFDLENBQUM7Z0JBQzVELENBQUM7WUFDSCxDQUFDO1lBRU8sb0JBQW9CO2dCQUMxQixNQUFNLGlCQUFpQixHQUFHLEtBQUssQ0FBQyxJQUFJLENBQUMsSUFBSSxDQUFDLFVBQVUsQ0FBQyxPQUFPLEVBQUUsQ0FBQyxDQUFDO2dCQUNoRSxLQUFLLE1BQU0sQ0FBQyxFQUFFLEVBQUUsUUFBUSxDQUFDLElBQUksaUJBQWlCLEVBQUUsQ0FBQztvQkFDL0MsTUFBTSxJQUFJLEdBQUcsSUFBSSxDQUFDLFNBQVMsQ0FBQyxHQUFHLENBQUMsRUFBRSxDQUFDLENBQUM7b0JBQ3BDLElBQUksQ0FBQyxJQUFJLEVBQUUsQ0FBQzt3QkFDVixJQUFJLENBQUMsTUFBTSxDQUFDLElBQUksQ0FBQyxrQkFBa0IsRUFBRSx5Q0FBeUMsRUFBRSxFQUFFLFlBQVksRUFBRSxFQUFFLEVBQUUsQ0FBQyxDQUFDO3dCQUN0RyxJQUFJLENBQUMsVUFBVSxDQUFDLE1BQU0sQ0FBQyxFQUFFLENBQUMsQ0FBQzt3QkFDM0IsU0FBUztvQkFDWCxDQUFDO29CQUVELE1BQU0sR0FBRyxHQUFHLElBQUksQ0FBQyxZQUFZLEVBQUUsQ0FBQztvQkFDaEMsTUFBTSxpQkFBaUIsR0FBRyxHQUFHLEdBQUcsUUFBUSxDQUFDLGFBQWEsQ0FBQztvQkFDdkQsSUFBSSxpQkFBaUIsSUFBSSxJQUFJLENBQUMsWUFBWSxFQUFFLENBQUM7d0JBQzNDLElBQUksQ0FBQyxNQUFNLENBQUMsSUFBSSxDQUFDLGtCQUFrQixFQUFFLDBDQUEwQyxFQUFFLEVBQUUsWUFBWSxFQUFFLEVBQUUsRUFBRSxDQUFDLENBQUM7d0JBQ3ZHLElBQUksQ0FBQyxVQUFVLENBQUMsTUFBTSxDQUFDLEVBQUUsQ0FBQyxDQUFDO3dCQUMzQixJQUFJLENBQUMsa0JBQWtCLENBQUMsSUFBSSxDQUFDLENBQUM7d0JBQzlCLElBQUksQ0FBQyxlQUFlLENBQUMsZUFBZSxDQUFDLElBQUksQ0FBQyxJQUFJLENBQUMsQ0FBQztvQkFDbEQsQ0FBQztnQkFDSCxDQUFDO1lBQ0gsQ0FBQztZQUVPLGtCQUFrQixDQUFDLEdBQWU7Z0JBQ3hDLElBQUksQ0FBQyxJQUFJLENBQUMsUUFBUSxDQUFDLEdBQUcsQ0FBQyxHQUFHLENBQUMsRUFBRSxDQUFDLEVBQUUsQ0FBQztvQkFDL0IsSUFBSSxDQUFDLFFBQVEsQ0FBQyxHQUFHLENBQUMsR0FBRyxDQUFDLEVBQUUsRUFBRSxvQkFBb0IsRUFBRSxDQUFDLENBQUM7Z0JBQ3BELENBQUM7Z0JBQ0QsSUFBSSxDQUFDLE1BQU0sQ0FBQyxHQUFHLENBQUMsSUFBSSxDQUFDLENBQUMsR0FBRyxDQUFDO29CQUN4QixXQUFXLEVBQUUsR0FBRyxDQUFDLFdBQVc7b0JBQzVCLEVBQUUsRUFBRSxHQUFHLENBQUMsRUFBRTtpQkFDWCxDQUFDLENBQUM7Z0JBQ0gsSUFBSSxDQUFDLFVBQVUsQ0FBQyxHQUFHLENBQUMsR0FBRyxDQUFDLEVBQUUsRUFBRSxJQUFJLEtBQUssRUFBRSxDQUFDLENBQUM7Z0JBQ3pDLElBQUksQ0FBQyxXQUFXLEdBQUcsSUFBSSxDQUFDLEdBQUcsQ0FBQyxJQUFJLENBQUMsV0FBVyxFQUFFLEdBQUcsQ0FBQyxXQUFXLENBQUMsQ0FBQztZQUNqRSxDQUFDO1lBRU8sVUFBVSxDQUFDLEdBQWU7Z0JBQ2hDLE9BQU8sR0FBRyxDQUFDLFdBQVcsR0FBRyxJQUFJLENBQUMsaUJBQWlCLEVBQUUsQ0FBQztZQUNwRCxDQUFDO1lBRU8saUJBQWlCO2dCQUN2QixPQUFPLElBQUksQ0FBQyxXQUFXLEdBQUcsSUFBSSxDQUFDLHlCQUF5QixDQUFDO1lBQzNELENBQUM7OzsyQ0E1WEQsS0FBSywyQ0FBb0IsR0FBZTtZQUN0QyxxREFBcUQ7WUFDckQsTUFBTSxTQUFTLEdBQUcsdUJBQUEsSUFBSSxvRUFBcUIsTUFBekIsSUFBSSxFQUFzQixHQUFHLENBQUMsRUFBRSxDQUFDLENBQUM7WUFDcEQsSUFBSSxJQUFJLENBQUMsU0FBUyxDQUFDLEdBQUcsQ0FBQyxHQUFHLENBQUMsRUFBRSxDQUFDLEVBQUUsQ0FBQztnQkFDL0IsTUFBTSxRQUFRLEdBQUcsSUFBSSxDQUFDLFNBQVMsQ0FBQyxHQUFHLENBQUMsR0FBRyxDQUFDLEVBQUUsQ0FBQyxDQUFDO2dCQUM1QyxNQUFNLENBQUMsZUFBZSxDQUFDLEdBQUcsRUFBRSxRQUFRLEVBQUUsMEJBQTBCLENBQUMsQ0FBQztnQkFDbEUsSUFBSSxDQUFDLE1BQU0sQ0FBQyxLQUFLLENBQUMsNEJBQTRCLEdBQUcsQ0FBQyxFQUFFLGdCQUFnQixHQUFHLENBQUMsV0FBVyxZQUFZLEVBQUU7b0JBQy9GLFlBQVksRUFBRSxHQUFHLENBQUMsRUFBRTtpQkFDckIsQ0FBQyxDQUFDO2dCQUNILE9BQU8sT0FBTyxDQUFDLE9BQU8sQ0FBQyxTQUFTLENBQUMsQ0FBQztZQUNwQyxDQUFDO1lBRUQsSUFBSSxJQUFJLENBQUMsVUFBVSxDQUFDLEdBQUcsQ0FBQyxFQUFFLENBQUM7Z0JBQ3pCLElBQUksQ0FBQyxNQUFNLENBQUMsSUFBSSxDQUFDLHlDQUF5QyxHQUFHLENBQUMsRUFBRSxnQkFBZ0IsR0FBRyxDQUFDLFdBQVcsRUFBRSxFQUFFO29CQUNqRyxZQUFZLEVBQUUsR0FBRyxDQUFDLEVBQUU7aUJBQ3JCLENBQUMsQ0FBQztnQkFDSCxNQUFNLElBQUksS0FBSyxDQUFDLDRCQUE0QixHQUFHLENBQUMsV0FBVyxvQkFBb0IsSUFBSSxDQUFDLFdBQVcsRUFBRSxDQUFDLENBQUM7WUFDckcsQ0FBQztZQUVELElBQUksQ0FBQyxNQUFNLENBQUMsSUFBSSxDQUFDLHNCQUFzQixHQUFHLENBQUMsRUFBRSxnQkFBZ0IsR0FBRyxDQUFDLFdBQVcsRUFBRSxFQUFFLEVBQUUsWUFBWSxFQUFFLEdBQUcsQ0FBQyxFQUFFLEVBQUUsQ0FBQyxDQUFDO1lBQzFHLElBQUksQ0FBQztnQkFDSCw2SEFBNkg7Z0JBQzdILElBQUksQ0FBQyxTQUFTLENBQUMsR0FBRyxDQUFDLEdBQUcsQ0FBQyxFQUFFLEVBQUUsR0FBRyxDQUFDLENBQUM7Z0JBQ2hDLE1BQU0sSUFBSSxDQUFDLFFBQVEsQ0FBQyxhQUFhLENBQUMsR0FBRyxDQUFDLENBQUM7Z0JBQ3ZDLElBQUksQ0FBQyxrQkFBa0IsQ0FBQyxHQUFHLENBQUMsQ0FBQztZQUMvQixDQUFDO1lBQUMsT0FBTyxHQUFHLEVBQUUsQ0FBQztnQkFDYixJQUFJLENBQUMsTUFBTSxDQUFDLEtBQUssQ0FBQyxpQ0FBaUMsR0FBRyxDQUFDLEVBQUUsS0FBSyxHQUFHLEVBQUUsRUFBRSxHQUFHLEVBQUUsRUFBRSxZQUFZLEVBQUUsR0FBRyxDQUFDLEVBQUUsRUFBRSxDQUFDLENBQUM7Z0JBQ3BHLElBQUksQ0FBQyxTQUFTLENBQUMsTUFBTSxDQUFDLEdBQUcsQ0FBQyxFQUFFLENBQUMsQ0FBQztnQkFDOUIsTUFBTSxHQUFHLENBQUM7WUFDWixDQUFDO1lBQ0QsT0FBTyxTQUFTLENBQUM7UUFDbkIsQ0FBQzswQ0FFRCxLQUFLLDBDQUFtQixFQUFnQjtZQUN0QyxJQUFJLENBQUMsSUFBSSxDQUFDLFNBQVMsQ0FBQyxHQUFHLENBQUMsRUFBRSxDQUFDLEVBQUUsQ0FBQztnQkFDNUIsSUFBSSxDQUFDLE1BQU0sQ0FBQyxJQUFJLENBQUMsNENBQTRDLEVBQUUsRUFBRSxFQUFFLEVBQUUsWUFBWSxFQUFFLEVBQUUsRUFBRSxDQUFDLENBQUM7Z0JBQ3pGLE9BQU87WUFDVCxDQUFDO1lBRUQsdUNBQXVDO1lBQ3ZDLElBQUksQ0FBQyxJQUFJLENBQUMsWUFBWSxDQUFDLEdBQUcsQ0FBQyxFQUFFLENBQUMsRUFBRSxDQUFDO2dCQUMvQixJQUFJLENBQUMsTUFBTSxDQUFDLElBQUksQ0FBQyxxQkFBcUIsRUFBRSxFQUFFLEVBQUUsRUFBRSxZQUFZLEVBQUUsRUFBRSxFQUFFLENBQUMsQ0FBQztnQkFDbEUsTUFBTSx1QkFBQSxJQUFJLHNFQUF1QixNQUEzQixJQUFJLEVBQXdCLEVBQUUsRUFBRSxTQUFTLEVBQUUsS0FBSyxDQUFDLENBQUM7WUFDMUQsQ0FBQztRQUNILENBQUM7eUZBWW9CLEVBQWdCO1lBQ25DLE1BQU0sTUFBTSxHQUFHLElBQUksQ0FBQyxZQUFZLENBQUMsR0FBRyxDQUFDLEVBQUUsQ0FBQyxDQUFDO1lBQ3pDLElBQUksTUFBTSxFQUFFLENBQUM7Z0JBQ1gsT0FBTyxNQUFNLENBQUM7WUFDaEIsQ0FBQztpQkFBTSxDQUFDO2dCQUNOLDJDQUEyQztnQkFDM0MsTUFBTSxJQUFJLEdBQUcsSUFBSSxDQUFDLFNBQVMsQ0FBQyxHQUFHLENBQUMsRUFBRSxDQUFDLENBQUM7Z0JBRXBDLElBQUksQ0FBQyxJQUFJLEVBQUUsQ0FBQztvQkFDVixPQUFPLEVBQUUsTUFBTSxFQUFFLFdBQVcsRUFBRSxDQUFDO2dCQUNqQyxDQUFDO2dCQUVELE9BQU8sRUFBRSxNQUFNLEVBQUUsSUFBSSxDQUFDLFVBQVUsQ0FBQyxHQUFHLENBQUMsRUFBRSxDQUFDLENBQUMsQ0FBQyxDQUFDLGFBQWEsQ0FBQyxDQUFDLENBQUMsVUFBVSxFQUFFLENBQUM7WUFDMUUsQ0FBQztRQUNILENBQUM7bUZBRWlCLEdBQW1CO1lBQ25DLE1BQU0sYUFBYSxHQUFHLEdBQUcsQ0FBQyxNQUFNLENBQUMsRUFBRSxDQUFDLEVBQUUsQ0FBQyxJQUFJLENBQUMsWUFBWSxDQUFDLEdBQUcsQ0FBQyxFQUFFLENBQUMsQ0FBQyxDQUFDO1lBQ2xFLE1BQU0sYUFBYSxHQUFHLElBQUksQ0FBQyx5QkFBeUIsQ0FBQztZQUNyRCxJQUFJLENBQUMseUJBQXlCLEdBQUcsRUFBRSxDQUFDO1lBQ3BDLE9BQU8sT0FBTyxDQUFDLE9BQU8sQ0FBQyxhQUFhLENBQUMsTUFBTSxDQUFDLGFBQWEsQ0FBQyxDQUFDLENBQUM7UUFDOUQsQ0FBQzs2RUFHYyxTQUEyQixFQUFFLFNBQVMsRUFBRSxFQUFFLEVBQUU7WUFDekQsTUFBTSxhQUFhLEdBQ2pCLEtBQUssQ0FBQyxPQUFPLENBQUMsTUFBTSxDQUFDLFNBQVMsQ0FBQyxJQUFJLE1BQU0sQ0FBQyxTQUFTLENBQUMsTUFBTSxHQUFHLENBQUM7Z0JBQzVELENBQUMsQ0FBQyxDQUFDLEdBQUcsTUFBTSxDQUFDLFNBQVMsQ0FBQztnQkFDdkIsQ0FBQyxDQUFDLE1BQU0sQ0FBQyxNQUFNLENBQUMsa0JBQWtCLENBQUMsQ0FBQyxNQUFNLENBQUMsQ0FBQyxDQUFDLEVBQTJCLEVBQUUsQ0FBQyxPQUFPLENBQUMsS0FBSyxRQUFRLENBQUMsQ0FBQztZQUN0RyxhQUFhLENBQUMsSUFBSSxDQUFDLG1CQUFtQixDQUFDLENBQUM7WUFFeEMsS0FBSyxNQUFNLFNBQVMsSUFBSSxhQUFhLEVBQUUsQ0FBQztnQkFDdEMsTUFBTSxLQUFLLEdBQUcsSUFBSSxDQUFDLE1BQU0sQ0FBQyxTQUFTLENBQUMsQ0FBQztnQkFDckMsSUFBSSxXQUEyQyxDQUFDO2dCQUNoRCxxRkFBcUY7Z0JBQ3JGLGdDQUFnQztnQkFDaEMscURBQXFEO2dCQUNyRCwwR0FBMEc7Z0JBQzFHLE9BQU8sQ0FBQyxXQUFXLEdBQUcsS0FBSyxDQUFDLFlBQVksRUFBRSxDQUFDLEVBQUUsQ0FBQztvQkFDNUMsTUFBTSxHQUFHLEdBQUcsSUFBSSxDQUFDLFNBQVMsQ0FBQyxHQUFHLENBQUMsV0FBVyxDQUFDLEVBQUUsQ0FBQyxDQUFDO29CQUMvQyxJQUFJLEdBQUcsSUFBSSxDQUFDLElBQUksQ0FBQyxVQUFVLENBQUMsR0FBRyxDQUFDLFdBQVcsQ0FBQyxFQUFFLENBQUMsSUFBSSxDQUFDLElBQUksQ0FBQyxZQUFZLENBQUMsR0FBRyxDQUFDLFdBQVcsQ0FBQyxFQUFFLENBQUMsRUFBRSxDQUFDO3dCQUMxRixNQUFNLElBQUksR0FBRyxJQUFJLENBQUMsWUFBWSxFQUFFLENBQUM7d0JBQ2pDLElBQUksQ0FBQyxVQUFVLENBQUMsR0FBRyxDQUFDLEdBQUcsQ0FBQyxFQUFFLEVBQUU7NEJBQzFCLEVBQUUsRUFBRSxHQUFHLENBQUMsRUFBRTs0QkFDVixTQUFTLEVBQUUsSUFBSTs0QkFDZixhQUFhLEVBQUUsSUFBSTt5QkFDcEIsQ0FBQyxDQUFDO3dCQUNILE1BQU0sVUFBVSxHQUFHLElBQUksQ0FBQyxVQUFVLENBQUMsR0FBRyxDQUFDLEdBQUcsQ0FBQyxFQUFFLENBQUMsQ0FBQzt3QkFDL0MsSUFBSSxVQUFVLEVBQUUsQ0FBQzs0QkFDZixJQUFJLENBQUMsZUFBZSxDQUFDLGFBQWEsQ0FBQyxHQUFHLENBQUMsSUFBSSxFQUFFLFVBQVUsQ0FBQyxDQUFDO3dCQUMzRCxDQUFDO3dCQUVELE9BQU8sRUFBRSxHQUFHLEVBQUUsSUFBSSxFQUFFLENBQUM7b0JBQ3ZCLENBQUM7Z0JBQ0gsQ0FBQztZQUNILENBQUM7WUFFRCxPQUFPLFNBQVMsQ0FBQztRQUNuQixDQUFDOytDQUVELEtBQUssK0NBQ0gsRUFBZ0IsRUFDaEIsR0FBVyxFQUNYLEtBQUssR0FBRyxLQUFLLEVBQ2IsTUFBeUI7WUFFekIsTUFBTSxJQUFJLEdBQUcsSUFBSSxDQUFDLFVBQVUsQ0FBQyxHQUFHLENBQUMsRUFBRSxDQUFDLENBQUM7WUFDckMsTUFBTSxJQUFJLEdBQUcsSUFBSSxDQUFDLFNBQVMsQ0FBQyxHQUFHLENBQUMsRUFBRSxDQUFDLENBQUM7WUFDcEMsTUFBTSxPQUFPLEdBQUcsSUFBSSxDQUFDLE9BQU8sQ0FBQyxHQUFHLENBQUMsRUFBRSxDQUFDLElBQUksQ0FBQyxDQUFDO1lBRTFDLElBQUksQ0FBQyxJQUFJLEVBQUUsQ0FBQztnQkFDVixJQUFJLENBQUMsTUFBTSxDQUFDLElBQUksQ0FBQyw2Q0FBNkMsRUFBRSxRQUFRLEdBQUcsRUFBRSxFQUFFLEVBQUUsV0FBVyxFQUFFLEVBQUUsRUFBRSxDQUFDLENBQUM7Z0JBQ3BHLE9BQU87WUFDVCxDQUFDO1lBRUQsSUFBSSxDQUFDLElBQUksRUFBRSxDQUFDO2dCQUNWLElBQUksQ0FBQyxNQUFNLENBQUMsSUFBSSxDQUFDLGtCQUFrQixFQUFFLFNBQVMsa0JBQWtCLENBQUMsSUFBSSxDQUFDLElBQUksQ0FBQyw2QkFBNkIsRUFBRTtvQkFDeEcsWUFBWSxFQUFFLEVBQUU7aUJBQ2pCLENBQUMsQ0FBQztZQUNMLENBQUM7aUJBQU0sQ0FBQztnQkFDTixJQUFJLENBQUMsVUFBVSxDQUFDLE1BQU0sQ0FBQyxFQUFFLENBQUMsQ0FBQztZQUM3QixDQUFDO1lBRUQsSUFBSSxJQUFJLENBQUMsWUFBWSxDQUFDLEdBQUcsQ0FBQyxFQUFFLENBQUMsRUFBRSxDQUFDO2dCQUM5QixJQUFJLENBQUMsTUFBTSxDQUFDLElBQUksQ0FBQyxrQkFBa0IsRUFBRSxxQ0FBcUMsR0FBRyxFQUFFLEVBQUU7b0JBQy9FLFlBQVksRUFBRSxFQUFFO2lCQUNqQixDQUFDLENBQUM7Z0JBQ0gsT0FBTyx1QkFBQSxJQUFJLDhEQUFlLE1BQW5CLElBQUksRUFBZ0IsTUFBTSxDQUFDLENBQUM7WUFDckMsQ0FBQztZQUVELElBQUksS0FBSyxJQUFJLE9BQU8sR0FBRyxDQUFDLEdBQUcsSUFBSSxDQUFDLFVBQVUsSUFBSSxDQUFDLElBQUksQ0FBQyxVQUFVLENBQUMsSUFBSSxDQUFDLEVBQUUsQ0FBQztnQkFDckUsSUFBSSxDQUFDLE1BQU0sQ0FBQyxJQUFJLENBQ2QsMkJBQTJCLEVBQUUsU0FBUyxrQkFBa0IsQ0FBQyxJQUFJLENBQUMsSUFBSSxDQUFDLFVBQVUsT0FBTyxHQUFHLENBQUMsUUFBUSxHQUFHLEVBQUUsRUFDckc7b0JBQ0UsWUFBWSxFQUFFLEVBQUU7aUJBQ2pCLENBQ0YsQ0FBQztnQkFFRixtQ0FBbUM7Z0JBQ25DLGdFQUFnRTtnQkFDaEUsTUFBTSxlQUFlLEdBQUcsdUJBQUEsSUFBSSw4REFBZSxNQUFuQixJQUFJLEVBQWdCLE1BQU0sQ0FBQyxDQUFDO2dCQUVwRCxJQUFJLENBQUMsT0FBTyxDQUFDLEdBQUcsQ0FBQyxFQUFFLEVBQUUsT0FBTyxHQUFHLENBQUMsQ0FBQyxDQUFDO2dCQUNsQyxJQUFJLENBQUMsa0JBQWtCLENBQUMsSUFBSSxDQUFDLENBQUM7Z0JBQzlCLElBQUksQ0FBQyxlQUFlLENBQUMsY0FBYyxDQUFDLElBQUksQ0FBQyxJQUFJLENBQUMsQ0FBQztnQkFFL0MsT0FBTyxlQUFlLENBQUM7WUFDekIsQ0FBQztZQUVELElBQUksQ0FBQyxNQUFNLENBQUMsSUFBSSxDQUNkLG9DQUFvQyxFQUFFLFNBQVMsa0JBQWtCLENBQUMsSUFBSSxDQUFDLElBQUksQ0FBQyxrQkFDMUUsT0FBTyxHQUFHLENBQ1osUUFBUSxHQUFHLEVBQUUsRUFDYjtnQkFDRSxZQUFZLEVBQUUsRUFBRTthQUNqQixDQUNGLENBQUM7WUFFRixvRUFBb0U7WUFDcEUsc0ZBQXNGO1lBQ3RGLE1BQU0sTUFBTSxHQUE0QixFQUFFLE1BQU0sRUFBRSxVQUFVLEVBQUUsTUFBTSxFQUFFLE1BQU0sQ0FBQyxHQUFHLENBQUMsRUFBRSxDQUFDO1lBQ3BGLElBQUksQ0FBQyxZQUFZLENBQUMsR0FBRyxDQUFDLEVBQUUsRUFBRSxNQUFNLENBQUMsQ0FBQztZQUNsQyxJQUFJLENBQUMsUUFBUSxDQUFDLEdBQUcsQ0FBQyxFQUFFLENBQUUsQ0FBQyxPQUFPLENBQUMsTUFBTSxDQUFDLENBQUM7WUFDdkMsSUFBSSxDQUFDLHlCQUF5QixDQUFDLElBQUksQ0FBQyxFQUFFLENBQUMsQ0FBQztZQUV4QyxJQUFJLENBQUMsZUFBZSxDQUFDLGVBQWUsQ0FBQyxJQUFJLENBQUMsSUFBSSxDQUFDLENBQUM7WUFDaEQsSUFBSSxJQUFJLEVBQUUsQ0FBQztnQkFDVCxNQUFNLFFBQVEsR0FBRyxJQUFJLENBQUMsWUFBWSxFQUFFLEdBQUcsSUFBSSxDQUFDLFNBQVMsQ0FBQztnQkFDdEQsSUFBSSxDQUFDLGVBQWUsQ0FBQyxpQkFBaUIsQ0FBQyxJQUFJLENBQUMsSUFBSSxFQUFFLFFBQVEsQ0FBQyxDQUFDO1lBQzlELENBQUM7WUFFRCxJQUFJLENBQUM7Z0JBQ0gsTUFBTSxJQUFJLENBQUMsUUFBUSxDQUFDLGtCQUFrQixDQUFDLEVBQUUsRUFBRSxHQUFHLENBQUMsQ0FBQztZQUNsRCxDQUFDO1lBQUMsT0FBTyxPQUFPLEVBQUUsQ0FBQztnQkFDakIsSUFBSSxDQUFDLE1BQU0sQ0FBQyxLQUFLLENBQUMsOENBQThDLEVBQUUsV0FBVyxHQUFHLEVBQUUsRUFBRSxPQUFPLEVBQUU7b0JBQzNGLFlBQVksRUFBRSxFQUFFO2lCQUNqQixDQUFDLENBQUM7Z0JBRUgsTUFBTSxPQUFPLENBQUM7WUFDaEIsQ0FBQztZQUVELE9BQU8sdUJBQUEsSUFBSSw4REFBZSxNQUFuQixJQUFJLEVBQWdCLE1BQU0sQ0FBQyxDQUFDO1FBQ3JDLENBQUM7bUdBR0MsRUFBZ0IsRUFDaEIsU0FBaUIsRUFDakIsTUFBeUI7WUFFekIsTUFBTSxHQUFHLEdBQUcsSUFBSSxDQUFDLFNBQVMsQ0FBQyxHQUFHLENBQUMsRUFBRSxDQUFDLENBQUM7WUFDbkMsSUFBSSxDQUFDLEdBQUcsRUFBRSxDQUFDO2dCQUNULElBQUksQ0FBQyxNQUFNLENBQUMsSUFBSSxDQUFDLGtCQUFrQixFQUFFLGlCQUFpQixFQUFFLEVBQUUsWUFBWSxFQUFFLEVBQUUsRUFBRSxDQUFDLENBQUM7Z0JBQzlFLE9BQU8sdUJBQUEsSUFBSSw4REFBZSxNQUFuQixJQUFJLEVBQWdCLE1BQU0sQ0FBQyxDQUFDO1lBQ3JDLENBQUM7WUFFRCxJQUFJLElBQUksQ0FBQyxZQUFZLENBQUMsR0FBRyxDQUFDLEVBQUUsQ0FBQyxFQUFFLENBQUM7Z0JBQzlCLElBQUksQ0FBQyxNQUFNLENBQUMsSUFBSSxDQUFDLGtCQUFrQixFQUFFLDZCQUE2QixFQUFFLEVBQUUsWUFBWSxFQUFFLEVBQUUsRUFBRSxDQUFDLENBQUM7Z0JBQzFGLE9BQU8sdUJBQUEsSUFBSSw4REFBZSxNQUFuQixJQUFJLEVBQWdCLE1BQU0sQ0FBQyxDQUFDO1lBQ3JDLENBQUM7WUFFRCxNQUFNLFFBQVEsR0FBRyxJQUFJLENBQUMsVUFBVSxDQUFDLEdBQUcsQ0FBQyxFQUFFLENBQUMsQ0FBQztZQUN6QyxNQUFNLEdBQUcsR0FBRyxJQUFJLENBQUMsWUFBWSxFQUFFLENBQUM7WUFDaEMsSUFBSSxDQUFDLFFBQVEsRUFBRSxDQUFDO2dCQUNkLElBQUksQ0FBQyxNQUFNLENBQUMsSUFBSSxDQUNkLGtCQUFrQixFQUFFLFNBQVMsa0JBQWtCLENBQUMsR0FBRyxDQUFDLElBQUksQ0FBQyxnREFBZ0QsRUFDekcsRUFBRSxZQUFZLEVBQUUsRUFBRSxFQUFFLENBQ3JCLENBQUM7Z0JBQ0YsdURBQXVEO2dCQUN2RCxrRkFBa0Y7Z0JBQ2xGLG1DQUFtQztnQkFDbkMsSUFBSSxDQUFDLFVBQVUsQ0FBQyxHQUFHLENBQUMsRUFBRSxFQUFFO29CQUN0QixFQUFFO29CQUNGLFNBQVM7b0JBQ1QsYUFBYSxFQUFFLElBQUksQ0FBQyxZQUFZLEVBQUU7aUJBQ25DLENBQUMsQ0FBQztnQkFDSCxPQUFPLFNBQVMsQ0FBQztZQUNuQixDQUFDO2lCQUFNLElBQUksU0FBUyxJQUFJLFFBQVEsQ0FBQyxTQUFTLEVBQUUsQ0FBQztnQkFDM0MsSUFBSSxTQUFTLEdBQUcsUUFBUSxDQUFDLFNBQVMsRUFBRSxDQUFDO29CQUNuQyxJQUFJLENBQUMsTUFBTSxDQUFDLElBQUksQ0FDZCxrQkFBa0IsRUFBRSxTQUFTLGtCQUFrQixDQUFDLEdBQUcsQ0FBQyxJQUFJLENBQUMsY0FBYyxTQUFTLDRCQUE0QixFQUM1RyxFQUFFLFlBQVksRUFBRSxFQUFFLEVBQUUsQ0FDckIsQ0FBQztnQkFDSixDQUFDO3FCQUFNLENBQUM7b0JBQ04sSUFBSSxDQUFDLE1BQU0sQ0FBQyxLQUFLLENBQUMsa0JBQWtCLEVBQUUsU0FBUyxrQkFBa0IsQ0FBQyxHQUFHLENBQUMsSUFBSSxDQUFDLFlBQVksRUFBRSxFQUFFLFlBQVksRUFBRSxFQUFFLEVBQUUsQ0FBQyxDQUFDO2dCQUNqSCxDQUFDO2dCQUNELFFBQVEsQ0FBQyxTQUFTLEdBQUcsU0FBUyxDQUFDO2dCQUMvQixRQUFRLENBQUMsYUFBYSxHQUFHLEdBQUcsQ0FBQztnQkFDN0IsT0FBTyxTQUFTLENBQUM7WUFDbkIsQ0FBQztZQUVELElBQUksQ0FBQyxNQUFNLENBQUMsSUFBSSxDQUNkLGtCQUFrQixFQUFFLFNBQ2xCLGtCQUFrQixDQUFDLEdBQUcsQ0FBQyxJQUFJLENBQzdCLDREQUE0RCxFQUM1RCxFQUFFLFlBQVksRUFBRSxFQUFFLEVBQUUsQ0FDckIsQ0FBQztZQUVGLE9BQU8sdUJBQUEsSUFBSSw4REFBZSxNQUFuQixJQUFJLEVBQWdCLE1BQU0sQ0FBQyxDQUFDO1FBQ3JDLENBQUM7aURBRUQsS0FBSyxpREFDSCxFQUFnQixFQUNoQixLQUFlLEVBQ2YsTUFBeUI7WUFFekIsTUFBTSxJQUFJLEdBQUcsSUFBSSxDQUFDLFVBQVUsQ0FBQyxHQUFHLENBQUMsRUFBRSxDQUFDLENBQUM7WUFDckMsTUFBTSxJQUFJLEdBQUcsSUFBSSxDQUFDLFNBQVMsQ0FBQyxHQUFHLENBQUMsRUFBRSxDQUFDLENBQUM7WUFDcEMsTUFBTSxPQUFPLEdBQUcsSUFBSSxDQUFDLE9BQU8sQ0FBQyxHQUFHLENBQUMsRUFBRSxDQUFDLElBQUksQ0FBQyxDQUFDO1lBQzFDLElBQUksQ0FBQyxJQUFJLEVBQUUsQ0FBQztnQkFDVixJQUFJLENBQUMsTUFBTSxDQUFDLElBQUksQ0FBQyxrQkFBa0IsRUFBRSxZQUFZLEVBQUUsRUFBRSxZQUFZLEVBQUUsRUFBRSxFQUFFLENBQUMsQ0FBQztnQkFDekUsT0FBTztZQUNULENBQUM7WUFFRCxJQUFJLENBQUMsSUFBSSxFQUFFLENBQUM7Z0JBQ1YsSUFBSSxDQUFDLE1BQU0sQ0FBQyxJQUFJLENBQUMsa0JBQWtCLEVBQUUsU0FBUyxrQkFBa0IsQ0FBQyxJQUFJLENBQUMsSUFBSSxDQUFDLDZCQUE2QixFQUFFO29CQUN4RyxZQUFZLEVBQUUsRUFBRTtpQkFDakIsQ0FBQyxDQUFDO1lBQ0wsQ0FBQztpQkFBTSxDQUFDO2dCQUNOLElBQUksQ0FBQyxVQUFVLENBQUMsTUFBTSxDQUFDLEVBQUUsQ0FBQyxDQUFDO1lBQzdCLENBQUM7WUFFRCxJQUFJLElBQUksQ0FBQyxZQUFZLENBQUMsR0FBRyxDQUFDLEVBQUUsQ0FBQyxFQUFFLENBQUM7Z0JBQzlCLElBQUksQ0FBQyxNQUFNLENBQUMsSUFBSSxDQUFDLGtCQUFrQixFQUFFLG1DQUFtQyxFQUFFLEVBQUUsWUFBWSxFQUFFLEVBQUUsRUFBRSxDQUFDLENBQUM7Z0JBQ2hHLE9BQU87WUFDVCxDQUFDO1lBRUQsSUFBSSxDQUFDLE1BQU0sQ0FBQyxJQUFJLENBQ2QsMkJBQTJCLEVBQUUsU0FBUyxrQkFBa0IsQ0FBQyxJQUFJLENBQUMsSUFBSSxDQUFDLGtCQUFrQixPQUFPLEdBQUcsQ0FBQyxFQUFFLEVBQ2xHLEVBQUUsWUFBWSxFQUFFLEVBQUUsRUFBRSxDQUNyQixDQUFDO1lBRUYsb0RBQW9EO1lBQ3BELDRFQUE0RTtZQUM1RSxtR0FBbUc7WUFDbkcsTUFBTSxNQUFNLEdBQTRCLEVBQUUsTUFBTSxFQUFFLFdBQVcsRUFBRSxLQUFLLEVBQUUsQ0FBQztZQUN2RSxJQUFJLENBQUMsWUFBWSxDQUFDLEdBQUcsQ0FBQyxFQUFFLEVBQUUsTUFBTSxDQUFDLENBQUM7WUFDbEMsSUFBSSxDQUFDLFFBQVEsQ0FBQyxHQUFHLENBQUMsRUFBRSxDQUFFLENBQUMsT0FBTyxDQUFDLE1BQU0sQ0FBQyxDQUFDO1lBQ3ZDLElBQUksQ0FBQyx5QkFBeUIsQ0FBQyxJQUFJLENBQUMsRUFBRSxDQUFDLENBQUM7WUFFeEMsSUFBSSxDQUFDLGVBQWUsQ0FBQyxlQUFlLENBQUMsSUFBSSxDQUFDLElBQUksQ0FBQyxDQUFDO1lBQ2hELElBQUksSUFBSSxFQUFFLENBQUM7Z0JBQ1QsTUFBTSxRQUFRLEdBQUcsSUFBSSxDQUFDLFlBQVksRUFBRSxHQUFHLElBQUksQ0FBQyxTQUFTLENBQUM7Z0JBQ3RELElBQUksQ0FBQyxlQUFlLENBQUMsaUJBQWlCLENBQUMsSUFBSSxDQUFDLElBQUksRUFBRSxRQUFRLENBQUMsQ0FBQztZQUM5RCxDQUFDO1lBRUQsSUFBSSxDQUFDO2dCQUNILE1BQU0sSUFBSSxDQUFDLFFBQVEsQ0FBQyxtQkFBbUIsQ0FBQyxFQUFFLEVBQUUsS0FBSyxDQUFDLENBQUM7WUFDckQsQ0FBQztZQUFDLE9BQU8sT0FBTyxFQUFFLENBQUM7Z0JBQ2pCLElBQUksQ0FBQyxNQUFNLENBQUMsS0FBSyxDQUFDLHdDQUF3QyxFQUFFLEVBQUUsRUFBRSxPQUFPLEVBQUU7b0JBQ3ZFLFlBQVksRUFBRSxFQUFFO2lCQUNqQixDQUFDLENBQUM7Z0JBRUgsTUFBTSxPQUFPLENBQUM7WUFDaEIsQ0FBQztZQUVELE9BQU8sdUJBQUEsSUFBSSw4REFBZSxNQUFuQixJQUFJLEVBQWdCLE1BQU0sQ0FBQyxDQUFDO1FBQ3JDLENBQUM7Ozt1Q0FFQSxTQUFTLENBQUMsMkJBQTJCLENBQUM7WUFDdkMsa0xBQWMsV0FBVyw2REFReEI7Ozs7O1NBM2ZVLGFBQWE7QUEyakIxQjs7Ozs7R0FLRztBQUNILFNBQVMsb0JBQW9CLENBQUMsQ0FBcUIsRUFBRSxDQUFxQjtJQUN4RSxJQUFJLENBQUMsQ0FBQyxXQUFXLEdBQUcsQ0FBQyxDQUFDLFdBQVcsRUFBRSxDQUFDO1FBQ2xDLE9BQU8sQ0FBQyxDQUFDLENBQUM7SUFDWixDQUFDO1NBQU0sSUFBSSxDQUFDLENBQUMsV0FBVyxHQUFHLENBQUMsQ0FBQyxXQUFXLEVBQUUsQ0FBQztRQUN6QyxPQUFPLENBQUMsQ0FBQztJQUNYLENBQUM7U0FBTSxDQUFDO1FBQ04sT0FBTyxDQUFDLENBQUM7SUFDWCxDQUFDO0FBQ0gsQ0FBQztBQUVEOzs7Ozs7O0dBT0c7QUFDSCxTQUFTLG1CQUFtQixDQUFDLENBQXFCLEVBQUUsQ0FBcUI7SUFDdkUsTUFBTSxRQUFRLEdBQUcsNkJBQTZCLENBQUMsT0FBTyxDQUFDLENBQUMsQ0FBQyxDQUFDO0lBQzFELE1BQU0sUUFBUSxHQUFHLDZCQUE2QixDQUFDLE9BQU8sQ0FBQyxDQUFDLENBQUMsQ0FBQztJQUMxRCxJQUFJLFFBQVEsS0FBSyxRQUFRLEVBQUUsQ0FBQztRQUMxQixPQUFPLENBQUMsQ0FBQztJQUNYLENBQUM7U0FBTSxJQUFJLFFBQVEsS0FBSyxDQUFDLENBQUMsRUFBRSxDQUFDO1FBQzNCLHlEQUF5RDtRQUN6RCwrQ0FBK0M7UUFDL0MsT0FBTyxDQUFDLENBQUM7SUFDWCxDQUFDO1NBQU0sSUFBSSxRQUFRLEtBQUssQ0FBQyxDQUFDLEVBQUUsQ0FBQztRQUMzQix5Q0FBeUM7UUFDekMsT0FBTyxDQUFDLENBQUMsQ0FBQztJQUNaLENBQUM7U0FBTSxJQUFJLFFBQVEsR0FBRyxRQUFRLEVBQUUsQ0FBQztRQUMvQixPQUFPLENBQUMsQ0FBQyxDQUFDO0lBQ1osQ0FBQztTQUFNLENBQUM7UUFDTixPQUFPLENBQUMsQ0FBQztJQUNYLENBQUM7QUFDSCxDQUFDO0FBRUQ7Ozs7OztHQU1HO0FBQ0gsTUFBTSw2QkFBNkIsR0FBeUI7SUFDMUQsa0JBQWtCLENBQUMsaUJBQWlCO0lBQ3BDLGtCQUFrQixDQUFDLDJCQUEyQjtJQUM5QyxrQkFBa0IsQ0FBQyxrQkFBa0I7SUFDckMsa0JBQWtCLENBQUMsV0FBVztJQUM5QixrQkFBa0IsQ0FBQyxZQUFZO0lBQy9CLGtCQUFrQixDQUFDLGtCQUFrQjtJQUNyQyxrQkFBa0IsQ0FBQyxtQkFBbUI7SUFDdEMsa0JBQWtCLENBQUMsU0FBUztJQUM1QixrQkFBa0IsQ0FBQyxVQUFVO0lBQzdCLGtCQUFrQixDQUFDLFdBQVc7SUFDOUIsa0JBQWtCLENBQUMsV0FBVztJQUM5QixrQkFBa0IsQ0FBQyx1QkFBdUI7Q0FDM0MsQ0FBQyJ9