@aztec/prover-client 0.75.0-commit.c03ba01a2a4122e43e90d5133ba017e54b90e9d2 → 0.75.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (138) hide show
  1. package/dest/bin/get-proof-inputs.d.ts +2 -0
  2. package/dest/bin/get-proof-inputs.d.ts.map +1 -0
  3. package/dest/bin/get-proof-inputs.js +16 -18
  4. package/dest/block_builder/index.d.ts +6 -0
  5. package/dest/block_builder/index.d.ts.map +1 -0
  6. package/dest/block_builder/index.js +1 -0
  7. package/dest/block_builder/light.d.ts +31 -0
  8. package/dest/block_builder/light.d.ts.map +1 -0
  9. package/dest/block_builder/light.js +13 -23
  10. package/dest/config.d.ts +17 -0
  11. package/dest/config.d.ts.map +1 -0
  12. package/dest/config.js +11 -9
  13. package/dest/index.d.ts +4 -0
  14. package/dest/index.d.ts.map +1 -0
  15. package/dest/index.js +1 -0
  16. package/dest/mocks/fixtures.d.ts +19 -0
  17. package/dest/mocks/fixtures.d.ts.map +1 -0
  18. package/dest/mocks/fixtures.js +26 -28
  19. package/dest/mocks/test_context.d.ts +49 -0
  20. package/dest/mocks/test_context.d.ts.map +1 -0
  21. package/dest/mocks/test_context.js +31 -55
  22. package/dest/orchestrator/block-building-helpers.d.ts +50 -0
  23. package/dest/orchestrator/block-building-helpers.d.ts.map +1 -0
  24. package/dest/orchestrator/block-building-helpers.js +90 -90
  25. package/dest/orchestrator/block-proving-state.d.ts +71 -0
  26. package/dest/orchestrator/block-proving-state.d.ts.map +1 -0
  27. package/dest/orchestrator/block-proving-state.js +70 -95
  28. package/dest/orchestrator/epoch-proving-state.d.ts +56 -0
  29. package/dest/orchestrator/epoch-proving-state.d.ts.map +1 -0
  30. package/dest/orchestrator/epoch-proving-state.js +40 -53
  31. package/dest/orchestrator/index.d.ts +2 -0
  32. package/dest/orchestrator/index.d.ts.map +1 -0
  33. package/dest/orchestrator/index.js +1 -0
  34. package/dest/orchestrator/orchestrator.d.ts +108 -0
  35. package/dest/orchestrator/orchestrator.d.ts.map +1 -0
  36. package/dest/orchestrator/orchestrator.js +653 -649
  37. package/dest/orchestrator/orchestrator_metrics.d.ts +8 -0
  38. package/dest/orchestrator/orchestrator_metrics.d.ts.map +1 -0
  39. package/dest/orchestrator/orchestrator_metrics.js +3 -4
  40. package/dest/orchestrator/tx-proving-state.d.ts +31 -0
  41. package/dest/orchestrator/tx-proving-state.d.ts.map +1 -0
  42. package/dest/orchestrator/tx-proving-state.js +51 -52
  43. package/dest/prover-agent/index.d.ts +4 -0
  44. package/dest/prover-agent/index.d.ts.map +1 -0
  45. package/dest/prover-agent/index.js +1 -0
  46. package/dest/prover-agent/memory-proving-queue.d.ts +82 -0
  47. package/dest/prover-agent/memory-proving-queue.d.ts.map +1 -0
  48. package/dest/prover-agent/memory-proving-queue.js +248 -237
  49. package/dest/prover-agent/prover-agent.d.ts +43 -0
  50. package/dest/prover-agent/prover-agent.d.ts.map +1 -0
  51. package/dest/prover-agent/prover-agent.js +187 -184
  52. package/dest/prover-agent/proving-error.d.ts +5 -0
  53. package/dest/prover-agent/proving-error.d.ts.map +1 -0
  54. package/dest/prover-agent/proving-error.js +1 -0
  55. package/dest/prover-agent/queue_metrics.d.ts +10 -0
  56. package/dest/prover-agent/queue_metrics.d.ts.map +1 -0
  57. package/dest/prover-agent/queue_metrics.js +5 -6
  58. package/dest/prover-agent/rpc.d.ts +11 -0
  59. package/dest/prover-agent/rpc.d.ts.map +1 -0
  60. package/dest/prover-agent/rpc.js +4 -6
  61. package/dest/prover-client/factory.d.ts +6 -0
  62. package/dest/prover-client/factory.d.ts.map +1 -0
  63. package/dest/prover-client/factory.js +1 -0
  64. package/dest/prover-client/index.d.ts +3 -0
  65. package/dest/prover-client/index.d.ts.map +1 -0
  66. package/dest/prover-client/index.js +1 -0
  67. package/dest/prover-client/prover-client.d.ts +42 -0
  68. package/dest/prover-client/prover-client.d.ts.map +1 -0
  69. package/dest/prover-client/prover-client.js +25 -30
  70. package/dest/prover-client/server-epoch-prover.d.ts +25 -0
  71. package/dest/prover-client/server-epoch-prover.d.ts.map +1 -0
  72. package/dest/prover-client/server-epoch-prover.js +4 -4
  73. package/dest/proving_broker/broker_prover_facade.d.ts +39 -0
  74. package/dest/proving_broker/broker_prover_facade.d.ts.map +1 -0
  75. package/dest/proving_broker/broker_prover_facade.js +59 -70
  76. package/dest/proving_broker/config.d.ts +61 -0
  77. package/dest/proving_broker/config.d.ts.map +1 -0
  78. package/dest/proving_broker/config.js +37 -22
  79. package/dest/proving_broker/factory.d.ts +5 -0
  80. package/dest/proving_broker/factory.d.ts.map +1 -0
  81. package/dest/proving_broker/factory.js +2 -1
  82. package/dest/proving_broker/fixtures.d.ts +5 -0
  83. package/dest/proving_broker/fixtures.d.ts.map +1 -0
  84. package/dest/proving_broker/fixtures.js +1 -0
  85. package/dest/proving_broker/index.d.ts +10 -0
  86. package/dest/proving_broker/index.d.ts.map +1 -0
  87. package/dest/proving_broker/index.js +1 -0
  88. package/dest/proving_broker/proof_store/factory.d.ts +6 -0
  89. package/dest/proving_broker/proof_store/factory.d.ts.map +1 -0
  90. package/dest/proving_broker/proof_store/factory.js +12 -9
  91. package/dest/proving_broker/proof_store/gcs_proof_store.d.ts +13 -0
  92. package/dest/proving_broker/proof_store/gcs_proof_store.d.ts.map +1 -0
  93. package/dest/proving_broker/proof_store/gcs_proof_store.js +7 -11
  94. package/dest/proving_broker/proof_store/index.d.ts +4 -0
  95. package/dest/proving_broker/proof_store/index.d.ts.map +1 -0
  96. package/dest/proving_broker/proof_store/index.js +1 -0
  97. package/dest/proving_broker/proof_store/inline_proof_store.d.ts +14 -0
  98. package/dest/proving_broker/proof_store/inline_proof_store.d.ts.map +1 -0
  99. package/dest/proving_broker/proof_store/inline_proof_store.js +7 -11
  100. package/dest/proving_broker/proof_store/proof_store.d.ts +35 -0
  101. package/dest/proving_broker/proof_store/proof_store.d.ts.map +1 -0
  102. package/dest/proving_broker/proof_store/proof_store.js +2 -3
  103. package/dest/proving_broker/proving_agent.d.ts +44 -0
  104. package/dest/proving_broker/proving_agent.d.ts.map +1 -0
  105. package/dest/proving_broker/proving_agent.js +110 -121
  106. package/dest/proving_broker/proving_agent_instrumentation.d.ts +8 -0
  107. package/dest/proving_broker/proving_agent_instrumentation.d.ts.map +1 -0
  108. package/dest/proving_broker/proving_agent_instrumentation.js +3 -3
  109. package/dest/proving_broker/proving_broker.d.ts +75 -0
  110. package/dest/proving_broker/proving_broker.d.ts.map +1 -0
  111. package/dest/proving_broker/proving_broker.js +451 -491
  112. package/dest/proving_broker/proving_broker_database/memory.d.ts +16 -0
  113. package/dest/proving_broker/proving_broker_database/memory.d.ts.map +1 -0
  114. package/dest/proving_broker/proving_broker_database/memory.js +13 -19
  115. package/dest/proving_broker/proving_broker_database/persisted.d.ts +21 -0
  116. package/dest/proving_broker/proving_broker_database/persisted.d.ts.map +1 -0
  117. package/dest/proving_broker/proving_broker_database/persisted.js +21 -41
  118. package/dest/proving_broker/proving_broker_database.d.ts +39 -0
  119. package/dest/proving_broker/proving_broker_database.d.ts.map +1 -0
  120. package/dest/proving_broker/proving_broker_database.js +2 -3
  121. package/dest/proving_broker/proving_broker_instrumentation.d.ts +25 -0
  122. package/dest/proving_broker/proving_broker_instrumentation.d.ts.map +1 -0
  123. package/dest/proving_broker/proving_broker_instrumentation.js +21 -28
  124. package/dest/proving_broker/proving_job_controller.d.ts +31 -0
  125. package/dest/proving_broker/proving_job_controller.d.ts.map +1 -0
  126. package/dest/proving_broker/proving_job_controller.js +62 -81
  127. package/dest/proving_broker/rpc.d.ts +11 -0
  128. package/dest/proving_broker/rpc.d.ts.map +1 -0
  129. package/dest/proving_broker/rpc.js +15 -23
  130. package/dest/test/mock_prover.d.ts +33 -0
  131. package/dest/test/mock_prover.d.ts.map +1 -0
  132. package/dest/test/mock_prover.js +9 -11
  133. package/package.json +11 -12
  134. package/src/index.ts +1 -1
  135. package/src/orchestrator/block-building-helpers.ts +1 -1
  136. package/src/proving_broker/proving_agent.ts +11 -30
  137. package/src/proving_broker/proving_broker.ts +27 -53
  138. package/src/proving_broker/rpc.ts +2 -8
@@ -1,526 +1,479 @@
1
- function _ts_decorate(decorators, target, key, desc) {
2
- var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
3
- if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
4
- else for(var i = decorators.length - 1; i >= 0; i--)if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
5
- return c > 3 && r && Object.defineProperty(target, key, r), r;
6
- }
7
- import { ProvingRequestType } from '@aztec/circuit-types';
1
+ import { __classPrivateFieldGet, __esDecorate, __runInitializers } from "tslib";
2
+ import { ProvingRequestType, } from '@aztec/circuit-types';
8
3
  import { createLogger } from '@aztec/foundation/log';
9
4
  import { RunningPromise, promiseWithResolvers } from '@aztec/foundation/promise';
10
5
  import { PriorityMemoryQueue, SerialQueue } from '@aztec/foundation/queue';
11
6
  import { Timer } from '@aztec/foundation/timer';
12
- import { getTelemetryClient, trackSpan } from '@aztec/telemetry-client';
7
+ import { getTelemetryClient, trackSpan, } from '@aztec/telemetry-client';
13
8
  import assert from 'assert';
14
9
  import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.js';
15
10
  /**
16
11
  * A broker that manages proof requests and distributes them to workers based on their priority.
17
12
  * It takes a backend that is responsible for storing and retrieving proof requests and results.
18
- */ export class ProvingBroker {
19
- database;
20
- logger;
21
- queues;
22
- // holds a copy of the database in memory in order to quickly fulfill requests
23
- // this is fine because this broker is the only one that can modify the database
24
- jobsCache;
25
- // as above, but for results
26
- resultsCache;
27
- // tracks when each job was enqueued
28
- enqueuedAt;
29
- // keeps track of which jobs are currently being processed
30
- // in the event of a crash this information is lost, but that's ok
31
- // the next time the broker starts it will recreate jobsCache and still
32
- // accept results from the workers
33
- inProgress;
34
- // keep track of which proving job has been retried
35
- retries;
36
- // a map of promises that will be resolved when a job is settled
37
- promises;
38
- cleanupPromise;
39
- msTimeSource;
40
- jobTimeoutMs;
41
- maxRetries;
42
- instrumentation;
43
- tracer;
44
- completedJobNotifications;
45
- /**
46
- * The broker keeps track of the highest epoch its seen.
47
- * This information is used for garbage collection: once it reaches the next epoch, it can start pruning the database of old state.
48
- * It is important that this value is initialised to zero. This ensures that we don't delete any old jobs until the current
49
- * process instance receives a job request informing it of the actual current highest epoch
50
- * Example:
51
- * proving epoch 11 - the broker will wipe all jobs for epochs 9 and lower
52
- * finished proving epoch 11 and got first job for epoch 12 -> the broker will wipe all settled jobs for epochs 10 and lower
53
- * reorged back to end of epoch 10 -> epoch 11 is skipped and epoch 12 starts -> the broker will wipe all settled jobs for epochs 10 and lower
54
- */ epochHeight;
55
- maxEpochsToKeepResultsFor;
56
- requestQueue;
57
- started;
58
- constructor(database, { jobTimeoutMs = 30_000, timeoutIntervalMs = 10_000, maxRetries = 3, maxEpochsToKeepResultsFor = 1 } = {}, client = getTelemetryClient(), logger = createLogger('prover-client:proving-broker')){
59
- this.database = database;
60
- this.logger = logger;
61
- this.queues = {
62
- [ProvingRequestType.PUBLIC_VM]: new PriorityMemoryQueue(provingJobComparator),
63
- [ProvingRequestType.TUBE_PROOF]: new PriorityMemoryQueue(provingJobComparator),
64
- [ProvingRequestType.PRIVATE_BASE_ROLLUP]: new PriorityMemoryQueue(provingJobComparator),
65
- [ProvingRequestType.PUBLIC_BASE_ROLLUP]: new PriorityMemoryQueue(provingJobComparator),
66
- [ProvingRequestType.MERGE_ROLLUP]: new PriorityMemoryQueue(provingJobComparator),
67
- [ProvingRequestType.ROOT_ROLLUP]: new PriorityMemoryQueue(provingJobComparator),
68
- [ProvingRequestType.BLOCK_MERGE_ROLLUP]: new PriorityMemoryQueue(provingJobComparator),
69
- [ProvingRequestType.BLOCK_ROOT_ROLLUP]: new PriorityMemoryQueue(provingJobComparator),
70
- [ProvingRequestType.SINGLE_TX_BLOCK_ROOT_ROLLUP]: new PriorityMemoryQueue(provingJobComparator),
71
- [ProvingRequestType.EMPTY_BLOCK_ROOT_ROLLUP]: new PriorityMemoryQueue(provingJobComparator),
72
- [ProvingRequestType.BASE_PARITY]: new PriorityMemoryQueue(provingJobComparator),
73
- [ProvingRequestType.ROOT_PARITY]: new PriorityMemoryQueue(provingJobComparator)
74
- };
75
- this.jobsCache = new Map();
76
- this.resultsCache = new Map();
77
- this.enqueuedAt = new Map();
78
- this.inProgress = new Map();
79
- this.retries = new Map();
80
- this.promises = new Map();
81
- this.msTimeSource = ()=>Date.now();
82
- this.completedJobNotifications = [];
83
- this.epochHeight = 0;
84
- this.maxEpochsToKeepResultsFor = 1;
85
- this.requestQueue = new SerialQueue();
86
- this.started = false;
87
- this.measureQueueDepth = (type)=>{
88
- return this.queues[type].length();
89
- };
90
- this.countActiveJobs = (type)=>{
91
- let count = 0;
92
- for (const { id } of this.inProgress.values()){
93
- const job = this.jobsCache.get(id);
94
- if (job?.type === type) {
95
- count++;
13
+ */
14
+ let ProvingBroker = (() => {
15
+ var _ProvingBroker_instances, _a, _ProvingBroker_enqueueProvingJob, _ProvingBroker_cancelProvingJob, _ProvingBroker_getProvingJobStatus, _ProvingBroker_getCompletedJobs, _ProvingBroker_getProvingJob, _ProvingBroker_reportProvingJobError, _ProvingBroker_reportProvingJobProgress, _ProvingBroker_reportProvingJobSuccess;
16
+ let _instanceExtraInitializers = [];
17
+ let _cleanupPass_decorators;
18
+ return _a = class ProvingBroker {
19
+ constructor(database, { jobTimeoutMs = 30000, timeoutIntervalMs = 10000, maxRetries = 3, maxEpochsToKeepResultsFor = 1, } = {}, client = getTelemetryClient(), logger = createLogger('prover-client:proving-broker')) {
20
+ _ProvingBroker_instances.add(this);
21
+ this.database = (__runInitializers(this, _instanceExtraInitializers), database);
22
+ this.logger = logger;
23
+ this.queues = {
24
+ [ProvingRequestType.PUBLIC_VM]: new PriorityMemoryQueue(provingJobComparator),
25
+ [ProvingRequestType.TUBE_PROOF]: new PriorityMemoryQueue(provingJobComparator),
26
+ [ProvingRequestType.PRIVATE_BASE_ROLLUP]: new PriorityMemoryQueue(provingJobComparator),
27
+ [ProvingRequestType.PUBLIC_BASE_ROLLUP]: new PriorityMemoryQueue(provingJobComparator),
28
+ [ProvingRequestType.MERGE_ROLLUP]: new PriorityMemoryQueue(provingJobComparator),
29
+ [ProvingRequestType.ROOT_ROLLUP]: new PriorityMemoryQueue(provingJobComparator),
30
+ [ProvingRequestType.BLOCK_MERGE_ROLLUP]: new PriorityMemoryQueue(provingJobComparator),
31
+ [ProvingRequestType.BLOCK_ROOT_ROLLUP]: new PriorityMemoryQueue(provingJobComparator),
32
+ [ProvingRequestType.SINGLE_TX_BLOCK_ROOT_ROLLUP]: new PriorityMemoryQueue(provingJobComparator),
33
+ [ProvingRequestType.EMPTY_BLOCK_ROOT_ROLLUP]: new PriorityMemoryQueue(provingJobComparator),
34
+ [ProvingRequestType.BASE_PARITY]: new PriorityMemoryQueue(provingJobComparator),
35
+ [ProvingRequestType.ROOT_PARITY]: new PriorityMemoryQueue(provingJobComparator),
36
+ };
37
+ // holds a copy of the database in memory in order to quickly fulfill requests
38
+ // this is fine because this broker is the only one that can modify the database
39
+ this.jobsCache = new Map();
40
+ // as above, but for results
41
+ this.resultsCache = new Map();
42
+ // tracks when each job was enqueued
43
+ this.enqueuedAt = new Map();
44
+ // keeps track of which jobs are currently being processed
45
+ // in the event of a crash this information is lost, but that's ok
46
+ // the next time the broker starts it will recreate jobsCache and still
47
+ // accept results from the workers
48
+ this.inProgress = new Map();
49
+ // keep track of which proving job has been retried
50
+ this.retries = new Map();
51
+ // a map of promises that will be resolved when a job is settled
52
+ this.promises = new Map();
53
+ this.msTimeSource = () => Date.now();
54
+ this.completedJobNotifications = [];
55
+ /**
56
+ * The broker keeps track of the highest epoch its seen.
57
+ * This information is used for garbage collection: once it reaches the next epoch, it can start pruning the database of old state.
58
+ * It is important that this value is initialised to zero. This ensures that we don't delete any old jobs until the current
59
+ * process instance receives a job request informing it of the actual current highest epoch
60
+ * Example:
61
+ * proving epoch 11 - the broker will wipe all jobs for epochs 9 and lower
62
+ * finished proving epoch 11 and got first job for epoch 12 -> the broker will wipe all settled jobs for epochs 10 and lower
63
+ * reorged back to end of epoch 10 -> epoch 11 is skipped and epoch 12 starts -> the broker will wipe all settled jobs for epochs 10 and lower
64
+ */
65
+ this.epochHeight = 0;
66
+ this.maxEpochsToKeepResultsFor = 1;
67
+ this.requestQueue = new SerialQueue();
68
+ this.started = false;
69
+ this.measureQueueDepth = (type) => {
70
+ return this.queues[type].length();
71
+ };
72
+ this.countActiveJobs = (type) => {
73
+ let count = 0;
74
+ for (const { id } of this.inProgress.values()) {
75
+ const job = this.jobsCache.get(id);
76
+ if (job?.type === type) {
77
+ count++;
78
+ }
79
+ }
80
+ return count;
81
+ };
82
+ this.tracer = client.getTracer('ProvingBroker');
83
+ this.instrumentation = new ProvingBrokerInstrumentation(client);
84
+ this.cleanupPromise = new RunningPromise(this.cleanupPass.bind(this), this.logger, timeoutIntervalMs);
85
+ this.jobTimeoutMs = jobTimeoutMs;
86
+ this.maxRetries = maxRetries;
87
+ this.maxEpochsToKeepResultsFor = maxEpochsToKeepResultsFor;
88
+ }
89
+ async start() {
90
+ if (this.started) {
91
+ this.logger.info('Proving Broker already started');
92
+ return Promise.resolve();
96
93
  }
94
+ this.logger.info('Proving Broker started');
95
+ for await (const [item, result] of this.database.allProvingJobs()) {
96
+ this.logger.info(`Restoring proving job id=${item.id} settled=${!!result}`, {
97
+ provingJobId: item.id,
98
+ status: result ? result.status : 'pending',
99
+ });
100
+ this.jobsCache.set(item.id, item);
101
+ this.promises.set(item.id, promiseWithResolvers());
102
+ if (result) {
103
+ this.promises.get(item.id).resolve(result);
104
+ this.resultsCache.set(item.id, result);
105
+ }
106
+ else {
107
+ this.enqueueJobInternal(item);
108
+ }
109
+ }
110
+ this.cleanupPromise.start();
111
+ this.requestQueue.start();
112
+ this.instrumentation.monitorQueueDepth(this.measureQueueDepth);
113
+ this.instrumentation.monitorActiveJobs(this.countActiveJobs);
114
+ this.started = true;
97
115
  }
98
- return count;
99
- };
100
- this.tracer = client.getTracer('ProvingBroker');
101
- this.instrumentation = new ProvingBrokerInstrumentation(client);
102
- this.cleanupPromise = new RunningPromise(this.cleanupPass.bind(this), this.logger, timeoutIntervalMs);
103
- this.jobTimeoutMs = jobTimeoutMs;
104
- this.maxRetries = maxRetries;
105
- this.maxEpochsToKeepResultsFor = maxEpochsToKeepResultsFor;
106
- }
107
- measureQueueDepth;
108
- countActiveJobs;
109
- async start() {
110
- if (this.started) {
111
- this.logger.info('Proving Broker already started');
112
- return Promise.resolve();
113
- }
114
- this.logger.info('Proving Broker started');
115
- for await (const [item, result] of this.database.allProvingJobs()){
116
- this.logger.info(`Restoring proving job id=${item.id} settled=${!!result}`, {
117
- provingJobId: item.id,
118
- status: result ? result.status : 'pending'
119
- });
120
- this.jobsCache.set(item.id, item);
121
- this.promises.set(item.id, promiseWithResolvers());
122
- if (result) {
123
- this.promises.get(item.id).resolve(result);
124
- this.resultsCache.set(item.id, result);
125
- } else {
126
- this.enqueueJobInternal(item);
116
+ async stop() {
117
+ if (!this.started) {
118
+ this.logger.warn('ProvingBroker not started');
119
+ return Promise.resolve();
120
+ }
121
+ await this.requestQueue.cancel();
122
+ await this.cleanupPromise.stop();
127
123
  }
128
- }
129
- this.cleanupPromise.start();
130
- this.requestQueue.start();
131
- this.instrumentation.monitorQueueDepth(this.measureQueueDepth);
132
- this.instrumentation.monitorActiveJobs(this.countActiveJobs);
133
- this.started = true;
134
- }
135
- async stop() {
136
- if (!this.started) {
137
- this.logger.warn('ProvingBroker not started');
138
- return Promise.resolve();
139
- }
140
- await this.requestQueue.cancel();
141
- await this.cleanupPromise.stop();
142
- }
143
- enqueueProvingJob(job) {
144
- return this.requestQueue.put(()=>this.#enqueueProvingJob(job));
145
- }
146
- cancelProvingJob(id) {
147
- return this.requestQueue.put(()=>this.#cancelProvingJob(id));
148
- }
149
- getProvingJobStatus(id) {
150
- return this.requestQueue.put(()=>this.#getProvingJobStatus(id));
151
- }
152
- getCompletedJobs(ids) {
153
- return this.requestQueue.put(()=>this.#getCompletedJobs(ids));
154
- }
155
- getProvingJob(filter) {
156
- return this.requestQueue.put(()=>this.#getProvingJob(filter));
157
- }
158
- reportProvingJobSuccess(id, value, filter) {
159
- return this.requestQueue.put(()=>this.#reportProvingJobSuccess(id, value, filter));
160
- }
161
- reportProvingJobError(id, err, retry = false, filter) {
162
- return this.requestQueue.put(()=>this.#reportProvingJobError(id, err, retry, filter));
163
- }
164
- reportProvingJobProgress(id, startedAt, filter) {
165
- return this.requestQueue.put(()=>this.#reportProvingJobProgress(id, startedAt, filter));
166
- }
167
- async #enqueueProvingJob(job) {
168
- // We return the job status at the start of this call
169
- const jobStatus = await this.#getProvingJobStatus(job.id);
170
- if (this.jobsCache.has(job.id)) {
171
- const existing = this.jobsCache.get(job.id);
172
- assert.deepStrictEqual(job, existing, 'Duplicate proving job ID');
173
- this.logger.debug(`Duplicate proving job id=${job.id} epochNumber=${job.epochNumber}. Ignoring`, {
174
- provingJobId: job.id
175
- });
176
- return jobStatus;
177
- }
178
- if (this.isJobStale(job)) {
179
- this.logger.warn(`Tried enqueueing stale proving job id=${job.id} epochNumber=${job.epochNumber}`, {
180
- provingJobId: job.id
181
- });
182
- throw new Error(`Epoch too old: job epoch ${job.epochNumber}, current epoch: ${this.epochHeight}`);
183
- }
184
- this.logger.info(`New proving job id=${job.id} epochNumber=${job.epochNumber}`, {
185
- provingJobId: job.id
186
- });
187
- try {
188
- // do this first so it acts as a "lock". If this job is enqueued again while we're saving it the if at the top will catch it.
189
- this.jobsCache.set(job.id, job);
190
- await this.database.addProvingJob(job);
191
- this.enqueueJobInternal(job);
192
- } catch (err) {
193
- this.logger.error(`Failed to save proving job id=${job.id}: ${err}`, err, {
194
- provingJobId: job.id
195
- });
196
- this.jobsCache.delete(job.id);
197
- throw err;
198
- }
199
- return jobStatus;
200
- }
201
- async #cancelProvingJob(id) {
202
- if (!this.jobsCache.has(id)) {
203
- this.logger.warn(`Can't cancel a job that doesn't exist id=${id}`, {
204
- provingJobId: id
205
- });
206
- return;
207
- }
208
- // notify listeners of the cancellation
209
- if (!this.resultsCache.has(id)) {
210
- this.logger.info(`Cancelling job id=${id}`, {
211
- provingJobId: id
212
- });
213
- await this.#reportProvingJobError(id, 'Aborted', false);
214
- }
215
- }
216
- cleanUpProvingJobState(ids) {
217
- for (const id of ids){
218
- this.jobsCache.delete(id);
219
- this.promises.delete(id);
220
- this.resultsCache.delete(id);
221
- this.inProgress.delete(id);
222
- this.retries.delete(id);
223
- }
224
- }
225
- #getProvingJobStatus(id) {
226
- const result = this.resultsCache.get(id);
227
- if (result) {
228
- return Promise.resolve(result);
229
- } else {
230
- // no result yet, check if we know the item
231
- const item = this.jobsCache.get(id);
232
- if (!item) {
233
- return Promise.resolve({
234
- status: 'not-found'
124
+ enqueueProvingJob(job) {
125
+ return this.requestQueue.put(() => __classPrivateFieldGet(this, _ProvingBroker_instances, "m", _ProvingBroker_enqueueProvingJob).call(this, job));
126
+ }
127
+ cancelProvingJob(id) {
128
+ return this.requestQueue.put(() => __classPrivateFieldGet(this, _ProvingBroker_instances, "m", _ProvingBroker_cancelProvingJob).call(this, id));
129
+ }
130
+ getProvingJobStatus(id) {
131
+ return this.requestQueue.put(() => __classPrivateFieldGet(this, _ProvingBroker_instances, "m", _ProvingBroker_getProvingJobStatus).call(this, id));
132
+ }
133
+ getCompletedJobs(ids) {
134
+ return this.requestQueue.put(() => __classPrivateFieldGet(this, _ProvingBroker_instances, "m", _ProvingBroker_getCompletedJobs).call(this, ids));
135
+ }
136
+ getProvingJob(filter) {
137
+ return this.requestQueue.put(() => __classPrivateFieldGet(this, _ProvingBroker_instances, "m", _ProvingBroker_getProvingJob).call(this, filter));
138
+ }
139
+ reportProvingJobSuccess(id, value) {
140
+ return this.requestQueue.put(() => __classPrivateFieldGet(this, _ProvingBroker_instances, "m", _ProvingBroker_reportProvingJobSuccess).call(this, id, value));
141
+ }
142
+ reportProvingJobError(id, err, retry = false) {
143
+ return this.requestQueue.put(() => __classPrivateFieldGet(this, _ProvingBroker_instances, "m", _ProvingBroker_reportProvingJobError).call(this, id, err, retry));
144
+ }
145
+ reportProvingJobProgress(id, startedAt, filter) {
146
+ return this.requestQueue.put(() => __classPrivateFieldGet(this, _ProvingBroker_instances, "m", _ProvingBroker_reportProvingJobProgress).call(this, id, startedAt, filter));
147
+ }
148
+ cleanUpProvingJobState(ids) {
149
+ for (const id of ids) {
150
+ this.jobsCache.delete(id);
151
+ this.promises.delete(id);
152
+ this.resultsCache.delete(id);
153
+ this.inProgress.delete(id);
154
+ this.retries.delete(id);
155
+ }
156
+ }
157
+ async cleanupPass() {
158
+ this.cleanupStaleJobs();
159
+ this.reEnqueueExpiredJobs();
160
+ const oldestEpochToKeep = this.oldestEpochToKeep();
161
+ if (oldestEpochToKeep > 0) {
162
+ await this.requestQueue.put(() => this.database.deleteAllProvingJobsOlderThanEpoch(oldestEpochToKeep));
163
+ this.logger.trace(`Deleted all epochs older than ${oldestEpochToKeep}`);
164
+ }
165
+ }
166
+ cleanupStaleJobs() {
167
+ const jobIds = Array.from(this.jobsCache.keys());
168
+ const jobsToClean = [];
169
+ for (const id of jobIds) {
170
+ const job = this.jobsCache.get(id);
171
+ if (this.isJobStale(job)) {
172
+ jobsToClean.push(id);
173
+ }
174
+ }
175
+ if (jobsToClean.length > 0) {
176
+ this.cleanUpProvingJobState(jobsToClean);
177
+ this.logger.info(`Cleaned up jobs=${jobsToClean.length}`);
178
+ }
179
+ }
180
+ reEnqueueExpiredJobs() {
181
+ const inProgressEntries = Array.from(this.inProgress.entries());
182
+ for (const [id, metadata] of inProgressEntries) {
183
+ const item = this.jobsCache.get(id);
184
+ if (!item) {
185
+ this.logger.warn(`Proving job id=${id} not found. Removing it from the queue.`, { provingJobId: id });
186
+ this.inProgress.delete(id);
187
+ continue;
188
+ }
189
+ const now = this.msTimeSource();
190
+ const msSinceLastUpdate = now - metadata.lastUpdatedAt;
191
+ if (msSinceLastUpdate >= this.jobTimeoutMs) {
192
+ this.logger.warn(`Proving job id=${id} timed out. Adding it back to the queue.`, { provingJobId: id });
193
+ this.inProgress.delete(id);
194
+ this.enqueueJobInternal(item);
195
+ this.instrumentation.incTimedOutJobs(item.type);
196
+ }
197
+ }
198
+ }
199
+ enqueueJobInternal(job) {
200
+ if (!this.promises.has(job.id)) {
201
+ this.promises.set(job.id, promiseWithResolvers());
202
+ }
203
+ this.queues[job.type].put({
204
+ epochNumber: job.epochNumber,
205
+ id: job.id,
235
206
  });
207
+ this.enqueuedAt.set(job.id, new Timer());
208
+ this.epochHeight = Math.max(this.epochHeight, job.epochNumber);
236
209
  }
237
- return Promise.resolve({
238
- status: this.inProgress.has(id) ? 'in-progress' : 'in-queue'
239
- });
240
- }
241
- }
242
- #getCompletedJobs(ids) {
243
- const completedJobs = ids.filter((id)=>this.resultsCache.has(id));
244
- const notifications = this.completedJobNotifications;
245
- this.completedJobNotifications = [];
246
- return Promise.resolve(notifications.concat(completedJobs));
247
- }
248
- // eslint-disable-next-line require-await
249
- #getProvingJob(filter = {
250
- allowList: []
251
- }) {
252
- const allowedProofs = Array.isArray(filter.allowList) && filter.allowList.length > 0 ? [
253
- ...filter.allowList
254
- ] : Object.values(ProvingRequestType).filter((x)=>typeof x === 'number');
255
- allowedProofs.sort(proofTypeComparator);
256
- for (const proofType of allowedProofs){
257
- const queue = this.queues[proofType];
258
- let enqueuedJob;
259
- // exhaust the queue and make sure we're not sending a job that's already in progress
260
- // or has already been completed
261
- // this can happen if the broker crashes and restarts
262
- // it's possible agents will report progress or results for jobs that are in the queue (after the restart)
263
- while(enqueuedJob = queue.getImmediate()){
264
- const job = this.jobsCache.get(enqueuedJob.id);
265
- if (job && !this.inProgress.has(enqueuedJob.id) && !this.resultsCache.has(enqueuedJob.id)) {
266
- const time = this.msTimeSource();
267
- this.inProgress.set(job.id, {
268
- id: job.id,
269
- startedAt: time,
270
- lastUpdatedAt: time
271
- });
272
- const enqueuedAt = this.enqueuedAt.get(job.id);
273
- if (enqueuedAt) {
274
- this.instrumentation.recordJobWait(job.type, enqueuedAt);
210
+ isJobStale(job) {
211
+ return job.epochNumber < this.oldestEpochToKeep();
212
+ }
213
+ oldestEpochToKeep() {
214
+ return this.epochHeight - this.maxEpochsToKeepResultsFor;
215
+ }
216
+ },
217
+ _ProvingBroker_instances = new WeakSet(),
218
+ _ProvingBroker_enqueueProvingJob = async function _ProvingBroker_enqueueProvingJob(job) {
219
+ // We return the job status at the start of this call
220
+ const jobStatus = await __classPrivateFieldGet(this, _ProvingBroker_instances, "m", _ProvingBroker_getProvingJobStatus).call(this, job.id);
221
+ if (this.jobsCache.has(job.id)) {
222
+ const existing = this.jobsCache.get(job.id);
223
+ assert.deepStrictEqual(job, existing, 'Duplicate proving job ID');
224
+ this.logger.debug(`Duplicate proving job id=${job.id} epochNumber=${job.epochNumber}. Ignoring`, {
225
+ provingJobId: job.id,
226
+ });
227
+ return jobStatus;
228
+ }
229
+ if (this.isJobStale(job)) {
230
+ this.logger.warn(`Tried enqueueing stale proving job id=${job.id} epochNumber=${job.epochNumber}`, {
231
+ provingJobId: job.id,
232
+ });
233
+ throw new Error(`Epoch too old: job epoch ${job.epochNumber}, current epoch: ${this.epochHeight}`);
234
+ }
235
+ this.logger.info(`New proving job id=${job.id} epochNumber=${job.epochNumber}`, { provingJobId: job.id });
236
+ try {
237
+ // do this first so it acts as a "lock". If this job is enqueued again while we're saving it the if at the top will catch it.
238
+ this.jobsCache.set(job.id, job);
239
+ await this.database.addProvingJob(job);
240
+ this.enqueueJobInternal(job);
241
+ }
242
+ catch (err) {
243
+ this.logger.error(`Failed to save proving job id=${job.id}: ${err}`, err, { provingJobId: job.id });
244
+ this.jobsCache.delete(job.id);
245
+ throw err;
246
+ }
247
+ return jobStatus;
248
+ },
249
+ _ProvingBroker_cancelProvingJob = async function _ProvingBroker_cancelProvingJob(id) {
250
+ if (!this.jobsCache.has(id)) {
251
+ this.logger.warn(`Can't cancel a job that doesn't exist id=${id}`, { provingJobId: id });
252
+ return;
253
+ }
254
+ // notify listeners of the cancellation
255
+ if (!this.resultsCache.has(id)) {
256
+ this.logger.info(`Cancelling job id=${id}`, { provingJobId: id });
257
+ await __classPrivateFieldGet(this, _ProvingBroker_instances, "m", _ProvingBroker_reportProvingJobError).call(this, id, 'Aborted', false);
258
+ }
259
+ },
260
+ _ProvingBroker_getProvingJobStatus = function _ProvingBroker_getProvingJobStatus(id) {
261
+ const result = this.resultsCache.get(id);
262
+ if (result) {
263
+ return Promise.resolve(result);
264
+ }
265
+ else {
266
+ // no result yet, check if we know the item
267
+ const item = this.jobsCache.get(id);
268
+ if (!item) {
269
+ return Promise.resolve({ status: 'not-found' });
270
+ }
271
+ return Promise.resolve({ status: this.inProgress.has(id) ? 'in-progress' : 'in-queue' });
272
+ }
273
+ },
274
+ _ProvingBroker_getCompletedJobs = function _ProvingBroker_getCompletedJobs(ids) {
275
+ const completedJobs = ids.filter(id => this.resultsCache.has(id));
276
+ const notifications = this.completedJobNotifications;
277
+ this.completedJobNotifications = [];
278
+ return Promise.resolve(notifications.concat(completedJobs));
279
+ },
280
+ _ProvingBroker_getProvingJob =
281
+ // eslint-disable-next-line require-await
282
+ async function _ProvingBroker_getProvingJob(filter = { allowList: [] }) {
283
+ const allowedProofs = Array.isArray(filter.allowList) && filter.allowList.length > 0
284
+ ? [...filter.allowList]
285
+ : Object.values(ProvingRequestType).filter((x) => typeof x === 'number');
286
+ allowedProofs.sort(proofTypeComparator);
287
+ for (const proofType of allowedProofs) {
288
+ const queue = this.queues[proofType];
289
+ let enqueuedJob;
290
+ // exhaust the queue and make sure we're not sending a job that's already in progress
291
+ // or has already been completed
292
+ // this can happen if the broker crashes and restarts
293
+ // it's possible agents will report progress or results for jobs that are in the queue (after the restart)
294
+ while ((enqueuedJob = queue.getImmediate())) {
295
+ const job = this.jobsCache.get(enqueuedJob.id);
296
+ if (job && !this.inProgress.has(enqueuedJob.id) && !this.resultsCache.has(enqueuedJob.id)) {
297
+ const time = this.msTimeSource();
298
+ this.inProgress.set(job.id, {
299
+ id: job.id,
300
+ startedAt: time,
301
+ lastUpdatedAt: time,
302
+ });
303
+ const enqueuedAt = this.enqueuedAt.get(job.id);
304
+ if (enqueuedAt) {
305
+ this.instrumentation.recordJobWait(job.type, enqueuedAt);
306
+ }
307
+ return { job, time };
275
308
  }
276
- return {
277
- job,
278
- time
279
- };
280
309
  }
281
310
  }
282
- }
283
- return undefined;
284
- }
285
- async #reportProvingJobError(id, err, retry = false, filter) {
286
- const info = this.inProgress.get(id);
287
- const item = this.jobsCache.get(id);
288
- const retries = this.retries.get(id) ?? 0;
289
- if (!item) {
290
- this.logger.warn(`Can't set error on unknown proving job id=${id} err=${err}`, {
291
- provingJoId: id
292
- });
293
- return;
294
- }
295
- if (!info) {
296
- this.logger.warn(`Proving job id=${id} type=${ProvingRequestType[item.type]} not in the in-progress set`, {
297
- provingJobId: id
298
- });
299
- } else {
300
- this.inProgress.delete(id);
301
- }
302
- if (this.resultsCache.has(id)) {
303
- this.logger.warn(`Proving job id=${id} is already settled, ignoring err=${err}`, {
304
- provingJobId: id
305
- });
306
- return this.#getProvingJob(filter);
307
- }
308
- if (retry && retries + 1 < this.maxRetries && !this.isJobStale(item)) {
309
- this.logger.info(`Retrying proving job id=${id} type=${ProvingRequestType[item.type]} retry=${retries + 1} err=${err}`, {
310
- provingJobId: id
311
- });
312
- // assign another job to this agent
313
- // do this first, before we put the failed job back in the queue
314
- const maybeAnotherJob = this.#getProvingJob(filter);
315
- this.retries.set(id, retries + 1);
316
- this.enqueueJobInternal(item);
317
- this.instrumentation.incRetriedJobs(item.type);
318
- return maybeAnotherJob;
319
- }
320
- this.logger.info(`Marking proving job as failed id=${id} type=${ProvingRequestType[item.type]} totalAttempts=${retries + 1} err=${err}`, {
321
- provingJobId: id
322
- });
323
- // save the result to the cache and notify clients of the job status
324
- // this should work even if our database breaks because the result is cached in memory
325
- const result = {
326
- status: 'rejected',
327
- reason: String(err)
328
- };
329
- this.resultsCache.set(id, result);
330
- this.promises.get(id).resolve(result);
331
- this.completedJobNotifications.push(id);
332
- this.instrumentation.incRejectedJobs(item.type);
333
- if (info) {
334
- const duration = this.msTimeSource() - info.startedAt;
335
- this.instrumentation.recordJobDuration(item.type, duration);
336
- }
337
- try {
338
- await this.database.setProvingJobError(id, err);
339
- } catch (saveErr) {
340
- this.logger.error(`Failed to save proving job error status id=${id} jobErr=${err}`, saveErr, {
341
- provingJobId: id
342
- });
343
- throw saveErr;
344
- }
345
- return this.#getProvingJob(filter);
346
- }
347
- #reportProvingJobProgress(id, startedAt, filter) {
348
- const job = this.jobsCache.get(id);
349
- if (!job) {
350
- this.logger.warn(`Proving job id=${id} does not exist`, {
351
- provingJobId: id
352
- });
353
- return this.#getProvingJob(filter);
354
- }
355
- if (this.resultsCache.has(id)) {
356
- this.logger.warn(`Proving job id=${id} has already been completed`, {
357
- provingJobId: id
358
- });
359
- return this.#getProvingJob(filter);
360
- }
361
- const metadata = this.inProgress.get(id);
362
- const now = this.msTimeSource();
363
- if (!metadata) {
364
- this.logger.warn(`Proving job id=${id} type=${ProvingRequestType[job.type]} not found in the in-progress cache, adding it`, {
365
- provingJobId: id
366
- });
367
- // the queue will still contain the item at this point!
368
- // we need to be careful when popping off the queue to make sure we're not sending
369
- // a job that's already in progress
370
- this.inProgress.set(id, {
371
- id,
372
- startedAt,
373
- lastUpdatedAt: this.msTimeSource()
374
- });
375
311
  return undefined;
376
- } else if (startedAt <= metadata.startedAt) {
377
- if (startedAt < metadata.startedAt) {
378
- this.logger.info(`Proving job id=${id} type=${ProvingRequestType[job.type]} startedAt=${startedAt} older agent has taken job`, {
379
- provingJobId: id
312
+ },
313
+ _ProvingBroker_reportProvingJobError = async function _ProvingBroker_reportProvingJobError(id, err, retry = false) {
314
+ const info = this.inProgress.get(id);
315
+ const item = this.jobsCache.get(id);
316
+ const retries = this.retries.get(id) ?? 0;
317
+ if (!item) {
318
+ this.logger.warn(`Can't set error on unknown proving job id=${id} err=${err}`, { provingJoId: id });
319
+ return;
320
+ }
321
+ if (!info) {
322
+ this.logger.warn(`Proving job id=${id} type=${ProvingRequestType[item.type]} not in the in-progress set`, {
323
+ provingJobId: id,
380
324
  });
381
- } else {
382
- this.logger.debug(`Proving job id=${id} type=${ProvingRequestType[job.type]} heartbeat`, {
383
- provingJobId: id
325
+ }
326
+ else {
327
+ this.inProgress.delete(id);
328
+ }
329
+ if (this.resultsCache.has(id)) {
330
+ this.logger.warn(`Proving job id=${id} is already settled, ignoring err=${err}`, {
331
+ provingJobId: id,
384
332
  });
333
+ return;
385
334
  }
386
- metadata.startedAt = startedAt;
387
- metadata.lastUpdatedAt = now;
388
- return undefined;
389
- }
390
- this.logger.warn(`Proving job id=${id} type=${ProvingRequestType[job.type]} already being worked on by another agent. Sending new one`, {
391
- provingJobId: id
392
- });
393
- return this.#getProvingJob(filter);
394
- }
395
- async #reportProvingJobSuccess(id, value, filter) {
396
- const info = this.inProgress.get(id);
397
- const item = this.jobsCache.get(id);
398
- const retries = this.retries.get(id) ?? 0;
399
- if (!item) {
400
- this.logger.warn(`Proving job id=${id} not found`, {
401
- provingJobId: id
402
- });
403
- return;
404
- }
405
- if (!info) {
406
- this.logger.warn(`Proving job id=${id} type=${ProvingRequestType[item.type]} not in the in-progress set`, {
407
- provingJobId: id
408
- });
409
- } else {
410
- this.inProgress.delete(id);
411
- }
412
- if (this.resultsCache.has(id)) {
413
- this.logger.warn(`Proving job id=${id} already settled, ignoring result`, {
414
- provingJobId: id
415
- });
416
- return;
417
- }
418
- this.logger.info(`Proving job complete id=${id} type=${ProvingRequestType[item.type]} totalAttempts=${retries + 1}`, {
419
- provingJobId: id
420
- });
421
- // save result to our local cache and notify clients
422
- // if save to database fails, that's ok because we have the result in memory
423
- // if the broker crashes and needs the result again, we're covered because we can just recompute it
424
- const result = {
425
- status: 'fulfilled',
426
- value
427
- };
428
- this.resultsCache.set(id, result);
429
- this.promises.get(id).resolve(result);
430
- this.completedJobNotifications.push(id);
431
- this.instrumentation.incResolvedJobs(item.type);
432
- if (info) {
433
- const duration = this.msTimeSource() - info.startedAt;
434
- this.instrumentation.recordJobDuration(item.type, duration);
435
- }
436
- try {
437
- await this.database.setProvingJobResult(id, value);
438
- } catch (saveErr) {
439
- this.logger.error(`Failed to save proving job result id=${id}`, saveErr, {
440
- provingJobId: id
335
+ if (retry && retries + 1 < this.maxRetries && !this.isJobStale(item)) {
336
+ this.logger.info(`Retrying proving job id=${id} type=${ProvingRequestType[item.type]} retry=${retries + 1} err=${err}`, {
337
+ provingJobId: id,
338
+ });
339
+ this.retries.set(id, retries + 1);
340
+ this.enqueueJobInternal(item);
341
+ this.instrumentation.incRetriedJobs(item.type);
342
+ return;
343
+ }
344
+ this.logger.info(`Marking proving job as failed id=${id} type=${ProvingRequestType[item.type]} totalAttempts=${retries + 1} err=${err}`, {
345
+ provingJobId: id,
441
346
  });
442
- throw saveErr;
443
- }
444
- return this.#getProvingJob(filter);
445
- }
446
- async cleanupPass() {
447
- this.cleanupStaleJobs();
448
- this.reEnqueueExpiredJobs();
449
- const oldestEpochToKeep = this.oldestEpochToKeep();
450
- if (oldestEpochToKeep > 0) {
451
- await this.requestQueue.put(()=>this.database.deleteAllProvingJobsOlderThanEpoch(oldestEpochToKeep));
452
- this.logger.trace(`Deleted all epochs older than ${oldestEpochToKeep}`);
453
- }
454
- }
455
- cleanupStaleJobs() {
456
- const jobIds = Array.from(this.jobsCache.keys());
457
- const jobsToClean = [];
458
- for (const id of jobIds){
347
+ // save the result to the cache and notify clients of the job status
348
+ // this should work even if our database breaks because the result is cached in memory
349
+ const result = { status: 'rejected', reason: String(err) };
350
+ this.resultsCache.set(id, result);
351
+ this.promises.get(id).resolve(result);
352
+ this.completedJobNotifications.push(id);
353
+ this.instrumentation.incRejectedJobs(item.type);
354
+ if (info) {
355
+ const duration = this.msTimeSource() - info.startedAt;
356
+ this.instrumentation.recordJobDuration(item.type, duration);
357
+ }
358
+ try {
359
+ await this.database.setProvingJobError(id, err);
360
+ }
361
+ catch (saveErr) {
362
+ this.logger.error(`Failed to save proving job error status id=${id} jobErr=${err}`, saveErr, {
363
+ provingJobId: id,
364
+ });
365
+ throw saveErr;
366
+ }
367
+ },
368
+ _ProvingBroker_reportProvingJobProgress = function _ProvingBroker_reportProvingJobProgress(id, startedAt, filter) {
459
369
  const job = this.jobsCache.get(id);
460
- if (this.isJobStale(job)) {
461
- jobsToClean.push(id);
370
+ if (!job) {
371
+ this.logger.warn(`Proving job id=${id} does not exist`, { provingJobId: id });
372
+ return filter ? __classPrivateFieldGet(this, _ProvingBroker_instances, "m", _ProvingBroker_getProvingJob).call(this, filter) : Promise.resolve(undefined);
462
373
  }
463
- }
464
- if (jobsToClean.length > 0) {
465
- this.cleanUpProvingJobState(jobsToClean);
466
- this.logger.info(`Cleaned up jobs=${jobsToClean.length}`);
467
- }
468
- }
469
- reEnqueueExpiredJobs() {
470
- const inProgressEntries = Array.from(this.inProgress.entries());
471
- for (const [id, metadata] of inProgressEntries){
374
+ if (this.resultsCache.has(id)) {
375
+ this.logger.warn(`Proving job id=${id} has already been completed`, { provingJobId: id });
376
+ return filter ? __classPrivateFieldGet(this, _ProvingBroker_instances, "m", _ProvingBroker_getProvingJob).call(this, filter) : Promise.resolve(undefined);
377
+ }
378
+ const metadata = this.inProgress.get(id);
379
+ const now = this.msTimeSource();
380
+ if (!metadata) {
381
+ this.logger.warn(`Proving job id=${id} type=${ProvingRequestType[job.type]} not found in the in-progress cache, adding it`, { provingJobId: id });
382
+ // the queue will still contain the item at this point!
383
+ // we need to be careful when popping off the queue to make sure we're not sending
384
+ // a job that's already in progress
385
+ this.inProgress.set(id, {
386
+ id,
387
+ startedAt,
388
+ lastUpdatedAt: this.msTimeSource(),
389
+ });
390
+ return Promise.resolve(undefined);
391
+ }
392
+ else if (startedAt <= metadata.startedAt) {
393
+ if (startedAt < metadata.startedAt) {
394
+ this.logger.info(`Proving job id=${id} type=${ProvingRequestType[job.type]} startedAt=${startedAt} older agent has taken job`, { provingJobId: id });
395
+ }
396
+ else {
397
+ this.logger.debug(`Proving job id=${id} type=${ProvingRequestType[job.type]} heartbeat`, { provingJobId: id });
398
+ }
399
+ metadata.startedAt = startedAt;
400
+ metadata.lastUpdatedAt = now;
401
+ return Promise.resolve(undefined);
402
+ }
403
+ else if (filter) {
404
+ this.logger.warn(`Proving job id=${id} type=${ProvingRequestType[job.type]} already being worked on by another agent. Sending new one`, { provingJobId: id });
405
+ return __classPrivateFieldGet(this, _ProvingBroker_instances, "m", _ProvingBroker_getProvingJob).call(this, filter);
406
+ }
407
+ else {
408
+ return Promise.resolve(undefined);
409
+ }
410
+ },
411
+ _ProvingBroker_reportProvingJobSuccess = async function _ProvingBroker_reportProvingJobSuccess(id, value) {
412
+ const info = this.inProgress.get(id);
472
413
  const item = this.jobsCache.get(id);
414
+ const retries = this.retries.get(id) ?? 0;
473
415
  if (!item) {
474
- this.logger.warn(`Proving job id=${id} not found. Removing it from the queue.`, {
475
- provingJobId: id
416
+ this.logger.warn(`Proving job id=${id} not found`, { provingJobId: id });
417
+ return;
418
+ }
419
+ if (!info) {
420
+ this.logger.warn(`Proving job id=${id} type=${ProvingRequestType[item.type]} not in the in-progress set`, {
421
+ provingJobId: id,
476
422
  });
423
+ }
424
+ else {
477
425
  this.inProgress.delete(id);
478
- continue;
479
426
  }
480
- const now = this.msTimeSource();
481
- const msSinceLastUpdate = now - metadata.lastUpdatedAt;
482
- if (msSinceLastUpdate >= this.jobTimeoutMs) {
483
- this.logger.warn(`Proving job id=${id} timed out. Adding it back to the queue.`, {
484
- provingJobId: id
427
+ if (this.resultsCache.has(id)) {
428
+ this.logger.warn(`Proving job id=${id} already settled, ignoring result`, { provingJobId: id });
429
+ return;
430
+ }
431
+ this.logger.info(`Proving job complete id=${id} type=${ProvingRequestType[item.type]} totalAttempts=${retries + 1}`, { provingJobId: id });
432
+ // save result to our local cache and notify clients
433
+ // if save to database fails, that's ok because we have the result in memory
434
+ // if the broker crashes and needs the result again, we're covered because we can just recompute it
435
+ const result = { status: 'fulfilled', value };
436
+ this.resultsCache.set(id, result);
437
+ this.promises.get(id).resolve(result);
438
+ this.completedJobNotifications.push(id);
439
+ this.instrumentation.incResolvedJobs(item.type);
440
+ if (info) {
441
+ const duration = this.msTimeSource() - info.startedAt;
442
+ this.instrumentation.recordJobDuration(item.type, duration);
443
+ }
444
+ try {
445
+ await this.database.setProvingJobResult(id, value);
446
+ }
447
+ catch (saveErr) {
448
+ this.logger.error(`Failed to save proving job result id=${id}`, saveErr, {
449
+ provingJobId: id,
485
450
  });
486
- this.inProgress.delete(id);
487
- this.enqueueJobInternal(item);
488
- this.instrumentation.incTimedOutJobs(item.type);
451
+ throw saveErr;
489
452
  }
490
- }
491
- }
492
- enqueueJobInternal(job) {
493
- if (!this.promises.has(job.id)) {
494
- this.promises.set(job.id, promiseWithResolvers());
495
- }
496
- this.queues[job.type].put({
497
- epochNumber: job.epochNumber,
498
- id: job.id
499
- });
500
- this.enqueuedAt.set(job.id, new Timer());
501
- this.epochHeight = Math.max(this.epochHeight, job.epochNumber);
502
- }
503
- isJobStale(job) {
504
- return job.epochNumber < this.oldestEpochToKeep();
505
- }
506
- oldestEpochToKeep() {
507
- return this.epochHeight - this.maxEpochsToKeepResultsFor;
508
- }
509
- }
510
- _ts_decorate([
511
- trackSpan('ProvingBroker.cleanupPass')
512
- ], ProvingBroker.prototype, "cleanupPass", null);
453
+ },
454
+ (() => {
455
+ const _metadata = typeof Symbol === "function" && Symbol.metadata ? Object.create(null) : void 0;
456
+ _cleanupPass_decorators = [trackSpan('ProvingBroker.cleanupPass')];
457
+ __esDecorate(_a, null, _cleanupPass_decorators, { kind: "method", name: "cleanupPass", static: false, private: false, access: { has: obj => "cleanupPass" in obj, get: obj => obj.cleanupPass }, metadata: _metadata }, null, _instanceExtraInitializers);
458
+ if (_metadata) Object.defineProperty(_a, Symbol.metadata, { enumerable: true, configurable: true, writable: true, value: _metadata });
459
+ })(),
460
+ _a;
461
+ })();
462
+ export { ProvingBroker };
513
463
  /**
514
464
  * Compares two proving jobs and selects which one's more important
515
465
  * @param a - A proving job
516
466
  * @param b - Another proving job
517
467
  * @returns A number indicating the relative priority of the two proving jobs
518
- */ function provingJobComparator(a, b) {
468
+ */
469
+ function provingJobComparator(a, b) {
519
470
  if (a.epochNumber < b.epochNumber) {
520
471
  return -1;
521
- } else if (a.epochNumber > b.epochNumber) {
472
+ }
473
+ else if (a.epochNumber > b.epochNumber) {
522
474
  return 1;
523
- } else {
475
+ }
476
+ else {
524
477
  return 0;
525
478
  }
526
479
  }
@@ -531,21 +484,26 @@ _ts_decorate([
531
484
  * @param a - A proof type
532
485
  * @param b - Another proof type
533
486
  * @returns A number indicating the relative priority of the two proof types
534
- */ function proofTypeComparator(a, b) {
487
+ */
488
+ function proofTypeComparator(a, b) {
535
489
  const indexOfA = PROOF_TYPES_IN_PRIORITY_ORDER.indexOf(a);
536
490
  const indexOfB = PROOF_TYPES_IN_PRIORITY_ORDER.indexOf(b);
537
491
  if (indexOfA === indexOfB) {
538
492
  return 0;
539
- } else if (indexOfA === -1) {
493
+ }
494
+ else if (indexOfA === -1) {
540
495
  // a is some new proof that didn't get added to the array
541
496
  // b is more important because we know about it
542
497
  return 1;
543
- } else if (indexOfB === -1) {
498
+ }
499
+ else if (indexOfB === -1) {
544
500
  // the opposite of the previous if branch
545
501
  return -1;
546
- } else if (indexOfA < indexOfB) {
502
+ }
503
+ else if (indexOfA < indexOfB) {
547
504
  return -1;
548
- } else {
505
+ }
506
+ else {
549
507
  return 1;
550
508
  }
551
509
  }
@@ -555,7 +513,8 @@ _ts_decorate([
555
513
  *
556
514
  * The aim is that this will speed up block proving as the closer we get to a block's root proof the more likely it
557
515
  * is to get picked up by agents
558
- */ const PROOF_TYPES_IN_PRIORITY_ORDER = [
516
+ */
517
+ const PROOF_TYPES_IN_PRIORITY_ORDER = [
559
518
  ProvingRequestType.BLOCK_ROOT_ROLLUP,
560
519
  ProvingRequestType.SINGLE_TX_BLOCK_ROOT_ROLLUP,
561
520
  ProvingRequestType.BLOCK_MERGE_ROLLUP,
@@ -567,5 +526,6 @@ _ts_decorate([
567
526
  ProvingRequestType.TUBE_PROOF,
568
527
  ProvingRequestType.ROOT_PARITY,
569
528
  ProvingRequestType.BASE_PARITY,
570
- ProvingRequestType.EMPTY_BLOCK_ROOT_ROLLUP
529
+ ProvingRequestType.EMPTY_BLOCK_ROOT_ROLLUP,
571
530
  ];
531
+ //# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoicHJvdmluZ19icm9rZXIuanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi9zcmMvcHJvdmluZ19icm9rZXIvcHJvdmluZ19icm9rZXIudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IjtBQUFBLE9BQU8sRUFTTCxrQkFBa0IsR0FDbkIsTUFBTSxzQkFBc0IsQ0FBQztBQUM5QixPQUFPLEVBQUUsWUFBWSxFQUFFLE1BQU0sdUJBQXVCLENBQUM7QUFDckQsT0FBTyxFQUE2QixjQUFjLEVBQUUsb0JBQW9CLEVBQUUsTUFBTSwyQkFBMkIsQ0FBQztBQUM1RyxPQUFPLEVBQUUsbUJBQW1CLEVBQUUsV0FBVyxFQUFFLE1BQU0seUJBQXlCLENBQUM7QUFDM0UsT0FBTyxFQUFFLEtBQUssRUFBRSxNQUFNLHlCQUF5QixDQUFDO0FBQ2hELE9BQU8sRUFJTCxrQkFBa0IsRUFDbEIsU0FBUyxHQUNWLE1BQU0seUJBQXlCLENBQUM7QUFFakMsT0FBTyxNQUFNLE1BQU0sUUFBUSxDQUFDO0FBRzVCLE9BQU8sRUFBd0IsNEJBQTRCLEVBQUUsTUFBTSxxQ0FBcUMsQ0FBQztBQWtCekc7OztHQUdHO0lBQ1UsYUFBYTs7OztzQkFBYixhQUFhO1lBa0V4QixZQUNVLFFBQStCLEVBQ3ZDLEVBQ0UsWUFBWSxHQUFHLEtBQU0sRUFDckIsaUJBQWlCLEdBQUcsS0FBTSxFQUMxQixVQUFVLEdBQUcsQ0FBQyxFQUNkLHlCQUF5QixHQUFHLENBQUMsTUFDRCxFQUFFLEVBQ2hDLFNBQTBCLGtCQUFrQixFQUFFLEVBQ3RDLFNBQVMsWUFBWSxDQUFDLDhCQUE4QixDQUFDOztnQkFSckQsYUFBUSxJQW5FUCxtREFBYSxFQW1FZCxRQUFRLEVBQXVCO2dCQVEvQixXQUFNLEdBQU4sTUFBTSxDQUErQztnQkExRXZELFdBQU0sR0FBa0I7b0JBQzlCLENBQUMsa0JBQWtCLENBQUMsU0FBUyxDQUFDLEVBQUUsSUFBSSxtQkFBbUIsQ0FBcUIsb0JBQW9CLENBQUM7b0JBQ2pHLENBQUMsa0JBQWtCLENBQUMsVUFBVSxDQUFDLEVBQUUsSUFBSSxtQkFBbUIsQ0FBcUIsb0JBQW9CLENBQUM7b0JBRWxHLENBQUMsa0JBQWtCLENBQUMsbUJBQW1CLENBQUMsRUFBRSxJQUFJLG1CQUFtQixDQUFxQixvQkFBb0IsQ0FBQztvQkFDM0csQ0FBQyxrQkFBa0IsQ0FBQyxrQkFBa0IsQ0FBQyxFQUFFLElBQUksbUJBQW1CLENBQXFCLG9CQUFvQixDQUFDO29CQUMxRyxDQUFDLGtCQUFrQixDQUFDLFlBQVksQ0FBQyxFQUFFLElBQUksbUJBQW1CLENBQXFCLG9CQUFvQixDQUFDO29CQUNwRyxDQUFDLGtCQUFrQixDQUFDLFdBQVcsQ0FBQyxFQUFFLElBQUksbUJBQW1CLENBQXFCLG9CQUFvQixDQUFDO29CQUVuRyxDQUFDLGtCQUFrQixDQUFDLGtCQUFrQixDQUFDLEVBQUUsSUFBSSxtQkFBbUIsQ0FBcUIsb0JBQW9CLENBQUM7b0JBQzFHLENBQUMsa0JBQWtCLENBQUMsaUJBQWlCLENBQUMsRUFBRSxJQUFJLG1CQUFtQixDQUFxQixvQkFBb0IsQ0FBQztvQkFDekcsQ0FBQyxrQkFBa0IsQ0FBQywyQkFBMkIsQ0FBQyxFQUFFLElBQUksbUJBQW1CLENBQXFCLG9CQUFvQixDQUFDO29CQUNuSCxDQUFDLGtCQUFrQixDQUFDLHVCQUF1QixDQUFDLEVBQUUsSUFBSSxtQkFBbUIsQ0FBcUIsb0JBQW9CLENBQUM7b0JBRS9HLENBQUMsa0JBQWtCLENBQUMsV0FBVyxDQUFDLEVBQUUsSUFBSSxtQkFBbUIsQ0FBcUIsb0JBQW9CLENBQUM7b0JBQ25HLENBQUMsa0JBQWtCLENBQUMsV0FBVyxDQUFDLEVBQUUsSUFBSSxtQkFBbUIsQ0FBcUIsb0JBQW9CLENBQUM7aUJBQ3BHLENBQUM7Z0JBRUYsOEVBQThFO2dCQUM5RSxnRkFBZ0Y7Z0JBQ3hFLGNBQVMsR0FBRyxJQUFJLEdBQUcsRUFBNEIsQ0FBQztnQkFDeEQsNEJBQTRCO2dCQUNwQixpQkFBWSxHQUFHLElBQUksR0FBRyxFQUF5QyxDQUFDO2dCQUV4RSxvQ0FBb0M7Z0JBQzVCLGVBQVUsR0FBRyxJQUFJLEdBQUcsRUFBdUIsQ0FBQztnQkFFcEQsMERBQTBEO2dCQUMxRCxrRUFBa0U7Z0JBQ2xFLHVFQUF1RTtnQkFDdkUsa0NBQWtDO2dCQUMxQixlQUFVLEdBQUcsSUFBSSxHQUFHLEVBQW9DLENBQUM7Z0JBRWpFLG1EQUFtRDtnQkFDM0MsWUFBTyxHQUFHLElBQUksR0FBRyxFQUF3QixDQUFDO2dCQUVsRCxnRUFBZ0U7Z0JBQ3hELGFBQVEsR0FBRyxJQUFJLEdBQUcsRUFBK0QsQ0FBQztnQkFHbEYsaUJBQVksR0FBRyxHQUFHLEVBQUUsQ0FBQyxJQUFJLENBQUMsR0FBRyxFQUFFLENBQUM7Z0JBT2hDLDhCQUF5QixHQUFtQixFQUFFLENBQUM7Z0JBRXZEOzs7Ozs7Ozs7bUJBU0c7Z0JBQ0ssZ0JBQVcsR0FBRyxDQUFDLENBQUM7Z0JBQ2hCLDhCQUF5QixHQUFHLENBQUMsQ0FBQztnQkFFOUIsaUJBQVksR0FBZ0IsSUFBSSxXQUFXLEVBQUUsQ0FBQztnQkFDOUMsWUFBTyxHQUFHLEtBQUssQ0FBQztnQkFxQmhCLHNCQUFpQixHQUFvQixDQUFDLElBQXdCLEVBQUUsRUFBRTtvQkFDeEUsT0FBTyxJQUFJLENBQUMsTUFBTSxDQUFDLElBQUksQ0FBQyxDQUFDLE1BQU0sRUFBRSxDQUFDO2dCQUNwQyxDQUFDLENBQUM7Z0JBRU0sb0JBQWUsR0FBb0IsQ0FBQyxJQUF3QixFQUFFLEVBQUU7b0JBQ3RFLElBQUksS0FBSyxHQUFHLENBQUMsQ0FBQztvQkFDZCxLQUFLLE1BQU0sRUFBRSxFQUFFLEVBQUUsSUFBSSxJQUFJLENBQUMsVUFBVSxDQUFDLE1BQU0sRUFBRSxFQUFFLENBQUM7d0JBQzlDLE1BQU0sR0FBRyxHQUFHLElBQUksQ0FBQyxTQUFTLENBQUMsR0FBRyxDQUFDLEVBQUUsQ0FBQyxDQUFDO3dCQUNuQyxJQUFJLEdBQUcsRUFBRSxJQUFJLEtBQUssSUFBSSxFQUFFLENBQUM7NEJBQ3ZCLEtBQUssRUFBRSxDQUFDO3dCQUNWLENBQUM7b0JBQ0gsQ0FBQztvQkFFRCxPQUFPLEtBQUssQ0FBQztnQkFDZixDQUFDLENBQUM7Z0JBdEJBLElBQUksQ0FBQyxNQUFNLEdBQUcsTUFBTSxDQUFDLFNBQVMsQ0FBQyxlQUFlLENBQUMsQ0FBQztnQkFDaEQsSUFBSSxDQUFDLGVBQWUsR0FBRyxJQUFJLDRCQUE0QixDQUFDLE1BQU0sQ0FBQyxDQUFDO2dCQUNoRSxJQUFJLENBQUMsY0FBYyxHQUFHLElBQUksY0FBYyxDQUFDLElBQUksQ0FBQyxXQUFXLENBQUMsSUFBSSxDQUFDLElBQUksQ0FBQyxFQUFFLElBQUksQ0FBQyxNQUFNLEVBQUUsaUJBQWlCLENBQUMsQ0FBQztnQkFDdEcsSUFBSSxDQUFDLFlBQVksR0FBRyxZQUFZLENBQUM7Z0JBQ2pDLElBQUksQ0FBQyxVQUFVLEdBQUcsVUFBVSxDQUFDO2dCQUM3QixJQUFJLENBQUMseUJBQXlCLEdBQUcseUJBQXlCLENBQUM7WUFDN0QsQ0FBQztZQWtCTSxLQUFLLENBQUMsS0FBSztnQkFDaEIsSUFBSSxJQUFJLENBQUMsT0FBTyxFQUFFLENBQUM7b0JBQ2pCLElBQUksQ0FBQyxNQUFNLENBQUMsSUFBSSxDQUFDLGdDQUFnQyxDQUFDLENBQUM7b0JBQ25ELE9BQU8sT0FBTyxDQUFDLE9BQU8sRUFBRSxDQUFDO2dCQUMzQixDQUFDO2dCQUNELElBQUksQ0FBQyxNQUFNLENBQUMsSUFBSSxDQUFDLHdCQUF3QixDQUFDLENBQUM7Z0JBQzNDLElBQUksS0FBSyxFQUFFLE1BQU0sQ0FBQyxJQUFJLEVBQUUsTUFBTSxDQUFDLElBQUksSUFBSSxDQUFDLFFBQVEsQ0FBQyxjQUFjLEVBQUUsRUFBRSxDQUFDO29CQUNsRSxJQUFJLENBQUMsTUFBTSxDQUFDLElBQUksQ0FBQyw0QkFBNEIsSUFBSSxDQUFDLEVBQUUsWUFBWSxDQUFDLENBQUMsTUFBTSxFQUFFLEVBQUU7d0JBQzFFLFlBQVksRUFBRSxJQUFJLENBQUMsRUFBRTt3QkFDckIsTUFBTSxFQUFFLE1BQU0sQ0FBQyxDQUFDLENBQUMsTUFBTSxDQUFDLE1BQU0sQ0FBQyxDQUFDLENBQUMsU0FBUztxQkFDM0MsQ0FBQyxDQUFDO29CQUVILElBQUksQ0FBQyxTQUFTLENBQUMsR0FBRyxDQUFDLElBQUksQ0FBQyxFQUFFLEVBQUUsSUFBSSxDQUFDLENBQUM7b0JBQ2xDLElBQUksQ0FBQyxRQUFRLENBQUMsR0FBRyxDQUFDLElBQUksQ0FBQyxFQUFFLEVBQUUsb0JBQW9CLEVBQUUsQ0FBQyxDQUFDO29CQUVuRCxJQUFJLE1BQU0sRUFBRSxDQUFDO3dCQUNYLElBQUksQ0FBQyxRQUFRLENBQUMsR0FBRyxDQUFDLElBQUksQ0FBQyxFQUFFLENBQUUsQ0FBQyxPQUFPLENBQUMsTUFBTSxDQUFDLENBQUM7d0JBQzVDLElBQUksQ0FBQyxZQUFZLENBQUMsR0FBRyxDQUFDLElBQUksQ0FBQyxFQUFFLEVBQUUsTUFBTSxDQUFDLENBQUM7b0JBQ3pDLENBQUM7eUJBQU0sQ0FBQzt3QkFDTixJQUFJLENBQUMsa0JBQWtCLENBQUMsSUFBSSxDQUFDLENBQUM7b0JBQ2hDLENBQUM7Z0JBQ0gsQ0FBQztnQkFFRCxJQUFJLENBQUMsY0FBYyxDQUFDLEtBQUssRUFBRSxDQUFDO2dCQUU1QixJQUFJLENBQUMsWUFBWSxDQUFDLEtBQUssRUFBRSxDQUFDO2dCQUUxQixJQUFJLENBQUMsZUFBZSxDQUFDLGlCQUFpQixDQUFDLElBQUksQ0FBQyxpQkFBaUIsQ0FBQyxDQUFDO2dCQUMvRCxJQUFJLENBQUMsZUFBZSxDQUFDLGlCQUFpQixDQUFDLElBQUksQ0FBQyxlQUFlLENBQUMsQ0FBQztnQkFFN0QsSUFBSSxDQUFDLE9BQU8sR0FBRyxJQUFJLENBQUM7WUFDdEIsQ0FBQztZQUVNLEtBQUssQ0FBQyxJQUFJO2dCQUNmLElBQUksQ0FBQyxJQUFJLENBQUMsT0FBTyxFQUFFLENBQUM7b0JBQ2xCLElBQUksQ0FBQyxNQUFNLENBQUMsSUFBSSxDQUFDLDJCQUEyQixDQUFDLENBQUM7b0JBQzlDLE9BQU8sT0FBTyxDQUFDLE9BQU8sRUFBRSxDQUFDO2dCQUMzQixDQUFDO2dCQUNELE1BQU0sSUFBSSxDQUFDLFlBQVksQ0FBQyxNQUFNLEVBQUUsQ0FBQztnQkFDakMsTUFBTSxJQUFJLENBQUMsY0FBYyxDQUFDLElBQUksRUFBRSxDQUFDO1lBQ25DLENBQUM7WUFFTSxpQkFBaUIsQ0FBQyxHQUFlO2dCQUN0QyxPQUFPLElBQUksQ0FBQyxZQUFZLENBQUMsR0FBRyxDQUFDLEdBQUcsRUFBRSxDQUFDLHVCQUFBLElBQUksa0VBQW1CLE1BQXZCLElBQUksRUFBb0IsR0FBRyxDQUFDLENBQUMsQ0FBQztZQUNuRSxDQUFDO1lBRU0sZ0JBQWdCLENBQUMsRUFBZ0I7Z0JBQ3RDLE9BQU8sSUFBSSxDQUFDLFlBQVksQ0FBQyxHQUFHLENBQUMsR0FBRyxFQUFFLENBQUMsdUJBQUEsSUFBSSxpRUFBa0IsTUFBdEIsSUFBSSxFQUFtQixFQUFFLENBQUMsQ0FBQyxDQUFDO1lBQ2pFLENBQUM7WUFFTSxtQkFBbUIsQ0FBQyxFQUFnQjtnQkFDekMsT0FBTyxJQUFJLENBQUMsWUFBWSxDQUFDLEdBQUcsQ0FBQyxHQUFHLEVBQUUsQ0FBQyx1QkFBQSxJQUFJLG9FQUFxQixNQUF6QixJQUFJLEVBQXNCLEVBQUUsQ0FBQyxDQUFDLENBQUM7WUFDcEUsQ0FBQztZQUVNLGdCQUFnQixDQUFDLEdBQW1CO2dCQUN6QyxPQUFPLElBQUksQ0FBQyxZQUFZLENBQUMsR0FBRyxDQUFDLEdBQUcsRUFBRSxDQUFDLHVCQUFBLElBQUksaUVBQWtCLE1BQXRCLElBQUksRUFBbUIsR0FBRyxDQUFDLENBQUMsQ0FBQztZQUNsRSxDQUFDO1lBRU0sYUFBYSxDQUFDLE1BQXlCO2dCQUM1QyxPQUFPLElBQUksQ0FBQyxZQUFZLENBQUMsR0FBRyxDQUFDLEdBQUcsRUFBRSxDQUFDLHVCQUFBLElBQUksOERBQWUsTUFBbkIsSUFBSSxFQUFnQixNQUFNLENBQUMsQ0FBQyxDQUFDO1lBQ2xFLENBQUM7WUFFTSx1QkFBdUIsQ0FBQyxFQUFnQixFQUFFLEtBQWU7Z0JBQzlELE9BQU8sSUFBSSxDQUFDLFlBQVksQ0FBQyxHQUFHLENBQUMsR0FBRyxFQUFFLENBQUMsdUJBQUEsSUFBSSx3RUFBeUIsTUFBN0IsSUFBSSxFQUEwQixFQUFFLEVBQUUsS0FBSyxDQUFDLENBQUMsQ0FBQztZQUMvRSxDQUFDO1lBRU0scUJBQXFCLENBQUMsRUFBZ0IsRUFBRSxHQUFXLEVBQUUsS0FBSyxHQUFHLEtBQUs7Z0JBQ3ZFLE9BQU8sSUFBSSxDQUFDLFlBQVksQ0FBQyxHQUFHLENBQUMsR0FBRyxFQUFFLENBQUMsdUJBQUEsSUFBSSxzRUFBdUIsTUFBM0IsSUFBSSxFQUF3QixFQUFFLEVBQUUsR0FBRyxFQUFFLEtBQUssQ0FBQyxDQUFDLENBQUM7WUFDbEYsQ0FBQztZQUVNLHdCQUF3QixDQUM3QixFQUFnQixFQUNoQixTQUFpQixFQUNqQixNQUF5QjtnQkFFekIsT0FBTyxJQUFJLENBQUMsWUFBWSxDQUFDLEdBQUcsQ0FBQyxHQUFHLEVBQUUsQ0FBQyx1QkFBQSxJQUFJLHlFQUEwQixNQUE5QixJQUFJLEVBQTJCLEVBQUUsRUFBRSxTQUFTLEVBQUUsTUFBTSxDQUFDLENBQUMsQ0FBQztZQUM1RixDQUFDO1lBZ0RPLHNCQUFzQixDQUFDLEdBQW1CO2dCQUNoRCxLQUFLLE1BQU0sRUFBRSxJQUFJLEdBQUcsRUFBRSxDQUFDO29CQUNyQixJQUFJLENBQUMsU0FBUyxDQUFDLE1BQU0sQ0FBQyxFQUFFLENBQUMsQ0FBQztvQkFDMUIsSUFBSSxDQUFDLFFBQVEsQ0FBQyxNQUFNLENBQUMsRUFBRSxDQUFDLENBQUM7b0JBQ3pCLElBQUksQ0FBQyxZQUFZLENBQUMsTUFBTSxDQUFDLEVBQUUsQ0FBQyxDQUFDO29CQUM3QixJQUFJLENBQUMsVUFBVSxDQUFDLE1BQU0sQ0FBQyxFQUFFLENBQUMsQ0FBQztvQkFDM0IsSUFBSSxDQUFDLE9BQU8sQ0FBQyxNQUFNLENBQUMsRUFBRSxDQUFDLENBQUM7Z0JBQzFCLENBQUM7WUFDSCxDQUFDO1lBcVBPLEtBQUssQ0FBQyxXQUFXO2dCQUN2QixJQUFJLENBQUMsZ0JBQWdCLEVBQUUsQ0FBQztnQkFDeEIsSUFBSSxDQUFDLG9CQUFvQixFQUFFLENBQUM7Z0JBQzVCLE1BQU0saUJBQWlCLEdBQUcsSUFBSSxDQUFDLGlCQUFpQixFQUFFLENBQUM7Z0JBQ25ELElBQUksaUJBQWlCLEdBQUcsQ0FBQyxFQUFFLENBQUM7b0JBQzFCLE1BQU0sSUFBSSxDQUFDLFlBQVksQ0FBQyxHQUFHLENBQUMsR0FBRyxFQUFFLENBQUMsSUFBSSxDQUFDLFFBQVEsQ0FBQyxrQ0FBa0MsQ0FBQyxpQkFBaUIsQ0FBQyxDQUFDLENBQUM7b0JBQ3ZHLElBQUksQ0FBQyxNQUFNLENBQUMsS0FBSyxDQUFDLGlDQUFpQyxpQkFBaUIsRUFBRSxDQUFDLENBQUM7Z0JBQzFFLENBQUM7WUFDSCxDQUFDO1lBRU8sZ0JBQWdCO2dCQUN0QixNQUFNLE1BQU0sR0FBRyxLQUFLLENBQUMsSUFBSSxDQUFDLElBQUksQ0FBQyxTQUFTLENBQUMsSUFBSSxFQUFFLENBQUMsQ0FBQztnQkFDakQsTUFBTSxXQUFXLEdBQW1CLEVBQUUsQ0FBQztnQkFDdkMsS0FBSyxNQUFNLEVBQUUsSUFBSSxNQUFNLEVBQUUsQ0FBQztvQkFDeEIsTUFBTSxHQUFHLEdBQUcsSUFBSSxDQUFDLFNBQVMsQ0FBQyxHQUFHLENBQUMsRUFBRSxDQUFFLENBQUM7b0JBQ3BDLElBQUksSUFBSSxDQUFDLFVBQVUsQ0FBQyxHQUFHLENBQUMsRUFBRSxDQUFDO3dCQUN6QixXQUFXLENBQUMsSUFBSSxDQUFDLEVBQUUsQ0FBQyxDQUFDO29CQUN2QixDQUFDO2dCQUNILENBQUM7Z0JBRUQsSUFBSSxXQUFXLENBQUMsTUFBTSxHQUFHLENBQUMsRUFBRSxDQUFDO29CQUMzQixJQUFJLENBQUMsc0JBQXNCLENBQUMsV0FBVyxDQUFDLENBQUM7b0JBQ3pDLElBQUksQ0FBQyxNQUFNLENBQUMsSUFBSSxDQUFDLG1CQUFtQixXQUFXLENBQUMsTUFBTSxFQUFFLENBQUMsQ0FBQztnQkFDNUQsQ0FBQztZQUNILENBQUM7WUFFTyxvQkFBb0I7Z0JBQzFCLE1BQU0saUJBQWlCLEdBQUcsS0FBSyxDQUFDLElBQUksQ0FBQyxJQUFJLENBQUMsVUFBVSxDQUFDLE9BQU8sRUFBRSxDQUFDLENBQUM7Z0JBQ2hFLEtBQUssTUFBTSxDQUFDLEVBQUUsRUFBRSxRQUFRLENBQUMsSUFBSSxpQkFBaUIsRUFBRSxDQUFDO29CQUMvQyxNQUFNLElBQUksR0FBRyxJQUFJLENBQUMsU0FBUyxDQUFDLEdBQUcsQ0FBQyxFQUFFLENBQUMsQ0FBQztvQkFDcEMsSUFBSSxDQUFDLElBQUksRUFBRSxDQUFDO3dCQUNWLElBQUksQ0FBQyxNQUFNLENBQUMsSUFBSSxDQUFDLGtCQUFrQixFQUFFLHlDQUF5QyxFQUFFLEVBQUUsWUFBWSxFQUFFLEVBQUUsRUFBRSxDQUFDLENBQUM7d0JBQ3RHLElBQUksQ0FBQyxVQUFVLENBQUMsTUFBTSxDQUFDLEVBQUUsQ0FBQyxDQUFDO3dCQUMzQixTQUFTO29CQUNYLENBQUM7b0JBRUQsTUFBTSxHQUFHLEdBQUcsSUFBSSxDQUFDLFlBQVksRUFBRSxDQUFDO29CQUNoQyxNQUFNLGlCQUFpQixHQUFHLEdBQUcsR0FBRyxRQUFRLENBQUMsYUFBYSxDQUFDO29CQUN2RCxJQUFJLGlCQUFpQixJQUFJLElBQUksQ0FBQyxZQUFZLEVBQUUsQ0FBQzt3QkFDM0MsSUFBSSxDQUFDLE1BQU0sQ0FBQyxJQUFJLENBQUMsa0JBQWtCLEVBQUUsMENBQTBDLEVBQUUsRUFBRSxZQUFZLEVBQUUsRUFBRSxFQUFFLENBQUMsQ0FBQzt3QkFDdkcsSUFBSSxDQUFDLFVBQVUsQ0FBQyxNQUFNLENBQUMsRUFBRSxDQUFDLENBQUM7d0JBQzNCLElBQUksQ0FBQyxrQkFBa0IsQ0FBQyxJQUFJLENBQUMsQ0FBQzt3QkFDOUIsSUFBSSxDQUFDLGVBQWUsQ0FBQyxlQUFlLENBQUMsSUFBSSxDQUFDLElBQUksQ0FBQyxDQUFDO29CQUNsRCxDQUFDO2dCQUNILENBQUM7WUFDSCxDQUFDO1lBRU8sa0JBQWtCLENBQUMsR0FBZTtnQkFDeEMsSUFBSSxDQUFDLElBQUksQ0FBQyxRQUFRLENBQUMsR0FBRyxDQUFDLEdBQUcsQ0FBQyxFQUFFLENBQUMsRUFBRSxDQUFDO29CQUMvQixJQUFJLENBQUMsUUFBUSxDQUFDLEdBQUcsQ0FBQyxHQUFHLENBQUMsRUFBRSxFQUFFLG9CQUFvQixFQUFFLENBQUMsQ0FBQztnQkFDcEQsQ0FBQztnQkFDRCxJQUFJLENBQUMsTUFBTSxDQUFDLEdBQUcsQ0FBQyxJQUFJLENBQUMsQ0FBQyxHQUFHLENBQUM7b0JBQ3hCLFdBQVcsRUFBRSxHQUFHLENBQUMsV0FBVztvQkFDNUIsRUFBRSxFQUFFLEdBQUcsQ0FBQyxFQUFFO2lCQUNYLENBQUMsQ0FBQztnQkFDSCxJQUFJLENBQUMsVUFBVSxDQUFDLEdBQUcsQ0FBQyxHQUFHLENBQUMsRUFBRSxFQUFFLElBQUksS0FBSyxFQUFFLENBQUMsQ0FBQztnQkFDekMsSUFBSSxDQUFDLFdBQVcsR0FBRyxJQUFJLENBQUMsR0FBRyxDQUFDLElBQUksQ0FBQyxXQUFXLEVBQUUsR0FBRyxDQUFDLFdBQVcsQ0FBQyxDQUFDO1lBQ2pFLENBQUM7WUFFTyxVQUFVLENBQUMsR0FBZTtnQkFDaEMsT0FBTyxHQUFHLENBQUMsV0FBVyxHQUFHLElBQUksQ0FBQyxpQkFBaUIsRUFBRSxDQUFDO1lBQ3BELENBQUM7WUFFTyxpQkFBaUI7Z0JBQ3ZCLE9BQU8sSUFBSSxDQUFDLFdBQVcsR0FBRyxJQUFJLENBQUMseUJBQXlCLENBQUM7WUFDM0QsQ0FBQzs7OzJDQTVXRCxLQUFLLDJDQUFvQixHQUFlO1lBQ3RDLHFEQUFxRDtZQUNyRCxNQUFNLFNBQVMsR0FBRyxNQUFNLHVCQUFBLElBQUksb0VBQXFCLE1BQXpCLElBQUksRUFBc0IsR0FBRyxDQUFDLEVBQUUsQ0FBQyxDQUFDO1lBQzFELElBQUksSUFBSSxDQUFDLFNBQVMsQ0FBQyxHQUFHLENBQUMsR0FBRyxDQUFDLEVBQUUsQ0FBQyxFQUFFLENBQUM7Z0JBQy9CLE1BQU0sUUFBUSxHQUFHLElBQUksQ0FBQyxTQUFTLENBQUMsR0FBRyxDQUFDLEdBQUcsQ0FBQyxFQUFFLENBQUMsQ0FBQztnQkFDNUMsTUFBTSxDQUFDLGVBQWUsQ0FBQyxHQUFHLEVBQUUsUUFBUSxFQUFFLDBCQUEwQixDQUFDLENBQUM7Z0JBQ2xFLElBQUksQ0FBQyxNQUFNLENBQUMsS0FBSyxDQUFDLDRCQUE0QixHQUFHLENBQUMsRUFBRSxnQkFBZ0IsR0FBRyxDQUFDLFdBQVcsWUFBWSxFQUFFO29CQUMvRixZQUFZLEVBQUUsR0FBRyxDQUFDLEVBQUU7aUJBQ3JCLENBQUMsQ0FBQztnQkFDSCxPQUFPLFNBQVMsQ0FBQztZQUNuQixDQUFDO1lBRUQsSUFBSSxJQUFJLENBQUMsVUFBVSxDQUFDLEdBQUcsQ0FBQyxFQUFFLENBQUM7Z0JBQ3pCLElBQUksQ0FBQyxNQUFNLENBQUMsSUFBSSxDQUFDLHlDQUF5QyxHQUFHLENBQUMsRUFBRSxnQkFBZ0IsR0FBRyxDQUFDLFdBQVcsRUFBRSxFQUFFO29CQUNqRyxZQUFZLEVBQUUsR0FBRyxDQUFDLEVBQUU7aUJBQ3JCLENBQUMsQ0FBQztnQkFDSCxNQUFNLElBQUksS0FBSyxDQUFDLDRCQUE0QixHQUFHLENBQUMsV0FBVyxvQkFBb0IsSUFBSSxDQUFDLFdBQVcsRUFBRSxDQUFDLENBQUM7WUFDckcsQ0FBQztZQUVELElBQUksQ0FBQyxNQUFNLENBQUMsSUFBSSxDQUFDLHNCQUFzQixHQUFHLENBQUMsRUFBRSxnQkFBZ0IsR0FBRyxDQUFDLFdBQVcsRUFBRSxFQUFFLEVBQUUsWUFBWSxFQUFFLEdBQUcsQ0FBQyxFQUFFLEVBQUUsQ0FBQyxDQUFDO1lBQzFHLElBQUksQ0FBQztnQkFDSCw2SEFBNkg7Z0JBQzdILElBQUksQ0FBQyxTQUFTLENBQUMsR0FBRyxDQUFDLEdBQUcsQ0FBQyxFQUFFLEVBQUUsR0FBRyxDQUFDLENBQUM7Z0JBQ2hDLE1BQU0sSUFBSSxDQUFDLFFBQVEsQ0FBQyxhQUFhLENBQUMsR0FBRyxDQUFDLENBQUM7Z0JBQ3ZDLElBQUksQ0FBQyxrQkFBa0IsQ0FBQyxHQUFHLENBQUMsQ0FBQztZQUMvQixDQUFDO1lBQUMsT0FBTyxHQUFHLEVBQUUsQ0FBQztnQkFDYixJQUFJLENBQUMsTUFBTSxDQUFDLEtBQUssQ0FBQyxpQ0FBaUMsR0FBRyxDQUFDLEVBQUUsS0FBSyxHQUFHLEVBQUUsRUFBRSxHQUFHLEVBQUUsRUFBRSxZQUFZLEVBQUUsR0FBRyxDQUFDLEVBQUUsRUFBRSxDQUFDLENBQUM7Z0JBQ3BHLElBQUksQ0FBQyxTQUFTLENBQUMsTUFBTSxDQUFDLEdBQUcsQ0FBQyxFQUFFLENBQUMsQ0FBQztnQkFDOUIsTUFBTSxHQUFHLENBQUM7WUFDWixDQUFDO1lBQ0QsT0FBTyxTQUFTLENBQUM7UUFDbkIsQ0FBQzswQ0FFRCxLQUFLLDBDQUFtQixFQUFnQjtZQUN0QyxJQUFJLENBQUMsSUFBSSxDQUFDLFNBQVMsQ0FBQyxHQUFHLENBQUMsRUFBRSxDQUFDLEVBQUUsQ0FBQztnQkFDNUIsSUFBSSxDQUFDLE1BQU0sQ0FBQyxJQUFJLENBQUMsNENBQTRDLEVBQUUsRUFBRSxFQUFFLEVBQUUsWUFBWSxFQUFFLEVBQUUsRUFBRSxDQUFDLENBQUM7Z0JBQ3pGLE9BQU87WUFDVCxDQUFDO1lBRUQsdUNBQXVDO1lBQ3ZDLElBQUksQ0FBQyxJQUFJLENBQUMsWUFBWSxDQUFDLEdBQUcsQ0FBQyxFQUFFLENBQUMsRUFBRSxDQUFDO2dCQUMvQixJQUFJLENBQUMsTUFBTSxDQUFDLElBQUksQ0FBQyxxQkFBcUIsRUFBRSxFQUFFLEVBQUUsRUFBRSxZQUFZLEVBQUUsRUFBRSxFQUFFLENBQUMsQ0FBQztnQkFDbEUsTUFBTSx1QkFBQSxJQUFJLHNFQUF1QixNQUEzQixJQUFJLEVBQXdCLEVBQUUsRUFBRSxTQUFTLEVBQUUsS0FBSyxDQUFDLENBQUM7WUFDMUQsQ0FBQztRQUNILENBQUM7eUZBWW9CLEVBQWdCO1lBQ25DLE1BQU0sTUFBTSxHQUFHLElBQUksQ0FBQyxZQUFZLENBQUMsR0FBRyxDQUFDLEVBQUUsQ0FBQyxDQUFDO1lBQ3pDLElBQUksTUFBTSxFQUFFLENBQUM7Z0JBQ1gsT0FBTyxPQUFPLENBQUMsT0FBTyxDQUFDLE1BQU0sQ0FBQyxDQUFDO1lBQ2pDLENBQUM7aUJBQU0sQ0FBQztnQkFDTiwyQ0FBMkM7Z0JBQzNDLE1BQU0sSUFBSSxHQUFHLElBQUksQ0FBQyxTQUFTLENBQUMsR0FBRyxDQUFDLEVBQUUsQ0FBQyxDQUFDO2dCQUVwQyxJQUFJLENBQUMsSUFBSSxFQUFFLENBQUM7b0JBQ1YsT0FBTyxPQUFPLENBQUMsT0FBTyxDQUFDLEVBQUUsTUFBTSxFQUFFLFdBQVcsRUFBRSxDQUFDLENBQUM7Z0JBQ2xELENBQUM7Z0JBRUQsT0FBTyxPQUFPLENBQUMsT0FBTyxDQUFDLEVBQUUsTUFBTSxFQUFFLElBQUksQ0FBQyxVQUFVLENBQUMsR0FBRyxDQUFDLEVBQUUsQ0FBQyxDQUFDLENBQUMsQ0FBQyxhQUFhLENBQUMsQ0FBQyxDQUFDLFVBQVUsRUFBRSxDQUFDLENBQUM7WUFDM0YsQ0FBQztRQUNILENBQUM7bUZBRWlCLEdBQW1CO1lBQ25DLE1BQU0sYUFBYSxHQUFHLEdBQUcsQ0FBQyxNQUFNLENBQUMsRUFBRSxDQUFDLEVBQUUsQ0FBQyxJQUFJLENBQUMsWUFBWSxDQUFDLEdBQUcsQ0FBQyxFQUFFLENBQUMsQ0FBQyxDQUFDO1lBQ2xFLE1BQU0sYUFBYSxHQUFHLElBQUksQ0FBQyx5QkFBeUIsQ0FBQztZQUNyRCxJQUFJLENBQUMseUJBQXlCLEdBQUcsRUFBRSxDQUFDO1lBQ3BDLE9BQU8sT0FBTyxDQUFDLE9BQU8sQ0FBQyxhQUFhLENBQUMsTUFBTSxDQUFDLGFBQWEsQ0FBQyxDQUFDLENBQUM7UUFDOUQsQ0FBQzs7UUFFRCx5Q0FBeUM7UUFDekMsS0FBSyx1Q0FDSCxTQUEyQixFQUFFLFNBQVMsRUFBRSxFQUFFLEVBQUU7WUFFNUMsTUFBTSxhQUFhLEdBQ2pCLEtBQUssQ0FBQyxPQUFPLENBQUMsTUFBTSxDQUFDLFNBQVMsQ0FBQyxJQUFJLE1BQU0sQ0FBQyxTQUFTLENBQUMsTUFBTSxHQUFHLENBQUM7Z0JBQzVELENBQUMsQ0FBQyxDQUFDLEdBQUcsTUFBTSxDQUFDLFNBQVMsQ0FBQztnQkFDdkIsQ0FBQyxDQUFDLE1BQU0sQ0FBQyxNQUFNLENBQUMsa0JBQWtCLENBQUMsQ0FBQyxNQUFNLENBQUMsQ0FBQyxDQUFDLEVBQTJCLEVBQUUsQ0FBQyxPQUFPLENBQUMsS0FBSyxRQUFRLENBQUMsQ0FBQztZQUN0RyxhQUFhLENBQUMsSUFBSSxDQUFDLG1CQUFtQixDQUFDLENBQUM7WUFFeEMsS0FBSyxNQUFNLFNBQVMsSUFBSSxhQUFhLEVBQUUsQ0FBQztnQkFDdEMsTUFBTSxLQUFLLEdBQUcsSUFBSSxDQUFDLE1BQU0sQ0FBQyxTQUFTLENBQUMsQ0FBQztnQkFDckMsSUFBSSxXQUEyQyxDQUFDO2dCQUNoRCxxRkFBcUY7Z0JBQ3JGLGdDQUFnQztnQkFDaEMscURBQXFEO2dCQUNyRCwwR0FBMEc7Z0JBQzFHLE9BQU8sQ0FBQyxXQUFXLEdBQUcsS0FBSyxDQUFDLFlBQVksRUFBRSxDQUFDLEVBQUUsQ0FBQztvQkFDNUMsTUFBTSxHQUFHLEdBQUcsSUFBSSxDQUFDLFNBQVMsQ0FBQyxHQUFHLENBQUMsV0FBVyxDQUFDLEVBQUUsQ0FBQyxDQUFDO29CQUMvQyxJQUFJLEdBQUcsSUFBSSxDQUFDLElBQUksQ0FBQyxVQUFVLENBQUMsR0FBRyxDQUFDLFdBQVcsQ0FBQyxFQUFFLENBQUMsSUFBSSxDQUFDLElBQUksQ0FBQyxZQUFZLENBQUMsR0FBRyxDQUFDLFdBQVcsQ0FBQyxFQUFFLENBQUMsRUFBRSxDQUFDO3dCQUMxRixNQUFNLElBQUksR0FBRyxJQUFJLENBQUMsWUFBWSxFQUFFLENBQUM7d0JBQ2pDLElBQUksQ0FBQyxVQUFVLENBQUMsR0FBRyxDQUFDLEdBQUcsQ0FBQyxFQUFFLEVBQUU7NEJBQzFCLEVBQUUsRUFBRSxHQUFHLENBQUMsRUFBRTs0QkFDVixTQUFTLEVBQUUsSUFBSTs0QkFDZixhQUFhLEVBQUUsSUFBSTt5QkFDcEIsQ0FBQyxDQUFDO3dCQUNILE1BQU0sVUFBVSxHQUFHLElBQUksQ0FBQyxVQUFVLENBQUMsR0FBRyxDQUFDLEdBQUcsQ0FBQyxFQUFFLENBQUMsQ0FBQzt3QkFDL0MsSUFBSSxVQUFVLEVBQUUsQ0FBQzs0QkFDZixJQUFJLENBQUMsZUFBZSxDQUFDLGFBQWEsQ0FBQyxHQUFHLENBQUMsSUFBSSxFQUFFLFVBQVUsQ0FBQyxDQUFDO3dCQUMzRCxDQUFDO3dCQUVELE9BQU8sRUFBRSxHQUFHLEVBQUUsSUFBSSxFQUFFLENBQUM7b0JBQ3ZCLENBQUM7Z0JBQ0gsQ0FBQztZQUNILENBQUM7WUFFRCxPQUFPLFNBQVMsQ0FBQztRQUNuQixDQUFDOytDQUVELEtBQUssK0NBQXdCLEVBQWdCLEVBQUUsR0FBVyxFQUFFLEtBQUssR0FBRyxLQUFLO1lBQ3ZFLE1BQU0sSUFBSSxHQUFHLElBQUksQ0FBQyxVQUFVLENBQUMsR0FBRyxDQUFDLEVBQUUsQ0FBQyxDQUFDO1lBQ3JDLE1BQU0sSUFBSSxHQUFHLElBQUksQ0FBQyxTQUFTLENBQUMsR0FBRyxDQUFDLEVBQUUsQ0FBQyxDQUFDO1lBQ3BDLE1BQU0sT0FBTyxHQUFHLElBQUksQ0FBQyxPQUFPLENBQUMsR0FBRyxDQUFDLEVBQUUsQ0FBQyxJQUFJLENBQUMsQ0FBQztZQUUxQyxJQUFJLENBQUMsSUFBSSxFQUFFLENBQUM7Z0JBQ1YsSUFBSSxDQUFDLE1BQU0sQ0FBQyxJQUFJLENBQUMsNkNBQTZDLEVBQUUsUUFBUSxHQUFHLEVBQUUsRUFBRSxFQUFFLFdBQVcsRUFBRSxFQUFFLEVBQUUsQ0FBQyxDQUFDO2dCQUNwRyxPQUFPO1lBQ1QsQ0FBQztZQUVELElBQUksQ0FBQyxJQUFJLEVBQUUsQ0FBQztnQkFDVixJQUFJLENBQUMsTUFBTSxDQUFDLElBQUksQ0FBQyxrQkFBa0IsRUFBRSxTQUFTLGtCQUFrQixDQUFDLElBQUksQ0FBQyxJQUFJLENBQUMsNkJBQTZCLEVBQUU7b0JBQ3hHLFlBQVksRUFBRSxFQUFFO2lCQUNqQixDQUFDLENBQUM7WUFDTCxDQUFDO2lCQUFNLENBQUM7Z0JBQ04sSUFBSSxDQUFDLFVBQVUsQ0FBQyxNQUFNLENBQUMsRUFBRSxDQUFDLENBQUM7WUFDN0IsQ0FBQztZQUVELElBQUksSUFBSSxDQUFDLFlBQVksQ0FBQyxHQUFHLENBQUMsRUFBRSxDQUFDLEVBQUUsQ0FBQztnQkFDOUIsSUFBSSxDQUFDLE1BQU0sQ0FBQyxJQUFJLENBQUMsa0JBQWtCLEVBQUUscUNBQXFDLEdBQUcsRUFBRSxFQUFFO29CQUMvRSxZQUFZLEVBQUUsRUFBRTtpQkFDakIsQ0FBQyxDQUFDO2dCQUNILE9BQU87WUFDVCxDQUFDO1lBRUQsSUFBSSxLQUFLLElBQUksT0FBTyxHQUFHLENBQUMsR0FBRyxJQUFJLENBQUMsVUFBVSxJQUFJLENBQUMsSUFBSSxDQUFDLFVBQVUsQ0FBQyxJQUFJLENBQUMsRUFBRSxDQUFDO2dCQUNyRSxJQUFJLENBQUMsTUFBTSxDQUFDLElBQUksQ0FDZCwyQkFBMkIsRUFBRSxTQUFTLGtCQUFrQixDQUFDLElBQUksQ0FBQyxJQUFJLENBQUMsVUFBVSxPQUFPLEdBQUcsQ0FBQyxRQUFRLEdBQUcsRUFBRSxFQUNyRztvQkFDRSxZQUFZLEVBQUUsRUFBRTtpQkFDakIsQ0FDRixDQUFDO2dCQUNGLElBQUksQ0FBQyxPQUFPLENBQUMsR0FBRyxDQUFDLEVBQUUsRUFBRSxPQUFPLEdBQUcsQ0FBQyxDQUFDLENBQUM7Z0JBQ2xDLElBQUksQ0FBQyxrQkFBa0IsQ0FBQyxJQUFJLENBQUMsQ0FBQztnQkFDOUIsSUFBSSxDQUFDLGVBQWUsQ0FBQyxjQUFjLENBQUMsSUFBSSxDQUFDLElBQUksQ0FBQyxDQUFDO2dCQUMvQyxPQUFPO1lBQ1QsQ0FBQztZQUVELElBQUksQ0FBQyxNQUFNLENBQUMsSUFBSSxDQUNkLG9DQUFvQyxFQUFFLFNBQVMsa0JBQWtCLENBQUMsSUFBSSxDQUFDLElBQUksQ0FBQyxrQkFDMUUsT0FBTyxHQUFHLENBQ1osUUFBUSxHQUFHLEVBQUUsRUFDYjtnQkFDRSxZQUFZLEVBQUUsRUFBRTthQUNqQixDQUNGLENBQUM7WUFFRixvRUFBb0U7WUFDcEUsc0ZBQXNGO1lBQ3RGLE1BQU0sTUFBTSxHQUE0QixFQUFFLE1BQU0sRUFBRSxVQUFVLEVBQUUsTUFBTSxFQUFFLE1BQU0sQ0FBQyxHQUFHLENBQUMsRUFBRSxDQUFDO1lBQ3BGLElBQUksQ0FBQyxZQUFZLENBQUMsR0FBRyxDQUFDLEVBQUUsRUFBRSxNQUFNLENBQUMsQ0FBQztZQUNsQyxJQUFJLENBQUMsUUFBUSxDQUFDLEdBQUcsQ0FBQyxFQUFFLENBQUUsQ0FBQyxPQUFPLENBQUMsTUFBTSxDQUFDLENBQUM7WUFDdkMsSUFBSSxDQUFDLHlCQUF5QixDQUFDLElBQUksQ0FBQyxFQUFFLENBQUMsQ0FBQztZQUV4QyxJQUFJLENBQUMsZUFBZSxDQUFDLGVBQWUsQ0FBQyxJQUFJLENBQUMsSUFBSSxDQUFDLENBQUM7WUFDaEQsSUFBSSxJQUFJLEVBQUUsQ0FBQztnQkFDVCxNQUFNLFFBQVEsR0FBRyxJQUFJLENBQUMsWUFBWSxFQUFFLEdBQUcsSUFBSSxDQUFDLFNBQVMsQ0FBQztnQkFDdEQsSUFBSSxDQUFDLGVBQWUsQ0FBQyxpQkFBaUIsQ0FBQyxJQUFJLENBQUMsSUFBSSxFQUFFLFFBQVEsQ0FBQyxDQUFDO1lBQzlELENBQUM7WUFFRCxJQUFJLENBQUM7Z0JBQ0gsTUFBTSxJQUFJLENBQUMsUUFBUSxDQUFDLGtCQUFrQixDQUFDLEVBQUUsRUFBRSxHQUFHLENBQUMsQ0FBQztZQUNsRCxDQUFDO1lBQUMsT0FBTyxPQUFPLEVBQUUsQ0FBQztnQkFDakIsSUFBSSxDQUFDLE1BQU0sQ0FBQyxLQUFLLENBQUMsOENBQThDLEVBQUUsV0FBVyxHQUFHLEVBQUUsRUFBRSxPQUFPLEVBQUU7b0JBQzNGLFlBQVksRUFBRSxFQUFFO2lCQUNqQixDQUFDLENBQUM7Z0JBRUgsTUFBTSxPQUFPLENBQUM7WUFDaEIsQ0FBQztRQUNILENBQUM7bUdBR0MsRUFBZ0IsRUFDaEIsU0FBaUIsRUFDakIsTUFBeUI7WUFFekIsTUFBTSxHQUFHLEdBQUcsSUFBSSxDQUFDLFNBQVMsQ0FBQyxHQUFHLENBQUMsRUFBRSxDQUFDLENBQUM7WUFDbkMsSUFBSSxDQUFDLEdBQUcsRUFBRSxDQUFDO2dCQUNULElBQUksQ0FBQyxNQUFNLENBQUMsSUFBSSxDQUFDLGtCQUFrQixFQUFFLGlCQUFpQixFQUFFLEVBQUUsWUFBWSxFQUFFLEVBQUUsRUFBRSxDQUFDLENBQUM7Z0JBQzlFLE9BQU8sTUFBTSxDQUFDLENBQUMsQ0FBQyx1QkFBQSxJQUFJLDhEQUFlLE1BQW5CLElBQUksRUFBZ0IsTUFBTSxDQUFDLENBQUMsQ0FBQyxDQUFDLE9BQU8sQ0FBQyxPQUFPLENBQUMsU0FBUyxDQUFDLENBQUM7WUFDM0UsQ0FBQztZQUVELElBQUksSUFBSSxDQUFDLFlBQVksQ0FBQyxHQUFHLENBQUMsRUFBRSxDQUFDLEVBQUUsQ0FBQztnQkFDOUIsSUFBSSxDQUFDLE1BQU0sQ0FBQyxJQUFJLENBQUMsa0JBQWtCLEVBQUUsNkJBQTZCLEVBQUUsRUFBRSxZQUFZLEVBQUUsRUFBRSxFQUFFLENBQUMsQ0FBQztnQkFDMUYsT0FBTyxNQUFNLENBQUMsQ0FBQyxDQUFDLHVCQUFBLElBQUksOERBQWUsTUFBbkIsSUFBSSxFQUFnQixNQUFNLENBQUMsQ0FBQyxDQUFDLENBQUMsT0FBTyxDQUFDLE9BQU8sQ0FBQyxTQUFTLENBQUMsQ0FBQztZQUMzRSxDQUFDO1lBRUQsTUFBTSxRQUFRLEdBQUcsSUFBSSxDQUFDLFVBQVUsQ0FBQyxHQUFHLENBQUMsRUFBRSxDQUFDLENBQUM7WUFDekMsTUFBTSxHQUFHLEdBQUcsSUFBSSxDQUFDLFlBQVksRUFBRSxDQUFDO1lBQ2hDLElBQUksQ0FBQyxRQUFRLEVBQUUsQ0FBQztnQkFDZCxJQUFJLENBQUMsTUFBTSxDQUFDLElBQUksQ0FDZCxrQkFBa0IsRUFBRSxTQUFTLGtCQUFrQixDQUFDLEdBQUcsQ0FBQyxJQUFJLENBQUMsZ0RBQWdELEVBQ3pHLEVBQUUsWUFBWSxFQUFFLEVBQUUsRUFBRSxDQUNyQixDQUFDO2dCQUNGLHVEQUF1RDtnQkFDdkQsa0ZBQWtGO2dCQUNsRixtQ0FBbUM7Z0JBQ25DLElBQUksQ0FBQyxVQUFVLENBQUMsR0FBRyxDQUFDLEVBQUUsRUFBRTtvQkFDdEIsRUFBRTtvQkFDRixTQUFTO29CQUNULGFBQWEsRUFBRSxJQUFJLENBQUMsWUFBWSxFQUFFO2lCQUNuQyxDQUFDLENBQUM7Z0JBQ0gsT0FBTyxPQUFPLENBQUMsT0FBTyxDQUFDLFNBQVMsQ0FBQyxDQUFDO1lBQ3BDLENBQUM7aUJBQU0sSUFBSSxTQUFTLElBQUksUUFBUSxDQUFDLFNBQVMsRUFBRSxDQUFDO2dCQUMzQyxJQUFJLFNBQVMsR0FBRyxRQUFRLENBQUMsU0FBUyxFQUFFLENBQUM7b0JBQ25DLElBQUksQ0FBQyxNQUFNLENBQUMsSUFBSSxDQUNkLGtCQUFrQixFQUFFLFNBQVMsa0JBQWtCLENBQUMsR0FBRyxDQUFDLElBQUksQ0FBQyxjQUFjLFNBQVMsNEJBQTRCLEVBQzVHLEVBQUUsWUFBWSxFQUFFLEVBQUUsRUFBRSxDQUNyQixDQUFDO2dCQUNKLENBQUM7cUJBQU0sQ0FBQztvQkFDTixJQUFJLENBQUMsTUFBTSxDQUFDLEtBQUssQ0FBQyxrQkFBa0IsRUFBRSxTQUFTLGtCQUFrQixDQUFDLEdBQUcsQ0FBQyxJQUFJLENBQUMsWUFBWSxFQUFFLEVBQUUsWUFBWSxFQUFFLEVBQUUsRUFBRSxDQUFDLENBQUM7Z0JBQ2pILENBQUM7Z0JBQ0QsUUFBUSxDQUFDLFNBQVMsR0FBRyxTQUFTLENBQUM7Z0JBQy9CLFFBQVEsQ0FBQyxhQUFhLEdBQUcsR0FBRyxDQUFDO2dCQUM3QixPQUFPLE9BQU8sQ0FBQyxPQUFPLENBQUMsU0FBUyxDQUFDLENBQUM7WUFDcEMsQ0FBQztpQkFBTSxJQUFJLE1BQU0sRUFBRSxDQUFDO2dCQUNsQixJQUFJLENBQUMsTUFBTSxDQUFDLElBQUksQ0FDZCxrQkFBa0IsRUFBRSxTQUNsQixrQkFBa0IsQ0FBQyxHQUFHLENBQUMsSUFBSSxDQUM3Qiw0REFBNEQsRUFDNUQsRUFBRSxZQUFZLEVBQUUsRUFBRSxFQUFFLENBQ3JCLENBQUM7Z0JBQ0YsT0FBTyx1QkFBQSxJQUFJLDhEQUFlLE1BQW5CLElBQUksRUFBZ0IsTUFBTSxDQUFDLENBQUM7WUFDckMsQ0FBQztpQkFBTSxDQUFDO2dCQUNOLE9BQU8sT0FBTyxDQUFDLE9BQU8sQ0FBQyxTQUFTLENBQUMsQ0FBQztZQUNwQyxDQUFDO1FBQ0gsQ0FBQztpREFFRCxLQUFLLGlEQUEwQixFQUFnQixFQUFFLEtBQWU7WUFDOUQsTUFBTSxJQUFJLEdBQUcsSUFBSSxDQUFDLFVBQVUsQ0FBQyxHQUFHLENBQUMsRUFBRSxDQUFDLENBQUM7WUFDckMsTUFBTSxJQUFJLEdBQUcsSUFBSSxDQUFDLFNBQVMsQ0FBQyxHQUFHLENBQUMsRUFBRSxDQUFDLENBQUM7WUFDcEMsTUFBTSxPQUFPLEdBQUcsSUFBSSxDQUFDLE9BQU8sQ0FBQyxHQUFHLENBQUMsRUFBRSxDQUFDLElBQUksQ0FBQyxDQUFDO1lBQzFDLElBQUksQ0FBQyxJQUFJLEVBQUUsQ0FBQztnQkFDVixJQUFJLENBQUMsTUFBTSxDQUFDLElBQUksQ0FBQyxrQkFBa0IsRUFBRSxZQUFZLEVBQUUsRUFBRSxZQUFZLEVBQUUsRUFBRSxFQUFFLENBQUMsQ0FBQztnQkFDekUsT0FBTztZQUNULENBQUM7WUFFRCxJQUFJLENBQUMsSUFBSSxFQUFFLENBQUM7Z0JBQ1YsSUFBSSxDQUFDLE1BQU0sQ0FBQyxJQUFJLENBQUMsa0JBQWtCLEVBQUUsU0FBUyxrQkFBa0IsQ0FBQyxJQUFJLENBQUMsSUFBSSxDQUFDLDZCQUE2QixFQUFFO29CQUN4RyxZQUFZLEVBQUUsRUFBRTtpQkFDakIsQ0FBQyxDQUFDO1lBQ0wsQ0FBQztpQkFBTSxDQUFDO2dCQUNOLElBQUksQ0FBQyxVQUFVLENBQUMsTUFBTSxDQUFDLEVBQUUsQ0FBQyxDQUFDO1lBQzdCLENBQUM7WUFFRCxJQUFJLElBQUksQ0FBQyxZQUFZLENBQUMsR0FBRyxDQUFDLEVBQUUsQ0FBQyxFQUFFLENBQUM7Z0JBQzlCLElBQUksQ0FBQyxNQUFNLENBQUMsSUFBSSxDQUFDLGtCQUFrQixFQUFFLG1DQUFtQyxFQUFFLEVBQUUsWUFBWSxFQUFFLEVBQUUsRUFBRSxDQUFDLENBQUM7Z0JBQ2hHLE9BQU87WUFDVCxDQUFDO1lBRUQsSUFBSSxDQUFDLE1BQU0sQ0FBQyxJQUFJLENBQ2QsMkJBQTJCLEVBQUUsU0FBUyxrQkFBa0IsQ0FBQyxJQUFJLENBQUMsSUFBSSxDQUFDLGtCQUFrQixPQUFPLEdBQUcsQ0FBQyxFQUFFLEVBQ2xHLEVBQUUsWUFBWSxFQUFFLEVBQUUsRUFBRSxDQUNyQixDQUFDO1lBRUYsb0RBQW9EO1lBQ3BELDRFQUE0RTtZQUM1RSxtR0FBbUc7WUFDbkcsTUFBTSxNQUFNLEdBQTRCLEVBQUUsTUFBTSxFQUFFLFdBQVcsRUFBRSxLQUFLLEVBQUUsQ0FBQztZQUN2RSxJQUFJLENBQUMsWUFBWSxDQUFDLEdBQUcsQ0FBQyxFQUFFLEVBQUUsTUFBTSxDQUFDLENBQUM7WUFDbEMsSUFBSSxDQUFDLFFBQVEsQ0FBQyxHQUFHLENBQUMsRUFBRSxDQUFFLENBQUMsT0FBTyxDQUFDLE1BQU0sQ0FBQyxDQUFDO1lBQ3ZDLElBQUksQ0FBQyx5QkFBeUIsQ0FBQyxJQUFJLENBQUMsRUFBRSxDQUFDLENBQUM7WUFFeEMsSUFBSSxDQUFDLGVBQWUsQ0FBQyxlQUFlLENBQUMsSUFBSSxDQUFDLElBQUksQ0FBQyxDQUFDO1lBQ2hELElBQUksSUFBSSxFQUFFLENBQUM7Z0JBQ1QsTUFBTSxRQUFRLEdBQUcsSUFBSSxDQUFDLFlBQVksRUFBRSxHQUFHLElBQUksQ0FBQyxTQUFTLENBQUM7Z0JBQ3RELElBQUksQ0FBQyxlQUFlLENBQUMsaUJBQWlCLENBQUMsSUFBSSxDQUFDLElBQUksRUFBRSxRQUFRLENBQUMsQ0FBQztZQUM5RCxDQUFDO1lBRUQsSUFBSSxDQUFDO2dCQUNILE1BQU0sSUFBSSxDQUFDLFFBQVEsQ0FBQyxtQkFBbUIsQ0FBQyxFQUFFLEVBQUUsS0FBSyxDQUFDLENBQUM7WUFDckQsQ0FBQztZQUFDLE9BQU8sT0FBTyxFQUFFLENBQUM7Z0JBQ2pCLElBQUksQ0FBQyxNQUFNLENBQUMsS0FBSyxDQUFDLHdDQUF3QyxFQUFFLEVBQUUsRUFBRSxPQUFPLEVBQUU7b0JBQ3ZFLFlBQVksRUFBRSxFQUFFO2lCQUNqQixDQUFDLENBQUM7Z0JBRUgsTUFBTSxPQUFPLENBQUM7WUFDaEIsQ0FBQztRQUNILENBQUM7Ozt1Q0FFQSxTQUFTLENBQUMsMkJBQTJCLENBQUM7WUFDdkMsa0xBQWMsV0FBVyw2REFReEI7Ozs7O1NBdGVVLGFBQWE7QUFzaUIxQjs7Ozs7R0FLRztBQUNILFNBQVMsb0JBQW9CLENBQUMsQ0FBcUIsRUFBRSxDQUFxQjtJQUN4RSxJQUFJLENBQUMsQ0FBQyxXQUFXLEdBQUcsQ0FBQyxDQUFDLFdBQVcsRUFBRSxDQUFDO1FBQ2xDLE9BQU8sQ0FBQyxDQUFDLENBQUM7SUFDWixDQUFDO1NBQU0sSUFBSSxDQUFDLENBQUMsV0FBVyxHQUFHLENBQUMsQ0FBQyxXQUFXLEVBQUUsQ0FBQztRQUN6QyxPQUFPLENBQUMsQ0FBQztJQUNYLENBQUM7U0FBTSxDQUFDO1FBQ04sT0FBTyxDQUFDLENBQUM7SUFDWCxDQUFDO0FBQ0gsQ0FBQztBQUVEOzs7Ozs7O0dBT0c7QUFDSCxTQUFTLG1CQUFtQixDQUFDLENBQXFCLEVBQUUsQ0FBcUI7SUFDdkUsTUFBTSxRQUFRLEdBQUcsNkJBQTZCLENBQUMsT0FBTyxDQUFDLENBQUMsQ0FBQyxDQUFDO0lBQzFELE1BQU0sUUFBUSxHQUFHLDZCQUE2QixDQUFDLE9BQU8sQ0FBQyxDQUFDLENBQUMsQ0FBQztJQUMxRCxJQUFJLFFBQVEsS0FBSyxRQUFRLEVBQUUsQ0FBQztRQUMxQixPQUFPLENBQUMsQ0FBQztJQUNYLENBQUM7U0FBTSxJQUFJLFFBQVEsS0FBSyxDQUFDLENBQUMsRUFBRSxDQUFDO1FBQzNCLHlEQUF5RDtRQUN6RCwrQ0FBK0M7UUFDL0MsT0FBTyxDQUFDLENBQUM7SUFDWCxDQUFDO1NBQU0sSUFBSSxRQUFRLEtBQUssQ0FBQyxDQUFDLEVBQUUsQ0FBQztRQUMzQix5Q0FBeUM7UUFDekMsT0FBTyxDQUFDLENBQUMsQ0FBQztJQUNaLENBQUM7U0FBTSxJQUFJLFFBQVEsR0FBRyxRQUFRLEVBQUUsQ0FBQztRQUMvQixPQUFPLENBQUMsQ0FBQyxDQUFDO0lBQ1osQ0FBQztTQUFNLENBQUM7UUFDTixPQUFPLENBQUMsQ0FBQztJQUNYLENBQUM7QUFDSCxDQUFDO0FBRUQ7Ozs7OztHQU1HO0FBQ0gsTUFBTSw2QkFBNkIsR0FBeUI7SUFDMUQsa0JBQWtCLENBQUMsaUJBQWlCO0lBQ3BDLGtCQUFrQixDQUFDLDJCQUEyQjtJQUM5QyxrQkFBa0IsQ0FBQyxrQkFBa0I7SUFDckMsa0JBQWtCLENBQUMsV0FBVztJQUM5QixrQkFBa0IsQ0FBQyxZQUFZO0lBQy9CLGtCQUFrQixDQUFDLGtCQUFrQjtJQUNyQyxrQkFBa0IsQ0FBQyxtQkFBbUI7SUFDdEMsa0JBQWtCLENBQUMsU0FBUztJQUM1QixrQkFBa0IsQ0FBQyxVQUFVO0lBQzdCLGtCQUFrQixDQUFDLFdBQVc7SUFDOUIsa0JBQWtCLENBQUMsV0FBVztJQUM5QixrQkFBa0IsQ0FBQyx1QkFBdUI7Q0FDM0MsQ0FBQyJ9