@aztec/prover-client 0.67.1 → 0.68.0

package/src/proving_broker/proving_broker.ts

@@ -14,7 +14,7 @@ import { createLogger } from '@aztec/foundation/log';
 import { type PromiseWithResolvers, RunningPromise, promiseWithResolvers } from '@aztec/foundation/promise';
 import { PriorityMemoryQueue } from '@aztec/foundation/queue';
 import { Timer } from '@aztec/foundation/timer';
-import { type TelemetryClient } from '@aztec/telemetry-client';
+import { type TelemetryClient, type Traceable, type Tracer, trackSpan } from '@aztec/telemetry-client';
 
 import assert from 'assert';
 
@@ -41,7 +41,7 @@ type EnqueuedProvingJob = Pick<ProvingJob, 'id' | 'epochNumber'>;
  * A broker that manages proof requests and distributes them to workers based on their priority.
  * It takes a backend that is responsible for storing and retrieving proof requests and results.
  */
-export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer {
+export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Traceable {
   private queues: ProvingQueues = {
     [ProvingRequestType.PUBLIC_VM]: new PriorityMemoryQueue<EnqueuedProvingJob>(provingJobComparator),
     [ProvingRequestType.TUBE_PROOF]: new PriorityMemoryQueue<EnqueuedProvingJob>(provingJobComparator),
@@ -87,6 +87,7 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer {
   private maxRetries: number;
 
   private instrumentation: ProvingBrokerInstrumentation;
+  public readonly tracer: Tracer;
 
   private maxParallelCleanUps: number;
 
@@ -115,8 +116,9 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer {
     }: ProofRequestBrokerConfig = {},
     private logger = createLogger('prover-client:proving-broker'),
   ) {
+    this.tracer = client.getTracer('ProvingBroker');
     this.instrumentation = new ProvingBrokerInstrumentation(client);
-    this.cleanupPromise = new RunningPromise(this.cleanupPass, timeoutIntervalMs);
+    this.cleanupPromise = new RunningPromise(this.cleanupPass.bind(this), this.logger, timeoutIntervalMs);
     this.jobTimeoutMs = jobTimeoutMs;
     this.maxRetries = maxRetries;
     this.maxEpochsToKeepResultsFor = maxEpochsToKeepResultsFor;
@@ -141,7 +143,10 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer {
 
   public start(): Promise<void> {
     for (const [item, result] of this.database.allProvingJobs()) {
-      this.logger.info(`Restoring proving job id=${item.id} settled=${!!result}`);
+      this.logger.info(`Restoring proving job id=${item.id} settled=${!!result}`, {
+        provingJobId: item.id,
+        status: result ? result.status : 'pending',
+      });
 
       this.jobsCache.set(item.id, item);
       this.promises.set(item.id, promiseWithResolvers());
@@ -150,7 +155,6 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer {
         this.promises.get(item.id)!.resolve(result);
         this.resultsCache.set(item.id, result);
       } else {
-        this.logger.debug(`Re-enqueuing proving job id=${item.id}`);
         this.enqueueJobInternal(item);
       }
     }
@@ -171,37 +175,66 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer {
     if (this.jobsCache.has(job.id)) {
       const existing = this.jobsCache.get(job.id);
       assert.deepStrictEqual(job, existing, 'Duplicate proving job ID');
+      this.logger.debug(`Duplicate proving job id=${job.id} epochNumber=${job.epochNumber}. Ignoring`, {
+        provingJobId: job.id,
+      });
       return;
     }
 
-    await this.database.addProvingJob(job);
-    this.jobsCache.set(job.id, job);
-    this.enqueueJobInternal(job);
+    if (this.isJobStale(job)) {
+      this.logger.warn(`Tried enqueueing stale proving job id=${job.id} epochNumber=${job.epochNumber}`, {
+        provingJobId: job.id,
+      });
+      throw new Error(`Epoch too old: job epoch ${job.epochNumber}, current epoch: ${this.epochHeight}`);
+    }
+
+    this.logger.info(`New proving job id=${job.id} epochNumber=${job.epochNumber}`, { provingJobId: job.id });
+    try {
+      // do this first so it acts as a "lock". If this job is enqueued again while we're saving it the if at the top will catch it.
+      this.jobsCache.set(job.id, job);
+      await this.database.addProvingJob(job);
+      this.enqueueJobInternal(job);
+    } catch (err) {
+      this.logger.error(`Failed to save proving job id=${job.id}: ${err}`, err, { provingJobId: job.id });
+      this.jobsCache.delete(job.id);
+      throw err;
+    }
   }
 
   public waitForJobToSettle(id: ProvingJobId): Promise<ProvingJobSettledResult> {
     const promiseWithResolvers = this.promises.get(id);
     if (!promiseWithResolvers) {
+      this.logger.warn(`Job id=${id} not found`, { provingJobId: id });
       return Promise.resolve({ status: 'rejected', reason: `Job ${id} not found` });
     }
     return promiseWithResolvers.promise;
   }
 
   public async cancelProvingJob(id: ProvingJobId): Promise<void> {
+    if (!this.jobsCache.has(id)) {
+      this.logger.warn(`Can't cancel a job that doesn't exist id=${id}`, { provingJobId: id });
+      return;
+    }
+
     // notify listeners of the cancellation
     if (!this.resultsCache.has(id)) {
-      this.logger.info(`Cancelling job id=${id}`);
+      this.logger.info(`Cancelling job id=${id}`, { provingJobId: id });
       await this.reportProvingJobError(id, 'Aborted', false);
     }
   }
 
-  public async cleanUpProvingJobState(id: ProvingJobId): Promise<void> {
+  private async cleanUpProvingJobState(id: ProvingJobId): Promise<void> {
+    if (!this.jobsCache.has(id)) {
+      this.logger.warn(`Can't clean up a job that doesn't exist id=${id}`, { provingJobId: id });
+      return;
+    }
+
     if (!this.resultsCache.has(id)) {
-      this.logger.warn(`Can't cleanup busy proving job: id=${id}`);
+      this.logger.warn(`Can't cleanup busy proving job: id=${id}`, { provingJobId: id });
       return;
     }
 
-    this.logger.debug(`Cleaning up state for job id=${id}`);
+    this.logger.debug(`Cleaning up state for job id=${id}`, { provingJobId: id });
     await this.database.deleteProvingJobAndResult(id);
     this.jobsCache.delete(id);
     this.promises.delete(id);
@@ -219,7 +252,7 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer {
       const item = this.jobsCache.get(id);
 
       if (!item) {
-        this.logger.warn(`Proving job id=${id} not found`);
+        this.logger.warn(`Proving job id=${id} not found`, { provingJobId: id });
         return Promise.resolve({ status: 'not-found' });
       }
 
@@ -272,45 +305,68 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer {
     const retries = this.retries.get(id) ?? 0;
 
     if (!item) {
-      this.logger.warn(`Proving job id=${id} not found`);
+      this.logger.warn(`Can't set error on unknown proving job id=${id} err=${err}`, { provingJoId: id });
       return;
     }
 
     if (!info) {
-      this.logger.warn(`Proving job id=${id} type=${ProvingRequestType[item.type]} not in the in-progress set`);
+      this.logger.warn(`Proving job id=${id} type=${ProvingRequestType[item.type]} not in the in-progress set`, {
+        provingJobId: id,
+      });
     } else {
       this.inProgress.delete(id);
     }
 
     if (this.resultsCache.has(id)) {
-      this.logger.warn(`Proving job id=${id} already is already settled, ignoring error`);
+      this.logger.warn(`Proving job id=${id} is already settled, ignoring err=${err}`, {
+        provingJobId: id,
+      });
       return;
     }
 
     if (retry && retries + 1 < this.maxRetries && !this.isJobStale(item)) {
-      this.logger.info(`Retrying proving job id=${id} type=${ProvingRequestType[item.type]} retry=${retries + 1}`);
+      this.logger.info(
+        `Retrying proving job id=${id} type=${ProvingRequestType[item.type]} retry=${retries + 1} err=${err}`,
+        {
+          provingJobId: id,
+        },
+      );
       this.retries.set(id, retries + 1);
       this.enqueueJobInternal(item);
       this.instrumentation.incRetriedJobs(item.type);
       return;
     }
 
-    this.logger.warn(
+    this.logger.info(
       `Marking proving job as failed id=${id} type=${ProvingRequestType[item.type]} totalAttempts=${
         retries + 1
       } err=${err}`,
+      {
+        provingJobId: id,
+      },
     );
 
-    await this.database.setProvingJobError(id, err);
-
+    // save the result to the cache and notify clients of the job status
+    // this should work even if our database breaks because the result is cached in memory
     const result: ProvingJobSettledResult = { status: 'rejected', reason: String(err) };
     this.resultsCache.set(id, result);
     this.promises.get(id)!.resolve(result);
+
     this.instrumentation.incRejectedJobs(item.type);
     if (info) {
       const duration = this.msTimeSource() - info.startedAt;
       this.instrumentation.recordJobDuration(item.type, duration);
     }
+
+    try {
+      await this.database.setProvingJobError(id, err);
+    } catch (saveErr) {
+      this.logger.error(`Failed to save proving job error status id=${id} jobErr=${err}`, saveErr, {
+        provingJobId: id,
+      });
+
+      throw saveErr;
+    }
   }
 
   reportProvingJobProgress(
@@ -320,12 +376,12 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer {
   ): Promise<{ job: ProvingJob; time: number } | undefined> {
     const job = this.jobsCache.get(id);
     if (!job) {
-      this.logger.warn(`Proving job id=${id} does not exist`);
+      this.logger.warn(`Proving job id=${id} does not exist`, { provingJobId: id });
       return filter ? this.getProvingJob(filter) : Promise.resolve(undefined);
     }
 
     if (this.resultsCache.has(id)) {
-      this.logger.warn(`Proving job id=${id} has already been completed`);
+      this.logger.warn(`Proving job id=${id} has already been completed`, { provingJobId: id });
       return filter ? this.getProvingJob(filter) : Promise.resolve(undefined);
     }
 
@@ -334,6 +390,7 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer {
     if (!metadata) {
       this.logger.warn(
         `Proving job id=${id} type=${ProvingRequestType[job.type]} not found in the in-progress cache, adding it`,
+        { provingJobId: id },
       );
       // the queue will still contain the item at this point!
       // we need to be careful when popping off the queue to make sure we're not sending
@@ -346,11 +403,12 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer {
       return Promise.resolve(undefined);
     } else if (startedAt <= metadata.startedAt) {
       if (startedAt < metadata.startedAt) {
-        this.logger.debug(
+        this.logger.info(
           `Proving job id=${id} type=${ProvingRequestType[job.type]} startedAt=${startedAt} older agent has taken job`,
+          { provingJobId: id },
         );
       } else {
-        this.logger.debug(`Proving job id=${id} type=${ProvingRequestType[job.type]} heartbeat`);
+        this.logger.debug(`Proving job id=${id} type=${ProvingRequestType[job.type]} heartbeat`, { provingJobId: id });
       }
       metadata.startedAt = startedAt;
       metadata.lastUpdatedAt = now;
@@ -360,6 +418,7 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer {
         `Proving job id=${id} type=${
           ProvingRequestType[job.type]
         } already being worked on by another agent. Sending new one`,
+        { provingJobId: id },
       );
       return this.getProvingJob(filter);
     } else {
@@ -372,37 +431,57 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer {
     const item = this.jobsCache.get(id);
     const retries = this.retries.get(id) ?? 0;
     if (!item) {
-      this.logger.warn(`Proving job id=${id} not found`);
+      this.logger.warn(`Proving job id=${id} not found`, { provingJobId: id });
       return;
     }
 
     if (!info) {
-      this.logger.warn(`Proving job id=${id} type=${ProvingRequestType[item.type]} not in the in-progress set`);
+      this.logger.warn(`Proving job id=${id} type=${ProvingRequestType[item.type]} not in the in-progress set`, {
+        provingJobId: id,
+      });
     } else {
       this.inProgress.delete(id);
     }
 
     if (this.resultsCache.has(id)) {
-      this.logger.warn(`Proving job id=${id} already settled, ignoring result`);
+      this.logger.warn(`Proving job id=${id} already settled, ignoring result`, { provingJobId: id });
       return;
     }
 
-    this.logger.debug(
+    this.logger.info(
       `Proving job complete id=${id} type=${ProvingRequestType[item.type]} totalAttempts=${retries + 1}`,
+      { provingJobId: id },
     );
 
-    await this.database.setProvingJobResult(id, value);
-
+    // save result to our local cache and notify clients
+    // if save to database fails, that's ok because we have the result in memory
+    // if the broker crashes and needs the result again, we're covered because we can just recompute it
     const result: ProvingJobSettledResult = { status: 'fulfilled', value };
     this.resultsCache.set(id, result);
     this.promises.get(id)!.resolve(result);
+
     this.instrumentation.incResolvedJobs(item.type);
+    if (info) {
+      const duration = this.msTimeSource() - info.startedAt;
+      this.instrumentation.recordJobDuration(item.type, duration);
+    }
+
+    try {
+      await this.database.setProvingJobResult(id, value);
+    } catch (saveErr) {
+      this.logger.error(`Failed to save proving job result id=${id}`, saveErr, {
+        provingJobId: id,
+      });
+
+      throw saveErr;
+    }
   }
 
-  private cleanupPass = async () => {
+  @trackSpan('ProvingBroker.cleanupPass')
+  private async cleanupPass() {
     await this.cleanupStaleJobs();
     await this.reEnqueueExpiredJobs();
-  };
+  }
 
   private async cleanupStaleJobs() {
     const jobIds = Array.from(this.jobsCache.keys());
@@ -416,7 +495,7 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer {
     }
 
     if (jobsToClean.length > 0) {
-      this.logger.info(`Cleaning up [${jobsToClean.join(',')}]`);
+      this.logger.info(`Cleaning up jobs=${jobsToClean.length}`);
       await asyncPool(this.maxParallelCleanUps, jobsToClean, async jobId => {
         await this.cleanUpProvingJobState(jobId);
       });
@@ -428,7 +507,7 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer {
     for (const [id, metadata] of inProgressEntries) {
       const item = this.jobsCache.get(id);
       if (!item) {
-        this.logger.warn(`Proving job id=${id} not found. Removing it from the queue.`);
+        this.logger.warn(`Proving job id=${id} not found. Removing it from the queue.`, { provingJobId: id });
         this.inProgress.delete(id);
         continue;
       }
@@ -440,7 +519,7 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer {
           // the job has timed out and it's also old, just cancel and move on
           await this.cancelProvingJob(item.id);
         } else {
-          this.logger.warn(`Proving job id=${id} timed out. Adding it back to the queue.`);
+          this.logger.warn(`Proving job id=${id} timed out. Adding it back to the queue.`, { provingJobId: id });
           this.inProgress.delete(id);
           this.enqueueJobInternal(item);
           this.instrumentation.incTimedOutJobs(item.type);
@@ -459,7 +538,6 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer {
     });
     this.enqueuedAt.set(job.id, new Timer());
     this.epochHeight = Math.max(this.epochHeight, job.epochNumber);
-    this.logger.debug(`Enqueued new proving job id=${job.id}`);
   }
 
   private isJobStale(job: ProvingJob) {

package/src/proving_broker/proving_job_controller.ts

@@ -30,6 +30,7 @@ export class ProvingJobController {
   constructor(
     private jobId: ProvingJobId,
     private inputs: ProvingJobInputs,
+    private epochNumber: number,
     private startedAt: number,
     private circuitProver: ServerCircuitProver,
     private onComplete: ProvingJobCompletionCallback,
@@ -100,51 +101,51 @@ export class ProvingJobController {
     const signal = this.abortController.signal;
     switch (type) {
       case ProvingRequestType.PUBLIC_VM: {
-        return await this.circuitProver.getAvmProof(inputs, signal);
+        return await this.circuitProver.getAvmProof(inputs, signal, this.epochNumber);
       }
 
       case ProvingRequestType.PRIVATE_BASE_ROLLUP: {
-        return await this.circuitProver.getPrivateBaseRollupProof(inputs, signal);
+        return await this.circuitProver.getPrivateBaseRollupProof(inputs, signal, this.epochNumber);
       }
 
       case ProvingRequestType.PUBLIC_BASE_ROLLUP: {
-        return await this.circuitProver.getPublicBaseRollupProof(inputs, signal);
+        return await this.circuitProver.getPublicBaseRollupProof(inputs, signal, this.epochNumber);
       }
 
       case ProvingRequestType.MERGE_ROLLUP: {
-        return await this.circuitProver.getMergeRollupProof(inputs, signal);
+        return await this.circuitProver.getMergeRollupProof(inputs, signal, this.epochNumber);
       }
 
       case ProvingRequestType.EMPTY_BLOCK_ROOT_ROLLUP: {
-        return await this.circuitProver.getEmptyBlockRootRollupProof(inputs, signal);
+        return await this.circuitProver.getEmptyBlockRootRollupProof(inputs, signal, this.epochNumber);
       }
 
       case ProvingRequestType.BLOCK_ROOT_ROLLUP: {
-        return await this.circuitProver.getBlockRootRollupProof(inputs, signal);
+        return await this.circuitProver.getBlockRootRollupProof(inputs, signal, this.epochNumber);
       }
 
       case ProvingRequestType.BLOCK_MERGE_ROLLUP: {
-        return await this.circuitProver.getBlockMergeRollupProof(inputs, signal);
+        return await this.circuitProver.getBlockMergeRollupProof(inputs, signal, this.epochNumber);
       }
 
       case ProvingRequestType.ROOT_ROLLUP: {
-        return await this.circuitProver.getRootRollupProof(inputs, signal);
+        return await this.circuitProver.getRootRollupProof(inputs, signal, this.epochNumber);
       }
 
       case ProvingRequestType.BASE_PARITY: {
-        return await this.circuitProver.getBaseParityProof(inputs, signal);
+        return await this.circuitProver.getBaseParityProof(inputs, signal, this.epochNumber);
       }
 
       case ProvingRequestType.ROOT_PARITY: {
-        return await this.circuitProver.getRootParityProof(inputs, signal);
+        return await this.circuitProver.getRootParityProof(inputs, signal, this.epochNumber);
       }
 
       case ProvingRequestType.PRIVATE_KERNEL_EMPTY: {
-        return await this.circuitProver.getEmptyPrivateKernelProof(inputs, signal);
+        return await this.circuitProver.getEmptyPrivateKernelProof(inputs, signal, this.epochNumber);
       }
 
       case ProvingRequestType.TUBE_PROOF: {
-        return await this.circuitProver.getTubeProof(inputs, signal);
+        return await this.circuitProver.getTubeProof(inputs, signal, this.epochNumber);
       }
 
       default: {

package/src/proving_broker/rpc.ts

@@ -28,7 +28,6 @@ const GetProvingJobResponse = z.object({
 export const ProvingJobProducerSchema: ApiSchemaFor<ProvingJobProducer> = {
   enqueueProvingJob: z.function().args(ProvingJob).returns(z.void()),
   getProvingJobStatus: z.function().args(ProvingJobId).returns(ProvingJobStatus),
-  cleanUpProvingJobState: z.function().args(ProvingJobId).returns(z.void()),
   cancelProvingJob: z.function().args(ProvingJobId).returns(z.void()),
   waitForJobToSettle: z.function().args(ProvingJobId).returns(ProvingJobSettledResult),
 };

package/src/test/mock_prover.ts

@@ -14,27 +14,29 @@ import {
   AVM_PROOF_LENGTH_IN_FIELDS,
   AVM_VERIFICATION_KEY_LENGTH_IN_FIELDS,
   type AvmCircuitInputs,
-  type BaseOrMergeRollupPublicInputs,
   type BaseParityInputs,
-  type BlockMergeRollupInputs,
-  type BlockRootOrBlockMergePublicInputs,
-  type BlockRootRollupInputs,
-  type EmptyBlockRootRollupInputs,
   type KernelCircuitPublicInputs,
-  type MergeRollupInputs,
   NESTED_RECURSIVE_PROOF_LENGTH,
-  type PrivateBaseRollupInputs,
   type PrivateKernelEmptyInputData,
-  type PublicBaseRollupInputs,
   RECURSIVE_PROOF_LENGTH,
   type RootParityInputs,
-  type RootRollupInputs,
-  type RootRollupPublicInputs,
   TUBE_PROOF_LENGTH,
   VerificationKeyData,
   makeEmptyRecursiveProof,
   makeRecursiveProof,
 } from '@aztec/circuits.js';
+import {
+  type BaseOrMergeRollupPublicInputs,
+  type BlockMergeRollupInputs,
+  type BlockRootOrBlockMergePublicInputs,
+  type BlockRootRollupInputs,
+  type EmptyBlockRootRollupInputs,
+  type MergeRollupInputs,
+  type PrivateBaseRollupInputs,
+  type PublicBaseRollupInputs,
+  type RootRollupInputs,
+  type RootRollupPublicInputs,
+} from '@aztec/circuits.js/rollup';
 import {
   makeBaseOrMergeRollupPublicInputs,
   makeBlockRootOrBlockMergeRollupPublicInputs,
@@ -58,8 +60,12 @@ export class TestBroker implements ProvingJobProducer {
     agentCount: number,
     prover: ServerCircuitProver,
     private proofStore: ProofStore = new InlineProofStore(),
+    agentPollInterval = 100,
   ) {
-    this.agents = times(agentCount, () => new ProvingAgent(this.broker, proofStore, prover, new NoopTelemetryClient()));
+    this.agents = times(
+      agentCount,
+      () => new ProvingAgent(this.broker, proofStore, prover, new NoopTelemetryClient(), undefined, agentPollInterval),
+    );
   }
 
   public async start() {
@@ -82,9 +88,6 @@ export class TestBroker implements ProvingJobProducer {
   getProvingJobStatus(id: ProvingJobId): Promise<ProvingJobStatus> {
     return this.broker.getProvingJobStatus(id);
   }
-  cleanUpProvingJobState(id: ProvingJobId): Promise<void> {
-    return this.broker.cleanUpProvingJobState(id);
-  }
   cancelProvingJob(id: string): Promise<void> {
     return this.broker.cancelProvingJob(id);
   }

package/dest/proving_broker/caching_broker_facade.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"caching_broker_facade.d.ts","sourceRoot":"","sources":["../../src/proving_broker/caching_broker_facade.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,uBAAuB,EAC5B,KAAK,WAAW,EAGhB,KAAK,kBAAkB,EAGvB,KAAK,6BAA6B,EAClC,KAAK,mBAAmB,EACzB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EACL,KAAK,0BAA0B,EAC/B,KAAK,gBAAgB,EACrB,KAAK,6BAA6B,EAClC,KAAK,gBAAgB,EACrB,KAAK,sBAAsB,EAC3B,KAAK,iCAAiC,EACtC,KAAK,qBAAqB,EAC1B,KAAK,0BAA0B,EAC/B,KAAK,yBAAyB,EAC9B,KAAK,iBAAiB,EACtB,KAAK,6BAA6B,EAClC,KAAK,kBAAkB,EACvB,KAAK,uBAAuB,EAC5B,KAAK,2BAA2B,EAChC,KAAK,sBAAsB,EAC3B,KAAK,sBAAsB,EAC3B,KAAK,gBAAgB,EACrB,KAAK,gBAAgB,EACrB,KAAK,sBAAsB,EAC3B,KAAK,iBAAiB,EACtB,KAAK,UAAU,EAChB,MAAM,oBAAoB,CAAC;AAK5B,OAAO,EAAoB,KAAK,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAMrE;;GAEG;AACH,qBAAa,mBAAoB,YAAW,mBAAmB;IAE3D,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,KAAK;IACb,OAAO,CAAC,UAAU;IAClB,OAAO,CAAC,aAAa;IACrB,OAAO,CAAC,cAAc;IACtB,OAAO,CAAC,GAAG;gBALH,MAAM,EAAE,kBAAkB,EAC1B,KAAK,GAAE,WAAuC,EAC9C,UAAU,GAAE,UAAmC,EAC/C,aAAa,SAAc,EAC3B,cAAc,SAAO,EACrB,GAAG,yCAAsD;YAGrD,oBAAoB;IAmGlC,WAAW,CACT,MAAM,EAAE,gBAAgB,EACxB,MAAM,CAAC,EAAE,WAAW,EACpB,WAAW,CAAC,EAAE,MAAM,GACnB,OAAO,CAAC,uBAAuB,CAAC,OAAO,0BAA0B,CAAC,CAAC;IAUtE,kBAAkB,CAChB,MAAM,EAAE,gBAAgB,EACxB,MAAM,CAAC,EAAE,WAAW,EACpB,WAAW,CAAC,EAAE,MAAM,GACnB,OAAO,CAAC,6BAA6B,CAAC,kBAAkB,EAAE,OAAO,sBAAsB,CAAC,CAAC;IAU5F,wBAAwB,CACtB,KAAK,EAAE,sBAAsB,EAC7B,MAAM,CAAC,EAAE,WAAW,EACpB,WAAW,CAAC,EAAE,MAAM,GACnB,OAAO,CAAC,6BAA6B,CAAC,iCAAiC,EAAE,OAAO,sBAAsB,CAAC,CAAC;IAU3G,uBAAuB,CACrB,KAAK,EAAE,qBAAqB,EAC5B,MAAM,CAAC,EAAE,WAAW,EACpB,WAAW,CAAC,EAAE,MAAM,GACnB,OAAO,CAAC,6BAA6B,CAAC,iCAAiC,EAAE,OAAO,sBAAsB,CAAC,CAAC;IAU3G,4BAA4B,CAC1B,KAAK,EAAE,0BAA0B,EACjC,MAAM,CAAC,EAAE,WAAW,EACpB,WAAW,CAAC,EAAE,MAAM,GACnB,OAAO,CAAC,6BAA6B,CAAC,iCAAiC,CAAC,CAAC;IAU5E,0BAA0B,CACxB,MAAM,EAAE,2BAA2B,EACnC,MAAM,CAAC,EAAE,WAAW,EACpB,WAAW,CAAC,EAAE,MAAM,GACnB,OAAO,CAAC,6BAA6B,CAAC,yBAAyB,EAAE,OAAO,sBAAsB,CAAC,CAAC;IAUnG,mBAAmB,CACjB,KAAK,EAAE,iBAAiB,EACxB,MAAM,CAAC,EAAE,WAAW,EACpB,WAAW,CAAC,EAAE,MAAM,GACnB,OAAO,CAAC,6BAA6B,CAAC,6BAA6B,EAAE,OAAO,sBAAsB,CAAC,CAAC;IASvG,yBAAyB,CACvB,eAAe,EAAE,uBAAuB,EACxC,MAAM,CAAC,EAAE,WAAW,EACpB,WAAW,CAAC,EAAE,MAAM,GACnB,OAAO,CAAC,6BAA6B,CAAC,6BAA6B,EAAE,OAAO,sBAAsB,CAAC,CAAC;IAUvG,wBAAwB,CACtB,MAAM,EAAE,sBAAsB,EAC9B,MAAM,CAAC,EAAE,WAAW,EACpB,WAAW,CAAC,EAAE,MAAM,GACnB,OAAO,CAAC,6BAA6B,CAAC,6BAA6B,EAAE,OAAO,sBAAsB,CAAC,CAAC;IAUvG,kBAAkB,CAChB,MAAM,EAAE,gBAAgB,EACxB,MAAM,CAAC,EAAE,WAAW,EACpB,WAAW,CAAC,EAAE,MAAM,GACnB,OAAO,CAAC,6BAA6B,CAAC,kBAAkB,EAAE,OAAO,6BAA6B,CAAC,CAAC;IAUnG,kBAAkB,CAChB,KAAK,EAAE,gBAAgB,EACvB,MAAM,CAAC,EAAE,WAAW,EACpB,WAAW,CAAC,EAAE,MAAM,GACnB,OAAO,CAAC,6BAA6B,CAAC,sBAAsB,EAAE,OAAO,sBAAsB,CAAC,CAAC;IAUhG,YAAY,CACV,SAAS,EAAE,UAAU,EACrB,MAAM,CAAC,EAAE,WAAW,EACpB,WAAW,CAAC,EAAE,MAAM,GACnB,OAAO,CAAC,uBAAuB,CAAC,OAAO,iBAAiB,CAAC,CAAC;IAU7D,OAAO,CAAC,UAAU;CAInB"}