@aztec/prover-client 0.0.1-commit.03f7ef2 → 0.0.1-commit.04d373f

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (150) hide show
  1. package/dest/config.d.ts +1 -1
  2. package/dest/config.d.ts.map +1 -1
  3. package/dest/config.js +16 -2
  4. package/dest/light/lightweight_checkpoint_builder.d.ts +22 -6
  5. package/dest/light/lightweight_checkpoint_builder.d.ts.map +1 -1
  6. package/dest/light/lightweight_checkpoint_builder.js +112 -28
  7. package/dest/mocks/fixtures.d.ts +1 -1
  8. package/dest/mocks/fixtures.d.ts.map +1 -1
  9. package/dest/mocks/fixtures.js +2 -1
  10. package/dest/mocks/test_context.d.ts +6 -2
  11. package/dest/mocks/test_context.d.ts.map +1 -1
  12. package/dest/mocks/test_context.js +31 -11
  13. package/dest/orchestrator/block-building-helpers.d.ts +5 -5
  14. package/dest/orchestrator/block-building-helpers.d.ts.map +1 -1
  15. package/dest/orchestrator/block-building-helpers.js +4 -4
  16. package/dest/orchestrator/block-proving-state.d.ts +4 -1
  17. package/dest/orchestrator/block-proving-state.d.ts.map +1 -1
  18. package/dest/orchestrator/block-proving-state.js +7 -0
  19. package/dest/orchestrator/checkpoint-proving-state.d.ts +24 -4
  20. package/dest/orchestrator/checkpoint-proving-state.d.ts.map +1 -1
  21. package/dest/orchestrator/checkpoint-proving-state.js +47 -5
  22. package/dest/orchestrator/checkpoint-sub-tree-orchestrator.d.ts +107 -0
  23. package/dest/orchestrator/checkpoint-sub-tree-orchestrator.d.ts.map +1 -0
  24. package/dest/orchestrator/checkpoint-sub-tree-orchestrator.js +151 -0
  25. package/dest/orchestrator/epoch-proving-context.d.ts +51 -0
  26. package/dest/orchestrator/epoch-proving-context.d.ts.map +1 -0
  27. package/dest/orchestrator/epoch-proving-context.js +81 -0
  28. package/dest/orchestrator/epoch-proving-state.d.ts +7 -6
  29. package/dest/orchestrator/epoch-proving-state.d.ts.map +1 -1
  30. package/dest/orchestrator/epoch-proving-state.js +37 -1
  31. package/dest/orchestrator/index.d.ts +4 -1
  32. package/dest/orchestrator/index.d.ts.map +1 -1
  33. package/dest/orchestrator/index.js +3 -0
  34. package/dest/orchestrator/orchestrator.d.ts +30 -25
  35. package/dest/orchestrator/orchestrator.d.ts.map +1 -1
  36. package/dest/orchestrator/orchestrator.js +549 -255
  37. package/dest/orchestrator/orchestrator_metrics.d.ts +1 -1
  38. package/dest/orchestrator/orchestrator_metrics.d.ts.map +1 -1
  39. package/dest/orchestrator/orchestrator_metrics.js +2 -6
  40. package/dest/orchestrator/proving-scheduler.d.ts +72 -0
  41. package/dest/orchestrator/proving-scheduler.d.ts.map +1 -0
  42. package/dest/orchestrator/proving-scheduler.js +117 -0
  43. package/dest/orchestrator/top-tree-orchestrator.d.ts +83 -0
  44. package/dest/orchestrator/top-tree-orchestrator.d.ts.map +1 -0
  45. package/dest/orchestrator/top-tree-orchestrator.js +182 -0
  46. package/dest/orchestrator/top-tree-proving-scheduler.d.ts +62 -0
  47. package/dest/orchestrator/top-tree-proving-scheduler.d.ts.map +1 -0
  48. package/dest/orchestrator/top-tree-proving-scheduler.js +73 -0
  49. package/dest/orchestrator/top-tree-proving-state.d.ts +61 -0
  50. package/dest/orchestrator/top-tree-proving-state.d.ts.map +1 -0
  51. package/dest/orchestrator/top-tree-proving-state.js +185 -0
  52. package/dest/orchestrator/tx-proving-state.d.ts +5 -4
  53. package/dest/orchestrator/tx-proving-state.d.ts.map +1 -1
  54. package/dest/orchestrator/tx-proving-state.js +6 -6
  55. package/dest/prover-client/factory.d.ts +3 -3
  56. package/dest/prover-client/factory.d.ts.map +1 -1
  57. package/dest/prover-client/prover-client.d.ts +66 -7
  58. package/dest/prover-client/prover-client.d.ts.map +1 -1
  59. package/dest/prover-client/prover-client.js +64 -11
  60. package/dest/proving_broker/broker_prover_facade.d.ts +7 -5
  61. package/dest/proving_broker/broker_prover_facade.d.ts.map +1 -1
  62. package/dest/proving_broker/broker_prover_facade.js +16 -22
  63. package/dest/proving_broker/config.d.ts +12 -64
  64. package/dest/proving_broker/config.d.ts.map +1 -1
  65. package/dest/proving_broker/config.js +22 -5
  66. package/dest/proving_broker/index.d.ts +2 -1
  67. package/dest/proving_broker/index.d.ts.map +1 -1
  68. package/dest/proving_broker/index.js +1 -0
  69. package/dest/proving_broker/proof_store/factory.d.ts +2 -5
  70. package/dest/proving_broker/proof_store/factory.d.ts.map +1 -1
  71. package/dest/proving_broker/proof_store/factory.js +7 -30
  72. package/dest/proving_broker/proof_store/file_store_proof_store.d.ts +18 -0
  73. package/dest/proving_broker/proof_store/file_store_proof_store.d.ts.map +1 -0
  74. package/dest/proving_broker/proof_store/file_store_proof_store.js +60 -0
  75. package/dest/proving_broker/proof_store/index.d.ts +2 -2
  76. package/dest/proving_broker/proof_store/index.d.ts.map +1 -1
  77. package/dest/proving_broker/proof_store/index.js +1 -1
  78. package/dest/proving_broker/proving_agent.d.ts +5 -9
  79. package/dest/proving_broker/proving_agent.d.ts.map +1 -1
  80. package/dest/proving_broker/proving_agent.js +4 -19
  81. package/dest/proving_broker/proving_broker.d.ts +8 -5
  82. package/dest/proving_broker/proving_broker.d.ts.map +1 -1
  83. package/dest/proving_broker/proving_broker.js +69 -21
  84. package/dest/proving_broker/proving_broker_database/persisted.d.ts +3 -2
  85. package/dest/proving_broker/proving_broker_database/persisted.d.ts.map +1 -1
  86. package/dest/proving_broker/proving_broker_database/persisted.js +391 -3
  87. package/dest/proving_broker/proving_broker_instrumentation.d.ts +3 -1
  88. package/dest/proving_broker/proving_broker_instrumentation.d.ts.map +1 -1
  89. package/dest/proving_broker/proving_broker_instrumentation.js +22 -35
  90. package/dest/proving_broker/proving_job_controller.d.ts +4 -3
  91. package/dest/proving_broker/proving_job_controller.d.ts.map +1 -1
  92. package/dest/proving_broker/proving_job_controller.js +6 -3
  93. package/dest/proving_broker/rpc.d.ts +6 -2
  94. package/dest/proving_broker/rpc.d.ts.map +1 -1
  95. package/dest/proving_broker/rpc.js +87 -23
  96. package/dest/test/mock_proof_store.d.ts +3 -3
  97. package/dest/test/mock_proof_store.d.ts.map +1 -1
  98. package/dest/test/mock_prover.d.ts +5 -5
  99. package/dest/test/mock_prover.d.ts.map +1 -1
  100. package/dest/test/mock_prover.js +3 -3
  101. package/package.json +19 -21
  102. package/src/config.ts +18 -2
  103. package/src/light/lightweight_checkpoint_builder.ts +159 -37
  104. package/src/mocks/fixtures.ts +2 -1
  105. package/src/mocks/test_context.ts +24 -11
  106. package/src/orchestrator/block-building-helpers.ts +4 -4
  107. package/src/orchestrator/block-proving-state.ts +9 -0
  108. package/src/orchestrator/checkpoint-proving-state.ts +65 -6
  109. package/src/orchestrator/checkpoint-sub-tree-orchestrator.ts +271 -0
  110. package/src/orchestrator/epoch-proving-context.ts +101 -0
  111. package/src/orchestrator/epoch-proving-state.ts +59 -9
  112. package/src/orchestrator/index.ts +8 -0
  113. package/src/orchestrator/orchestrator.ts +165 -293
  114. package/src/orchestrator/orchestrator_metrics.ts +2 -6
  115. package/src/orchestrator/proving-scheduler.ts +156 -0
  116. package/src/orchestrator/top-tree-orchestrator.ts +314 -0
  117. package/src/orchestrator/top-tree-proving-scheduler.ts +154 -0
  118. package/src/orchestrator/top-tree-proving-state.ts +220 -0
  119. package/src/orchestrator/tx-proving-state.ts +8 -11
  120. package/src/prover-client/factory.ts +6 -2
  121. package/src/prover-client/prover-client.ts +157 -24
  122. package/src/proving_broker/broker_prover_facade.ts +25 -26
  123. package/src/proving_broker/config.ts +25 -2
  124. package/src/proving_broker/index.ts +1 -0
  125. package/src/proving_broker/proof_store/factory.ts +10 -32
  126. package/src/proving_broker/proof_store/file_store_proof_store.ts +78 -0
  127. package/src/proving_broker/proof_store/index.ts +1 -1
  128. package/src/proving_broker/proving_agent.ts +6 -19
  129. package/src/proving_broker/proving_broker.ts +65 -16
  130. package/src/proving_broker/proving_broker_database/persisted.ts +17 -3
  131. package/src/proving_broker/proving_broker_instrumentation.ts +23 -35
  132. package/src/proving_broker/proving_job_controller.ts +9 -3
  133. package/src/proving_broker/rpc.ts +46 -20
  134. package/src/test/mock_prover.ts +1 -8
  135. package/dest/block-factory/index.d.ts +0 -2
  136. package/dest/block-factory/index.d.ts.map +0 -1
  137. package/dest/block-factory/index.js +0 -1
  138. package/dest/block-factory/light.d.ts +0 -38
  139. package/dest/block-factory/light.d.ts.map +0 -1
  140. package/dest/block-factory/light.js +0 -108
  141. package/dest/proving_broker/proof_store/gcs_proof_store.d.ts +0 -14
  142. package/dest/proving_broker/proof_store/gcs_proof_store.d.ts.map +0 -1
  143. package/dest/proving_broker/proof_store/gcs_proof_store.js +0 -52
  144. package/dest/proving_broker/proving_agent_instrumentation.d.ts +0 -8
  145. package/dest/proving_broker/proving_agent_instrumentation.d.ts.map +0 -1
  146. package/dest/proving_broker/proving_agent_instrumentation.js +0 -16
  147. package/src/block-factory/index.ts +0 -1
  148. package/src/block-factory/light.ts +0 -137
  149. package/src/proving_broker/proof_store/gcs_proof_store.ts +0 -76
  150. package/src/proving_broker/proving_agent_instrumentation.ts +0 -21
@@ -0,0 +1,60 @@
1
+ import { jsonParseWithSchema, jsonStringify } from '@aztec/foundation/json-rpc';
2
+ import { ProvingJobResult, getProvingJobInputClassFor } from '@aztec/stdlib/interfaces/server';
3
+ import { ProvingRequestType } from '@aztec/stdlib/proofs';
4
+ const INPUTS_PATH = 'inputs';
5
+ const OUTPUTS_PATH = 'outputs';
6
+ /**
7
+ * A proof store implementation backed by a generic FileStore.
8
+ * Supports any storage backend (GCS, S3, local filesystem) via the FileStore abstraction.
9
+ */ export class FileStoreProofStore {
10
+ fileStore;
11
+ constructor(fileStore){
12
+ this.fileStore = fileStore;
13
+ }
14
+ async saveProofInput(id, type, inputs) {
15
+ const path = `${INPUTS_PATH}/${ProvingRequestType[type]}/${id}`;
16
+ const uri = await this.fileStore.save(path, inputs.toBuffer());
17
+ return uri;
18
+ }
19
+ async saveProofOutput(id, type, result) {
20
+ const jobResult = {
21
+ type,
22
+ result
23
+ };
24
+ const json = jsonStringify(jobResult);
25
+ const path = `${OUTPUTS_PATH}/${ProvingRequestType[type]}/${id}.json`;
26
+ const uri = await this.fileStore.save(path, Buffer.from(json, 'utf-8'));
27
+ return uri;
28
+ }
29
+ async getProofInput(uri) {
30
+ try {
31
+ const buffer = await this.fileStore.read(uri);
32
+ const type = this.extractTypeFromUri(uri);
33
+ const inputs = getProvingJobInputClassFor(type).fromBuffer(buffer);
34
+ return {
35
+ inputs,
36
+ type
37
+ };
38
+ } catch (err) {
39
+ throw new Error(`Error getting proof input at ${uri}: ${err}`);
40
+ }
41
+ }
42
+ async getProofOutput(uri) {
43
+ try {
44
+ const buffer = await this.fileStore.read(uri);
45
+ return jsonParseWithSchema(buffer.toString('utf-8'), ProvingJobResult);
46
+ } catch (err) {
47
+ throw new Error(`Error getting proof output at ${uri}: ${err}`);
48
+ }
49
+ }
50
+ extractTypeFromUri(uri) {
51
+ const url = new URL(uri);
52
+ const pathParts = url.pathname.split('/').filter(Boolean);
53
+ const typeString = pathParts.at(-2);
54
+ const type = typeString ? ProvingRequestType[typeString] : undefined;
55
+ if (type === undefined) {
56
+ throw new Error(`Unrecognized proof type ${typeString} in URI ${uri}`);
57
+ }
58
+ return type;
59
+ }
60
+ }
@@ -1,5 +1,5 @@
1
1
  export * from './proof_store.js';
2
2
  export * from './inline_proof_store.js';
3
3
  export * from './factory.js';
4
- export * from './gcs_proof_store.js';
5
- //# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiaW5kZXguZC50cyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uLy4uL3NyYy9wcm92aW5nX2Jyb2tlci9wcm9vZl9zdG9yZS9pbmRleC50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSxjQUFjLGtCQUFrQixDQUFDO0FBQ2pDLGNBQWMseUJBQXlCLENBQUM7QUFDeEMsY0FBYyxjQUFjLENBQUM7QUFDN0IsY0FBYyxzQkFBc0IsQ0FBQyJ9
4
+ export * from './file_store_proof_store.js';
5
+ //# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiaW5kZXguZC50cyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uLy4uL3NyYy9wcm92aW5nX2Jyb2tlci9wcm9vZl9zdG9yZS9pbmRleC50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSxjQUFjLGtCQUFrQixDQUFDO0FBQ2pDLGNBQWMseUJBQXlCLENBQUM7QUFDeEMsY0FBYyxjQUFjLENBQUM7QUFDN0IsY0FBYyw2QkFBNkIsQ0FBQyJ9
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/proving_broker/proof_store/index.ts"],"names":[],"mappings":"AAAA,cAAc,kBAAkB,CAAC;AACjC,cAAc,yBAAyB,CAAC;AACxC,cAAc,cAAc,CAAC;AAC7B,cAAc,sBAAsB,CAAC"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/proving_broker/proof_store/index.ts"],"names":[],"mappings":"AAAA,cAAc,kBAAkB,CAAC;AACjC,cAAc,yBAAyB,CAAC;AACxC,cAAc,cAAc,CAAC;AAC7B,cAAc,6BAA6B,CAAC"}
@@ -1,4 +1,4 @@
1
1
  export * from './proof_store.js';
2
2
  export * from './inline_proof_store.js';
3
3
  export * from './factory.js';
4
- export * from './gcs_proof_store.js';
4
+ export * from './file_store_proof_store.js';
@@ -1,11 +1,11 @@
1
+ import { type LoggerBindings } from '@aztec/foundation/log';
1
2
  import type { ProverAgentStatus, ProvingJobConsumer, ServerCircuitProver } from '@aztec/stdlib/interfaces/server';
2
3
  import { ProvingRequestType } from '@aztec/stdlib/proofs';
3
- import { type TelemetryClient, type Traceable, type Tracer } from '@aztec/telemetry-client';
4
4
  import type { ProofStore } from './proof_store/index.js';
5
5
  /**
6
6
  * A helper class that encapsulates a circuit prover and connects it to a job source.
7
7
  */
8
- export declare class ProvingAgent implements Traceable {
8
+ export declare class ProvingAgent {
9
9
  /** The source of proving jobs */
10
10
  private broker;
11
11
  /** Database holding proof inputs and outputs */
@@ -16,11 +16,9 @@ export declare class ProvingAgent implements Traceable {
16
16
  private proofAllowList;
17
17
  /** How long to wait between jobs */
18
18
  private pollIntervalMs;
19
- private log;
20
19
  private currentJobController?;
21
20
  private runningPromise;
22
- private instrumentation;
23
- readonly tracer: Tracer;
21
+ private log;
24
22
  constructor(
25
23
  /** The source of proving jobs */
26
24
  broker: ProvingJobConsumer,
@@ -31,9 +29,7 @@ export declare class ProvingAgent implements Traceable {
31
29
  /** Optional list of allowed proof types to build */
32
30
  proofAllowList?: Array<ProvingRequestType>,
33
31
  /** How long to wait between jobs */
34
- pollIntervalMs?: number,
35
- /** A telemetry client through which to emit metrics */
36
- client?: TelemetryClient, log?: import("@aztec/foundation/log").Logger);
32
+ pollIntervalMs?: number, bindings?: LoggerBindings);
37
33
  setCircuitProver(circuitProver: ServerCircuitProver): void;
38
34
  isRunning(): boolean;
39
35
  start(): void;
@@ -43,4 +39,4 @@ export declare class ProvingAgent implements Traceable {
43
39
  private startJob;
44
40
  private reportResult;
45
41
  }
46
- //# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoicHJvdmluZ19hZ2VudC5kLnRzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vLi4vc3JjL3Byb3ZpbmdfYnJva2VyL3Byb3ZpbmdfYWdlbnQudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBS0EsT0FBTyxLQUFLLEVBRVYsaUJBQWlCLEVBQ2pCLGtCQUFrQixFQUlsQixtQkFBbUIsRUFDcEIsTUFBTSxpQ0FBaUMsQ0FBQztBQUN6QyxPQUFPLEVBQUUsa0JBQWtCLEVBQUUsTUFBTSxzQkFBc0IsQ0FBQztBQUMxRCxPQUFPLEVBQ0wsS0FBSyxlQUFlLEVBQ3BCLEtBQUssU0FBUyxFQUNkLEtBQUssTUFBTSxFQUdaLE1BQU0seUJBQXlCLENBQUM7QUFFakMsT0FBTyxLQUFLLEVBQUUsVUFBVSxFQUFFLE1BQU0sd0JBQXdCLENBQUM7QUFJekQ7O0dBRUc7QUFDSCxxQkFBYSxZQUFhLFlBQVcsU0FBUztJQVExQyxpQ0FBaUM7SUFDakMsT0FBTyxDQUFDLE1BQU07SUFDZCxnREFBZ0Q7SUFDaEQsT0FBTyxDQUFDLFVBQVU7SUFDbEIsaURBQWlEO0lBQ2pELE9BQU8sQ0FBQyxhQUFhO0lBQ3JCLG9EQUFvRDtJQUNwRCxPQUFPLENBQUMsY0FBYztJQUN0QixvQ0FBb0M7SUFDcEMsT0FBTyxDQUFDLGNBQWM7SUFHdEIsT0FBTyxDQUFDLEdBQUc7SUFuQmIsT0FBTyxDQUFDLG9CQUFvQixDQUFDLENBQXVCO0lBQ3BELE9BQU8sQ0FBQyxjQUFjLENBQWlCO0lBQ3ZDLE9BQU8sQ0FBQyxlQUFlLENBQThCO0lBRXJELFNBQWdCLE1BQU0sRUFBRSxNQUFNLENBQUM7SUFFL0I7SUFDRSxpQ0FBaUM7SUFDekIsTUFBTSxFQUFFLGtCQUFrQjtJQUNsQyxnREFBZ0Q7SUFDeEMsVUFBVSxFQUFFLFVBQVU7SUFDOUIsaURBQWlEO0lBQ3pDLGFBQWEsRUFBRSxtQkFBbUI7SUFDMUMsb0RBQW9EO0lBQzVDLGNBQWMsR0FBRSxLQUFLLENBQUMsa0JBQWtCLENBQU07SUFDdEQsb0NBQW9DO0lBQzVCLGNBQWMsU0FBTztJQUM3Qix1REFBdUQ7SUFDdkQsTUFBTSxHQUFFLGVBQXNDLEVBQ3RDLEdBQUcseUNBQThDLEVBSzFEO0lBRU0sZ0JBQWdCLENBQUMsYUFBYSxFQUFFLG1CQUFtQixHQUFHLElBQUksQ0FFaEU7SUFFTSxTQUFTLElBQUksT0FBTyxDQUUxQjtJQUVNLEtBQUssSUFBSSxJQUFJLENBRW5CO0lBRVksSUFBSSxJQUFJLE9BQU8sQ0FBQyxJQUFJLENBQUMsQ0FHakM7SUFFTSxTQUFTLElBQUksaUJBQWlCLENBV3BDO1lBR2EsSUFBSTtZQXFESixRQUFRO1lBdURSLFlBQVk7Q0EwQjNCIn0=
42
+ //# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoicHJvdmluZ19hZ2VudC5kLnRzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vLi4vc3JjL3Byb3ZpbmdfYnJva2VyL3Byb3ZpbmdfYWdlbnQudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQ0EsT0FBTyxFQUFlLEtBQUssY0FBYyxFQUFnQixNQUFNLHVCQUF1QixDQUFDO0FBSXZGLE9BQU8sS0FBSyxFQUVWLGlCQUFpQixFQUNqQixrQkFBa0IsRUFJbEIsbUJBQW1CLEVBQ3BCLE1BQU0saUNBQWlDLENBQUM7QUFDekMsT0FBTyxFQUFFLGtCQUFrQixFQUFFLE1BQU0sc0JBQXNCLENBQUM7QUFFMUQsT0FBTyxLQUFLLEVBQUUsVUFBVSxFQUFFLE1BQU0sd0JBQXdCLENBQUM7QUFHekQ7O0dBRUc7QUFDSCxxQkFBYSxZQUFZO0lBTXJCLGlDQUFpQztJQUNqQyxPQUFPLENBQUMsTUFBTTtJQUNkLGdEQUFnRDtJQUNoRCxPQUFPLENBQUMsVUFBVTtJQUNsQixpREFBaUQ7SUFDakQsT0FBTyxDQUFDLGFBQWE7SUFDckIsb0RBQW9EO0lBQ3BELE9BQU8sQ0FBQyxjQUFjO0lBQ3RCLG9DQUFvQztJQUNwQyxPQUFPLENBQUMsY0FBYztJQWR4QixPQUFPLENBQUMsb0JBQW9CLENBQUMsQ0FBdUI7SUFDcEQsT0FBTyxDQUFDLGNBQWMsQ0FBaUI7SUFDdkMsT0FBTyxDQUFDLEdBQUcsQ0FBUztJQUVwQjtJQUNFLGlDQUFpQztJQUN6QixNQUFNLEVBQUUsa0JBQWtCO0lBQ2xDLGdEQUFnRDtJQUN4QyxVQUFVLEVBQUUsVUFBVTtJQUM5QixpREFBaUQ7SUFDekMsYUFBYSxFQUFFLG1CQUFtQjtJQUMxQyxvREFBb0Q7SUFDNUMsY0FBYyxHQUFFLEtBQUssQ0FBQyxrQkFBa0IsQ0FBTTtJQUN0RCxvQ0FBb0M7SUFDNUIsY0FBYyxTQUFPLEVBQzdCLFFBQVEsQ0FBQyxFQUFFLGNBQWMsRUFJMUI7SUFFTSxnQkFBZ0IsQ0FBQyxhQUFhLEVBQUUsbUJBQW1CLEdBQUcsSUFBSSxDQUVoRTtJQUVNLFNBQVMsSUFBSSxPQUFPLENBRTFCO0lBRU0sS0FBSyxJQUFJLElBQUksQ0FFbkI7SUFFWSxJQUFJLElBQUksT0FBTyxDQUFDLElBQUksQ0FBQyxDQUdqQztJQUVNLFNBQVMsSUFBSSxpQkFBaUIsQ0FXcEM7WUFFYSxJQUFJO1lBcURKLFFBQVE7WUF3RFIsWUFBWTtDQTBCM0IifQ==
@@ -1 +1 @@
1
- {"version":3,"file":"proving_agent.d.ts","sourceRoot":"","sources":["../../src/proving_broker/proving_agent.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAEV,iBAAiB,EACjB,kBAAkB,EAIlB,mBAAmB,EACpB,MAAM,iCAAiC,CAAC;AACzC,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAC1D,OAAO,EACL,KAAK,eAAe,EACpB,KAAK,SAAS,EACd,KAAK,MAAM,EAGZ,MAAM,yBAAyB,CAAC;AAEjC,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAIzD;;GAEG;AACH,qBAAa,YAAa,YAAW,SAAS;IAQ1C,iCAAiC;IACjC,OAAO,CAAC,MAAM;IACd,gDAAgD;IAChD,OAAO,CAAC,UAAU;IAClB,iDAAiD;IACjD,OAAO,CAAC,aAAa;IACrB,oDAAoD;IACpD,OAAO,CAAC,cAAc;IACtB,oCAAoC;IACpC,OAAO,CAAC,cAAc;IAGtB,OAAO,CAAC,GAAG;IAnBb,OAAO,CAAC,oBAAoB,CAAC,CAAuB;IACpD,OAAO,CAAC,cAAc,CAAiB;IACvC,OAAO,CAAC,eAAe,CAA8B;IAErD,SAAgB,MAAM,EAAE,MAAM,CAAC;IAE/B;IACE,iCAAiC;IACzB,MAAM,EAAE,kBAAkB;IAClC,gDAAgD;IACxC,UAAU,EAAE,UAAU;IAC9B,iDAAiD;IACzC,aAAa,EAAE,mBAAmB;IAC1C,oDAAoD;IAC5C,cAAc,GAAE,KAAK,CAAC,kBAAkB,CAAM;IACtD,oCAAoC;IAC5B,cAAc,SAAO;IAC7B,uDAAuD;IACvD,MAAM,GAAE,eAAsC,EACtC,GAAG,yCAA8C,EAK1D;IAEM,gBAAgB,CAAC,aAAa,EAAE,mBAAmB,GAAG,IAAI,CAEhE;IAEM,SAAS,IAAI,OAAO,CAE1B;IAEM,KAAK,IAAI,IAAI,CAEnB;IAEY,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAGjC;IAEM,SAAS,IAAI,iBAAiB,CAWpC;YAGa,IAAI;YAqDJ,QAAQ;YAuDR,YAAY;CA0B3B"}
1
+ {"version":3,"file":"proving_agent.d.ts","sourceRoot":"","sources":["../../src/proving_broker/proving_agent.ts"],"names":[],"mappings":"AACA,OAAO,EAAe,KAAK,cAAc,EAAgB,MAAM,uBAAuB,CAAC;AAIvF,OAAO,KAAK,EAEV,iBAAiB,EACjB,kBAAkB,EAIlB,mBAAmB,EACpB,MAAM,iCAAiC,CAAC;AACzC,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAE1D,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAGzD;;GAEG;AACH,qBAAa,YAAY;IAMrB,iCAAiC;IACjC,OAAO,CAAC,MAAM;IACd,gDAAgD;IAChD,OAAO,CAAC,UAAU;IAClB,iDAAiD;IACjD,OAAO,CAAC,aAAa;IACrB,oDAAoD;IACpD,OAAO,CAAC,cAAc;IACtB,oCAAoC;IACpC,OAAO,CAAC,cAAc;IAdxB,OAAO,CAAC,oBAAoB,CAAC,CAAuB;IACpD,OAAO,CAAC,cAAc,CAAiB;IACvC,OAAO,CAAC,GAAG,CAAS;IAEpB;IACE,iCAAiC;IACzB,MAAM,EAAE,kBAAkB;IAClC,gDAAgD;IACxC,UAAU,EAAE,UAAU;IAC9B,iDAAiD;IACzC,aAAa,EAAE,mBAAmB;IAC1C,oDAAoD;IAC5C,cAAc,GAAE,KAAK,CAAC,kBAAkB,CAAM;IACtD,oCAAoC;IAC5B,cAAc,SAAO,EAC7B,QAAQ,CAAC,EAAE,cAAc,EAI1B;IAEM,gBAAgB,CAAC,aAAa,EAAE,mBAAmB,GAAG,IAAI,CAEhE;IAEM,SAAS,IAAI,OAAO,CAE1B;IAEM,KAAK,IAAI,IAAI,CAEnB;IAEY,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAGjC;IAEM,SAAS,IAAI,iBAAiB,CAWpC;YAEa,IAAI;YAqDJ,QAAQ;YAwDR,YAAY;CA0B3B"}
@@ -1,17 +1,9 @@
1
- function _ts_decorate(decorators, target, key, desc) {
2
- var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
3
- if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
4
- else for(var i = decorators.length - 1; i >= 0; i--)if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
5
- return c > 3 && r && Object.defineProperty(target, key, r), r;
6
- }
7
1
  import { AbortError } from '@aztec/foundation/error';
8
2
  import { createLogger } from '@aztec/foundation/log';
9
3
  import { RunningPromise } from '@aztec/foundation/running-promise';
10
4
  import { truncate } from '@aztec/foundation/string';
11
5
  import { ProvingError } from '@aztec/stdlib/errors';
12
6
  import { ProvingRequestType } from '@aztec/stdlib/proofs';
13
- import { getTelemetryClient, trackSpan } from '@aztec/telemetry-client';
14
- import { ProvingAgentInstrumentation } from './proving_agent_instrumentation.js';
15
7
  import { ProvingJobController, ProvingJobControllerStatus } from './proving_job_controller.js';
16
8
  /**
17
9
  * A helper class that encapsulates a circuit prover and connects it to a job source.
@@ -21,20 +13,16 @@ import { ProvingJobController, ProvingJobControllerStatus } from './proving_job_
21
13
  circuitProver;
22
14
  proofAllowList;
23
15
  pollIntervalMs;
24
- log;
25
16
  currentJobController;
26
17
  runningPromise;
27
- instrumentation;
28
- tracer;
29
- constructor(/** The source of proving jobs */ broker, /** Database holding proof inputs and outputs */ proofStore, /** The prover implementation to defer jobs to */ circuitProver, /** Optional list of allowed proof types to build */ proofAllowList = [], /** How long to wait between jobs */ pollIntervalMs = 1000, /** A telemetry client through which to emit metrics */ client = getTelemetryClient(), log = createLogger('prover-client:proving-agent')){
18
+ log;
19
+ constructor(/** The source of proving jobs */ broker, /** Database holding proof inputs and outputs */ proofStore, /** The prover implementation to defer jobs to */ circuitProver, /** Optional list of allowed proof types to build */ proofAllowList = [], /** How long to wait between jobs */ pollIntervalMs = 1000, bindings){
30
20
  this.broker = broker;
31
21
  this.proofStore = proofStore;
32
22
  this.circuitProver = circuitProver;
33
23
  this.proofAllowList = proofAllowList;
34
24
  this.pollIntervalMs = pollIntervalMs;
35
- this.log = log;
36
- this.tracer = client.getTracer('ProvingAgent');
37
- this.instrumentation = new ProvingAgentInstrumentation(client);
25
+ this.log = createLogger('prover-client:proving-agent', bindings);
38
26
  this.runningPromise = new RunningPromise(this.work.bind(this), this.log, this.pollIntervalMs);
39
27
  }
40
28
  setCircuitProver(circuitProver) {
@@ -136,7 +124,7 @@ import { ProvingJobController, ProvingJobControllerStatus } from './proving_job_
136
124
  // trigger a run of the main work loop when proving completes
137
125
  // no need to await this here. The controller will stay alive (in DONE state) until the result is send to the broker
138
126
  void this.runningPromise.trigger();
139
- });
127
+ }, this.log.getBindings());
140
128
  if (abortedProofJobId) {
141
129
  this.log.info(`Aborting job id=${abortedProofJobId} type=${abortedProofName} to start new job id=${this.currentJobController.getJobId()} type=${this.currentJobController.getProofTypeName()} inputsUri=${truncate(job.inputsUri)}`);
142
130
  } else {
@@ -165,6 +153,3 @@ import { ProvingJobController, ProvingJobControllerStatus } from './proving_job_
165
153
  return maybeJob;
166
154
  }
167
155
  }
168
- _ts_decorate([
169
- trackSpan('ProvingAgent.safeWork')
170
- ], ProvingAgent.prototype, "work", null);
@@ -1,4 +1,5 @@
1
- import { type GetProvingJobResponse, type ProofUri, type ProvingJob, type ProvingJobConsumer, type ProvingJobFilter, type ProvingJobId, type ProvingJobProducer, type ProvingJobStatus } from '@aztec/stdlib/interfaces/server';
1
+ import { EpochNumber } from '@aztec/foundation/branded-types';
2
+ import { type GetProvingJobResponse, type ProofUri, type ProvingJob, type ProvingJobBrokerDebug, type ProvingJobConsumer, type ProvingJobFilter, type ProvingJobId, type ProvingJobProducer, type ProvingJobStatus } from '@aztec/stdlib/interfaces/server';
2
3
  import { ProvingRequestType } from '@aztec/stdlib/proofs';
3
4
  import { type TelemetryClient, type Traceable, type Tracer } from '@aztec/telemetry-client';
4
5
  import { type ProverBrokerConfig } from './config.js';
@@ -7,7 +8,7 @@ import type { ProvingBrokerDatabase } from './proving_broker_database.js';
7
8
  * A broker that manages proof requests and distributes them to workers based on their priority.
8
9
  * It takes a backend that is responsible for storing and retrieving proof requests and results.
9
10
  */
10
- export declare class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Traceable {
11
+ export declare class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, ProvingJobBrokerDebug, Traceable {
11
12
  #private;
12
13
  private database;
13
14
  private logger;
@@ -38,7 +39,8 @@ export declare class ProvingBroker implements ProvingJobProducer, ProvingJobCons
38
39
  private epochHeight;
39
40
  private maxEpochsToKeepResultsFor;
40
41
  private started;
41
- constructor(database: ProvingBrokerDatabase, { proverBrokerJobTimeoutMs, proverBrokerPollIntervalMs, proverBrokerJobMaxRetries, proverBrokerMaxEpochsToKeepResultsFor }?: Required<Pick<ProverBrokerConfig, 'proverBrokerJobTimeoutMs' | 'proverBrokerPollIntervalMs' | 'proverBrokerJobMaxRetries' | 'proverBrokerMaxEpochsToKeepResultsFor'>>, client?: TelemetryClient, logger?: import("@aztec/foundation/log").Logger);
42
+ private debugReplayEnabled;
43
+ constructor(database: ProvingBrokerDatabase, { proverBrokerJobTimeoutMs, proverBrokerPollIntervalMs, proverBrokerJobMaxRetries, proverBrokerMaxEpochsToKeepResultsFor, proverBrokerDebugReplayEnabled }?: Required<Pick<ProverBrokerConfig, 'proverBrokerJobTimeoutMs' | 'proverBrokerPollIntervalMs' | 'proverBrokerJobMaxRetries' | 'proverBrokerMaxEpochsToKeepResultsFor' | 'proverBrokerDebugReplayEnabled'>>, client?: TelemetryClient, logger?: import("@aztec/foundation/log").Logger);
42
44
  private measureQueueDepth;
43
45
  private countActiveJobs;
44
46
  start(): Promise<void>;
@@ -54,9 +56,10 @@ export declare class ProvingBroker implements ProvingJobProducer, ProvingJobCons
54
56
  job: ProvingJob;
55
57
  time: number;
56
58
  } | undefined>;
59
+ replayProvingJob(jobId: ProvingJobId, type: ProvingRequestType, epochNumber: EpochNumber, inputsUri: ProofUri): Promise<ProvingJobStatus>;
57
60
  private cleanUpProvingJobState;
58
61
  private cleanupPass;
59
- private cleanupStaleJobs;
62
+ private cleanupJobsOlderThanEpoch;
60
63
  private reEnqueueExpiredJobs;
61
64
  private enqueueJobInternal;
62
65
  private isJobStale;
@@ -70,4 +73,4 @@ export declare class ProvingBroker implements ProvingJobProducer, ProvingJobCons
70
73
  * is to get picked up by agents
71
74
  */
72
75
  export declare const PROOF_TYPES_IN_PRIORITY_ORDER: ProvingRequestType[];
73
- //# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoicHJvdmluZ19icm9rZXIuZC50cyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uL3NyYy9wcm92aW5nX2Jyb2tlci9wcm92aW5nX2Jyb2tlci50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFLQSxPQUFPLEVBQ0wsS0FBSyxxQkFBcUIsRUFDMUIsS0FBSyxRQUFRLEVBQ2IsS0FBSyxVQUFVLEVBQ2YsS0FBSyxrQkFBa0IsRUFDdkIsS0FBSyxnQkFBZ0IsRUFDckIsS0FBSyxZQUFZLEVBQ2pCLEtBQUssa0JBQWtCLEVBRXZCLEtBQUssZ0JBQWdCLEVBRXRCLE1BQU0saUNBQWlDLENBQUM7QUFDekMsT0FBTyxFQUFFLGtCQUFrQixFQUFFLE1BQU0sc0JBQXNCLENBQUM7QUFDMUQsT0FBTyxFQUNMLEtBQUssZUFBZSxFQUNwQixLQUFLLFNBQVMsRUFDZCxLQUFLLE1BQU0sRUFHWixNQUFNLHlCQUF5QixDQUFDO0FBSWpDLE9BQU8sRUFBRSxLQUFLLGtCQUFrQixFQUE2QixNQUFNLGFBQWEsQ0FBQztBQUNqRixPQUFPLEtBQUssRUFBRSxxQkFBcUIsRUFBRSxNQUFNLDhCQUE4QixDQUFDO0FBVzFFOzs7R0FHRztBQUNILHFCQUFhLGFBQWMsWUFBVyxrQkFBa0IsRUFBRSxrQkFBa0IsRUFBRSxTQUFTOztJQStFbkYsT0FBTyxDQUFDLFFBQVE7SUFnQmhCLE9BQU8sQ0FBQyxNQUFNO0lBOUZoQixPQUFPLENBQUMsTUFBTSxDQTZCWjtJQUlGLE9BQU8sQ0FBQyxTQUFTLENBQXVDO0lBRXhELE9BQU8sQ0FBQyxZQUFZLENBQW9EO0lBR3hFLE9BQU8sQ0FBQyxVQUFVLENBQWtDO0lBTXBELE9BQU8sQ0FBQyxVQUFVLENBQStDO0lBR2pFLE9BQU8sQ0FBQyxPQUFPLENBQW1DO0lBR2xELE9BQU8sQ0FBQyxRQUFRLENBQTBFO0lBRTFGLE9BQU8sQ0FBQyxjQUFjLENBQWlCO0lBQ3ZDLE9BQU8sQ0FBQyxZQUFZLENBQW9CO0lBQ3hDLE9BQU8sQ0FBQyxZQUFZLENBQVM7SUFDN0IsT0FBTyxDQUFDLFVBQVUsQ0FBUztJQUUzQixPQUFPLENBQUMsZUFBZSxDQUErQjtJQUN0RCxTQUFnQixNQUFNLEVBQUUsTUFBTSxDQUFDO0lBRS9CLE9BQU8sQ0FBQyx5QkFBeUIsQ0FBc0I7SUFFdkQ7Ozs7Ozs7OztPQVNHO0lBQ0gsT0FBTyxDQUFDLFdBQVcsQ0FBSztJQUN4QixPQUFPLENBQUMseUJBQXlCLENBQUs7SUFFdEMsT0FBTyxDQUFDLE9BQU8sQ0FBUztJQUV4QixZQUNVLFFBQVEsRUFBRSxxQkFBcUIsRUFDdkMsRUFDRSx3QkFBd0IsRUFDeEIsMEJBQTBCLEVBQzFCLHlCQUF5QixFQUN6QixxQ0FBcUMsRUFDdEMsR0FBRSxRQUFRLENBQ1QsSUFBSSxDQUNGLGtCQUFrQixFQUNoQiwwQkFBMEIsR0FDMUIsNEJBQTRCLEdBQzVCLDJCQUEyQixHQUMzQix1Q0FBdUMsQ0FDMUMsQ0FDMEIsRUFDN0IsTUFBTSxHQUFFLGVBQXNDLEVBQ3RDLE1BQU0seUNBQStDLEVBUTlEO0lBRUQsT0FBTyxDQUFDLGlCQUFpQixDQUV2QjtJQUVGLE9BQU8sQ0FBQyxlQUFlLENBVXJCO0lBRVcsS0FBSyxJQUFJLE9BQU8sQ0FBQyxJQUFJLENBQUMsQ0E2QmxDO0lBRVksSUFBSSxJQUFJLE9BQU8sQ0FBQyxJQUFJLENBQUMsQ0FNakM7SUFFTSxpQkFBaUIsQ0FBQyxHQUFHLEVBQUUsVUFBVSxHQUFHLE9BQU8sQ0FBQyxnQkFBZ0IsQ0FBQyxDQUVuRTtJQUVNLGdCQUFnQixDQUFDLEVBQUUsRUFBRSxZQUFZLEdBQUcsT0FBTyxDQUFDLElBQUksQ0FBQyxDQUV2RDtJQUVNLG1CQUFtQixDQUFDLEVBQUUsRUFBRSxZQUFZLEdBQUcsT0FBTyxDQUFDLGdCQUFnQixDQUFDLENBRXRFO0lBRU0sZ0JBQWdCLENBQUMsR0FBRyxFQUFFLFlBQVksRUFBRSxHQUFHLE9BQU8sQ0FBQyxZQUFZLEVBQUUsQ0FBQyxDQUVwRTtJQUVNLGFBQWEsQ0FBQyxNQUFNLENBQUMsRUFBRSxnQkFBZ0IsR0FBRyxPQUFPLENBQUMscUJBQXFCLEdBQUcsU0FBUyxDQUFDLENBRTFGO0lBRU0sdUJBQXVCLENBQzVCLEVBQUUsRUFBRSxZQUFZLEVBQ2hCLEtBQUssRUFBRSxRQUFRLEVBQ2YsTUFBTSxDQUFDLEVBQUUsZ0JBQWdCLEdBQ3hCLE9BQU8sQ0FBQyxxQkFBcUIsR0FBRyxTQUFTLENBQUMsQ0FFNUM7SUFFTSxxQkFBcUIsQ0FDMUIsRUFBRSxFQUFFLFlBQVksRUFDaEIsR0FBRyxFQUFFLE1BQU0sRUFDWCxLQUFLLFVBQVEsRUFDYixNQUFNLENBQUMsRUFBRSxnQkFBZ0IsR0FDeEIsT0FBTyxDQUFDLHFCQUFxQixHQUFHLFNBQVMsQ0FBQyxDQUU1QztJQUVNLHdCQUF3QixDQUM3QixFQUFFLEVBQUUsWUFBWSxFQUNoQixTQUFTLEVBQUUsTUFBTSxFQUNqQixNQUFNLENBQUMsRUFBRSxnQkFBZ0IsR0FDeEIsT0FBTyxDQUFDO1FBQUUsR0FBRyxFQUFFLFVBQVUsQ0FBQztRQUFDLElBQUksRUFBRSxNQUFNLENBQUE7S0FBRSxHQUFHLFNBQVMsQ0FBQyxDQUV4RDtJQWtERCxPQUFPLENBQUMsc0JBQXNCO1lBK1FoQixXQUFXO0lBVXpCLE9BQU8sQ0FBQyxnQkFBZ0I7SUFnQnhCLE9BQU8sQ0FBQyxvQkFBb0I7SUFxQjVCLE9BQU8sQ0FBQyxrQkFBa0I7SUFZMUIsT0FBTyxDQUFDLFVBQVU7SUFJbEIsT0FBTyxDQUFDLGlCQUFpQjtDQUcxQjtBQWlERDs7Ozs7O0dBTUc7QUFDSCxlQUFPLE1BQU0sNkJBQTZCLEVBQUUsa0JBQWtCLEVBbUI3RCxDQUFDIn0=
76
+ //# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoicHJvdmluZ19icm9rZXIuZC50cyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uL3NyYy9wcm92aW5nX2Jyb2tlci9wcm92aW5nX2Jyb2tlci50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSxPQUFPLEVBQUUsV0FBVyxFQUFFLE1BQU0saUNBQWlDLENBQUM7QUFLOUQsT0FBTyxFQUNMLEtBQUsscUJBQXFCLEVBQzFCLEtBQUssUUFBUSxFQUNiLEtBQUssVUFBVSxFQUNmLEtBQUsscUJBQXFCLEVBQzFCLEtBQUssa0JBQWtCLEVBQ3ZCLEtBQUssZ0JBQWdCLEVBQ3JCLEtBQUssWUFBWSxFQUNqQixLQUFLLGtCQUFrQixFQUV2QixLQUFLLGdCQUFnQixFQUV0QixNQUFNLGlDQUFpQyxDQUFDO0FBQ3pDLE9BQU8sRUFBRSxrQkFBa0IsRUFBRSxNQUFNLHNCQUFzQixDQUFDO0FBQzFELE9BQU8sRUFBRSxLQUFLLGVBQWUsRUFBRSxLQUFLLFNBQVMsRUFBRSxLQUFLLE1BQU0sRUFBc0IsTUFBTSx5QkFBeUIsQ0FBQztBQUloSCxPQUFPLEVBQUUsS0FBSyxrQkFBa0IsRUFBNkIsTUFBTSxhQUFhLENBQUM7QUFDakYsT0FBTyxLQUFLLEVBQUUscUJBQXFCLEVBQUUsTUFBTSw4QkFBOEIsQ0FBQztBQVcxRTs7O0dBR0c7QUFDSCxxQkFBYSxhQUFjLFlBQVcsa0JBQWtCLEVBQUUsa0JBQWtCLEVBQUUscUJBQXFCLEVBQUUsU0FBUzs7SUFpRjFHLE9BQU8sQ0FBQyxRQUFRO0lBa0JoQixPQUFPLENBQUMsTUFBTTtJQWxHaEIsT0FBTyxDQUFDLE1BQU0sQ0E2Qlo7SUFJRixPQUFPLENBQUMsU0FBUyxDQUF1QztJQUV4RCxPQUFPLENBQUMsWUFBWSxDQUFvRDtJQUd4RSxPQUFPLENBQUMsVUFBVSxDQUFrQztJQU1wRCxPQUFPLENBQUMsVUFBVSxDQUErQztJQUdqRSxPQUFPLENBQUMsT0FBTyxDQUFtQztJQUdsRCxPQUFPLENBQUMsUUFBUSxDQUEwRTtJQUUxRixPQUFPLENBQUMsY0FBYyxDQUFpQjtJQUN2QyxPQUFPLENBQUMsWUFBWSxDQUFvQjtJQUN4QyxPQUFPLENBQUMsWUFBWSxDQUFTO0lBQzdCLE9BQU8sQ0FBQyxVQUFVLENBQVM7SUFFM0IsT0FBTyxDQUFDLGVBQWUsQ0FBK0I7SUFDdEQsU0FBZ0IsTUFBTSxFQUFFLE1BQU0sQ0FBQztJQUUvQixPQUFPLENBQUMseUJBQXlCLENBQXNCO0lBRXZEOzs7Ozs7Ozs7T0FTRztJQUNILE9BQU8sQ0FBQyxXQUFXLENBQUs7SUFDeEIsT0FBTyxDQUFDLHlCQUF5QixDQUFLO0lBRXRDLE9BQU8sQ0FBQyxPQUFPLENBQVM7SUFFeEIsT0FBTyxDQUFDLGtCQUFrQixDQUFVO0lBRXBDLFlBQ1UsUUFBUSxFQUFFLHFCQUFxQixFQUN2QyxFQUNFLHdCQUF3QixFQUN4QiwwQkFBMEIsRUFDMUIseUJBQXlCLEVBQ3pCLHFDQUFxQyxFQUNyQyw4QkFBOEIsRUFDL0IsR0FBRSxRQUFRLENBQ1QsSUFBSSxDQUNGLGtCQUFrQixFQUNoQiwwQkFBMEIsR0FDMUIsNEJBQTRCLEdBQzVCLDJCQUEyQixHQUMzQix1Q0FBdUMsR0FDdkMsZ0NBQWdDLENBQ25DLENBQzBCLEVBQzdCLE1BQU0sR0FBRSxlQUFzQyxFQUN0QyxNQUFNLHlDQUErQyxFQVM5RDtJQUVELE9BQU8sQ0FBQyxpQkFBaUIsQ0FFdkI7SUFFRixPQUFPLENBQUMsZUFBZSxDQVVyQjtJQUVXLEtBQUssSUFBSSxPQUFPLENBQUMsSUFBSSxDQUFDLENBNkJsQztJQUVZLElBQUksSUFBSSxPQUFPLENBQUMsSUFBSSxDQUFDLENBTWpDO0lBRU0saUJBQWlCLENBQUMsR0FBRyxFQUFFLFVBQVUsR0FBRyxPQUFPLENBQUMsZ0JBQWdCLENBQUMsQ0FFbkU7SUFFTSxnQkFBZ0IsQ0FBQyxFQUFFLEVBQUUsWUFBWSxHQUFHLE9BQU8sQ0FBQyxJQUFJLENBQUMsQ0FFdkQ7SUFFTSxtQkFBbUIsQ0FBQyxFQUFFLEVBQUUsWUFBWSxHQUFHLE9BQU8sQ0FBQyxnQkFBZ0IsQ0FBQyxDQUV0RTtJQUVNLGdCQUFnQixDQUFDLEdBQUcsRUFBRSxZQUFZLEVBQUUsR0FBRyxPQUFPLENBQUMsWUFBWSxFQUFFLENBQUMsQ0FFcEU7SUFFTSxhQUFhLENBQUMsTUFBTSxDQUFDLEVBQUUsZ0JBQWdCLEdBQUcsT0FBTyxDQUFDLHFCQUFxQixHQUFHLFNBQVMsQ0FBQyxDQUUxRjtJQUVNLHVCQUF1QixDQUM1QixFQUFFLEVBQUUsWUFBWSxFQUNoQixLQUFLLEVBQUUsUUFBUSxFQUNmLE1BQU0sQ0FBQyxFQUFFLGdCQUFnQixHQUN4QixPQUFPLENBQUMscUJBQXFCLEdBQUcsU0FBUyxDQUFDLENBRTVDO0lBRU0scUJBQXFCLENBQzFCLEVBQUUsRUFBRSxZQUFZLEVBQ2hCLEdBQUcsRUFBRSxNQUFNLEVBQ1gsS0FBSyxVQUFRLEVBQ2IsTUFBTSxDQUFDLEVBQUUsZ0JBQWdCLEdBQ3hCLE9BQU8sQ0FBQyxxQkFBcUIsR0FBRyxTQUFTLENBQUMsQ0FFNUM7SUFFTSx3QkFBd0IsQ0FDN0IsRUFBRSxFQUFFLFlBQVksRUFDaEIsU0FBUyxFQUFFLE1BQU0sRUFDakIsTUFBTSxDQUFDLEVBQUUsZ0JBQWdCLEdBQ3hCLE9BQU8sQ0FBQztRQUFFLEdBQUcsRUFBRSxVQUFVLENBQUM7UUFBQyxJQUFJLEVBQUUsTUFBTSxDQUFBO0tBQUUsR0FBRyxTQUFTLENBQUMsQ0FFeEQ7SUFFWSxnQkFBZ0IsQ0FDM0IsS0FBSyxFQUFFLFlBQVksRUFDbkIsSUFBSSxFQUFFLGtCQUFrQixFQUN4QixXQUFXLEVBQUUsV0FBVyxFQUN4QixTQUFTLEVBQUUsUUFBUSxHQUNsQixPQUFPLENBQUMsZ0JBQWdCLENBQUMsQ0FnQjNCO0lBa0RELE9BQU8sQ0FBQyxzQkFBc0I7WUF5UmhCLFdBQVc7SUFVekIsT0FBTyxDQUFDLHlCQUF5QjtJQWdCakMsT0FBTyxDQUFDLG9CQUFvQjtJQXFDNUIsT0FBTyxDQUFDLGtCQUFrQjtJQVkxQixPQUFPLENBQUMsVUFBVTtJQUlsQixPQUFPLENBQUMsaUJBQWlCO0NBRzFCO0FBaUREOzs7Ozs7R0FNRztBQUNILGVBQU8sTUFBTSw2QkFBNkIsRUFBRSxrQkFBa0IsRUFtQjdELENBQUMifQ==
@@ -1 +1 @@
1
- {"version":3,"file":"proving_broker.d.ts","sourceRoot":"","sources":["../../src/proving_broker/proving_broker.ts"],"names":[],"mappings":"AAKA,OAAO,EACL,KAAK,qBAAqB,EAC1B,KAAK,QAAQ,EACb,KAAK,UAAU,EACf,KAAK,kBAAkB,EACvB,KAAK,gBAAgB,EACrB,KAAK,YAAY,EACjB,KAAK,kBAAkB,EAEvB,KAAK,gBAAgB,EAEtB,MAAM,iCAAiC,CAAC;AACzC,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAC1D,OAAO,EACL,KAAK,eAAe,EACpB,KAAK,SAAS,EACd,KAAK,MAAM,EAGZ,MAAM,yBAAyB,CAAC;AAIjC,OAAO,EAAE,KAAK,kBAAkB,EAA6B,MAAM,aAAa,CAAC;AACjF,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,8BAA8B,CAAC;AAW1E;;;GAGG;AACH,qBAAa,aAAc,YAAW,kBAAkB,EAAE,kBAAkB,EAAE,SAAS;;IA+EnF,OAAO,CAAC,QAAQ;IAgBhB,OAAO,CAAC,MAAM;IA9FhB,OAAO,CAAC,MAAM,CA6BZ;IAIF,OAAO,CAAC,SAAS,CAAuC;IAExD,OAAO,CAAC,YAAY,CAAoD;IAGxE,OAAO,CAAC,UAAU,CAAkC;IAMpD,OAAO,CAAC,UAAU,CAA+C;IAGjE,OAAO,CAAC,OAAO,CAAmC;IAGlD,OAAO,CAAC,QAAQ,CAA0E;IAE1F,OAAO,CAAC,cAAc,CAAiB;IACvC,OAAO,CAAC,YAAY,CAAoB;IACxC,OAAO,CAAC,YAAY,CAAS;IAC7B,OAAO,CAAC,UAAU,CAAS;IAE3B,OAAO,CAAC,eAAe,CAA+B;IACtD,SAAgB,MAAM,EAAE,MAAM,CAAC;IAE/B,OAAO,CAAC,yBAAyB,CAAsB;IAEvD;;;;;;;;;OASG;IACH,OAAO,CAAC,WAAW,CAAK;IACxB,OAAO,CAAC,yBAAyB,CAAK;IAEtC,OAAO,CAAC,OAAO,CAAS;IAExB,YACU,QAAQ,EAAE,qBAAqB,EACvC,EACE,wBAAwB,EACxB,0BAA0B,EAC1B,yBAAyB,EACzB,qCAAqC,EACtC,GAAE,QAAQ,CACT,IAAI,CACF,kBAAkB,EAChB,0BAA0B,GAC1B,4BAA4B,GAC5B,2BAA2B,GAC3B,uCAAuC,CAC1C,CAC0B,EAC7B,MAAM,GAAE,eAAsC,EACtC,MAAM,yCAA+C,EAQ9D;IAED,OAAO,CAAC,iBAAiB,CAEvB;IAEF,OAAO,CAAC,eAAe,CAUrB;IAEW,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CA6BlC;IAEY,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAMjC;IAEM,iBAAiB,CAAC,GAAG,EAAE,UAAU,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAEnE;IAEM,gBAAgB,CAAC,EAAE,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAEvD;IAEM,mBAAmB,CAAC,EAAE,EAAE,YAAY,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAEtE;IAEM,gBAAgB,CAAC,GAAG,EAAE,YAAY,EAAE,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC,CAEpE;IAEM,aAAa,CAAC,MAAM,CAAC,EAAE,gBAAgB,GAAG,OAAO,CAAC,qBAAqB,GAAG,SAAS,CAAC,CAE1F;IAEM,uBAAuB,CAC5B,EAAE,EAAE,YAAY,EAChB,KAAK,EAAE,QAAQ,EACf,MAAM,CAAC,EAAE,gBAAgB,GACxB,OAAO,CAAC,qBAAqB,GAAG,SAAS,CAAC,CAE5C;IAEM,qBAAqB,CAC1B,EAAE,EAAE,YAAY,EAChB,GAAG,EAAE,MAAM,EACX,KAAK,UAAQ,EACb,MAAM,CAAC,EAAE,gBAAgB,GACxB,OAAO,CAAC,qBAAqB,GAAG,SAAS,CAAC,CAE5C;IAEM,wBAAwB,CAC7B,EAAE,EAAE,YAAY,EAChB,SAAS,EAAE,MAAM,EACjB,MAAM,CAAC,EAAE,gBAAgB,GACxB,OAAO,CAAC;QAAE,GAAG,EAAE,UAAU,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,SAAS,CAAC,CAExD;IAkDD,OAAO,CAAC,sBAAsB;YA+QhB,WAAW;IAUzB,OAAO,CAAC,gBAAgB;IAgBxB,OAAO,CAAC,oBAAoB;IAqB5B,OAAO,CAAC,kBAAkB;IAY1B,OAAO,CAAC,UAAU;IAIlB,OAAO,CAAC,iBAAiB;CAG1B;AAiDD;;;;;;GAMG;AACH,eAAO,MAAM,6BAA6B,EAAE,kBAAkB,EAmB7D,CAAC"}
1
+ {"version":3,"file":"proving_broker.d.ts","sourceRoot":"","sources":["../../src/proving_broker/proving_broker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iCAAiC,CAAC;AAK9D,OAAO,EACL,KAAK,qBAAqB,EAC1B,KAAK,QAAQ,EACb,KAAK,UAAU,EACf,KAAK,qBAAqB,EAC1B,KAAK,kBAAkB,EACvB,KAAK,gBAAgB,EACrB,KAAK,YAAY,EACjB,KAAK,kBAAkB,EAEvB,KAAK,gBAAgB,EAEtB,MAAM,iCAAiC,CAAC;AACzC,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAC1D,OAAO,EAAE,KAAK,eAAe,EAAE,KAAK,SAAS,EAAE,KAAK,MAAM,EAAsB,MAAM,yBAAyB,CAAC;AAIhH,OAAO,EAAE,KAAK,kBAAkB,EAA6B,MAAM,aAAa,CAAC;AACjF,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,8BAA8B,CAAC;AAW1E;;;GAGG;AACH,qBAAa,aAAc,YAAW,kBAAkB,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,SAAS;;IAiF1G,OAAO,CAAC,QAAQ;IAkBhB,OAAO,CAAC,MAAM;IAlGhB,OAAO,CAAC,MAAM,CA6BZ;IAIF,OAAO,CAAC,SAAS,CAAuC;IAExD,OAAO,CAAC,YAAY,CAAoD;IAGxE,OAAO,CAAC,UAAU,CAAkC;IAMpD,OAAO,CAAC,UAAU,CAA+C;IAGjE,OAAO,CAAC,OAAO,CAAmC;IAGlD,OAAO,CAAC,QAAQ,CAA0E;IAE1F,OAAO,CAAC,cAAc,CAAiB;IACvC,OAAO,CAAC,YAAY,CAAoB;IACxC,OAAO,CAAC,YAAY,CAAS;IAC7B,OAAO,CAAC,UAAU,CAAS;IAE3B,OAAO,CAAC,eAAe,CAA+B;IACtD,SAAgB,MAAM,EAAE,MAAM,CAAC;IAE/B,OAAO,CAAC,yBAAyB,CAAsB;IAEvD;;;;;;;;;OASG;IACH,OAAO,CAAC,WAAW,CAAK;IACxB,OAAO,CAAC,yBAAyB,CAAK;IAEtC,OAAO,CAAC,OAAO,CAAS;IAExB,OAAO,CAAC,kBAAkB,CAAU;IAEpC,YACU,QAAQ,EAAE,qBAAqB,EACvC,EACE,wBAAwB,EACxB,0BAA0B,EAC1B,yBAAyB,EACzB,qCAAqC,EACrC,8BAA8B,EAC/B,GAAE,QAAQ,CACT,IAAI,CACF,kBAAkB,EAChB,0BAA0B,GAC1B,4BAA4B,GAC5B,2BAA2B,GAC3B,uCAAuC,GACvC,gCAAgC,CACnC,CAC0B,EAC7B,MAAM,GAAE,eAAsC,EACtC,MAAM,yCAA+C,EAS9D;IAED,OAAO,CAAC,iBAAiB,CAEvB;IAEF,OAAO,CAAC,eAAe,CAUrB;IAEW,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CA6BlC;IAEY,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAMjC;IAEM,iBAAiB,CAAC,GAAG,EAAE,UAAU,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAEnE;IAEM,gBAAgB,CAAC,EAAE,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAEvD;IAEM,mBAAmB,CAAC,EAAE,EAAE,YAAY,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAEtE;IAEM,gBAAgB,CAAC,GAAG,EAAE,YAAY,EAAE,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC,CAEpE;IAEM,aAAa,CAAC,MAAM,CAAC,EAAE,gBAAgB,GAAG,OAAO,CAAC,qBAAqB,GAAG,SAAS,CAAC,CAE1F;IAEM,uBAAuB,CAC5B,EAAE,EAAE,YAAY,EAChB,KAAK,EAAE,QAAQ,EACf,MAAM,CAAC,EAAE,gBAAgB,GACxB,OAAO,CAAC,qBAAqB,GAAG,SAAS,CAAC,CAE5C;IAEM,qBAAqB,CAC1B,EAAE,EAAE,YAAY,EAChB,GAAG,EAAE,MAAM,EACX,KAAK,UAAQ,EACb,MAAM,CAAC,EAAE,gBAAgB,GACxB,OAAO,CAAC,qBAAqB,GAAG,SAAS,CAAC,CAE5C;IAEM,wBAAwB,CAC7B,EAAE,EAAE,YAAY,EAChB,SAAS,EAAE,MAAM,EACjB,MAAM,CAAC,EAAE,gBAAgB,GACxB,OAAO,CAAC;QAAE,GAAG,EAAE,UAAU,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,SAAS,CAAC,CAExD;IAEY,gBAAgB,CAC3B,KAAK,EAAE,YAAY,EACnB,IAAI,EAAE,kBAAkB,EACxB,WAAW,EAAE,WAAW,EACxB,SAAS,EAAE,QAAQ,GAClB,OAAO,CAAC,gBAAgB,CAAC,CAgB3B;IAkDD,OAAO,CAAC,sBAAsB;YAyRhB,WAAW;IAUzB,OAAO,CAAC,yBAAyB;IAgBjC,OAAO,CAAC,oBAAoB;IAqC5B,OAAO,CAAC,kBAAkB;IAY1B,OAAO,CAAC,UAAU;IAIlB,OAAO,CAAC,iBAAiB;CAG1B;AAiDD;;;;;;GAMG;AACH,eAAO,MAAM,6BAA6B,EAAE,kBAAkB,EAmB7D,CAAC"}
@@ -1,9 +1,3 @@
1
- function _ts_decorate(decorators, target, key, desc) {
2
- var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
3
- if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
4
- else for(var i = decorators.length - 1; i >= 0; i--)if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
5
- return c > 3 && r && Object.defineProperty(target, key, r), r;
6
- }
7
1
  import { EpochNumber } from '@aztec/foundation/branded-types';
8
2
  import { createLogger } from '@aztec/foundation/log';
9
3
  import { RunningPromise, promiseWithResolvers } from '@aztec/foundation/promise';
@@ -11,7 +5,7 @@ import { PriorityMemoryQueue } from '@aztec/foundation/queue';
11
5
  import { Timer } from '@aztec/foundation/timer';
12
6
  import { tryStop } from '@aztec/stdlib/interfaces/server';
13
7
  import { ProvingRequestType } from '@aztec/stdlib/proofs';
14
- import { getTelemetryClient, trackSpan } from '@aztec/telemetry-client';
8
+ import { getTelemetryClient } from '@aztec/telemetry-client';
15
9
  import assert from 'assert';
16
10
  import { defaultProverBrokerConfig } from './config.js';
17
11
  import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.js';
@@ -57,7 +51,8 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
57
51
  */ epochHeight;
58
52
  maxEpochsToKeepResultsFor;
59
53
  started;
60
- constructor(database, { proverBrokerJobTimeoutMs, proverBrokerPollIntervalMs, proverBrokerJobMaxRetries, proverBrokerMaxEpochsToKeepResultsFor } = defaultProverBrokerConfig, client = getTelemetryClient(), logger = createLogger('prover-client:proving-broker')){
54
+ debugReplayEnabled;
55
+ constructor(database, { proverBrokerJobTimeoutMs, proverBrokerPollIntervalMs, proverBrokerJobMaxRetries, proverBrokerMaxEpochsToKeepResultsFor, proverBrokerDebugReplayEnabled } = defaultProverBrokerConfig, client = getTelemetryClient(), logger = createLogger('prover-client:proving-broker')){
61
56
  this.database = database;
62
57
  this.logger = logger;
63
58
  this.queues = {
@@ -110,6 +105,7 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
110
105
  this.jobTimeoutMs = proverBrokerJobTimeoutMs;
111
106
  this.maxRetries = proverBrokerJobMaxRetries;
112
107
  this.maxEpochsToKeepResultsFor = proverBrokerMaxEpochsToKeepResultsFor;
108
+ this.debugReplayEnabled = proverBrokerDebugReplayEnabled ?? false;
113
109
  }
114
110
  measureQueueDepth;
115
111
  countActiveJobs;
@@ -169,6 +165,32 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
169
165
  reportProvingJobProgress(id, startedAt, filter) {
170
166
  return Promise.resolve(this.#reportProvingJobProgress(id, startedAt, filter));
171
167
  }
168
+ async replayProvingJob(jobId, type, epochNumber, inputsUri) {
169
+ if (!this.debugReplayEnabled) {
170
+ throw new Error('Debug replay not enabled. Set PROVER_BROKER_DEBUG_REPLAY_ENABLED=true');
171
+ }
172
+ this.logger.info(`Replaying proving job`, {
173
+ provingJobId: jobId,
174
+ epochNumber,
175
+ inputsUri
176
+ });
177
+ // Clear existing state and enqueue
178
+ this.cleanUpProvingJobState([
179
+ jobId
180
+ ]);
181
+ const job = {
182
+ id: jobId,
183
+ type,
184
+ epochNumber,
185
+ inputsUri
186
+ };
187
+ this.jobsCache.set(jobId, job);
188
+ await this.database.addProvingJob(job);
189
+ this.enqueueJobInternal(job);
190
+ return {
191
+ status: 'in-queue'
192
+ };
193
+ }
172
194
  async #enqueueProvingJob(job) {
173
195
  // We return the job status at the start of this call
174
196
  const jobStatus = this.#getProvingJobStatus(job.id);
@@ -217,18 +239,27 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
217
239
  this.logger.info(`Cancelling job id=${id}`, {
218
240
  provingJobId: id
219
241
  });
220
- await this.#reportProvingJobError(id, 'Aborted', false);
242
+ await this.#reportProvingJobError(id, 'Aborted', false, undefined, true);
221
243
  }
222
244
  }
223
245
  cleanUpProvingJobState(ids) {
246
+ const idsToClean = new Set(ids);
224
247
  for (const id of ids){
225
248
  this.jobsCache.delete(id);
249
+ const deferred = this.promises.get(id);
250
+ if (deferred) {
251
+ deferred.resolve({
252
+ status: 'rejected',
253
+ reason: 'Proving job cleaned up'
254
+ });
255
+ }
226
256
  this.promises.delete(id);
227
257
  this.resultsCache.delete(id);
228
258
  this.inProgress.delete(id);
229
259
  this.retries.delete(id);
230
260
  this.enqueuedAt.delete(id);
231
261
  }
262
+ this.completedJobNotifications = this.completedJobNotifications.filter((id)=>!idsToClean.has(id));
232
263
  }
233
264
  #getProvingJobStatus(id) {
234
265
  const result = this.resultsCache.get(id);
@@ -291,7 +322,7 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
291
322
  }
292
323
  return undefined;
293
324
  }
294
- async #reportProvingJobError(id, err, retry = false, filter) {
325
+ async #reportProvingJobError(id, err, retry = false, filter, aborted = false) {
295
326
  const info = this.inProgress.get(id);
296
327
  const item = this.jobsCache.get(id);
297
328
  const retries = this.retries.get(id) ?? 0;
@@ -338,7 +369,11 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
338
369
  this.resultsCache.set(id, result);
339
370
  this.promises.get(id).resolve(result);
340
371
  this.completedJobNotifications.push(id);
341
- this.instrumentation.incRejectedJobs(item.type);
372
+ if (aborted) {
373
+ this.instrumentation.incAbortedJobs(item.type);
374
+ } else {
375
+ this.instrumentation.incRejectedJobs(item.type);
376
+ }
342
377
  if (info) {
343
378
  const duration = this.msTimeSource() - info.startedAt;
344
379
  this.instrumentation.recordJobDuration(item.type, duration);
@@ -453,20 +488,20 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
453
488
  return this.#getProvingJob(filter);
454
489
  }
455
490
  async cleanupPass() {
456
- this.cleanupStaleJobs();
457
491
  this.reEnqueueExpiredJobs();
458
492
  const oldestEpochToKeep = this.oldestEpochToKeep();
459
493
  if (oldestEpochToKeep > 0) {
494
+ this.cleanupJobsOlderThanEpoch(EpochNumber(oldestEpochToKeep));
460
495
  await this.database.deleteAllProvingJobsOlderThanEpoch(EpochNumber(oldestEpochToKeep));
461
496
  this.logger.trace(`Deleted all epochs older than ${oldestEpochToKeep}`);
462
497
  }
463
498
  }
464
- cleanupStaleJobs() {
499
+ cleanupJobsOlderThanEpoch(epochNumber) {
465
500
  const jobIds = Array.from(this.jobsCache.keys());
466
501
  const jobsToClean = [];
467
502
  for (const id of jobIds){
468
503
  const job = this.jobsCache.get(id);
469
- if (this.isJobStale(job)) {
504
+ if (job.epochNumber < epochNumber) {
470
505
  jobsToClean.push(id);
471
506
  }
472
507
  }
@@ -489,12 +524,28 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
489
524
  const now = this.msTimeSource();
490
525
  const msSinceLastUpdate = now - metadata.lastUpdatedAt;
491
526
  if (msSinceLastUpdate >= this.jobTimeoutMs) {
492
- this.logger.warn(`Proving job id=${id} timed out. Adding it back to the queue.`, {
493
- provingJobId: id
494
- });
495
527
  this.inProgress.delete(id);
496
- this.enqueueJobInternal(item);
497
528
  this.instrumentation.incTimedOutJobs(item.type);
529
+ const retries = this.retries.get(id) ?? 0;
530
+ if (retries + 1 < this.maxRetries && !this.isJobStale(item)) {
531
+ this.logger.warn(`Proving job id=${id} timed out. Re-enqueueing (retry ${retries + 1}/${this.maxRetries}).`, {
532
+ provingJobId: id
533
+ });
534
+ this.retries.set(id, retries + 1);
535
+ this.enqueueJobInternal(item);
536
+ } else {
537
+ this.logger.error(`Proving job id=${id} timed out after ${retries + 1} attempts. Marking as failed.`, {
538
+ provingJobId: id
539
+ });
540
+ const result = {
541
+ status: 'rejected',
542
+ reason: 'Timed out'
543
+ };
544
+ this.resultsCache.set(id, result);
545
+ this.promises.get(id)?.resolve(result);
546
+ this.completedJobNotifications.push(id);
547
+ this.instrumentation.incRejectedJobs(item.type);
548
+ }
498
549
  }
499
550
  }
500
551
  }
@@ -516,9 +567,6 @@ import { ProvingBrokerInstrumentation } from './proving_broker_instrumentation.j
516
567
  return this.epochHeight - this.maxEpochsToKeepResultsFor;
517
568
  }
518
569
  }
519
- _ts_decorate([
520
- trackSpan('ProvingBroker.cleanupPass')
521
- ], ProvingBroker.prototype, "cleanupPass", null);
522
570
  /**
523
571
  * Compares two proving jobs and selects which one's more important
524
572
  * @param a - A proving job
@@ -1,7 +1,7 @@
1
1
  import { EpochNumber } from '@aztec/foundation/branded-types';
2
2
  import { type Logger } from '@aztec/foundation/log';
3
3
  import { type ProofUri, ProvingJob, type ProvingJobId, ProvingJobSettledResult } from '@aztec/stdlib/interfaces/server';
4
- import { type TelemetryClient } from '@aztec/telemetry-client';
4
+ import { type TelemetryClient, type Tracer } from '@aztec/telemetry-client';
5
5
  import type { ProverBrokerConfig } from '../config.js';
6
6
  import type { ProvingBrokerDatabase } from '../proving_broker_database.js';
7
7
  export declare class KVBrokerDatabase implements ProvingBrokerDatabase {
@@ -10,6 +10,7 @@ export declare class KVBrokerDatabase implements ProvingBrokerDatabase {
10
10
  private logger;
11
11
  private metrics;
12
12
  private batchQueue;
13
+ readonly tracer: Tracer;
13
14
  private constructor();
14
15
  commitWrites(items: Array<ProvingJob | [ProvingJobId, ProvingJobSettledResult]>, epochNumber: number): Promise<void>;
15
16
  private estimateSize;
@@ -23,4 +24,4 @@ export declare class KVBrokerDatabase implements ProvingBrokerDatabase {
23
24
  setProvingJobResult(id: ProvingJobId, value: ProofUri): Promise<void>;
24
25
  private getEpochDatabase;
25
26
  }
26
- //# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoicGVyc2lzdGVkLmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi8uLi9zcmMvcHJvdmluZ19icm9rZXIvcHJvdmluZ19icm9rZXJfZGF0YWJhc2UvcGVyc2lzdGVkLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLE9BQU8sRUFBRSxXQUFXLEVBQUUsTUFBTSxpQ0FBaUMsQ0FBQztBQUU5RCxPQUFPLEVBQUUsS0FBSyxNQUFNLEVBQWdCLE1BQU0sdUJBQXVCLENBQUM7QUFJbEUsT0FBTyxFQUNMLEtBQUssUUFBUSxFQUNiLFVBQVUsRUFDVixLQUFLLFlBQVksRUFDakIsdUJBQXVCLEVBRXhCLE1BQU0saUNBQWlDLENBQUM7QUFDekMsT0FBTyxFQUEyQixLQUFLLGVBQWUsRUFBc0IsTUFBTSx5QkFBeUIsQ0FBQztBQUs1RyxPQUFPLEtBQUssRUFBRSxrQkFBa0IsRUFBRSxNQUFNLGNBQWMsQ0FBQztBQUN2RCxPQUFPLEtBQUssRUFBRSxxQkFBcUIsRUFBRSxNQUFNLCtCQUErQixDQUFDO0FBd0QzRSxxQkFBYSxnQkFBaUIsWUFBVyxxQkFBcUI7SUFNMUQsT0FBTyxDQUFDLE1BQU07SUFDZCxPQUFPLENBQUMsTUFBTTtJQUVkLE9BQU8sQ0FBQyxNQUFNO0lBUmhCLE9BQU8sQ0FBQyxPQUFPLENBQWM7SUFFN0IsT0FBTyxDQUFDLFVBQVUsQ0FBMkU7SUFFN0YsT0FBTyxlQW9CTjtJQUdZLFlBQVksQ0FBQyxLQUFLLEVBQUUsS0FBSyxDQUFDLFVBQVUsR0FBRyxDQUFDLFlBQVksRUFBRSx1QkFBdUIsQ0FBQyxDQUFDLEVBQUUsV0FBVyxFQUFFLE1BQU0saUJBTWhIO1lBRWEsWUFBWTtJQVUxQixPQUFvQixHQUFHLENBQ3JCLE1BQU0sRUFBRSxrQkFBa0IsRUFDMUIsTUFBTSxHQUFFLGVBQXNDLEVBQzlDLE1BQU0sU0FBd0QsNkJBOEIvRDtJQUVELE9BQU8sQ0FBQyxLQUFLO0lBSVAsS0FBSyxJQUFJLE9BQU8sQ0FBQyxJQUFJLENBQUMsQ0FLM0I7SUFFSyxrQ0FBa0MsQ0FBQyxXQUFXLEVBQUUsV0FBVyxHQUFHLE9BQU8sQ0FBQyxJQUFJLENBQUMsQ0FXaEY7SUFFRCxhQUFhLENBQUMsR0FBRyxFQUFFLFVBQVUsR0FBRyxPQUFPLENBQUMsSUFBSSxDQUFDLENBRTVDO0lBRU0sY0FBYyxJQUFJLHFCQUFxQixDQUFDLENBQUMsVUFBVSxFQUFFLHVCQUF1QixHQUFHLFNBQVMsQ0FBQyxDQUFDLENBS2hHO0lBRUQsa0JBQWtCLENBQUMsRUFBRSxFQUFFLFlBQVksRUFBRSxNQUFNLEVBQUUsTUFBTSxHQUFHLE9BQU8sQ0FBQyxJQUFJLENBQUMsQ0FFbEU7SUFFRCxtQkFBbUIsQ0FBQyxFQUFFLEVBQUUsWUFBWSxFQUFFLEtBQUssRUFBRSxRQUFRLEdBQUcsT0FBTyxDQUFDLElBQUksQ0FBQyxDQUVwRTtZQUVhLGdCQUFnQjtDQW9CL0IifQ==
27
+ //# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoicGVyc2lzdGVkLmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi8uLi9zcmMvcHJvdmluZ19icm9rZXIvcHJvdmluZ19icm9rZXJfZGF0YWJhc2UvcGVyc2lzdGVkLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLE9BQU8sRUFBRSxXQUFXLEVBQUUsTUFBTSxpQ0FBaUMsQ0FBQztBQUU5RCxPQUFPLEVBQUUsS0FBSyxNQUFNLEVBQWdCLE1BQU0sdUJBQXVCLENBQUM7QUFJbEUsT0FBTyxFQUNMLEtBQUssUUFBUSxFQUNiLFVBQVUsRUFDVixLQUFLLFlBQVksRUFDakIsdUJBQXVCLEVBRXhCLE1BQU0saUNBQWlDLENBQUM7QUFDekMsT0FBTyxFQUdMLEtBQUssZUFBZSxFQUNwQixLQUFLLE1BQU0sRUFHWixNQUFNLHlCQUF5QixDQUFDO0FBS2pDLE9BQU8sS0FBSyxFQUFFLGtCQUFrQixFQUFFLE1BQU0sY0FBYyxDQUFDO0FBQ3ZELE9BQU8sS0FBSyxFQUFFLHFCQUFxQixFQUFFLE1BQU0sK0JBQStCLENBQUM7QUF3RDNFLHFCQUFhLGdCQUFpQixZQUFXLHFCQUFxQjtJQVExRCxPQUFPLENBQUMsTUFBTTtJQUNkLE9BQU8sQ0FBQyxNQUFNO0lBRWQsT0FBTyxDQUFDLE1BQU07SUFWaEIsT0FBTyxDQUFDLE9BQU8sQ0FBYztJQUU3QixPQUFPLENBQUMsVUFBVSxDQUEyRTtJQUU3RixTQUFnQixNQUFNLEVBQUUsTUFBTSxDQUFDO0lBRS9CLE9BQU8sZUFzQk47SUFHWSxZQUFZLENBQUMsS0FBSyxFQUFFLEtBQUssQ0FBQyxVQUFVLEdBQUcsQ0FBQyxZQUFZLEVBQUUsdUJBQXVCLENBQUMsQ0FBQyxFQUFFLFdBQVcsRUFBRSxNQUFNLGlCQU1oSDtZQUVhLFlBQVk7SUFVMUIsT0FBb0IsR0FBRyxDQUNyQixNQUFNLEVBQUUsa0JBQWtCLEVBQzFCLE1BQU0sR0FBRSxlQUFzQyxFQUM5QyxNQUFNLFNBQXdELDZCQThCL0Q7SUFFRCxPQUFPLENBQUMsS0FBSztJQUlQLEtBQUssSUFBSSxPQUFPLENBQUMsSUFBSSxDQUFDLENBSzNCO0lBS0ssa0NBQWtDLENBQUMsV0FBVyxFQUFFLFdBQVcsR0FBRyxPQUFPLENBQUMsSUFBSSxDQUFDLENBV2hGO0lBRUQsYUFBYSxDQUFDLEdBQUcsRUFBRSxVQUFVLEdBQUcsT0FBTyxDQUFDLElBQUksQ0FBQyxDQUU1QztJQUVNLGNBQWMsSUFBSSxxQkFBcUIsQ0FBQyxDQUFDLFVBQVUsRUFBRSx1QkFBdUIsR0FBRyxTQUFTLENBQUMsQ0FBQyxDQUtoRztJQUVELGtCQUFrQixDQUFDLEVBQUUsRUFBRSxZQUFZLEVBQUUsTUFBTSxFQUFFLE1BQU0sR0FBRyxPQUFPLENBQUMsSUFBSSxDQUFDLENBRWxFO0lBRUQsbUJBQW1CLENBQUMsRUFBRSxFQUFFLFlBQVksRUFBRSxLQUFLLEVBQUUsUUFBUSxHQUFHLE9BQU8sQ0FBQyxJQUFJLENBQUMsQ0FFcEU7WUFFYSxnQkFBZ0I7Q0FvQi9CIn0=
@@ -1 +1 @@
1
- {"version":3,"file":"persisted.d.ts","sourceRoot":"","sources":["../../../src/proving_broker/proving_broker_database/persisted.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iCAAiC,CAAC;AAE9D,OAAO,EAAE,KAAK,MAAM,EAAgB,MAAM,uBAAuB,CAAC;AAIlE,OAAO,EACL,KAAK,QAAQ,EACb,UAAU,EACV,KAAK,YAAY,EACjB,uBAAuB,EAExB,MAAM,iCAAiC,CAAC;AACzC,OAAO,EAA2B,KAAK,eAAe,EAAsB,MAAM,yBAAyB,CAAC;AAK5G,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAC;AACvD,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,+BAA+B,CAAC;AAwD3E,qBAAa,gBAAiB,YAAW,qBAAqB;IAM1D,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,MAAM;IAEd,OAAO,CAAC,MAAM;IARhB,OAAO,CAAC,OAAO,CAAc;IAE7B,OAAO,CAAC,UAAU,CAA2E;IAE7F,OAAO,eAoBN;IAGY,YAAY,CAAC,KAAK,EAAE,KAAK,CAAC,UAAU,GAAG,CAAC,YAAY,EAAE,uBAAuB,CAAC,CAAC,EAAE,WAAW,EAAE,MAAM,iBAMhH;YAEa,YAAY;IAU1B,OAAoB,GAAG,CACrB,MAAM,EAAE,kBAAkB,EAC1B,MAAM,GAAE,eAAsC,EAC9C,MAAM,SAAwD,6BA8B/D;IAED,OAAO,CAAC,KAAK;IAIP,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAK3B;IAEK,kCAAkC,CAAC,WAAW,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAWhF;IAED,aAAa,CAAC,GAAG,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAE5C;IAEM,cAAc,IAAI,qBAAqB,CAAC,CAAC,UAAU,EAAE,uBAAuB,GAAG,SAAS,CAAC,CAAC,CAKhG;IAED,kBAAkB,CAAC,EAAE,EAAE,YAAY,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAElE;IAED,mBAAmB,CAAC,EAAE,EAAE,YAAY,EAAE,KAAK,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CAEpE;YAEa,gBAAgB;CAoB/B"}
1
+ {"version":3,"file":"persisted.d.ts","sourceRoot":"","sources":["../../../src/proving_broker/proving_broker_database/persisted.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iCAAiC,CAAC;AAE9D,OAAO,EAAE,KAAK,MAAM,EAAgB,MAAM,uBAAuB,CAAC;AAIlE,OAAO,EACL,KAAK,QAAQ,EACb,UAAU,EACV,KAAK,YAAY,EACjB,uBAAuB,EAExB,MAAM,iCAAiC,CAAC;AACzC,OAAO,EAGL,KAAK,eAAe,EACpB,KAAK,MAAM,EAGZ,MAAM,yBAAyB,CAAC;AAKjC,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAC;AACvD,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,+BAA+B,CAAC;AAwD3E,qBAAa,gBAAiB,YAAW,qBAAqB;IAQ1D,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,MAAM;IAEd,OAAO,CAAC,MAAM;IAVhB,OAAO,CAAC,OAAO,CAAc;IAE7B,OAAO,CAAC,UAAU,CAA2E;IAE7F,SAAgB,MAAM,EAAE,MAAM,CAAC;IAE/B,OAAO,eAsBN;IAGY,YAAY,CAAC,KAAK,EAAE,KAAK,CAAC,UAAU,GAAG,CAAC,YAAY,EAAE,uBAAuB,CAAC,CAAC,EAAE,WAAW,EAAE,MAAM,iBAMhH;YAEa,YAAY;IAU1B,OAAoB,GAAG,CACrB,MAAM,EAAE,kBAAkB,EAC1B,MAAM,GAAE,eAAsC,EAC9C,MAAM,SAAwD,6BA8B/D;IAED,OAAO,CAAC,KAAK;IAIP,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAK3B;IAKK,kCAAkC,CAAC,WAAW,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAWhF;IAED,aAAa,CAAC,GAAG,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAE5C;IAEM,cAAc,IAAI,qBAAqB,CAAC,CAAC,UAAU,EAAE,uBAAuB,GAAG,SAAS,CAAC,CAAC,CAKhG;IAED,kBAAkB,CAAC,EAAE,EAAE,YAAY,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAElE;IAED,mBAAmB,CAAC,EAAE,EAAE,YAAY,EAAE,KAAK,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CAEpE;YAEa,gBAAgB;CAoB/B"}