@aztec/prover-node 5.0.0-private.20260319 → 5.0.0-rc.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. package/README.md +518 -0
  2. package/dest/actions/download-epoch-proving-job.js +1 -1
  3. package/dest/actions/rerun-epoch-proving-job.d.ts +4 -3
  4. package/dest/actions/rerun-epoch-proving-job.d.ts.map +1 -1
  5. package/dest/actions/rerun-epoch-proving-job.js +103 -21
  6. package/dest/bin/run-failed-epoch.js +1 -3
  7. package/dest/checkpoint-store.d.ts +87 -0
  8. package/dest/checkpoint-store.d.ts.map +1 -0
  9. package/dest/checkpoint-store.js +186 -0
  10. package/dest/config.d.ts +1 -1
  11. package/dest/config.d.ts.map +1 -1
  12. package/dest/config.js +1 -1
  13. package/dest/factory.d.ts +1 -1
  14. package/dest/factory.d.ts.map +1 -1
  15. package/dest/factory.js +22 -8
  16. package/dest/index.d.ts +2 -1
  17. package/dest/index.d.ts.map +1 -1
  18. package/dest/index.js +1 -0
  19. package/dest/job/checkpoint-prover.d.ts +134 -0
  20. package/dest/job/checkpoint-prover.d.ts.map +1 -0
  21. package/dest/job/checkpoint-prover.js +353 -0
  22. package/dest/job/epoch-session.d.ts +146 -0
  23. package/dest/job/epoch-session.d.ts.map +1 -0
  24. package/dest/job/epoch-session.js +720 -0
  25. package/dest/job/top-tree-job.d.ts +82 -0
  26. package/dest/job/top-tree-job.d.ts.map +1 -0
  27. package/dest/job/top-tree-job.js +152 -0
  28. package/dest/metrics.d.ts +35 -8
  29. package/dest/metrics.d.ts.map +1 -1
  30. package/dest/metrics.js +86 -14
  31. package/dest/monitors/epoch-monitor.js +6 -2
  32. package/dest/proof-publishing-service.d.ts +161 -0
  33. package/dest/proof-publishing-service.d.ts.map +1 -0
  34. package/dest/proof-publishing-service.js +335 -0
  35. package/dest/prover-node-publisher.d.ts +22 -15
  36. package/dest/prover-node-publisher.d.ts.map +1 -1
  37. package/dest/prover-node-publisher.js +197 -60
  38. package/dest/prover-node.d.ts +105 -67
  39. package/dest/prover-node.d.ts.map +1 -1
  40. package/dest/prover-node.js +482 -224
  41. package/dest/prover-publisher-factory.d.ts +2 -2
  42. package/dest/prover-publisher-factory.d.ts.map +1 -1
  43. package/dest/prover-publisher-factory.js +3 -3
  44. package/dest/session-manager.d.ts +158 -0
  45. package/dest/session-manager.d.ts.map +1 -0
  46. package/dest/session-manager.js +452 -0
  47. package/dest/test/index.d.ts +7 -6
  48. package/dest/test/index.d.ts.map +1 -1
  49. package/package.json +23 -23
  50. package/src/actions/download-epoch-proving-job.ts +1 -1
  51. package/src/actions/rerun-epoch-proving-job.ts +114 -28
  52. package/src/bin/run-failed-epoch.ts +1 -2
  53. package/src/checkpoint-store.ts +218 -0
  54. package/src/config.ts +2 -1
  55. package/src/factory.ts +18 -10
  56. package/src/index.ts +1 -0
  57. package/src/job/checkpoint-prover.ts +468 -0
  58. package/src/job/epoch-session.ts +436 -0
  59. package/src/job/top-tree-job.ts +227 -0
  60. package/src/metrics.ts +102 -23
  61. package/src/monitors/epoch-monitor.ts +2 -2
  62. package/src/proof-publishing-service.ts +427 -0
  63. package/src/prover-node-publisher.ts +231 -77
  64. package/src/prover-node.ts +552 -251
  65. package/src/prover-publisher-factory.ts +3 -3
  66. package/src/session-manager.ts +552 -0
  67. package/src/test/index.ts +6 -6
  68. package/dest/job/epoch-proving-job.d.ts +0 -63
  69. package/dest/job/epoch-proving-job.d.ts.map +0 -1
  70. package/dest/job/epoch-proving-job.js +0 -762
  71. package/src/job/epoch-proving-job.ts +0 -465
package/src/metrics.ts CHANGED
@@ -18,47 +18,48 @@ import {
18
18
 
19
19
  import { formatEther, formatUnits } from 'viem';
20
20
 
21
+ import type { CheckpointStore } from './checkpoint-store.js';
22
+ import type { SessionManager } from './session-manager.js';
23
+
21
24
  export class ProverNodeJobMetrics {
22
- proverEpochExecutionDuration: Histogram;
23
25
  provingJobDuration: Histogram;
24
26
  provingJobCheckpoints: Gauge;
25
27
  provingJobBlocks: Gauge;
26
28
  provingJobTransactions: Gauge;
27
29
 
28
30
  private blobProcessingDuration: Gauge;
29
- private chonkVerifierDuration: Gauge;
30
31
  private blockProcessingDuration: Histogram;
31
32
  private checkpointProcessingDuration: Histogram;
32
- private allCheckpointsProcessingDuration: Gauge;
33
+ private checkpointProvingDuration: Histogram;
34
+ private checkpointBlocks: Histogram;
35
+ private checkpointTransactions: Histogram;
36
+
37
+ /** Observable gauges for live state. Registered via `observeState(...)` once the
38
+ * CheckpointStore and SessionManager are available. */
39
+ private activeCheckpoints: ObservableGauge | undefined;
40
+ private activeEpochSessions: ObservableGauge | undefined;
41
+ private stateObserver: ((observer: BatchObservableResult) => void) | undefined;
42
+ private stateObservedMetrics: ObservableGauge[] = [];
33
43
 
34
44
  constructor(
35
45
  private meter: Meter,
36
46
  public readonly tracer: Tracer,
37
47
  private logger = createLogger('prover-node:publisher:metrics'),
38
48
  ) {
39
- this.proverEpochExecutionDuration = this.meter.createHistogram(Metrics.PROVER_NODE_EXECUTION_DURATION);
40
49
  this.provingJobDuration = this.meter.createHistogram(Metrics.PROVER_NODE_JOB_DURATION);
41
50
  this.provingJobCheckpoints = this.meter.createGauge(Metrics.PROVER_NODE_JOB_CHECKPOINTS);
42
51
  this.provingJobBlocks = this.meter.createGauge(Metrics.PROVER_NODE_JOB_BLOCKS);
43
52
  this.provingJobTransactions = this.meter.createGauge(Metrics.PROVER_NODE_JOB_TRANSACTIONS);
44
53
 
45
54
  this.blobProcessingDuration = this.meter.createGauge(Metrics.PROVER_NODE_BLOB_PROCESSING_LAST_DURATION);
46
- this.chonkVerifierDuration = this.meter.createGauge(Metrics.PROVER_NODE_CHONK_VERIFIER_LAST_DURATION);
47
55
  this.blockProcessingDuration = this.meter.createHistogram(Metrics.PROVER_NODE_BLOCK_PROCESSING_DURATION);
48
56
  this.checkpointProcessingDuration = this.meter.createHistogram(Metrics.PROVER_NODE_CHECKPOINT_PROCESSING_DURATION);
49
- this.allCheckpointsProcessingDuration = this.meter.createGauge(
50
- Metrics.PROVER_NODE_ALL_CHECKPOINTS_PROCESSING_LAST_DURATION,
51
- );
57
+ this.checkpointProvingDuration = this.meter.createHistogram(Metrics.PROVER_NODE_CHECKPOINT_PROVING_DURATION);
58
+ this.checkpointBlocks = this.meter.createHistogram(Metrics.PROVER_NODE_CHECKPOINT_BLOCKS);
59
+ this.checkpointTransactions = this.meter.createHistogram(Metrics.PROVER_NODE_CHECKPOINT_TRANSACTIONS);
52
60
  }
53
61
 
54
- public recordProvingJob(
55
- executionTimeMs: number,
56
- totalTimeMs: number,
57
- numCheckpoints: number,
58
- numBlocks: number,
59
- numTxs: number,
60
- ) {
61
- this.proverEpochExecutionDuration.record(Math.ceil(executionTimeMs));
62
+ public recordProvingJob(totalTimeMs: number, numCheckpoints: number, numBlocks: number, numTxs: number) {
62
63
  this.provingJobDuration.record(totalTimeMs / 1000);
63
64
  this.provingJobCheckpoints.record(Math.floor(numCheckpoints));
64
65
  this.provingJobBlocks.record(Math.floor(numBlocks));
@@ -69,20 +70,61 @@ export class ProverNodeJobMetrics {
69
70
  this.blobProcessingDuration.record(Math.ceil(durationMs));
70
71
  }
71
72
 
72
- public recordChonkVerifier(durationMs: number) {
73
- this.chonkVerifierDuration.record(Math.ceil(durationMs));
74
- }
75
-
76
73
  public recordBlockProcessing(durationMs: number) {
77
74
  this.blockProcessingDuration.record(Math.ceil(durationMs));
78
75
  }
79
76
 
80
- public recordCheckpointProcessing(durationMs: number) {
77
+ public recordCheckpointProcessing(durationMs: number, numBlocks: number, numTxs: number) {
81
78
  this.checkpointProcessingDuration.record(Math.ceil(durationMs));
79
+ this.checkpointBlocks.record(Math.floor(numBlocks));
80
+ this.checkpointTransactions.record(Math.floor(numTxs));
82
81
  }
83
82
 
84
- public recordAllCheckpointsProcessing(durationMs: number) {
85
- this.allCheckpointsProcessingDuration.record(Math.ceil(durationMs));
83
+ public recordCheckpointProving(durationMs: number) {
84
+ this.checkpointProvingDuration.record(Math.ceil(durationMs));
85
+ }
86
+
87
+ /**
88
+ * Registers observable gauges for the prover-node's live state: how many canonical
89
+ * checkpoint provers are in the store, and how many epoch sessions are live (broken
90
+ * down by kind). Idempotent — repeated calls re-arm with the latest references.
91
+ *
92
+ * Call this once the `SessionManager` has been constructed (i.e. inside `ProverNode.start()`).
93
+ */
94
+ public observeState(checkpointStore: CheckpointStore, sessionManager: SessionManager): void {
95
+ this.stopObservingState();
96
+ this.activeCheckpoints = this.meter.createObservableGauge(Metrics.PROVER_NODE_ACTIVE_CHECKPOINTS);
97
+ this.activeEpochSessions = this.meter.createObservableGauge(Metrics.PROVER_NODE_ACTIVE_EPOCH_SESSIONS);
98
+ this.stateObserver = (observer: BatchObservableResult) => {
99
+ observer.observe(this.activeCheckpoints!, checkpointStore.listCanonical().length);
100
+ let full = 0;
101
+ let partial = 0;
102
+ for (const session of sessionManager.allSessions()) {
103
+ if (session.isTerminal()) {
104
+ continue;
105
+ }
106
+ if (session.getKind() === 'full') {
107
+ full++;
108
+ } else {
109
+ partial++;
110
+ }
111
+ }
112
+ observer.observe(this.activeEpochSessions!, full, { [Attributes.EPOCH_SESSION_KIND]: 'full' });
113
+ observer.observe(this.activeEpochSessions!, partial, { [Attributes.EPOCH_SESSION_KIND]: 'partial' });
114
+ };
115
+ this.stateObservedMetrics = [this.activeCheckpoints, this.activeEpochSessions];
116
+ this.meter.addBatchObservableCallback(this.stateObserver, this.stateObservedMetrics);
117
+ }
118
+
119
+ /** Tears down the observable callback registered by `observeState`. Idempotent. */
120
+ public stopObservingState(): void {
121
+ if (this.stateObserver) {
122
+ this.meter.removeBatchObservableCallback(this.stateObserver, this.stateObservedMetrics);
123
+ this.stateObserver = undefined;
124
+ this.stateObservedMetrics = [];
125
+ this.activeCheckpoints = undefined;
126
+ this.activeEpochSessions = undefined;
127
+ }
86
128
  }
87
129
  }
88
130
 
@@ -140,6 +182,13 @@ export class ProverNodeRewardsMetrics {
140
182
  };
141
183
  }
142
184
 
185
+ export type EstimatedSubmitProofStats = {
186
+ gasLimit: bigint;
187
+ baseFeePerGas: bigint;
188
+ maxPriorityFeePerGas: bigint;
189
+ estimatedTotalFee: bigint;
190
+ };
191
+
143
192
  export class ProverNodePublisherMetrics {
144
193
  gasPrice: Histogram;
145
194
  txCount: UpDownCounter;
@@ -151,6 +200,10 @@ export class ProverNodePublisherMetrics {
151
200
  txBlobDataGasCost: Histogram;
152
201
  txTotalFee: Histogram;
153
202
 
203
+ private txGasEstimated: Histogram;
204
+ private gasPriceEstimated: Histogram;
205
+ private txTotalFeeEstimated: Histogram;
206
+
154
207
  private senderBalance: Gauge;
155
208
  private meter: Meter;
156
209
 
@@ -182,6 +235,12 @@ export class ProverNodePublisherMetrics {
182
235
 
183
236
  this.txTotalFee = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_TOTAL_FEE);
184
237
 
238
+ this.txGasEstimated = this.meter.createHistogram(Metrics.PROVER_NODE_ESTIMATED_SUBMISSION_GAS);
239
+
240
+ this.gasPriceEstimated = this.meter.createHistogram(Metrics.PROVER_NODE_ESTIMATED_SUBMISSION_GAS_PRICE);
241
+
242
+ this.txTotalFeeEstimated = this.meter.createHistogram(Metrics.PROVER_NODE_ESTIMATED_SUBMISSION_TOTAL_FEE);
243
+
185
244
  this.senderBalance = this.meter.createGauge(Metrics.L1_PUBLISHER_BALANCE);
186
245
  }
187
246
 
@@ -196,6 +255,26 @@ export class ProverNodePublisherMetrics {
196
255
  this.recordTx(durationMs, stats);
197
256
  }
198
257
 
258
+ public recordEstimatedSubmitProof(stats: EstimatedSubmitProofStats) {
259
+ const attributes = { [Attributes.L1_TX_TYPE]: 'submitProof' } as const;
260
+
261
+ this.txGasEstimated.record(Number(stats.gasLimit), attributes);
262
+
263
+ try {
264
+ this.gasPriceEstimated.record(
265
+ parseInt(formatEther(stats.baseFeePerGas + stats.maxPriorityFeePerGas, 'gwei'), 10),
266
+ );
267
+ } catch {
268
+ // ignore
269
+ }
270
+
271
+ try {
272
+ this.txTotalFeeEstimated.record(parseFloat(formatEther(stats.estimatedTotalFee)));
273
+ } catch {
274
+ // ignore
275
+ }
276
+ }
277
+
199
278
  public recordSenderBalance(wei: bigint, senderAddress: string) {
200
279
  const eth = parseFloat(formatEther(wei, 'wei'));
201
280
  this.senderBalance.record(eth, {
@@ -96,9 +96,9 @@ export class EpochMonitor implements Traceable {
96
96
  }
97
97
 
98
98
  private async getEpochNumberToProve() {
99
- const lastBlockProven = await this.l2BlockSource.getProvenBlockNumber();
99
+ const lastBlockProven = (await this.l2BlockSource.getBlockNumber({ tag: 'proven' })) ?? BlockNumber.ZERO;
100
100
  const firstBlockToProve = BlockNumber(lastBlockProven + 1);
101
- const firstBlockHeaderToProve = await this.l2BlockSource.getBlockHeader(firstBlockToProve);
101
+ const firstBlockHeaderToProve = (await this.l2BlockSource.getBlockData({ number: firstBlockToProve }))?.header;
102
102
  if (!firstBlockHeaderToProve) {
103
103
  return { epochToProve: undefined, blockNumber: firstBlockToProve };
104
104
  }
@@ -0,0 +1,427 @@
1
+ import type { BatchedBlob } from '@aztec/blob-lib';
2
+ import type { ViemCommitteeAttestation } from '@aztec/ethereum/contracts';
3
+ import { BlockNumber, type CheckpointNumber, type EpochNumber } from '@aztec/foundation/branded-types';
4
+ import { type Logger, type LoggerBindings, createLogger } from '@aztec/foundation/log';
5
+ import { promiseWithResolvers } from '@aztec/foundation/promise';
6
+ import { SerialQueue } from '@aztec/foundation/queue';
7
+ import type { DateProvider } from '@aztec/foundation/timer';
8
+ import type { L2BlockSource } from '@aztec/stdlib/block';
9
+ import type { Proof } from '@aztec/stdlib/proofs';
10
+ import type { CheckpointHeader, RootRollupPublicInputs } from '@aztec/stdlib/rollup';
11
+
12
+ import type { ProverNodePublisher } from './prover-node-publisher.js';
13
+ import type { ProverPublisherFactory } from './prover-publisher-factory.js';
14
+
15
+ /** A single proof candidate offered to the publishing service by an `EpochSession`. */
16
+ export type PublishCandidate = {
17
+ /** Stable id; matches the originating session so `withdraw` can target this entry. */
18
+ id: string;
19
+ epoch: EpochNumber;
20
+ /**
21
+ * Full vs partial. A `partial` candidate is an early-finish optimisation: if the chain's
22
+ * proven tip catches up to or past its `endBlock` before it publishes, it's superseded —
23
+ * publishing would be wasted L1 gas. A `full` candidate covers the entire epoch and is
24
+ * useful to publish even after some other prover-node has already submitted (the rollup
25
+ * contract records the submission per prover-id), so it is never auto-superseded by the
26
+ * proven tip alone.
27
+ */
28
+ kind: 'full' | 'partial';
29
+ /** First L2 block in the candidate's range. */
30
+ startBlock: BlockNumber;
31
+ /** Last L2 block in the candidate's range. */
32
+ endBlock: BlockNumber;
33
+ /**
34
+ * Wall-clock time after which the candidate is no longer worth publishing — typically
35
+ * the L1 proof-submission window deadline. If the candidate is still queued at this
36
+ * time it resolves as `'expired'`. If it's already in flight, the publish runs to
37
+ * completion (the L1 tx may still mine; the deadline only governs whether the service
38
+ * will start a publish). `undefined` disables the per-candidate timer.
39
+ */
40
+ deadline: Date | undefined;
41
+ /** Everything `ProverNodePublisher.submitEpochProof` needs. */
42
+ fromCheckpoint: CheckpointNumber;
43
+ toCheckpoint: CheckpointNumber;
44
+ publicInputs: RootRollupPublicInputs;
45
+ proof: Proof;
46
+ batchedBlobInputs: BatchedBlob;
47
+ attestations: ViemCommitteeAttestation[];
48
+ /** Committee-attested checkpoint headers for the range, supplying the L1-verified fee recipient/value. */
49
+ headers: CheckpointHeader[];
50
+ };
51
+
52
+ /** Terminal outcome for a candidate. The promise from `submit()` resolves with one of these. */
53
+ export type PublishOutcome = 'published' | 'superseded' | 'failed' | 'withdrawn' | 'expired';
54
+
55
+ /** Subset of `ProverPublisherFactory` the service uses — single async `create()` call. */
56
+ export type PublisherFactoryLike = Pick<ProverPublisherFactory, 'create'>;
57
+
58
+ /** Subset of `ProverNodePublisher` the service drives — one publish per fresh publisher. */
59
+ export type PublisherLike = Pick<ProverNodePublisher, 'submitEpochProof' | 'analyzeEpochProofSubmission'>;
60
+
61
+ /** Config for the publishing service. */
62
+ export type ProofPublishingServiceConfig = {
63
+ /** When true, submitting a candidate runs `analyzeEpochProofSubmission` instead of publishing. */
64
+ skipSubmitProof: boolean;
65
+ };
66
+
67
+ export type ProofPublishingServiceDeps = {
68
+ publisherFactory: PublisherFactoryLike;
69
+ l2BlockSource: Pick<L2BlockSource, 'getBlockNumber'>;
70
+ dateProvider: DateProvider;
71
+ config: ProofPublishingServiceConfig;
72
+ bindings?: LoggerBindings;
73
+ };
74
+
75
+ /** Per-epoch bucket: live candidates, their pending-outcome resolvers, and expiry timers. */
76
+ type EpochBucket = {
77
+ candidates: Map<string, PublishCandidate>;
78
+ resolvers: Map<string, (outcome: PublishOutcome) => void>;
79
+ expiryTimers: Map<string, NodeJS.Timeout>;
80
+ };
81
+
82
+ /**
83
+ * Backoff after a transient `publisherFactory.create()` failure. The candidate stays
84
+ * in the queue and the drain is re-scheduled after this delay; if the failure persists
85
+ * the candidate's own `deadline` timer caps the total wait.
86
+ */
87
+ const PUBLISHER_ACQUIRE_RETRY_DELAY_MS = 1_000;
88
+
89
+ /**
90
+ * Central owner of L1 proof submission. Sessions offer their proofs here as
91
+ * `PublishCandidate`s; the service serialises one publish at a time, picks the
92
+ * longest candidate per epoch as the winner, and resolves the rest as
93
+ * `'superseded'` without spending L1 gas.
94
+ *
95
+ * Construction-time invariants:
96
+ * - Every publish runs against a freshly-created `ProverNodePublisher` from the factory.
97
+ * - Only one publish is ever in flight (`SerialQueue` drain) — no defensive locks.
98
+ * - Once an L1 publish starts, it runs to completion. `withdraw` is a queue-only
99
+ * operation: it removes a candidate that has not yet started publishing. An in-flight
100
+ * candidate is left alone and its outcome (`'published'` / `'failed'`) is reported as
101
+ * usual — the originating session has already moved to a terminal state via `cancel()`
102
+ * and ignores the late outcome.
103
+ *
104
+ * Eligibility for publication is decided against the proven block number read inside
105
+ * the drain (so the value is consistent with the publish that runs on the same drain
106
+ * pass): a candidate is eligible when its predecessor block is proven and (for partial
107
+ * candidates) the candidate's range extends past the proven tip. `onChainProven` is a
108
+ * wake-up signal; it does not pass state into the drain.
109
+ */
110
+ export class ProofPublishingService {
111
+ private readonly log: Logger;
112
+ private readonly epochs: Map<EpochNumber, EpochBucket> = new Map();
113
+ /**
114
+ * One drain task at a time. Submits, withdrawals, chain-proven advances, and prunes
115
+ * all schedule a `drain` here, so the eligibility re-check and the L1 publish never
116
+ * interleave.
117
+ *
118
+ * Protected so unit tests can `await drainQueue.syncPoint()` to wait for pending
119
+ * drain work to settle deterministically (no sleeps).
120
+ */
121
+ protected readonly drainQueue = new SerialQueue();
122
+ /** Tracks the candidate currently being published. Set while drain is awaiting the L1 publish. */
123
+ private inFlight: { id: string } | undefined;
124
+ private stopped = false;
125
+
126
+ constructor(private readonly deps: ProofPublishingServiceDeps) {
127
+ this.log = createLogger('prover-node:proof-publishing-service', deps.bindings);
128
+ this.drainQueue.start();
129
+ }
130
+
131
+ /**
132
+ * Offers a proof candidate to the service. The returned promise resolves once the
133
+ * service settles the candidate's fate: `'published'` if it wins and L1 accepts it,
134
+ * `'superseded'` if a longer candidate for the same epoch wins, `'failed'` if the
135
+ * L1 submission errored, `'withdrawn'` if the originating session cancelled,
136
+ * `'expired'` if the candidate's `deadline` elapsed before publishing started.
137
+ */
138
+ public submit(candidate: PublishCandidate): Promise<PublishOutcome> {
139
+ if (this.stopped) {
140
+ return Promise.resolve<PublishOutcome>('withdrawn');
141
+ }
142
+ const { promise, resolve } = promiseWithResolvers<PublishOutcome>();
143
+ let bucket = this.epochs.get(candidate.epoch);
144
+ if (!bucket) {
145
+ bucket = { candidates: new Map(), resolvers: new Map(), expiryTimers: new Map() };
146
+ this.epochs.set(candidate.epoch, bucket);
147
+ }
148
+ bucket.candidates.set(candidate.id, candidate);
149
+ bucket.resolvers.set(candidate.id, resolve);
150
+ this.scheduleExpiry(bucket, candidate);
151
+ this.log.info(`Candidate proof ${candidate.id} submitted for publishing`, {
152
+ candidateId: candidate.id,
153
+ epoch: candidate.epoch,
154
+ startBlock: candidate.startBlock,
155
+ endBlock: candidate.endBlock,
156
+ deadline: candidate.deadline?.toISOString(),
157
+ });
158
+ this.scheduleDrain();
159
+ return promise;
160
+ }
161
+
162
+ /**
163
+ * Pulls a queued candidate from the bucket and resolves its promise as `'withdrawn'`.
164
+ * If the candidate is already being published, the publish runs to completion and the
165
+ * outcome reports whatever L1 returned — callers that cancelled mid-publish must rely
166
+ * on their own terminal-state check to ignore the late outcome. No-op if the candidate
167
+ * is unknown.
168
+ */
169
+ public withdraw(candidateId: string): void {
170
+ if (this.inFlight?.id === candidateId) {
171
+ this.log.debug(`Withdraw for in-flight candidate ${candidateId} ignored; publish will run to completion`, {
172
+ candidateId,
173
+ });
174
+ return;
175
+ }
176
+ for (const bucket of this.epochs.values()) {
177
+ if (bucket.candidates.has(candidateId)) {
178
+ this.log.info(`Candidate ${candidateId} withdrawn`, { candidateId });
179
+ this.resolveCandidate(bucket, candidateId, 'withdrawn');
180
+ this.scheduleDrain();
181
+ return;
182
+ }
183
+ }
184
+ }
185
+
186
+ /**
187
+ * Signals that the L1 proven tip has advanced and the queue should be re-evaluated.
188
+ * The drain reads the proven block number from `l2BlockSource` itself rather than
189
+ * relying on the value passed here — that way the eligibility check uses a value read
190
+ * inside the serial drain, not one captured by a concurrent caller of `onChainProven`.
191
+ */
192
+ public onChainProven(_provenBlock: BlockNumber): void {
193
+ this.scheduleDrain();
194
+ }
195
+
196
+ /**
197
+ * Stops accepting new submissions, waits for any in-flight publish to settle, and
198
+ * resolves remaining queued candidates as `'withdrawn'`.
199
+ */
200
+ public async stop(): Promise<void> {
201
+ this.stopped = true;
202
+ await this.drainQueue.end();
203
+ // Anything still parked in a bucket never ran through drain — resolve it as withdrawn so
204
+ // callers awaiting `submit()` aren't left hanging.
205
+ for (const bucket of Array.from(this.epochs.values())) {
206
+ for (const id of Array.from(bucket.candidates.keys())) {
207
+ this.resolveCandidate(bucket, id, 'withdrawn');
208
+ }
209
+ }
210
+ this.epochs.clear();
211
+ }
212
+
213
+ // ---------------- drain ----------------
214
+
215
+ private scheduleDrain(): void {
216
+ if (this.stopped) {
217
+ return;
218
+ }
219
+ void this.drainQueue
220
+ .put(() => this.drain())
221
+ .catch(err => {
222
+ this.log.error(`Drain task threw`, err);
223
+ });
224
+ }
225
+
226
+ private async drain(): Promise<void> {
227
+ if (this.stopped) {
228
+ return;
229
+ }
230
+ // Read the proven block number afresh inside the serial drain so the eligibility
231
+ // check is consistent with the publish that follows it on the same drain pass.
232
+ const proven = await this.readProvenBlockNumber();
233
+
234
+ // Process epochs in ascending order: the proven tip advances monotonically, so the lower
235
+ // epoch is the natural next eligible candidate.
236
+ const orderedEpochs = Array.from(this.epochs.keys()).sort((a, b) => Number(a) - Number(b));
237
+ for (const epoch of orderedEpochs) {
238
+ const bucket = this.epochs.get(epoch)!;
239
+ const eligible = this.pickEpochWinner(bucket, proven);
240
+ if (!eligible) {
241
+ continue;
242
+ }
243
+ await this.publishWinner(epoch, eligible.winner, bucket);
244
+ }
245
+
246
+ // Drop empty buckets
247
+ for (const [key, bucket] of Array.from(this.epochs.entries())) {
248
+ if (bucket.candidates.size === 0) {
249
+ this.epochs.delete(key);
250
+ }
251
+ }
252
+ }
253
+
254
+ /**
255
+ * Picks the winning candidate for a given epoch. Partial candidates whose `endBlock` is
256
+ * already proven on-chain resolve `'superseded'`.
257
+ * Full candidates are never auto-superseded by the proven tip — multiple prover-nodes
258
+ * legitimately submit redundant full epoch proofs (one per prover-id) and L1 records each.
259
+ * Among the remaining candidates with their predecessor proven, the one with the highest
260
+ * `endBlock` wins; the others resolve `'superseded'`.
261
+ */
262
+ private pickEpochWinner(bucket: EpochBucket, proven: BlockNumber): { winner: PublishCandidate } | undefined {
263
+ const now = this.deps.dateProvider.now();
264
+ // Resolve any candidate whose deadline has already passed.
265
+ for (const candidate of Array.from(bucket.candidates.values())) {
266
+ if (candidate.deadline && candidate.deadline.getTime() <= now) {
267
+ this.resolveCandidate(bucket, candidate.id, 'expired');
268
+ }
269
+ }
270
+ // Drop partial candidates the proven chain has already caught up to.
271
+ for (const candidate of Array.from(bucket.candidates.values())) {
272
+ if (candidate.kind === 'partial' && candidate.endBlock <= proven) {
273
+ this.resolveCandidate(bucket, candidate.id, 'superseded');
274
+ }
275
+ }
276
+
277
+ const remaining = Array.from(bucket.candidates.values()).filter(c => c.startBlock - 1 <= proven);
278
+ if (remaining.length === 0) {
279
+ return undefined;
280
+ }
281
+ const winner = remaining.reduce((best, c) => (c.endBlock > best.endBlock ? c : best));
282
+ // Every other same-epoch candidate is superseded by the winner.
283
+ for (const candidate of remaining) {
284
+ if (candidate.id !== winner.id) {
285
+ this.resolveCandidate(bucket, candidate.id, 'superseded');
286
+ }
287
+ }
288
+ return { winner };
289
+ }
290
+
291
+ private async publishWinner(epoch: EpochNumber, winner: PublishCandidate, bucket: EpochBucket): Promise<void> {
292
+ let publisher: PublisherLike;
293
+ try {
294
+ publisher = await this.deps.publisherFactory.create();
295
+ } catch (err) {
296
+ // Treat this as transient: the publisher pool may be temporarily exhausted
297
+ // (every signer busy, funding tx in flight, etc.). Leave the candidate queued and
298
+ // schedule another drain after a short backoff. If the failure persists past the
299
+ // candidate's deadline the expiry timer will resolve it as `'expired'`.
300
+ this.log.warn(`Failed to acquire publisher for candidate ${winner.id}; retrying`, {
301
+ candidateId: winner.id,
302
+ epoch: winner.epoch,
303
+ retryDelayMs: PUBLISHER_ACQUIRE_RETRY_DELAY_MS,
304
+ err,
305
+ });
306
+ setTimeout(() => this.scheduleDrain(), PUBLISHER_ACQUIRE_RETRY_DELAY_MS);
307
+ return;
308
+ }
309
+
310
+ this.inFlight = { id: winner.id };
311
+ this.log.info(`Publishing candidate ${winner.id}`, {
312
+ candidateId: winner.id,
313
+ epoch: winner.epoch,
314
+ startBlock: winner.startBlock,
315
+ endBlock: winner.endBlock,
316
+ fromCheckpoint: winner.fromCheckpoint,
317
+ toCheckpoint: winner.toCheckpoint,
318
+ });
319
+
320
+ const outcome = await this.runPublish(winner, publisher);
321
+ this.inFlight = undefined;
322
+ this.resolveCandidate(bucket, winner.id, outcome);
323
+
324
+ if (bucket.candidates.size === 0) {
325
+ this.epochs.delete(epoch);
326
+ }
327
+ }
328
+
329
+ private async runPublish(candidate: PublishCandidate, publisher: PublisherLike): Promise<PublishOutcome> {
330
+ const submitArgs = {
331
+ epochNumber: candidate.epoch,
332
+ fromCheckpoint: candidate.fromCheckpoint,
333
+ toCheckpoint: candidate.toCheckpoint,
334
+ publicInputs: candidate.publicInputs,
335
+ proof: candidate.proof,
336
+ batchedBlobInputs: candidate.batchedBlobInputs,
337
+ attestations: candidate.attestations,
338
+ headers: candidate.headers,
339
+ // Stop the L1 tx retrying past the candidate's submission-window deadline.
340
+ deadline: candidate.deadline,
341
+ };
342
+
343
+ if (this.deps.config.skipSubmitProof) {
344
+ try {
345
+ await publisher.analyzeEpochProofSubmission(submitArgs);
346
+ return 'published';
347
+ } catch (err) {
348
+ this.log.warn(`Failed to analyze estimated L1 fees for candidate ${candidate.id}`, {
349
+ err,
350
+ candidateId: candidate.id,
351
+ epoch: candidate.epoch,
352
+ });
353
+ // Analyze-mode failures are recorded but the session shouldn't enter `failed` —
354
+ // the operator opted out of submission. Match the previous EpochSession behaviour.
355
+ return 'published';
356
+ }
357
+ }
358
+
359
+ try {
360
+ const success = await publisher.submitEpochProof(submitArgs);
361
+ return success ? 'published' : 'failed';
362
+ } catch (err) {
363
+ this.log.error(`Error publishing candidate ${candidate.id}`, err, {
364
+ candidateId: candidate.id,
365
+ epoch: candidate.epoch,
366
+ });
367
+ return 'failed';
368
+ }
369
+ }
370
+
371
+ private resolveCandidate(bucket: EpochBucket, id: string, outcome: PublishOutcome): void {
372
+ const resolve = bucket.resolvers.get(id);
373
+ const timer = bucket.expiryTimers.get(id);
374
+ if (timer) {
375
+ clearTimeout(timer);
376
+ bucket.expiryTimers.delete(id);
377
+ }
378
+ bucket.candidates.delete(id);
379
+ bucket.resolvers.delete(id);
380
+ if (resolve) {
381
+ this.log.info(`Candidate ${id} resolved as ${outcome}`, { candidateId: id, outcome });
382
+ resolve(outcome);
383
+ }
384
+ }
385
+
386
+ /**
387
+ * Arms a per-candidate expiry timer if the candidate carries a deadline. When the timer
388
+ * fires, the candidate resolves as `'expired'` — unless it is already in flight, in
389
+ * which case the publish runs to completion (the timer becomes a no-op). The timer is
390
+ * cleared by `resolveCandidate` whenever the candidate settles for any other reason.
391
+ */
392
+ private scheduleExpiry(bucket: EpochBucket, candidate: PublishCandidate): void {
393
+ if (!candidate.deadline) {
394
+ return;
395
+ }
396
+ const delay = Math.max(candidate.deadline.getTime() - this.deps.dateProvider.now(), 0);
397
+ const timer = setTimeout(() => this.handleExpiry(candidate.id), delay);
398
+ bucket.expiryTimers.set(candidate.id, timer);
399
+ }
400
+
401
+ /**
402
+ * Protected so unit tests can drive the deadline path without waiting on the real
403
+ * `setTimeout` to fire. Production code calls this only via the per-candidate timer
404
+ * armed in `scheduleExpiry`.
405
+ */
406
+ protected handleExpiry(candidateId: string): void {
407
+ if (this.inFlight?.id === candidateId) {
408
+ this.log.debug(`Expiry for in-flight candidate ${candidateId} ignored; publish will run to completion`, {
409
+ candidateId,
410
+ });
411
+ return;
412
+ }
413
+ for (const bucket of this.epochs.values()) {
414
+ if (bucket.candidates.has(candidateId)) {
415
+ this.log.info(`Candidate ${candidateId} expired before publishing`, { candidateId });
416
+ this.resolveCandidate(bucket, candidateId, 'expired');
417
+ this.scheduleDrain();
418
+ return;
419
+ }
420
+ }
421
+ }
422
+
423
+ private async readProvenBlockNumber(): Promise<BlockNumber> {
424
+ const proven = await this.deps.l2BlockSource.getBlockNumber({ tag: 'proven' });
425
+ return BlockNumber(proven ?? 0);
426
+ }
427
+ }