@aztec/sequencer-client 0.0.1-commit.b1c78909e → 0.0.1-commit.b2a5d0dd1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. package/dest/client/sequencer-client.d.ts +6 -12
  2. package/dest/client/sequencer-client.d.ts.map +1 -1
  3. package/dest/client/sequencer-client.js +30 -80
  4. package/dest/config.d.ts +4 -3
  5. package/dest/config.d.ts.map +1 -1
  6. package/dest/config.js +17 -13
  7. package/dest/global_variable_builder/fee_predictor.d.ts +37 -0
  8. package/dest/global_variable_builder/fee_predictor.d.ts.map +1 -0
  9. package/dest/global_variable_builder/fee_predictor.js +128 -0
  10. package/dest/global_variable_builder/fee_provider.d.ts +21 -0
  11. package/dest/global_variable_builder/fee_provider.d.ts.map +1 -0
  12. package/dest/global_variable_builder/fee_provider.js +58 -0
  13. package/dest/global_variable_builder/global_builder.d.ts +15 -14
  14. package/dest/global_variable_builder/global_builder.d.ts.map +1 -1
  15. package/dest/global_variable_builder/global_builder.js +16 -50
  16. package/dest/global_variable_builder/index.d.ts +4 -2
  17. package/dest/global_variable_builder/index.d.ts.map +1 -1
  18. package/dest/global_variable_builder/index.js +2 -0
  19. package/dest/publisher/config.d.ts +13 -1
  20. package/dest/publisher/config.d.ts.map +1 -1
  21. package/dest/publisher/config.js +19 -4
  22. package/dest/publisher/l1_tx_failed_store/failed_tx_store.d.ts +3 -4
  23. package/dest/publisher/l1_tx_failed_store/failed_tx_store.d.ts.map +1 -1
  24. package/dest/publisher/sequencer-publisher-factory.d.ts +3 -5
  25. package/dest/publisher/sequencer-publisher-factory.d.ts.map +1 -1
  26. package/dest/publisher/sequencer-publisher-factory.js +2 -3
  27. package/dest/publisher/sequencer-publisher.d.ts +51 -43
  28. package/dest/publisher/sequencer-publisher.d.ts.map +1 -1
  29. package/dest/publisher/sequencer-publisher.js +135 -111
  30. package/dest/sequencer/chain_state_overrides.d.ts +25 -0
  31. package/dest/sequencer/chain_state_overrides.d.ts.map +1 -0
  32. package/dest/sequencer/chain_state_overrides.js +39 -0
  33. package/dest/sequencer/checkpoint_proposal_job.d.ts +40 -8
  34. package/dest/sequencer/checkpoint_proposal_job.d.ts.map +1 -1
  35. package/dest/sequencer/checkpoint_proposal_job.js +441 -183
  36. package/dest/sequencer/checkpoint_proposal_job_metrics.d.ts +34 -0
  37. package/dest/sequencer/checkpoint_proposal_job_metrics.d.ts.map +1 -0
  38. package/dest/sequencer/checkpoint_proposal_job_metrics.js +72 -0
  39. package/dest/sequencer/checkpoint_voter.d.ts +1 -2
  40. package/dest/sequencer/checkpoint_voter.d.ts.map +1 -1
  41. package/dest/sequencer/checkpoint_voter.js +2 -5
  42. package/dest/sequencer/events.d.ts +7 -1
  43. package/dest/sequencer/events.d.ts.map +1 -1
  44. package/dest/sequencer/metrics.d.ts +11 -10
  45. package/dest/sequencer/metrics.d.ts.map +1 -1
  46. package/dest/sequencer/metrics.js +38 -20
  47. package/dest/sequencer/sequencer.d.ts +21 -6
  48. package/dest/sequencer/sequencer.d.ts.map +1 -1
  49. package/dest/sequencer/sequencer.js +125 -60
  50. package/dest/sequencer/timetable.d.ts +14 -1
  51. package/dest/sequencer/timetable.d.ts.map +1 -1
  52. package/dest/sequencer/timetable.js +45 -36
  53. package/dest/test/mock_checkpoint_builder.d.ts +4 -4
  54. package/dest/test/mock_checkpoint_builder.d.ts.map +1 -1
  55. package/package.json +27 -28
  56. package/src/client/sequencer-client.ts +42 -108
  57. package/src/config.ts +20 -12
  58. package/src/global_variable_builder/README.md +44 -0
  59. package/src/global_variable_builder/fee_predictor.ts +172 -0
  60. package/src/global_variable_builder/fee_provider.ts +75 -0
  61. package/src/global_variable_builder/global_builder.ts +26 -62
  62. package/src/global_variable_builder/index.ts +3 -1
  63. package/src/publisher/config.ts +38 -4
  64. package/src/publisher/l1_tx_failed_store/failed_tx_store.ts +3 -1
  65. package/src/publisher/sequencer-publisher-factory.ts +3 -6
  66. package/src/publisher/sequencer-publisher.ts +202 -158
  67. package/src/sequencer/README.md +83 -13
  68. package/src/sequencer/chain_state_overrides.ts +87 -0
  69. package/src/sequencer/checkpoint_proposal_job.ts +558 -204
  70. package/src/sequencer/checkpoint_proposal_job_metrics.ts +128 -0
  71. package/src/sequencer/checkpoint_voter.ts +1 -12
  72. package/src/sequencer/events.ts +6 -1
  73. package/src/sequencer/metrics.ts +49 -25
  74. package/src/sequencer/sequencer.ts +185 -66
  75. package/src/sequencer/timetable.ts +57 -45
  76. package/src/test/mock_checkpoint_builder.ts +3 -3
@@ -1,4 +1,5 @@
1
1
  import type { EpochCache } from '@aztec/epoch-cache';
2
+ import type { SimulationOverridesPlan } from '@aztec/ethereum/contracts';
2
3
  import {
3
4
  BlockNumber,
4
5
  CheckpointNumber,
@@ -17,6 +18,7 @@ import { EthAddress } from '@aztec/foundation/eth-address';
17
18
  import { Signature } from '@aztec/foundation/eth-signature';
18
19
  import { filter } from '@aztec/foundation/iterator';
19
20
  import { type Logger, type LoggerBindings, createLogger } from '@aztec/foundation/log';
21
+ import { retryUntil } from '@aztec/foundation/retry';
20
22
  import { sleep, sleepUntil } from '@aztec/foundation/sleep';
21
23
  import { type DateProvider, Timer } from '@aztec/foundation/timer';
22
24
  import { type TypedEventEmitter, isErrorClass, unfreeze } from '@aztec/foundation/types';
@@ -29,18 +31,24 @@ import {
29
31
  type L2BlockSink,
30
32
  type L2BlockSource,
31
33
  MaliciousCommitteeAttestationsAndSigners,
34
+ type ValidateCheckpointResult,
32
35
  } from '@aztec/stdlib/block';
33
- import { type Checkpoint, validateCheckpoint } from '@aztec/stdlib/checkpoint';
34
- import { getSlotStartBuildTimestamp } from '@aztec/stdlib/epoch-helpers';
36
+ import { type Checkpoint, type ProposedCheckpointData, validateCheckpoint } from '@aztec/stdlib/checkpoint';
37
+ import { computeQuorum, getSlotStartBuildTimestamp, getTimestampForSlot } from '@aztec/stdlib/epoch-helpers';
35
38
  import { Gas } from '@aztec/stdlib/gas';
36
39
  import {
37
- NoValidTxsError,
38
- type PublicProcessorLimits,
40
+ type BlockBuilderOptions,
41
+ InsufficientValidTxsError,
39
42
  type ResolvedSequencerConfig,
40
43
  type WorldStateSynchronizer,
41
44
  } from '@aztec/stdlib/interfaces/server';
42
45
  import { type L1ToL2MessageSource, computeInHashFromL1ToL2Messages } from '@aztec/stdlib/messaging';
43
- import type { BlockProposalOptions, CheckpointProposal, CheckpointProposalOptions } from '@aztec/stdlib/p2p';
46
+ import type {
47
+ BlockProposal,
48
+ BlockProposalOptions,
49
+ CheckpointProposal,
50
+ CheckpointProposalOptions,
51
+ } from '@aztec/stdlib/p2p';
44
52
  import { orderAttestations, trimAttestations } from '@aztec/stdlib/p2p';
45
53
  import type { L2BlockBuiltStats } from '@aztec/stdlib/stats';
46
54
  import { type FailedTx, Tx } from '@aztec/stdlib/tx';
@@ -51,6 +59,11 @@ import { DutyAlreadySignedError, SlashingProtectionError } from '@aztec/validato
51
59
 
52
60
  import type { GlobalVariableBuilder } from '../global_variable_builder/global_builder.js';
53
61
  import type { InvalidateCheckpointRequest, SequencerPublisher } from '../publisher/sequencer-publisher.js';
62
+ import {
63
+ buildPipelinedParentSimulationOverridesPlan,
64
+ buildSubmissionSimulationOverridesPlan,
65
+ } from './chain_state_overrides.js';
66
+ import type { CheckpointProposalJobMetricsRecorder } from './checkpoint_proposal_job_metrics.js';
54
67
  import { CheckpointVoter } from './checkpoint_voter.js';
55
68
  import { SequencerInterruptedError } from './errors.js';
56
69
  import type { SequencerEvents } from './events.js';
@@ -62,6 +75,20 @@ import { SequencerState } from './utils.js';
62
75
  /** How much time to sleep while waiting for min transactions to accumulate for a block */
63
76
  const TXS_POLLING_MS = 500;
64
77
 
78
+ /** Result from proposeCheckpoint when a checkpoint was successfully built and broadcast. */
79
+ type CheckpointProposalBroadcast = {
80
+ checkpoint: Checkpoint;
81
+ proposal: CheckpointProposal;
82
+ blockProposedAt: number;
83
+ };
84
+
85
+ /** Result after attestation collection and signing, ready for L1 submission. */
86
+ type CheckpointProposalResult = {
87
+ checkpoint: Checkpoint;
88
+ attestations: CommitteeAttestationsAndSigners;
89
+ attestationsSignature: Signature;
90
+ };
91
+
65
92
  /**
66
93
  * Handles the execution of a checkpoint proposal after the initial preparation phase.
67
94
  * This includes building blocks, collecting attestations, and publishing the checkpoint to L1,
@@ -71,9 +98,16 @@ const TXS_POLLING_MS = 500;
71
98
  export class CheckpointProposalJob implements Traceable {
72
99
  protected readonly log: Logger;
73
100
 
101
+ /** Tracks the fire-and-forget L1 submission promise so it can be awaited during shutdown. */
102
+ private pendingL1Submission: Promise<void> | undefined;
103
+
104
+ /** Pipelined parent chain state used while building and later submitting this checkpoint. */
105
+ private pipelinedParentSimulationOverridesPlan?: SimulationOverridesPlan;
106
+
74
107
  constructor(
75
- private readonly epoch: EpochNumber,
76
- private readonly slot: SlotNumber,
108
+ private readonly slotNow: SlotNumber,
109
+ private readonly targetSlot: SlotNumber,
110
+ private readonly targetEpoch: EpochNumber,
77
111
  private readonly checkpointNumber: CheckpointNumber,
78
112
  private readonly syncedToBlockNumber: BlockNumber,
79
113
  // TODO(palla/mbps): Can we remove the proposer in favor of attestorAddress? Need to check fisherman-node flows.
@@ -96,17 +130,31 @@ export class CheckpointProposalJob implements Traceable {
96
130
  private readonly epochCache: EpochCache,
97
131
  private readonly dateProvider: DateProvider,
98
132
  private readonly metrics: SequencerMetrics,
99
- private readonly eventEmitter: TypedEventEmitter<SequencerEvents>,
133
+ private readonly checkpointMetrics: CheckpointProposalJobMetricsRecorder,
134
+ protected readonly eventEmitter: TypedEventEmitter<SequencerEvents>,
100
135
  private readonly setStateFn: (state: SequencerState, slot?: SlotNumber) => void,
101
136
  public readonly tracer: Tracer,
102
137
  bindings?: LoggerBindings,
138
+ private readonly proposedCheckpointData?: ProposedCheckpointData,
103
139
  ) {
104
- this.log = createLogger('sequencer:checkpoint-proposal', { ...bindings, instanceId: `slot-${slot}` });
140
+ this.log = createLogger('sequencer:checkpoint-proposal', {
141
+ ...bindings,
142
+ instanceId: `slot-${this.slotNow}`,
143
+ });
144
+ }
145
+
146
+ /** Awaits the pending L1 submission if one is in progress. Call during shutdown. */
147
+ public async awaitPendingSubmission(): Promise<void> {
148
+ this.log.info('Awaiting pending L1 payload submission');
149
+ await this.pendingL1Submission;
105
150
  }
106
151
 
107
152
  /**
108
153
  * Executes the checkpoint proposal job.
109
- * Returns the published checkpoint if successful, undefined otherwise.
154
+ * Builds blocks, assembles checkpoint, and broadcasts the proposal (blocking).
155
+ * Attestation collection, signing, and L1 submission are backgrounded so the
156
+ * work loop can return to IDLE immediately for consecutive slot proposals.
157
+ * Returns the built checkpoint if successful, undefined otherwise.
110
158
  */
111
159
  @trackSpan('CheckpointProposalJob.execute')
112
160
  public async execute(): Promise<Checkpoint | undefined> {
@@ -114,7 +162,7 @@ export class CheckpointProposalJob implements Traceable {
114
162
  // In fisherman mode, we simulate slashing but don't actually publish to L1
115
163
  // These are constant for the whole slot, so we only enqueue them once
116
164
  const votesPromises = new CheckpointVoter(
117
- this.slot,
165
+ this.targetSlot,
118
166
  this.publisher,
119
167
  this.attestorAddress,
120
168
  this.validatorClient,
@@ -125,33 +173,271 @@ export class CheckpointProposalJob implements Traceable {
125
173
  this.log,
126
174
  ).enqueueVotes();
127
175
 
128
- // Build and propose the checkpoint. This will enqueue the request on the publisher if a checkpoint is built.
129
- const checkpoint = await this.proposeCheckpoint();
130
-
131
- // Wait until the voting promises have resolved, so all requests are enqueued (not sent)
132
- await Promise.all(votesPromises);
176
+ // Build blocks, assemble checkpoint, and broadcast proposal (BLOCKING).
177
+ // Returns after broadcast — attestation collection is deferred.
178
+ const broadcast = await this.proposeCheckpoint();
133
179
 
134
- if (checkpoint) {
135
- this.metrics.recordCheckpointProposalSuccess();
180
+ if (!broadcast) {
181
+ await Promise.all(votesPromises);
182
+ // Still submit votes even without a checkpoint
183
+ if (!this.config.fishermanMode) {
184
+ this.pendingL1Submission = this.publisher.sendRequestsAt(this.dateProvider.nowAsDate()).then(() => {});
185
+ }
186
+ return undefined;
136
187
  }
137
188
 
189
+ const { checkpoint } = broadcast;
190
+ this.metrics.recordCheckpointProposalSuccess();
191
+
138
192
  // Do not post anything to L1 if we are fishermen, but do perform L1 fee analysis
139
193
  if (this.config.fishermanMode) {
140
194
  await this.handleCheckpointEndAsFisherman(checkpoint);
141
- return;
195
+ return checkpoint;
142
196
  }
143
197
 
144
- // Then send everything to L1
145
- const l1Response = await this.publisher.sendRequests();
146
- const proposedAction = l1Response?.successfulActions.find(a => a === 'propose');
147
- if (proposedAction) {
148
- this.eventEmitter.emit('checkpoint-published', { checkpoint: this.checkpointNumber, slot: this.slot });
149
- const coinbase = checkpoint?.header.coinbase;
150
- await this.metrics.incFilledSlot(this.publisher.getSenderAddress().toString(), coinbase);
151
- return checkpoint;
152
- } else if (checkpoint) {
153
- this.eventEmitter.emit('checkpoint-publish-failed', { ...l1Response, slot: this.slot });
154
- return undefined;
198
+ // Background the attestation signing → L1 pipeline so the work loop is unblocked
199
+ this.pendingL1Submission = this.waitForAttestationsAndEnqueueSubmissionAsync(broadcast, votesPromises);
200
+
201
+ // Return the built checkpoint immediately — the work loop is now unblocked
202
+ return checkpoint;
203
+ }
204
+
205
+ /**
206
+ * Background pipeline: collects attestations, signs them, enqueues the checkpoint, and submits to L1.
207
+ * Runs as a fire-and-forget task stored in `pendingL1Submission` so the work loop is unblocked.
208
+ */
209
+ private async waitForAttestationsAndEnqueueSubmissionAsync(
210
+ broadcast: CheckpointProposalBroadcast,
211
+ votesPromises: Promise<unknown>[],
212
+ ): Promise<void> {
213
+ const { checkpoint } = broadcast;
214
+ const isPipelining = this.epochCache.isProposerPipeliningEnabled();
215
+
216
+ try {
217
+ // Wait for all votes actions, enqueued at the beginning, to resolve
218
+ await Promise.all(votesPromises);
219
+
220
+ // Try to collect attestations from the committee
221
+ const signedAttestations = await this.getSignedCommitteeAttestations(broadcast);
222
+
223
+ // If pipelining, wait for the previous checkpoint to land on L1 before submitting,
224
+ // so we can check it matches the proposed checkpoint we used as parent, and has valid attestations.
225
+ if (signedAttestations && (!isPipelining || (await this.waitForValidParentCheckpointOnL1()))) {
226
+ await this.enqueueCheckpointForSubmission({ checkpoint, ...signedAttestations });
227
+ }
228
+
229
+ // If we failed to collect attestations, at least check if we need to issue an invalidation
230
+ // Note that if we are not pipelining, we enqueued the invalidation at the beginning
231
+ if (!signedAttestations && isPipelining && (await this.waitForSyncedL2SlotNumber(this.slotNow))) {
232
+ const validationStatus = await this.l2BlockSource.getPendingChainValidationStatus();
233
+ if (!validationStatus.valid) {
234
+ this.log.warn(
235
+ `Checkpoint ${validationStatus.checkpoint.checkpointNumber} has invalid attestations, enqueuing invalidation in spite of attestation collection failure`,
236
+ { checkpoint: validationStatus.checkpoint, reason: validationStatus.reason },
237
+ );
238
+ await this.enqueueInvalidation(validationStatus);
239
+ }
240
+ }
241
+
242
+ // Send whatever was enqueued: votes + (propose | invalidation | nothing).
243
+ // Compute the earliest time to submit: pipeline slot start when pipelining, now otherwise.
244
+ const submitAfter = isPipelining
245
+ ? new Date(Number(getTimestampForSlot(this.targetSlot, this.l1Constants)) * 1000)
246
+ : new Date(this.dateProvider.now());
247
+
248
+ const l1Response = await this.publisher.sendRequestsAt(submitAfter);
249
+ const proposedAction = l1Response?.successfulActions.find(a => a === 'propose');
250
+ if (proposedAction) {
251
+ this.eventEmitter.emit('checkpoint-published', { checkpoint: this.checkpointNumber, slot: this.targetSlot });
252
+ const coinbase = checkpoint.header.coinbase;
253
+ await this.metrics.incFilledSlot(this.publisher.getSenderAddress().toString(), coinbase);
254
+ } else {
255
+ this.eventEmitter.emit('checkpoint-publish-failed', { ...l1Response, slot: this.targetSlot });
256
+ if (isPipelining) {
257
+ this.metrics.recordPipelineDiscard();
258
+ }
259
+ }
260
+ } catch (err) {
261
+ if (err instanceof SequencerInterruptedError) {
262
+ return;
263
+ }
264
+ this.log.error(`Background attestation/L1 pipeline failed for slot ${this.targetSlot}`, err);
265
+ this.eventEmitter.emit('checkpoint-publish-failed', { slot: this.targetSlot });
266
+ if (isPipelining) {
267
+ this.metrics.recordPipelineDiscard();
268
+ }
269
+ }
270
+ }
271
+
272
+ /** Enqueues the checkpoint for L1 submission. Called after pipeline sleep in execute(). */
273
+ private async enqueueCheckpointForSubmission(result: CheckpointProposalResult): Promise<void> {
274
+ const { checkpoint, attestations, attestationsSignature } = result;
275
+
276
+ this.setStateFn(SequencerState.PUBLISHING_CHECKPOINT, this.targetSlot);
277
+ const aztecSlotDuration = this.l1Constants.slotDuration;
278
+ const submissionSlotStart = Number(getTimestampForSlot(this.targetSlot, this.l1Constants));
279
+ const txTimeoutAt = new Date((submissionSlotStart + aztecSlotDuration) * 1000);
280
+
281
+ // If we have been configured to potentially skip publishing checkpoint then roll the dice here
282
+ if (
283
+ this.config.skipPublishingCheckpointsPercent !== undefined &&
284
+ this.config.skipPublishingCheckpointsPercent > 0
285
+ ) {
286
+ const roll = Math.max(0, randomInt(100));
287
+ if (roll < this.config.skipPublishingCheckpointsPercent) {
288
+ this.log.warn(
289
+ `Skipping publishing proposal for checkpoint ${checkpoint.number}. Configured percentage: ${this.config.skipPublishingCheckpointsPercent}, generated value: ${roll}`,
290
+ );
291
+ return;
292
+ }
293
+ }
294
+
295
+ const isPipelining = this.epochCache.isProposerPipeliningEnabled();
296
+ const submissionSimulationOverridesPlan = buildSubmissionSimulationOverridesPlan({
297
+ pipelinedParentPlan: this.pipelinedParentSimulationOverridesPlan,
298
+ invalidateToPendingCheckpointNumber: this.invalidateCheckpoint?.forcePendingCheckpointNumber,
299
+ lastArchiveRoot: checkpoint.header.lastArchiveRoot,
300
+ pipeliningEnabled: isPipelining,
301
+ });
302
+
303
+ await this.publisher.enqueueProposeCheckpoint(checkpoint, attestations, attestationsSignature, {
304
+ txTimeoutAt,
305
+ ...(submissionSimulationOverridesPlan ? { simulationOverridesPlan: submissionSimulationOverridesPlan } : {}),
306
+ });
307
+ }
308
+
309
+ /**
310
+ * Wait until the archiver syncs past the given L2 slot number.
311
+ * The deadline is the end of `this.targetSlot`, beyond which any pipelined work would miss its
312
+ * L1 submission window and is no longer useful.
313
+ */
314
+ private async waitForSyncedL2SlotNumber(waitForSlot: SlotNumber): Promise<boolean> {
315
+ const targetSlotStart = Number(getTimestampForSlot(this.targetSlot, this.l1Constants));
316
+ const targetSlotEndMs = (targetSlotStart + this.l1Constants.slotDuration) * 1000;
317
+ const syncDelayTolerance = this.l1Constants.ethereumSlotDuration * 2 * 1000;
318
+ const timeoutSeconds = Math.max(0.1, (targetSlotEndMs + syncDelayTolerance - this.dateProvider.now()) / 1000);
319
+
320
+ try {
321
+ return await retryUntil(
322
+ async () => {
323
+ const syncedSlot = await this.l2BlockSource.getSyncedL2SlotNumber();
324
+ return syncedSlot !== undefined && syncedSlot >= waitForSlot;
325
+ },
326
+ `archiver sync past slot ${waitForSlot}`,
327
+ timeoutSeconds,
328
+ 0.2,
329
+ );
330
+ } catch {
331
+ this.log.warn(
332
+ `Archiver did not sync L1 past slot ${waitForSlot} before slot ${this.targetSlot} expired, discarding pipelined work`,
333
+ { checkpointNumber: this.checkpointNumber },
334
+ );
335
+ this.emitPipelinedCheckpointDiscarded('archiver-sync-timeout');
336
+ return false;
337
+ }
338
+ }
339
+
340
+ /**
341
+ * Waits for the parent checkpoint to land on L1 before submitting a pipelined checkpoint.
342
+ * Polls until the archiver has synced L1 past the parent's slot, then verifies:
343
+ * - If we built on a proposed parent: it must have landed on L1 with matching hash and valid attestations.
344
+ * - If we built without a proposed parent: no new checkpoint must have appeared for that slot.
345
+ * If the parent has invalid attestations, enqueues an invalidation. Returns whether to proceed with the proposal.
346
+ */
347
+ protected async waitForValidParentCheckpointOnL1(): Promise<boolean> {
348
+ const parentCheckpointNumber = CheckpointNumber(this.checkpointNumber - 1);
349
+
350
+ // Wait until archiver has synced L1 past the parent's slot (slotNow)
351
+ if (!(await this.waitForSyncedL2SlotNumber(this.slotNow))) {
352
+ return false;
353
+ }
354
+
355
+ const tips = await this.l2BlockSource.getL2Tips();
356
+ const checkpointedNumber = tips.checkpointed.checkpoint.number;
357
+
358
+ // We built on top of a proposed checkpoint. Verify it landed on L1 as expected.
359
+ if (this.proposedCheckpointData) {
360
+ // After syncing from L1 we see the chain tip has invalid attestations. This means the parent checkpoint was posted
361
+ // with invalid attestations, or it built on top of something with invalid attestations and didnt invalidate them.
362
+ // Either way, we thought our parent would be valid, so we have to throw away our work. But at least we'll try and
363
+ // invalidate on L1 so we clean up the chain for the next proposer. And we'll slash them, but that's handled elsewhere.
364
+ const validationStatus = await this.l2BlockSource.getPendingChainValidationStatus();
365
+ if (!validationStatus.valid) {
366
+ this.log.warn(
367
+ `Parent checkpoint ${parentCheckpointNumber} has invalid attestations, discarding pipelined work`,
368
+ { checkpointNumber: this.checkpointNumber, reason: validationStatus.reason },
369
+ );
370
+ this.emitPipelinedCheckpointDiscarded('parent-invalid-attestations');
371
+ await this.enqueueInvalidation(validationStatus);
372
+ return false;
373
+ }
374
+
375
+ // The pending chain is valid. But did the parent checkpoint land on L1 at all?
376
+ if (checkpointedNumber < parentCheckpointNumber) {
377
+ this.log.warn(`Parent checkpoint ${parentCheckpointNumber} did not land on L1, discarding pipelined work`, {
378
+ checkpointNumber: this.checkpointNumber,
379
+ checkpointedNumber,
380
+ });
381
+ this.emitPipelinedCheckpointDiscarded('parent-not-on-l1');
382
+ return false;
383
+ }
384
+
385
+ // It landed. But is it the one we were expecting?
386
+ const expectedHash = this.proposedCheckpointData.header.hash().toString();
387
+ if (tips.checkpointed.checkpoint.hash !== expectedHash) {
388
+ this.log.warn(`Parent checkpoint ${parentCheckpointNumber} hash mismatch on L1, discarding pipelined work`, {
389
+ checkpointNumber: this.checkpointNumber,
390
+ expectedHash,
391
+ actualHash: tips.checkpointed.checkpoint.hash,
392
+ });
393
+ this.emitPipelinedCheckpointDiscarded('parent-hash-mismatch');
394
+ return false;
395
+ }
396
+
397
+ return true;
398
+ } else {
399
+ // We didn't see a proposed checkpoint at build time, so we built on checkpointed parent from two slots ago.
400
+ // But if a new checkpoint for the previous slot appeared on L1 in the meantime, our checkpoint assumed the wrong parent,
401
+ // so we have to discard our work. This can happen if we're somehow cut off from p2p and fail to see the checkpoint
402
+ // proposal for the previous slot.
403
+ if (checkpointedNumber > parentCheckpointNumber) {
404
+ this.log.warn(
405
+ `Unexpected checkpoint ${checkpointedNumber} landed on L1 after we built on top of parent ${parentCheckpointNumber}, discarding pipelined work`,
406
+ { checkpointNumber: this.checkpointNumber, checkpointedNumber },
407
+ );
408
+ this.emitPipelinedCheckpointDiscarded('unexpected-parent-appeared');
409
+ return false;
410
+ }
411
+
412
+ return true;
413
+ }
414
+ }
415
+
416
+ /** Emits the pipelined-checkpoint-discarded event and records the metric. */
417
+ private emitPipelinedCheckpointDiscarded(reason: string): void {
418
+ this.metrics.recordPipelineParentCheckpointMismatch(reason);
419
+ this.eventEmitter.emit('pipelined-checkpoint-discarded', {
420
+ slot: this.targetSlot,
421
+ checkpointNumber: this.checkpointNumber,
422
+ reason,
423
+ });
424
+ }
425
+
426
+ /** Simulates and enqueues an invalidation request for the invalid parent checkpoint. */
427
+ private async enqueueInvalidation(validationStatus: ValidateCheckpointResult): Promise<void> {
428
+ if (this.config.skipInvalidateBlockAsProposer) {
429
+ this.log.warn(`Skipping checkpoint invalidation as proposer due to test configuration`);
430
+ return;
431
+ }
432
+ const invalidateRequest = await this.publisher.simulateInvalidateCheckpoint(validationStatus);
433
+ if (invalidateRequest) {
434
+ const submissionSlotStart = Number(getTimestampForSlot(this.targetSlot, this.l1Constants));
435
+ const txTimeoutAt = new Date((submissionSlotStart + this.l1Constants.slotDuration) * 1000);
436
+ this.publisher.enqueueInvalidateCheckpoint(invalidateRequest, { txTimeoutAt });
437
+ } else {
438
+ this.log.info(`Invalidation simulation returned undefined, checkpoint may have been removed already`, {
439
+ checkpointNumber: this.checkpointNumber,
440
+ });
155
441
  }
156
442
  }
157
443
 
@@ -159,29 +445,59 @@ export class CheckpointProposalJob implements Traceable {
159
445
  return {
160
446
  // nullish operator needed for tests
161
447
  [Attributes.COINBASE]: this.validatorClient.getCoinbaseForAttestor(this.attestorAddress)?.toString(),
162
- [Attributes.SLOT_NUMBER]: this.slot,
448
+ [Attributes.SLOT_NUMBER]: this.targetSlot,
163
449
  };
164
450
  })
165
- private async proposeCheckpoint(): Promise<Checkpoint | undefined> {
451
+ private async proposeCheckpoint(): Promise<CheckpointProposalBroadcast | undefined> {
166
452
  try {
453
+ const now = this.dateProvider.now();
454
+ if (this.epochCache.isProposerPipeliningEnabled() && this.proposedCheckpointData) {
455
+ // Measure against the wall-clock slot whose build window we are currently using.
456
+ // In pipelining mode `targetSlot` is intentionally one slot ahead, which makes the
457
+ // target-slot boundary a full slot away from the actual build start time.
458
+ const slotBoundaryMs = Number(getTimestampForSlot(this.slotNow, this.l1Constants)) * 1000;
459
+ this.checkpointMetrics.recordPipelinedCheckpointBuildStartOffsetFromSlotBoundary(now - slotBoundaryMs);
460
+ }
461
+ this.checkpointMetrics.startCheckpointTiming(now);
462
+
167
463
  // Get operator configured coinbase and fee recipient for this attestor
168
464
  const coinbase = this.validatorClient.getCoinbaseForAttestor(this.attestorAddress);
169
465
  const feeRecipient = this.validatorClient.getFeeRecipientForAttestor(this.attestorAddress);
170
466
 
171
467
  // Start the checkpoint
172
- this.setStateFn(SequencerState.INITIALIZING_CHECKPOINT, this.slot);
173
- this.metrics.incOpenSlot(this.slot, this.proposer?.toString() ?? 'unknown');
468
+ this.setStateFn(SequencerState.INITIALIZING_CHECKPOINT, this.targetSlot);
469
+ this.log.info(`Starting checkpoint proposal`, {
470
+ buildSlot: this.slotNow,
471
+ submissionSlot: this.targetSlot,
472
+ pipelining: this.epochCache.isProposerPipeliningEnabled(),
473
+ proposer: this.proposer?.toString(),
474
+ coinbase: coinbase.toString(),
475
+ });
476
+ this.metrics.incOpenSlot(this.targetSlot, this.proposer?.toString() ?? 'unknown');
174
477
 
175
478
  // Enqueues checkpoint invalidation (constant for the whole slot)
176
479
  if (this.invalidateCheckpoint && !this.config.skipInvalidateBlockAsProposer) {
177
480
  this.publisher.enqueueInvalidateCheckpoint(this.invalidateCheckpoint);
178
481
  }
179
482
 
180
- // Create checkpoint builder for the slot
483
+ // Create checkpoint builder for the slot.
484
+ // When pipelining, force the proposed checkpoint number and fee header to our parent so the
485
+ // fee computation sees the same chain tip that L1 will see once the previous pipelined checkpoint lands.
486
+ const isPipelining = this.epochCache.isProposerPipeliningEnabled();
487
+ this.pipelinedParentSimulationOverridesPlan = isPipelining
488
+ ? await buildPipelinedParentSimulationOverridesPlan({
489
+ checkpointNumber: this.checkpointNumber,
490
+ proposedCheckpointData: this.proposedCheckpointData,
491
+ rollup: this.publisher.rollupContract,
492
+ log: this.log,
493
+ })
494
+ : undefined;
495
+
181
496
  const checkpointGlobalVariables = await this.globalsBuilder.buildCheckpointGlobalVariables(
182
497
  coinbase,
183
498
  feeRecipient,
184
- this.slot,
499
+ this.targetSlot,
500
+ this.pipelinedParentSimulationOverridesPlan,
185
501
  );
186
502
 
187
503
  // Collect L1 to L2 messages for the checkpoint and compute their hash
@@ -189,7 +505,7 @@ export class CheckpointProposalJob implements Traceable {
189
505
  const inHash = computeInHashFromL1ToL2Messages(l1ToL2Messages);
190
506
 
191
507
  // Collect the out hashes of all the checkpoints before this one in the same epoch
192
- const previousCheckpointOutHashes = (await this.l2BlockSource.getCheckpointsDataForEpoch(this.epoch))
508
+ const previousCheckpointOutHashes = (await this.l2BlockSource.getCheckpointsDataForEpoch(this.targetEpoch))
193
509
  .filter(c => c.checkpointNumber < this.checkpointNumber)
194
510
  .map(c => c.checkpointOutHash);
195
511
 
@@ -222,7 +538,7 @@ export class CheckpointProposalJob implements Traceable {
222
538
  };
223
539
 
224
540
  let blocksInCheckpoint: L2Block[] = [];
225
- let blockPendingBroadcast: { block: L2Block; txs: Tx[] } | undefined = undefined;
541
+ let blockPendingBroadcast: BlockProposal | undefined = undefined;
226
542
  const checkpointBuildTimer = new Timer();
227
543
 
228
544
  try {
@@ -246,8 +562,8 @@ export class CheckpointProposalJob implements Traceable {
246
562
  }
247
563
 
248
564
  if (blocksInCheckpoint.length === 0) {
249
- this.log.warn(`No blocks were built for slot ${this.slot}`, { slot: this.slot });
250
- this.eventEmitter.emit('checkpoint-empty', { slot: this.slot });
565
+ this.log.warn(`No blocks were built for slot ${this.targetSlot}`, { slot: this.targetSlot });
566
+ this.eventEmitter.emit('checkpoint-empty', { slot: this.targetSlot });
251
567
  return undefined;
252
568
  }
253
569
 
@@ -255,17 +571,18 @@ export class CheckpointProposalJob implements Traceable {
255
571
  if (minBlocksForCheckpoint !== undefined && blocksInCheckpoint.length < minBlocksForCheckpoint) {
256
572
  this.log.warn(
257
573
  `Checkpoint has fewer blocks than minimum (${blocksInCheckpoint.length} < ${minBlocksForCheckpoint}), skipping proposal`,
258
- { slot: this.slot, blocksBuilt: blocksInCheckpoint.length, minBlocksForCheckpoint },
574
+ { slot: this.targetSlot, blocksBuilt: blocksInCheckpoint.length, minBlocksForCheckpoint },
259
575
  );
260
576
  return undefined;
261
577
  }
262
578
 
263
579
  // Assemble and broadcast the checkpoint proposal, including the last block that was not
264
580
  // broadcasted yet, and wait to collect the committee attestations.
265
- this.setStateFn(SequencerState.ASSEMBLING_CHECKPOINT, this.slot);
581
+ this.setStateFn(SequencerState.ASSEMBLING_CHECKPOINT, this.targetSlot);
266
582
  const checkpoint = await checkpointBuilder.completeCheckpoint();
267
583
 
268
- // Final validation round for the checkpoint before we propose it, just for safety
584
+ // Final validation: per-block limits are only checked if the operator set them explicitly.
585
+ // Otherwise, checkpoint-level budgets were already enforced by the redistribution logic.
269
586
  try {
270
587
  validateCheckpoint(checkpoint, {
271
588
  rollupManaLimit: this.l1Constants.rollupManaLimit,
@@ -275,113 +592,60 @@ export class CheckpointProposalJob implements Traceable {
275
592
  maxTxsPerCheckpoint: this.config.maxTxsPerCheckpoint,
276
593
  });
277
594
  } catch (err) {
278
- this.log.error(`Built an invalid checkpoint at slot ${this.slot} (skipping proposal)`, err, {
595
+ this.log.error(`Built an invalid checkpoint at slot ${this.slotNow} (skipping proposal)`, err, {
279
596
  checkpoint: checkpoint.header.toInspect(),
280
597
  });
281
598
  return undefined;
282
599
  }
283
600
 
284
601
  // Record checkpoint-level build metrics
285
- this.metrics.recordCheckpointBuild(
602
+ this.checkpointMetrics.recordCheckpointBuild(
286
603
  checkpointBuildTimer.ms(),
287
604
  blocksInCheckpoint.length,
288
605
  checkpoint.getStats().txCount,
289
606
  Number(checkpoint.header.totalManaUsed.toBigInt()),
290
607
  );
291
608
 
292
- // Do not collect attestations nor publish to L1 in fisherman mode
609
+ // In fisherman mode, return the checkpoint without broadcasting or collecting attestations
293
610
  if (this.config.fishermanMode) {
294
611
  this.log.info(
295
- `Built checkpoint for slot ${this.slot} with ${blocksInCheckpoint.length} blocks. ` +
612
+ `Built checkpoint for slot ${this.targetSlot} with ${blocksInCheckpoint.length} blocks. ` +
296
613
  `Skipping proposal in fisherman mode.`,
297
614
  {
298
- slot: this.slot,
615
+ slot: this.targetSlot,
299
616
  checkpoint: checkpoint.header.toInspect(),
300
617
  blocksBuilt: blocksInCheckpoint.length,
301
618
  },
302
619
  );
303
620
  this.metrics.recordCheckpointSuccess();
304
- return checkpoint;
621
+ // Return a broadcast result with a dummy proposal — fisherman mode skips attestation collection
622
+ return { checkpoint, proposal: undefined!, blockProposedAt: this.dateProvider.now() };
305
623
  }
306
624
 
307
- // Include the block pending broadcast in the checkpoint proposal if any
308
- const lastBlock = blockPendingBroadcast && {
309
- blockHeader: blockPendingBroadcast.block.header,
310
- indexWithinCheckpoint: blockPendingBroadcast.block.indexWithinCheckpoint,
311
- txs: blockPendingBroadcast.txs,
312
- };
313
-
314
625
  // Create the checkpoint proposal and broadcast it
315
626
  const proposal = await this.validatorClient.createCheckpointProposal(
316
627
  checkpoint.header,
317
628
  checkpoint.archive.root,
629
+ this.checkpointNumber,
318
630
  feeAssetPriceModifier,
319
- lastBlock,
631
+ blockPendingBroadcast,
320
632
  this.proposer,
321
633
  checkpointProposalOptions,
322
634
  );
323
635
 
324
636
  const blockProposedAt = this.dateProvider.now();
325
637
  await this.p2pClient.broadcastCheckpointProposal(proposal);
638
+ this.checkpointMetrics.noteCheckpointBroadcast(this.dateProvider.now());
326
639
 
327
- this.setStateFn(SequencerState.COLLECTING_ATTESTATIONS, this.slot);
328
- const attestations = await this.waitForAttestations(proposal);
329
- const blockAttestedAt = this.dateProvider.now();
330
-
331
- this.metrics.recordCheckpointAttestationDelay(blockAttestedAt - blockProposedAt);
332
-
333
- // Proposer must sign over the attestations before pushing them to L1
334
- const signer = this.proposer ?? this.publisher.getSenderAddress();
335
- let attestationsSignature: Signature;
336
- try {
337
- attestationsSignature = await this.validatorClient.signAttestationsAndSigners(
338
- attestations,
339
- signer,
340
- this.slot,
341
- this.checkpointNumber,
342
- );
343
- } catch (err) {
344
- // We shouldn't really get here since we yield to another HA node
345
- // as soon as we see these errors when creating block or checkpoint proposals.
346
- if (this.handleHASigningError(err, 'Attestations signature')) {
347
- return undefined;
348
- }
349
- throw err;
350
- }
351
-
352
- // Enqueue publishing the checkpoint to L1
353
- this.setStateFn(SequencerState.PUBLISHING_CHECKPOINT, this.slot);
354
- const aztecSlotDuration = this.l1Constants.slotDuration;
355
- const slotStartBuildTimestamp = this.getSlotStartBuildTimestamp();
356
- const txTimeoutAt = new Date((slotStartBuildTimestamp + aztecSlotDuration) * 1000);
357
-
358
- // If we have been configured to potentially skip publishing checkpoint then roll the dice here
359
- if (
360
- this.config.skipPublishingCheckpointsPercent !== undefined &&
361
- this.config.skipPublishingCheckpointsPercent > 0
362
- ) {
363
- const result = Math.max(0, randomInt(100));
364
- if (result < this.config.skipPublishingCheckpointsPercent) {
365
- this.log.warn(
366
- `Skipping publishing proposal for checkpoint ${checkpoint.number}. Configured percentage: ${this.config.skipPublishingCheckpointsPercent}, generated value: ${result}`,
367
- );
368
- return checkpoint;
369
- }
370
- }
371
-
372
- await this.publisher.enqueueProposeCheckpoint(checkpoint, attestations, attestationsSignature, {
373
- txTimeoutAt,
374
- forcePendingCheckpointNumber: this.invalidateCheckpoint?.forcePendingCheckpointNumber,
375
- });
376
-
377
- return checkpoint;
640
+ // Return immediately after broadcast — attestation collection happens in the background
641
+ return { checkpoint, proposal, blockProposedAt };
378
642
  } catch (err) {
379
643
  if (err && (err instanceof DutyAlreadySignedError || err instanceof SlashingProtectionError)) {
380
644
  // swallow this error. It's already been logged by a function deeper in the stack
381
645
  return undefined;
382
646
  }
383
647
 
384
- this.log.error(`Error building checkpoint at slot ${this.slot}`, err);
648
+ this.log.error(`Error building checkpoint at slot ${this.targetSlot}`, err);
385
649
  return undefined;
386
650
  }
387
651
  }
@@ -397,14 +661,14 @@ export class CheckpointProposalJob implements Traceable {
397
661
  blockProposalOptions: BlockProposalOptions,
398
662
  ): Promise<{
399
663
  blocksInCheckpoint: L2Block[];
400
- blockPendingBroadcast: { block: L2Block; txs: Tx[] } | undefined;
664
+ blockPendingBroadcast: BlockProposal | undefined;
401
665
  }> {
402
666
  const blocksInCheckpoint: L2Block[] = [];
403
667
  const txHashesAlreadyIncluded = new Set<string>();
404
668
  const initialBlockNumber = BlockNumber(this.syncedToBlockNumber + 1);
405
669
 
406
670
  // Last block in the checkpoint will usually be flagged as pending broadcast, so we send it along with the checkpoint proposal
407
- let blockPendingBroadcast: { block: L2Block; txs: Tx[] } | undefined = undefined;
671
+ let blockPendingBroadcast: BlockProposal | undefined = undefined;
408
672
 
409
673
  while (true) {
410
674
  const blocksBuilt = blocksInCheckpoint.length;
@@ -416,7 +680,7 @@ export class CheckpointProposalJob implements Traceable {
416
680
 
417
681
  if (!timingInfo.canStart) {
418
682
  this.log.debug(`Not enough time left in slot to start another block`, {
419
- slot: this.slot,
683
+ slot: this.targetSlot,
420
684
  blocksBuilt,
421
685
  secondsIntoSlot,
422
686
  });
@@ -437,22 +701,23 @@ export class CheckpointProposalJob implements Traceable {
437
701
  txHashesAlreadyIncluded,
438
702
  });
439
703
 
440
- // TODO(palla/mbps): Review these conditions. We may want to keep trying in some scenarios.
441
- if (!buildResult && timingInfo.isLastBlock) {
442
- // If no block was produced due to not enough txs and this was the last subslot, exit
443
- break;
444
- } else if (!buildResult && timingInfo.deadline !== undefined) {
445
- // But if there is still time for more blocks, wait until the next subslot and try again
704
+ // If we failed to build the block due to insufficient txs, we try again if there is still time left in the slot
705
+ if ('failure' in buildResult) {
706
+ // If this was the last subslot, or we're running with a single block per slot, we're done
707
+ if (timingInfo.isLastBlock || timingInfo.deadline === undefined) {
708
+ break;
709
+ }
710
+ // Otherwise, if there is still time for more blocks, we wait until the next subslot and try again
446
711
  await this.waitUntilNextSubslot(timingInfo.deadline);
447
712
  continue;
448
- } else if (!buildResult) {
449
- // Exit if there is no possibility of building more blocks
450
- break;
451
- } else if ('error' in buildResult) {
452
- // If there was an error building the block, just exit the loop and give up the rest of the slot
713
+ }
714
+
715
+ // If there was an error building the block, we just exit the loop and give up the rest of the slot.
716
+ // We don't want to risk building more blocks if something went wrong.
717
+ if ('error' in buildResult) {
453
718
  if (!(buildResult.error instanceof SequencerInterruptedError)) {
454
- this.log.warn(`Halting block building for slot ${this.slot}`, {
455
- slot: this.slot,
719
+ this.log.warn(`Halting block building for slot ${this.targetSlot}`, {
720
+ slot: this.targetSlot,
456
721
  blocksBuilt,
457
722
  error: buildResult.error,
458
723
  });
@@ -461,61 +726,80 @@ export class CheckpointProposalJob implements Traceable {
461
726
  }
462
727
 
463
728
  const { block, usedTxs } = buildResult;
464
- blocksInCheckpoint.push(block);
465
-
466
- // Sync the proposed block to the archiver to make it available
467
- // Note that the checkpoint builder uses its own fork so it should not need to wait for this syncing
468
- // Eventually we should refactor the checkpoint builder to not need a separate long-lived fork
469
- // Fire and forget - don't block the critical path, but log errors
470
- this.syncProposedBlockToArchiver(block).catch(err => {
471
- this.log.error(`Failed to sync proposed block ${block.number} to archiver`, { blockNumber: block.number, err });
729
+ this.checkpointMetrics.noteCheckpointBlockBuilt(this.dateProvider.now(), {
730
+ isFirstBlock: blocksBuilt === 0,
731
+ isLastBlock: timingInfo.isLastBlock,
472
732
  });
473
733
 
734
+ blocksInCheckpoint.push(block);
474
735
  usedTxs.forEach(tx => txHashesAlreadyIncluded.add(tx.txHash.toString()));
475
736
 
476
- // If this is the last block, exit the loop now so we start collecting attestations
737
+ // Sign the block proposal. This will throw if HA signing fails.
738
+ const proposal = await this.createBlockProposal(block, inHash, usedTxs, blockProposalOptions);
739
+
740
+ // Sync the proposed block to the archiver to make it available, only after we've managed to sign the proposal,
741
+ // so we avoid polluting our archive with a block that would fail.
742
+ // We wait for the sync to succeed, as this helps catch consistency errors, even if it means we lose some time for block-building.
743
+ // If this throws, we abort the entire checkpoint.
744
+ await this.syncProposedBlockToArchiver(block);
745
+
746
+ // If this is the last block, do not broadcast it, since it will be included in the checkpoint proposal.
477
747
  if (timingInfo.isLastBlock) {
478
- this.log.verbose(`Completed final block ${blockNumber} for slot ${this.slot}`, {
479
- slot: this.slot,
748
+ this.log.verbose(`Completed final block ${blockNumber} for slot ${this.targetSlot}`, {
749
+ slot: this.targetSlot,
480
750
  blockNumber,
481
751
  blocksBuilt,
482
752
  });
483
- blockPendingBroadcast = { block, txs: usedTxs };
753
+
754
+ blockPendingBroadcast = proposal;
484
755
  break;
485
756
  }
486
757
 
487
- // For non-last blocks, broadcast the block proposal (unless we're in fisherman mode)
488
- // If the block is the last one, we'll broadcast it along with the checkpoint at the end of the loop
489
- if (!this.config.fishermanMode) {
490
- const proposal = await this.validatorClient.createBlockProposal(
491
- block.header,
492
- block.indexWithinCheckpoint,
493
- inHash,
494
- block.archive.root,
495
- usedTxs,
496
- this.proposer,
497
- blockProposalOptions,
498
- );
499
- await this.p2pClient.broadcastProposal(proposal);
500
- }
758
+ // Once we have a signed proposal and the archiver agreed with our proposed block, then we broadcast it.
759
+ proposal && (await this.p2pClient.broadcastProposal(proposal));
501
760
 
502
761
  // Wait until the next block's start time
503
762
  await this.waitUntilNextSubslot(timingInfo.deadline);
504
763
  }
505
764
 
506
- this.log.verbose(`Block building loop completed for slot ${this.slot}`, {
507
- slot: this.slot,
765
+ this.log.verbose(`Block building loop completed for slot ${this.targetSlot}`, {
766
+ slot: this.targetSlot,
508
767
  blocksBuilt: blocksInCheckpoint.length,
509
768
  });
510
769
 
511
770
  return { blocksInCheckpoint, blockPendingBroadcast };
512
771
  }
513
772
 
773
+ /** Creates a block proposal for a given block via the validator client (unless in fisherman mode) */
774
+ private createBlockProposal(
775
+ block: L2Block,
776
+ inHash: Fr,
777
+ usedTxs: Tx[],
778
+ blockProposalOptions: BlockProposalOptions,
779
+ ): Promise<BlockProposal | undefined> {
780
+ if (this.config.fishermanMode) {
781
+ this.log.info(`Skipping block proposal for block ${block.number} in fisherman mode`);
782
+ return Promise.resolve(undefined);
783
+ }
784
+ return this.validatorClient.createBlockProposal(
785
+ block.header,
786
+ this.checkpointNumber,
787
+ block.indexWithinCheckpoint,
788
+ inHash,
789
+ block.archive.root,
790
+ usedTxs,
791
+ this.proposer,
792
+ blockProposalOptions,
793
+ );
794
+ }
795
+
514
796
  /** Sleeps until it is time to produce the next block in the slot */
515
797
  @trackSpan('CheckpointProposalJob.waitUntilNextSubslot')
516
798
  private async waitUntilNextSubslot(nextSubslotStart: number) {
517
- this.setStateFn(SequencerState.WAITING_UNTIL_NEXT_BLOCK, this.slot);
518
- this.log.verbose(`Waiting until time for the next block at ${nextSubslotStart}s into slot`, { slot: this.slot });
799
+ this.setStateFn(SequencerState.WAITING_UNTIL_NEXT_BLOCK, this.targetSlot);
800
+ this.log.verbose(`Waiting until time for the next block at ${nextSubslotStart}s into slot`, {
801
+ slot: this.targetSlot,
802
+ });
519
803
  await this.waitUntilTimeInSlot(nextSubslotStart);
520
804
  }
521
805
 
@@ -531,12 +815,14 @@ export class CheckpointProposalJob implements Traceable {
531
815
  buildDeadline: Date | undefined;
532
816
  txHashesAlreadyIncluded: Set<string>;
533
817
  },
534
- ): Promise<{ block: L2Block; usedTxs: Tx[] } | { error: Error } | undefined> {
818
+ ): Promise<
819
+ { block: L2Block; usedTxs: Tx[] } | { failure: 'insufficient-txs' | 'insufficient-valid-txs' } | { error: Error }
820
+ > {
535
821
  const { blockTimestamp, forceCreate, blockNumber, indexWithinCheckpoint, buildDeadline, txHashesAlreadyIncluded } =
536
822
  opts;
537
823
 
538
824
  this.log.verbose(
539
- `Preparing block ${blockNumber} index ${indexWithinCheckpoint} at checkpoint ${this.checkpointNumber} for slot ${this.slot}`,
825
+ `Preparing block ${blockNumber} index ${indexWithinCheckpoint} at checkpoint ${this.checkpointNumber} for slot ${this.targetSlot}`,
540
826
  { ...checkpointBuilder.getConstantData(), ...opts },
541
827
  );
542
828
 
@@ -545,12 +831,12 @@ export class CheckpointProposalJob implements Traceable {
545
831
  const { availableTxs, canStartBuilding, minTxs } = await this.waitForMinTxs(opts);
546
832
  if (!canStartBuilding) {
547
833
  this.log.warn(
548
- `Not enough txs to build block ${blockNumber} at index ${indexWithinCheckpoint} in slot ${this.slot} (got ${availableTxs} txs but needs ${minTxs})`,
549
- { blockNumber, slot: this.slot, indexWithinCheckpoint },
834
+ `Not enough txs to build block ${blockNumber} at index ${indexWithinCheckpoint} in slot ${this.targetSlot} (got ${availableTxs} txs but needs ${minTxs})`,
835
+ { blockNumber, slot: this.targetSlot, indexWithinCheckpoint },
550
836
  );
551
- this.eventEmitter.emit('block-tx-count-check-failed', { minTxs, availableTxs, slot: this.slot });
837
+ this.eventEmitter.emit('block-tx-count-check-failed', { minTxs, availableTxs, slot: this.targetSlot });
552
838
  this.metrics.recordBlockProposalFailed('insufficient_txs');
553
- return undefined;
839
+ return { failure: 'insufficient-txs' };
554
840
  }
555
841
 
556
842
  // Create iterator to pending txs. We filter out txs already included in previous blocks in the checkpoint
@@ -561,14 +847,16 @@ export class CheckpointProposalJob implements Traceable {
561
847
  );
562
848
 
563
849
  this.log.debug(
564
- `Building block ${blockNumber} at index ${indexWithinCheckpoint} for slot ${this.slot} with ${availableTxs} available txs`,
565
- { slot: this.slot, blockNumber, indexWithinCheckpoint },
850
+ `Building block ${blockNumber} at index ${indexWithinCheckpoint} for slot ${this.targetSlot} with ${availableTxs} available txs`,
851
+ { slot: this.targetSlot, blockNumber, indexWithinCheckpoint },
566
852
  );
567
- this.setStateFn(SequencerState.CREATING_BLOCK, this.slot);
853
+ this.setStateFn(SequencerState.CREATING_BLOCK, this.targetSlot);
568
854
 
569
- // Per-block limits derived at startup by computeBlockLimits(), further capped
855
+ // Per-block limits are operator overrides (from SEQ_MAX_L2_BLOCK_GAS etc.) further capped
570
856
  // by remaining checkpoint-level budgets inside CheckpointBuilder before each block is built.
571
- const blockBuilderOptions: PublicProcessorLimits = {
857
+ // minValidTxs is passed into the builder so it can reject the block *before* updating state.
858
+ const minValidTxs = forceCreate ? 0 : (this.config.minValidTxsPerBlock ?? minTxs);
859
+ const blockBuilderOptions: BlockBuilderOptions = {
572
860
  maxTransactions: this.config.maxTxsPerBlock,
573
861
  maxBlockGas:
574
862
  this.config.maxL2BlockGas !== undefined || this.config.maxDABlockGas !== undefined
@@ -576,9 +864,14 @@ export class CheckpointProposalJob implements Traceable {
576
864
  : undefined,
577
865
  deadline: buildDeadline,
578
866
  isBuildingProposal: true,
867
+ minValidTxs,
868
+ maxBlocksPerCheckpoint: this.timetable.maxNumberOfBlocks,
869
+ perBlockAllocationMultiplier: this.config.perBlockAllocationMultiplier,
579
870
  };
580
871
 
581
- // Actually build the block by executing txs
872
+ // Actually build the block by executing txs. The builder throws InsufficientValidTxsError
873
+ // if the number of successfully processed txs is below minValidTxs, ensuring state is not
874
+ // updated for blocks that will be discarded.
582
875
  const buildResult = await this.buildSingleBlockWithCheckpointBuilder(
583
876
  checkpointBuilder,
584
877
  pendingTxs,
@@ -590,22 +883,27 @@ export class CheckpointProposalJob implements Traceable {
590
883
  // If any txs failed during execution, drop them from the mempool so we don't pick them up again
591
884
  await this.dropFailedTxsFromP2P(buildResult.failedTxs);
592
885
 
593
- // Check if we have created a block with enough txs. If there were invalid txs in the pool, or if execution took
594
- // too long, then we may not get to minTxsPerBlock after executing public functions.
595
- const minValidTxs = this.config.minValidTxsPerBlock ?? minTxs;
596
- const numTxs = buildResult.status === 'no-valid-txs' ? 0 : buildResult.numTxs;
597
- if (buildResult.status === 'no-valid-txs' || (!forceCreate && numTxs < minValidTxs)) {
886
+ if (buildResult.status === 'insufficient-valid-txs') {
598
887
  this.log.warn(
599
- `Block ${blockNumber} at index ${indexWithinCheckpoint} on slot ${this.slot} has too few valid txs to be proposed`,
600
- { slot: this.slot, blockNumber, numTxs, indexWithinCheckpoint, minValidTxs, buildResult: buildResult.status },
888
+ `Block ${blockNumber} at index ${indexWithinCheckpoint} on slot ${this.targetSlot} has too few valid txs to be proposed`,
889
+ {
890
+ slot: this.targetSlot,
891
+ blockNumber,
892
+ numTxs: buildResult.processedCount,
893
+ indexWithinCheckpoint,
894
+ minValidTxs,
895
+ },
601
896
  );
602
- this.eventEmitter.emit('block-build-failed', { reason: `Insufficient valid txs`, slot: this.slot });
897
+ this.eventEmitter.emit('block-build-failed', {
898
+ reason: `Insufficient valid txs`,
899
+ slot: this.targetSlot,
900
+ });
603
901
  this.metrics.recordBlockProposalFailed('insufficient_valid_txs');
604
- return undefined;
902
+ return { failure: 'insufficient-valid-txs' };
605
903
  }
606
904
 
607
905
  // Block creation succeeded, emit stats and metrics
608
- const { block, publicProcessorDuration, usedTxs, blockBuildDuration } = buildResult;
906
+ const { block, publicProcessorDuration, usedTxs, blockBuildDuration, numTxs } = buildResult;
609
907
 
610
908
  const blockStats = {
611
909
  eventName: 'l2-block-built',
@@ -619,30 +917,39 @@ export class CheckpointProposalJob implements Traceable {
619
917
  const manaPerSec = block.header.totalManaUsed.toNumberUnsafe() / (blockBuildDuration / 1000);
620
918
 
621
919
  this.log.info(
622
- `Built block ${block.number} at checkpoint ${this.checkpointNumber} for slot ${this.slot} with ${numTxs} txs`,
920
+ `Built block ${block.number} at checkpoint ${this.checkpointNumber} for slot ${this.targetSlot} with ${numTxs} txs`,
623
921
  { blockHash, txHashes, manaPerSec, ...blockStats },
624
922
  );
625
923
 
626
- this.eventEmitter.emit('block-proposed', { blockNumber: block.number, slot: this.slot });
627
- this.metrics.recordBuiltBlock(blockBuildDuration, block.header.totalManaUsed.toNumberUnsafe());
924
+ // `slot` is the target/submission slot (may be one ahead when pipelining),
925
+ // `buildSlot` is the wall-clock slot during which the block was actually built.
926
+ this.eventEmitter.emit('block-proposed', {
927
+ blockNumber: block.number,
928
+ slot: this.targetSlot,
929
+ buildSlot: this.slotNow,
930
+ });
931
+ this.metrics.recordBuiltBlock(blockBuildDuration, block.header.totalManaUsed.toNumberUnsafe(), this.targetSlot);
628
932
 
629
933
  return { block, usedTxs };
630
934
  } catch (err: any) {
631
- this.eventEmitter.emit('block-build-failed', { reason: err.message, slot: this.slot });
632
- this.log.error(`Error building block`, err, { blockNumber, slot: this.slot });
935
+ this.eventEmitter.emit('block-build-failed', {
936
+ reason: err.message,
937
+ slot: this.targetSlot,
938
+ });
939
+ this.log.error(`Error building block`, err, { blockNumber, slot: this.targetSlot });
633
940
  this.metrics.recordBlockProposalFailed(err.name || 'unknown_error');
634
941
  this.metrics.recordFailedBlock();
635
942
  return { error: err };
636
943
  }
637
944
  }
638
945
 
639
- /** Uses the checkpoint builder to build a block, catching specific txs */
946
+ /** Uses the checkpoint builder to build a block, catching InsufficientValidTxsError. */
640
947
  private async buildSingleBlockWithCheckpointBuilder(
641
948
  checkpointBuilder: CheckpointBuilder,
642
949
  pendingTxs: AsyncIterable<Tx>,
643
950
  blockNumber: BlockNumber,
644
951
  blockTimestamp: bigint,
645
- blockBuilderOptions: PublicProcessorLimits,
952
+ blockBuilderOptions: BlockBuilderOptions,
646
953
  ) {
647
954
  try {
648
955
  const workTimer = new Timer();
@@ -650,8 +957,12 @@ export class CheckpointProposalJob implements Traceable {
650
957
  const blockBuildDuration = workTimer.ms();
651
958
  return { ...result, blockBuildDuration, status: 'success' as const };
652
959
  } catch (err: unknown) {
653
- if (isErrorClass(err, NoValidTxsError)) {
654
- return { failedTxs: err.failedTxs, status: 'no-valid-txs' as const };
960
+ if (isErrorClass(err, InsufficientValidTxsError)) {
961
+ return {
962
+ failedTxs: err.failedTxs,
963
+ processedCount: err.processedCount,
964
+ status: 'insufficient-valid-txs' as const,
965
+ };
655
966
  }
656
967
  throw err;
657
968
  }
@@ -685,10 +996,10 @@ export class CheckpointProposalJob implements Traceable {
685
996
  }
686
997
 
687
998
  // Wait a bit before checking again
688
- this.setStateFn(SequencerState.WAITING_FOR_TXS, this.slot);
999
+ this.setStateFn(SequencerState.WAITING_FOR_TXS, this.targetSlot);
689
1000
  this.log.verbose(
690
- `Waiting for enough txs to build block ${blockNumber} at index ${indexWithinCheckpoint} in slot ${this.slot} (have ${availableTxs} but need ${minTxs})`,
691
- { blockNumber, slot: this.slot, indexWithinCheckpoint },
1001
+ `Waiting for enough txs to build block ${blockNumber} at index ${indexWithinCheckpoint} in slot ${this.targetSlot} (have ${availableTxs} but need ${minTxs})`,
1002
+ { blockNumber, slot: this.targetSlot, indexWithinCheckpoint },
692
1003
  );
693
1004
  await this.waitForTxsPollingInterval();
694
1005
  availableTxs = await this.p2pClient.getPendingTxCount();
@@ -697,12 +1008,44 @@ export class CheckpointProposalJob implements Traceable {
697
1008
  return { canStartBuilding: true, availableTxs, minTxs };
698
1009
  }
699
1010
 
1011
+ private async getSignedCommitteeAttestations(
1012
+ broadcast: CheckpointProposalBroadcast,
1013
+ ): Promise<{ attestations: CommitteeAttestationsAndSigners; attestationsSignature: Signature } | undefined> {
1014
+ const { proposal, blockProposedAt } = broadcast;
1015
+ this.setStateFn(SequencerState.COLLECTING_ATTESTATIONS, this.targetSlot);
1016
+ const attestations = await this.waitForAttestations(proposal);
1017
+ if (!attestations) {
1018
+ return undefined;
1019
+ }
1020
+ this.checkpointMetrics.recordCheckpointAttestationDelay(this.dateProvider.now() - blockProposedAt);
1021
+
1022
+ // Proposer must sign over the attestations before pushing them to L1
1023
+ const signer = this.proposer ?? this.publisher.getSenderAddress();
1024
+ try {
1025
+ const attestationsSignature = await this.validatorClient.signAttestationsAndSigners(
1026
+ attestations,
1027
+ signer,
1028
+ this.targetSlot,
1029
+ this.checkpointNumber,
1030
+ );
1031
+ return { attestations, attestationsSignature };
1032
+ } catch (err) {
1033
+ if (this.handleHASigningError(err, 'Attestations signature')) {
1034
+ return;
1035
+ }
1036
+ this.log.error(`Error signing attestations for checkpoint proposal at slot ${proposal.slotNumber}`, err);
1037
+ return undefined;
1038
+ }
1039
+ }
1040
+
700
1041
  /**
701
1042
  * Waits for enough attestations to be collected via p2p.
702
1043
  * This is run after all blocks for the checkpoint have been built.
703
1044
  */
704
1045
  @trackSpan('CheckpointProposalJob.waitForAttestations')
705
- private async waitForAttestations(proposal: CheckpointProposal): Promise<CommitteeAttestationsAndSigners> {
1046
+ private async waitForAttestations(
1047
+ proposal: CheckpointProposal,
1048
+ ): Promise<CommitteeAttestationsAndSigners | undefined> {
706
1049
  if (this.config.fishermanMode) {
707
1050
  this.log.debug('Skipping attestation collection in fisherman mode');
708
1051
  return CommitteeAttestationsAndSigners.empty();
@@ -720,16 +1063,16 @@ export class CheckpointProposalJob implements Traceable {
720
1063
  this.log.debug(`Attesting committee length is ${committee.length}`, { committee });
721
1064
  }
722
1065
 
723
- const numberOfRequiredAttestations = Math.floor((committee.length * 2) / 3) + 1;
1066
+ const numberOfRequiredAttestations = computeQuorum(committee.length);
724
1067
 
725
1068
  if (this.config.skipCollectingAttestations) {
726
1069
  this.log.warn('Skipping attestation collection as per config (attesting with own keys only)');
727
- const attestations = await this.validatorClient?.collectOwnAttestations(proposal);
1070
+ const attestations = await this.validatorClient?.collectOwnAttestations(proposal, this.checkpointNumber);
728
1071
  return new CommitteeAttestationsAndSigners(orderAttestations(attestations ?? [], committee));
729
1072
  }
730
1073
 
731
1074
  const attestationTimeAllowed = this.config.enforceTimeTable
732
- ? this.timetable.getMaxAllowedTime(SequencerState.PUBLISHING_CHECKPOINT)!
1075
+ ? this.timetable.getCheckpointAttestationDeadline()
733
1076
  : this.l1Constants.slotDuration;
734
1077
  const attestationDeadline = new Date((this.getSlotStartBuildTimestamp() + attestationTimeAllowed) * 1000);
735
1078
 
@@ -742,6 +1085,7 @@ export class CheckpointProposalJob implements Traceable {
742
1085
  proposal,
743
1086
  numberOfRequiredAttestations,
744
1087
  attestationDeadline,
1088
+ this.checkpointNumber,
745
1089
  );
746
1090
 
747
1091
  collectedAttestationsCount = attestations.length;
@@ -775,8 +1119,14 @@ export class CheckpointProposalJob implements Traceable {
775
1119
  } catch (err) {
776
1120
  if (err && err instanceof AttestationTimeoutError) {
777
1121
  collectedAttestationsCount = err.collectedCount;
1122
+ this.log.error(
1123
+ `Timeout while waiting for attestations for checkpoint proposal at slot ${proposal.slotNumber} (collected ${collectedAttestationsCount}/${numberOfRequiredAttestations})`,
1124
+ err,
1125
+ );
1126
+ } else {
1127
+ this.log.error(`Error collecting attestations for checkpoint proposal at slot ${proposal.slotNumber}`, err);
778
1128
  }
779
- throw err;
1129
+ return undefined;
780
1130
  } finally {
781
1131
  this.metrics.recordCollectedAttestations(collectedAttestationsCount, collectAttestationsTimer.ms());
782
1132
  }
@@ -868,9 +1218,13 @@ export class CheckpointProposalJob implements Traceable {
868
1218
  * Adds the proposed block to the archiver so it's available via P2P.
869
1219
  * Gossip doesn't echo messages back to the sender, so the proposer's archiver/world-state
870
1220
  * would never receive its own block without this explicit sync.
1221
+ *
1222
+ * In fisherman mode we skip this push: the fisherman builds blocks locally for validation
1223
+ * and fee analysis only, and pushing them to the archiver causes spurious reorg cascades
1224
+ * whenever the real proposer's block arrives from L1.
871
1225
  */
872
1226
  private async syncProposedBlockToArchiver(block: L2Block): Promise<void> {
873
- if (this.config.skipPushProposedBlocksToArchiver !== false) {
1227
+ if (this.config.skipPushProposedBlocksToArchiver || this.config.fishermanMode) {
874
1228
  this.log.warn(`Skipping push of proposed block ${block.number} to archiver`, {
875
1229
  blockNumber: block.number,
876
1230
  slot: block.header.globalVariables.slotNumber,
@@ -888,19 +1242,19 @@ export class CheckpointProposalJob implements Traceable {
888
1242
  private async handleCheckpointEndAsFisherman(checkpoint: Checkpoint | undefined) {
889
1243
  // Perform L1 fee analysis before clearing requests
890
1244
  // The callback is invoked asynchronously after the next block is mined
891
- const feeAnalysis = await this.publisher.analyzeL1Fees(this.slot, analysis =>
1245
+ const feeAnalysis = await this.publisher.analyzeL1Fees(this.targetSlot, analysis =>
892
1246
  this.metrics.recordFishermanFeeAnalysis(analysis),
893
1247
  );
894
1248
 
895
1249
  if (checkpoint) {
896
- this.log.info(`Validation checkpoint building SUCCEEDED for slot ${this.slot}`, {
1250
+ this.log.info(`Validation checkpoint building SUCCEEDED for slot ${this.targetSlot}`, {
897
1251
  ...checkpoint.toCheckpointInfo(),
898
1252
  ...checkpoint.getStats(),
899
1253
  feeAnalysisId: feeAnalysis?.id,
900
1254
  });
901
1255
  } else {
902
- this.log.warn(`Validation block building FAILED for slot ${this.slot}`, {
903
- slot: this.slot,
1256
+ this.log.warn(`Validation block building FAILED for slot ${this.targetSlot}`, {
1257
+ slot: this.targetSlot,
904
1258
  feeAnalysisId: feeAnalysis?.id,
905
1259
  });
906
1260
  this.metrics.recordCheckpointProposalFailed('block_build_failed');
@@ -914,15 +1268,15 @@ export class CheckpointProposalJob implements Traceable {
914
1268
  */
915
1269
  private handleHASigningError(err: any, errorContext: string): boolean {
916
1270
  if (err instanceof DutyAlreadySignedError) {
917
- this.log.info(`${errorContext} for slot ${this.slot} already signed by another HA node, yielding`, {
918
- slot: this.slot,
1271
+ this.log.info(`${errorContext} for slot ${this.targetSlot} already signed by another HA node, yielding`, {
1272
+ slot: this.targetSlot,
919
1273
  signedByNode: err.signedByNode,
920
1274
  });
921
1275
  return true;
922
1276
  }
923
1277
  if (err instanceof SlashingProtectionError) {
924
- this.log.info(`${errorContext} for slot ${this.slot} blocked by slashing protection, yielding`, {
925
- slot: this.slot,
1278
+ this.log.info(`${errorContext} for slot ${this.targetSlot} blocked by slashing protection, yielding`, {
1279
+ slot: this.targetSlot,
926
1280
  existingMessageHash: err.existingMessageHash,
927
1281
  attemptedMessageHash: err.attemptedMessageHash,
928
1282
  });
@@ -945,7 +1299,7 @@ export class CheckpointProposalJob implements Traceable {
945
1299
  }
946
1300
 
947
1301
  private getSlotStartBuildTimestamp(): number {
948
- return getSlotStartBuildTimestamp(this.slot, this.l1Constants);
1302
+ return getSlotStartBuildTimestamp(this.slotNow, this.l1Constants);
949
1303
  }
950
1304
 
951
1305
  private getSecondsIntoSlot(): number {