@aztec/sequencer-client 0.0.1-commit.04852196a → 0.0.1-commit.04d373f

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (105) hide show
  1. package/README.md +282 -21
  2. package/dest/client/sequencer-client.d.ts +8 -14
  3. package/dest/client/sequencer-client.d.ts.map +1 -1
  4. package/dest/client/sequencer-client.js +35 -84
  5. package/dest/config.d.ts +7 -3
  6. package/dest/config.d.ts.map +1 -1
  7. package/dest/config.js +43 -15
  8. package/dest/global_variable_builder/fee_predictor.d.ts +37 -0
  9. package/dest/global_variable_builder/fee_predictor.d.ts.map +1 -0
  10. package/dest/global_variable_builder/fee_predictor.js +128 -0
  11. package/dest/global_variable_builder/fee_provider.d.ts +21 -0
  12. package/dest/global_variable_builder/fee_provider.d.ts.map +1 -0
  13. package/dest/global_variable_builder/fee_provider.js +58 -0
  14. package/dest/global_variable_builder/global_builder.d.ts +14 -14
  15. package/dest/global_variable_builder/global_builder.d.ts.map +1 -1
  16. package/dest/global_variable_builder/global_builder.js +16 -50
  17. package/dest/global_variable_builder/index.d.ts +4 -2
  18. package/dest/global_variable_builder/index.d.ts.map +1 -1
  19. package/dest/global_variable_builder/index.js +2 -0
  20. package/dest/index.d.ts +2 -2
  21. package/dest/index.d.ts.map +1 -1
  22. package/dest/index.js +1 -1
  23. package/dest/publisher/config.d.ts +15 -3
  24. package/dest/publisher/config.d.ts.map +1 -1
  25. package/dest/publisher/config.js +19 -4
  26. package/dest/publisher/l1_tx_failed_store/failed_tx_store.d.ts +3 -4
  27. package/dest/publisher/l1_tx_failed_store/failed_tx_store.d.ts.map +1 -1
  28. package/dest/publisher/sequencer-bundle-simulator.d.ts +96 -0
  29. package/dest/publisher/sequencer-bundle-simulator.d.ts.map +1 -0
  30. package/dest/publisher/sequencer-bundle-simulator.js +198 -0
  31. package/dest/publisher/sequencer-publisher-factory.d.ts +3 -5
  32. package/dest/publisher/sequencer-publisher-factory.d.ts.map +1 -1
  33. package/dest/publisher/sequencer-publisher-factory.js +2 -3
  34. package/dest/publisher/sequencer-publisher.d.ts +73 -65
  35. package/dest/publisher/sequencer-publisher.d.ts.map +1 -1
  36. package/dest/publisher/sequencer-publisher.js +316 -523
  37. package/dest/sequencer/automine/automine_factory.d.ts +54 -0
  38. package/dest/sequencer/automine/automine_factory.d.ts.map +1 -0
  39. package/dest/sequencer/automine/automine_factory.js +84 -0
  40. package/dest/sequencer/automine/automine_sequencer.d.ts +151 -0
  41. package/dest/sequencer/automine/automine_sequencer.d.ts.map +1 -0
  42. package/dest/sequencer/automine/automine_sequencer.js +506 -0
  43. package/dest/sequencer/chain_state_overrides.d.ts +61 -0
  44. package/dest/sequencer/chain_state_overrides.d.ts.map +1 -0
  45. package/dest/sequencer/chain_state_overrides.js +98 -0
  46. package/dest/sequencer/checkpoint_proposal_job.d.ts +54 -9
  47. package/dest/sequencer/checkpoint_proposal_job.d.ts.map +1 -1
  48. package/dest/sequencer/checkpoint_proposal_job.js +723 -199
  49. package/dest/sequencer/checkpoint_proposal_job_metrics.d.ts +34 -0
  50. package/dest/sequencer/checkpoint_proposal_job_metrics.d.ts.map +1 -0
  51. package/dest/sequencer/checkpoint_proposal_job_metrics.js +72 -0
  52. package/dest/sequencer/checkpoint_voter.d.ts +1 -2
  53. package/dest/sequencer/checkpoint_voter.d.ts.map +1 -1
  54. package/dest/sequencer/checkpoint_voter.js +2 -5
  55. package/dest/sequencer/events.d.ts +47 -2
  56. package/dest/sequencer/events.d.ts.map +1 -1
  57. package/dest/sequencer/index.d.ts +3 -1
  58. package/dest/sequencer/index.d.ts.map +1 -1
  59. package/dest/sequencer/index.js +2 -0
  60. package/dest/sequencer/metrics.d.ts +13 -10
  61. package/dest/sequencer/metrics.d.ts.map +1 -1
  62. package/dest/sequencer/metrics.js +45 -20
  63. package/dest/sequencer/sequencer.d.ts +53 -13
  64. package/dest/sequencer/sequencer.d.ts.map +1 -1
  65. package/dest/sequencer/sequencer.js +332 -118
  66. package/dest/sequencer/timetable.d.ts +14 -1
  67. package/dest/sequencer/timetable.d.ts.map +1 -1
  68. package/dest/sequencer/timetable.js +45 -36
  69. package/dest/sequencer/types.d.ts +2 -5
  70. package/dest/sequencer/types.d.ts.map +1 -1
  71. package/dest/test/mock_checkpoint_builder.d.ts +4 -4
  72. package/dest/test/mock_checkpoint_builder.d.ts.map +1 -1
  73. package/dest/test/utils.d.ts +1 -1
  74. package/dest/test/utils.d.ts.map +1 -1
  75. package/dest/test/utils.js +7 -6
  76. package/package.json +27 -28
  77. package/src/client/sequencer-client.ts +54 -114
  78. package/src/config.ts +52 -12
  79. package/src/global_variable_builder/README.md +44 -0
  80. package/src/global_variable_builder/fee_predictor.ts +172 -0
  81. package/src/global_variable_builder/fee_provider.ts +75 -0
  82. package/src/global_variable_builder/global_builder.ts +25 -62
  83. package/src/global_variable_builder/index.ts +3 -1
  84. package/src/index.ts +10 -1
  85. package/src/publisher/config.ts +40 -6
  86. package/src/publisher/l1_tx_failed_store/failed_tx_store.ts +3 -1
  87. package/src/publisher/sequencer-bundle-simulator.ts +254 -0
  88. package/src/publisher/sequencer-publisher-factory.ts +3 -6
  89. package/src/publisher/sequencer-publisher.ts +375 -568
  90. package/src/sequencer/README.md +152 -450
  91. package/src/sequencer/automine/README.md +46 -0
  92. package/src/sequencer/automine/automine_factory.ts +145 -0
  93. package/src/sequencer/automine/automine_sequencer.ts +592 -0
  94. package/src/sequencer/chain_state_overrides.ts +162 -0
  95. package/src/sequencer/checkpoint_proposal_job.ts +871 -220
  96. package/src/sequencer/checkpoint_proposal_job_metrics.ts +128 -0
  97. package/src/sequencer/checkpoint_voter.ts +1 -12
  98. package/src/sequencer/events.ts +50 -2
  99. package/src/sequencer/index.ts +2 -0
  100. package/src/sequencer/metrics.ts +57 -24
  101. package/src/sequencer/sequencer.ts +413 -131
  102. package/src/sequencer/timetable.ts +57 -45
  103. package/src/sequencer/types.ts +2 -5
  104. package/src/test/mock_checkpoint_builder.ts +3 -3
  105. package/src/test/utils.ts +28 -10
@@ -1,4 +1,5 @@
1
1
  import type { EpochCache } from '@aztec/epoch-cache';
2
+ import type { SimulationOverridesPlan } from '@aztec/ethereum/contracts';
2
3
  import {
3
4
  BlockNumber,
4
5
  CheckpointNumber,
@@ -17,6 +18,7 @@ import { EthAddress } from '@aztec/foundation/eth-address';
17
18
  import { Signature } from '@aztec/foundation/eth-signature';
18
19
  import { filter } from '@aztec/foundation/iterator';
19
20
  import { type Logger, type LoggerBindings, createLogger } from '@aztec/foundation/log';
21
+ import { retryUntil } from '@aztec/foundation/retry';
20
22
  import { sleep, sleepUntil } from '@aztec/foundation/sleep';
21
23
  import { type DateProvider, Timer } from '@aztec/foundation/timer';
22
24
  import { type TypedEventEmitter, isErrorClass, unfreeze } from '@aztec/foundation/types';
@@ -29,18 +31,32 @@ import {
29
31
  type L2BlockSink,
30
32
  type L2BlockSource,
31
33
  MaliciousCommitteeAttestationsAndSigners,
34
+ type ProposedCheckpointSink,
35
+ type ValidateCheckpointResult,
32
36
  } from '@aztec/stdlib/block';
33
- import { type Checkpoint, validateCheckpoint } from '@aztec/stdlib/checkpoint';
34
- import { getSlotStartBuildTimestamp } from '@aztec/stdlib/epoch-helpers';
37
+ import {
38
+ type Checkpoint,
39
+ type ProposedCheckpointData,
40
+ getPreviousCheckpointOutHashes,
41
+ validateCheckpoint,
42
+ } from '@aztec/stdlib/checkpoint';
43
+ import { computeQuorum, getSlotStartBuildTimestamp, getTimestampForSlot } from '@aztec/stdlib/epoch-helpers';
35
44
  import { Gas } from '@aztec/stdlib/gas';
36
45
  import {
37
- NoValidTxsError,
38
- type PublicProcessorLimits,
46
+ type BlockBuilderOptions,
47
+ InsufficientValidTxsError,
39
48
  type ResolvedSequencerConfig,
40
49
  type WorldStateSynchronizer,
41
50
  } from '@aztec/stdlib/interfaces/server';
42
51
  import { type L1ToL2MessageSource, computeInHashFromL1ToL2Messages } from '@aztec/stdlib/messaging';
43
- import type { BlockProposalOptions, CheckpointProposal, CheckpointProposalOptions } from '@aztec/stdlib/p2p';
52
+ import type {
53
+ BlockProposal,
54
+ BlockProposalOptions,
55
+ CheckpointAttestation,
56
+ CheckpointProposal,
57
+ CheckpointProposalOptions,
58
+ CoordinationSignatureContext,
59
+ } from '@aztec/stdlib/p2p';
44
60
  import { orderAttestations, trimAttestations } from '@aztec/stdlib/p2p';
45
61
  import type { L2BlockBuiltStats } from '@aztec/stdlib/stats';
46
62
  import { type FailedTx, Tx } from '@aztec/stdlib/tx';
@@ -51,6 +67,8 @@ import { DutyAlreadySignedError, SlashingProtectionError } from '@aztec/validato
51
67
 
52
68
  import type { GlobalVariableBuilder } from '../global_variable_builder/global_builder.js';
53
69
  import type { InvalidateCheckpointRequest, SequencerPublisher } from '../publisher/sequencer-publisher.js';
70
+ import { buildCheckpointSimulationOverridesPlan } from './chain_state_overrides.js';
71
+ import type { CheckpointProposalJobMetricsRecorder } from './checkpoint_proposal_job_metrics.js';
54
72
  import { CheckpointVoter } from './checkpoint_voter.js';
55
73
  import { SequencerInterruptedError } from './errors.js';
56
74
  import type { SequencerEvents } from './events.js';
@@ -62,6 +80,20 @@ import { SequencerState } from './utils.js';
62
80
  /** How much time to sleep while waiting for min transactions to accumulate for a block */
63
81
  const TXS_POLLING_MS = 500;
64
82
 
83
+ /** Result from proposeCheckpoint when a checkpoint was successfully built and broadcast. */
84
+ type CheckpointProposalBroadcast = {
85
+ checkpoint: Checkpoint;
86
+ proposal: CheckpointProposal;
87
+ blockProposedAt: number;
88
+ };
89
+
90
+ /** Result after attestation collection and signing, ready for L1 submission. */
91
+ type CheckpointProposalResult = {
92
+ checkpoint: Checkpoint;
93
+ attestations: CommitteeAttestationsAndSigners;
94
+ attestationsSignature: Signature;
95
+ };
96
+
65
97
  /**
66
98
  * Handles the execution of a checkpoint proposal after the initial preparation phase.
67
99
  * This includes building blocks, collecting attestations, and publishing the checkpoint to L1,
@@ -70,12 +102,30 @@ const TXS_POLLING_MS = 500;
70
102
  */
71
103
  export class CheckpointProposalJob implements Traceable {
72
104
  protected readonly log: Logger;
105
+ private readonly checkpointEventLog: Logger;
106
+
107
+ /** Tracks the fire-and-forget L1 submission promise so it can be awaited during shutdown. */
108
+ private pendingL1Submission: Promise<void> | undefined;
109
+
110
+ /**
111
+ * Chain state overrides built once per slot in proposeCheckpoint after the checkpoint is
112
+ * complete. Carries the pending parent override (archive + slot + fee header) for pipelining,
113
+ * or the invalidation pending override when rolling back. Consumed by
114
+ * publisher.validateBlockHeader before broadcast.
115
+ */
116
+ private checkpointSimulationOverridesPlan?: SimulationOverridesPlan;
117
+
118
+ private getSignatureContext(): CoordinationSignatureContext {
119
+ return this.signatureContext;
120
+ }
73
121
 
74
122
  constructor(
75
- private readonly epoch: EpochNumber,
76
- private readonly slot: SlotNumber,
123
+ private readonly slotNow: SlotNumber,
124
+ private readonly targetSlot: SlotNumber,
125
+ private readonly targetEpoch: EpochNumber,
77
126
  private readonly checkpointNumber: CheckpointNumber,
78
127
  private readonly syncedToBlockNumber: BlockNumber,
128
+ private readonly checkpointedCheckpointNumber: CheckpointNumber,
79
129
  // TODO(palla/mbps): Can we remove the proposer in favor of attestorAddress? Need to check fisherman-node flows.
80
130
  private readonly proposer: EthAddress | undefined,
81
131
  private readonly publisher: SequencerPublisher,
@@ -88,25 +138,51 @@ export class CheckpointProposalJob implements Traceable {
88
138
  private readonly l1ToL2MessageSource: L1ToL2MessageSource,
89
139
  private readonly l2BlockSource: L2BlockSource,
90
140
  private readonly checkpointsBuilder: FullNodeCheckpointsBuilder,
91
- private readonly blockSink: L2BlockSink,
141
+ private readonly blockSink: L2BlockSink & ProposedCheckpointSink,
92
142
  private readonly l1Constants: SequencerRollupConstants,
143
+ private readonly signatureContext: CoordinationSignatureContext,
93
144
  protected config: ResolvedSequencerConfig,
94
145
  protected timetable: SequencerTimetable,
95
146
  private readonly slasherClient: SlasherClientInterface | undefined,
96
147
  private readonly epochCache: EpochCache,
97
148
  private readonly dateProvider: DateProvider,
98
149
  private readonly metrics: SequencerMetrics,
99
- private readonly eventEmitter: TypedEventEmitter<SequencerEvents>,
150
+ private readonly checkpointMetrics: CheckpointProposalJobMetricsRecorder,
151
+ protected readonly eventEmitter: TypedEventEmitter<SequencerEvents>,
100
152
  private readonly setStateFn: (state: SequencerState, slot?: SlotNumber) => void,
101
153
  public readonly tracer: Tracer,
102
154
  bindings?: LoggerBindings,
155
+ private readonly proposedCheckpointData?: ProposedCheckpointData,
103
156
  ) {
104
- this.log = createLogger('sequencer:checkpoint-proposal', { ...bindings, instanceId: `slot-${slot}` });
157
+ this.log = createLogger('sequencer:checkpoint-proposal', {
158
+ ...bindings,
159
+ instanceId: `slot-${this.slotNow}`,
160
+ });
161
+ this.checkpointEventLog = createLogger('sequencer:checkpoint-events', {
162
+ ...bindings,
163
+ instanceId: `slot-${this.slotNow}`,
164
+ });
165
+ }
166
+
167
+ /** Awaits the pending L1 submission if one is in progress. Call during shutdown. */
168
+ public async awaitPendingSubmission(): Promise<void> {
169
+ this.log.info('Awaiting pending L1 payload submission');
170
+ await this.pendingL1Submission;
171
+ }
172
+
173
+ private logCheckpointEvent(eventName: string, message: string, fields: Record<string, unknown>): void {
174
+ this.checkpointEventLog.debug(message, {
175
+ eventName: `sequencer-checkpoint-${eventName}`,
176
+ ...fields,
177
+ });
105
178
  }
106
179
 
107
180
  /**
108
181
  * Executes the checkpoint proposal job.
109
- * Returns the published checkpoint if successful, undefined otherwise.
182
+ * Builds blocks, assembles checkpoint, and broadcasts the proposal (blocking).
183
+ * Attestation collection, signing, and L1 submission are backgrounded so the
184
+ * work loop can return to IDLE immediately for consecutive slot proposals.
185
+ * Returns the built checkpoint if successful, undefined otherwise.
110
186
  */
111
187
  @trackSpan('CheckpointProposalJob.execute')
112
188
  public async execute(): Promise<Checkpoint | undefined> {
@@ -114,7 +190,7 @@ export class CheckpointProposalJob implements Traceable {
114
190
  // In fisherman mode, we simulate slashing but don't actually publish to L1
115
191
  // These are constant for the whole slot, so we only enqueue them once
116
192
  const votesPromises = new CheckpointVoter(
117
- this.slot,
193
+ this.targetSlot,
118
194
  this.publisher,
119
195
  this.attestorAddress,
120
196
  this.validatorClient,
@@ -125,33 +201,302 @@ export class CheckpointProposalJob implements Traceable {
125
201
  this.log,
126
202
  ).enqueueVotes();
127
203
 
128
- // Build and propose the checkpoint. This will enqueue the request on the publisher if a checkpoint is built.
129
- const checkpoint = await this.proposeCheckpoint();
130
-
131
- // Wait until the voting promises have resolved, so all requests are enqueued (not sent)
132
- await Promise.all(votesPromises);
133
-
134
- if (checkpoint) {
135
- this.metrics.recordCheckpointProposalSuccess();
204
+ // Build blocks, assemble checkpoint, and broadcast proposal (BLOCKING).
205
+ // Returns after broadcast — attestation collection is deferred.
206
+ const broadcast = await this.proposeCheckpoint();
207
+
208
+ if (!broadcast) {
209
+ await Promise.all(votesPromises);
210
+ // Still submit votes even without a checkpoint.
211
+ // Under proposer pipelining, vote-offenses signatures are EIP-712-bound to `targetSlot`
212
+ // (the pipelined slot in which the multicall is expected to mine). Submitting at the
213
+ // wall-clock time would let the multicall mine in a different L2 slot, causing
214
+ // signature verification to fail silently inside Multicall3. Delay submission to the
215
+ // start of `targetSlot` so the tx mines in the slot the vote was signed for.
216
+ if (!this.config.fishermanMode) {
217
+ this.pendingL1Submission = this.publisher.sendRequestsAt(this.targetSlot).then(() => {});
218
+ }
219
+ return undefined;
136
220
  }
137
221
 
222
+ const { checkpoint } = broadcast;
223
+ this.metrics.recordCheckpointProposalSuccess();
224
+
138
225
  // Do not post anything to L1 if we are fishermen, but do perform L1 fee analysis
139
226
  if (this.config.fishermanMode) {
140
227
  await this.handleCheckpointEndAsFisherman(checkpoint);
141
- return;
228
+ return checkpoint;
142
229
  }
143
230
 
144
- // Then send everything to L1
145
- const l1Response = await this.publisher.sendRequests();
146
- const proposedAction = l1Response?.successfulActions.find(a => a === 'propose');
147
- if (proposedAction) {
148
- this.eventEmitter.emit('checkpoint-published', { checkpoint: this.checkpointNumber, slot: this.slot });
149
- const coinbase = checkpoint?.header.coinbase;
150
- await this.metrics.incFilledSlot(this.publisher.getSenderAddress().toString(), coinbase);
151
- return checkpoint;
152
- } else if (checkpoint) {
153
- this.eventEmitter.emit('checkpoint-publish-failed', { ...l1Response, slot: this.slot });
154
- return undefined;
231
+ // Background the attestation signing → L1 pipeline so the work loop is unblocked
232
+ this.pendingL1Submission = this.waitForAttestationsAndEnqueueSubmissionAsync(broadcast, votesPromises);
233
+
234
+ // Return the built checkpoint immediately — the work loop is now unblocked
235
+ return checkpoint;
236
+ }
237
+
238
+ /**
239
+ * Background pipeline: collects attestations, signs them, enqueues the checkpoint, and submits to L1.
240
+ * Runs as a fire-and-forget task stored in `pendingL1Submission` so the work loop is unblocked.
241
+ */
242
+ private async waitForAttestationsAndEnqueueSubmissionAsync(
243
+ broadcast: CheckpointProposalBroadcast,
244
+ votesPromises: Promise<unknown>[],
245
+ ): Promise<void> {
246
+ const { checkpoint } = broadcast;
247
+ const isPipelining = this.epochCache.isProposerPipeliningEnabled();
248
+
249
+ try {
250
+ // Wait for all votes actions, enqueued at the beginning, to resolve
251
+ await Promise.all(votesPromises);
252
+
253
+ // Try to collect attestations from the committee
254
+ const signedAttestations = await this.getSignedCommitteeAttestations(broadcast);
255
+
256
+ // If pipelining, wait for the previous checkpoint to land on L1 before submitting,
257
+ // so we can check it matches the proposed checkpoint we used as parent, and has valid attestations.
258
+ if (signedAttestations && (!isPipelining || (await this.waitForValidParentCheckpointOnL1()))) {
259
+ await this.enqueueCheckpointForSubmission({ checkpoint, ...signedAttestations });
260
+ }
261
+
262
+ // If we failed to collect attestations, at least check if we need to issue an invalidation
263
+ // Note that if we are not pipelining, we enqueued the invalidation at the beginning
264
+ if (!signedAttestations && isPipelining && (await this.waitForSyncedL2SlotNumber(this.slotNow))) {
265
+ const validationStatus = await this.l2BlockSource.getPendingChainValidationStatus();
266
+ if (!validationStatus.valid) {
267
+ this.log.warn(
268
+ `Checkpoint ${validationStatus.checkpoint.checkpointNumber} has invalid attestations, enqueuing invalidation in spite of attestation collection failure`,
269
+ { checkpoint: validationStatus.checkpoint, reason: validationStatus.reason },
270
+ );
271
+ await this.enqueueInvalidation(validationStatus);
272
+ }
273
+ }
274
+
275
+ // Send whatever was enqueued: votes + (propose | invalidation | nothing).
276
+ const l1Response = await this.publisher.sendRequestsAt(this.targetSlot);
277
+ const proposedAction = l1Response?.successfulActions.find(a => a === 'propose');
278
+ if (proposedAction) {
279
+ this.logCheckpointEvent('published', `Checkpoint published for slot ${this.targetSlot}`, {
280
+ slot: this.targetSlot,
281
+ checkpointNumber: this.checkpointNumber,
282
+ successfulActions: l1Response?.successfulActions,
283
+ sentActions: l1Response?.sentActions,
284
+ });
285
+ this.eventEmitter.emit('checkpoint-published', { checkpoint: this.checkpointNumber, slot: this.targetSlot });
286
+ const coinbase = checkpoint.header.coinbase;
287
+ await this.metrics.incFilledSlot(this.publisher.getSenderAddress().toString(), coinbase);
288
+ } else {
289
+ this.logCheckpointEvent('publish-failed', `Checkpoint publish failed for slot ${this.targetSlot}`, {
290
+ slot: this.targetSlot,
291
+ checkpointNumber: this.checkpointNumber,
292
+ successfulActions: l1Response?.successfulActions,
293
+ failedActions: l1Response?.failedActions,
294
+ sentActions: l1Response?.sentActions,
295
+ expiredActions: l1Response?.expiredActions,
296
+ reason: 'propose_action_not_successful',
297
+ });
298
+ this.log.warn(`Checkpoint publish failed for slot ${this.targetSlot}`, {
299
+ slot: this.targetSlot,
300
+ checkpointNumber: this.checkpointNumber,
301
+ successfulActions: l1Response?.successfulActions,
302
+ failedActions: l1Response?.failedActions,
303
+ sentActions: l1Response?.sentActions,
304
+ expiredActions: l1Response?.expiredActions,
305
+ reason: 'propose_action_not_successful',
306
+ });
307
+ this.eventEmitter.emit('checkpoint-publish-failed', { ...l1Response, slot: this.targetSlot });
308
+ if (isPipelining) {
309
+ this.metrics.recordPipelineDiscard();
310
+ }
311
+ }
312
+ } catch (err) {
313
+ if (err instanceof SequencerInterruptedError) {
314
+ return;
315
+ }
316
+ this.logCheckpointEvent('publish-failed', `Checkpoint publish failed for slot ${this.targetSlot}`, {
317
+ slot: this.targetSlot,
318
+ checkpointNumber: this.checkpointNumber,
319
+ reason: err instanceof Error ? err.message : String(err),
320
+ });
321
+ this.log.error(`Background attestation/L1 pipeline failed for slot ${this.targetSlot}`, err, {
322
+ slot: this.targetSlot,
323
+ checkpointNumber: this.checkpointNumber,
324
+ reason: err instanceof Error ? err.message : String(err),
325
+ });
326
+ this.eventEmitter.emit('checkpoint-publish-failed', { slot: this.targetSlot });
327
+ if (isPipelining) {
328
+ this.metrics.recordPipelineDiscard();
329
+ }
330
+ }
331
+ }
332
+
333
+ /** Enqueues the checkpoint for L1 submission. Called after pipeline sleep in execute(). */
334
+ private async enqueueCheckpointForSubmission(result: CheckpointProposalResult): Promise<void> {
335
+ const { checkpoint, attestations, attestationsSignature } = result;
336
+
337
+ this.setStateFn(SequencerState.PUBLISHING_CHECKPOINT, this.targetSlot);
338
+ const aztecSlotDuration = this.l1Constants.slotDuration;
339
+ const submissionSlotStart = Number(getTimestampForSlot(this.targetSlot, this.l1Constants));
340
+ const txTimeoutAt = new Date((submissionSlotStart + aztecSlotDuration) * 1000);
341
+
342
+ // If we have been configured to potentially skip publishing checkpoint then roll the dice here
343
+ if (
344
+ this.config.skipPublishingCheckpointsPercent !== undefined &&
345
+ this.config.skipPublishingCheckpointsPercent > 0
346
+ ) {
347
+ const roll = Math.max(0, randomInt(100));
348
+ if (roll < this.config.skipPublishingCheckpointsPercent) {
349
+ this.log.warn(
350
+ `Skipping publishing proposal for checkpoint ${checkpoint.number}. Configured percentage: ${this.config.skipPublishingCheckpointsPercent}, generated value: ${roll}`,
351
+ );
352
+ return;
353
+ }
354
+ }
355
+
356
+ await this.publisher.enqueueProposeCheckpoint(checkpoint, attestations, attestationsSignature, {
357
+ txTimeoutAt,
358
+ });
359
+ }
360
+
361
+ /**
362
+ * Wait until the archiver syncs past the given L2 slot number.
363
+ * The deadline is the end of `this.targetSlot`, beyond which any pipelined work would miss its
364
+ * L1 submission window and is no longer useful.
365
+ */
366
+ private async waitForSyncedL2SlotNumber(waitForSlot: SlotNumber): Promise<boolean> {
367
+ const targetSlotStart = Number(getTimestampForSlot(this.targetSlot, this.l1Constants));
368
+ const targetSlotEndMs = (targetSlotStart + this.l1Constants.slotDuration) * 1000;
369
+ const syncDelayTolerance = this.l1Constants.ethereumSlotDuration * 2 * 1000;
370
+ const timeoutSeconds = Math.max(0.1, (targetSlotEndMs + syncDelayTolerance - this.dateProvider.now()) / 1000);
371
+
372
+ try {
373
+ return await retryUntil(
374
+ async () => {
375
+ const syncedSlot = await this.l2BlockSource.getSyncedL2SlotNumber();
376
+ return syncedSlot !== undefined && syncedSlot >= waitForSlot;
377
+ },
378
+ `archiver sync past slot ${waitForSlot}`,
379
+ timeoutSeconds,
380
+ 0.2,
381
+ );
382
+ } catch {
383
+ this.log.warn(
384
+ `Archiver did not sync L1 past slot ${waitForSlot} before slot ${this.targetSlot} expired, discarding pipelined work`,
385
+ { checkpointNumber: this.checkpointNumber },
386
+ );
387
+ this.emitPipelinedCheckpointDiscarded('archiver-sync-timeout');
388
+ return false;
389
+ }
390
+ }
391
+
392
+ /**
393
+ * Waits for the parent checkpoint to land on L1 before submitting a pipelined checkpoint.
394
+ * Polls until the archiver has synced L1 past the parent's slot, then verifies:
395
+ * - If we built on a proposed parent: it must have landed on L1 with matching hash and valid attestations.
396
+ * - If we built without a proposed parent: no new checkpoint must have appeared for that slot.
397
+ * If the parent has invalid attestations, enqueues an invalidation. Returns whether to proceed with the proposal.
398
+ */
399
+ protected async waitForValidParentCheckpointOnL1(): Promise<boolean> {
400
+ if (this.config.skipWaitForValidParentCheckpointOnL1) {
401
+ this.log.warn(`Skipping waitForValidParentCheckpointOnL1 due to test configuration`, {
402
+ checkpointNumber: this.checkpointNumber,
403
+ });
404
+ return true;
405
+ }
406
+
407
+ const parentCheckpointNumber = CheckpointNumber(this.checkpointNumber - 1);
408
+
409
+ // Wait until archiver has synced L1 past the parent's slot (slotNow)
410
+ if (!(await this.waitForSyncedL2SlotNumber(this.slotNow))) {
411
+ return false;
412
+ }
413
+
414
+ const tips = await this.l2BlockSource.getL2Tips();
415
+ const checkpointedNumber = tips.checkpointed.checkpoint.number;
416
+
417
+ // We built on top of a proposed checkpoint. Verify it landed on L1 as expected.
418
+ if (this.proposedCheckpointData) {
419
+ // After syncing from L1 we see the chain tip has invalid attestations. This means the parent checkpoint was posted
420
+ // with invalid attestations, or it built on top of something with invalid attestations and didnt invalidate them.
421
+ // Either way, we thought our parent would be valid, so we have to throw away our work. But at least we'll try and
422
+ // invalidate on L1 so we clean up the chain for the next proposer. And we'll slash them, but that's handled elsewhere.
423
+ const validationStatus = await this.l2BlockSource.getPendingChainValidationStatus();
424
+ if (!validationStatus.valid) {
425
+ this.log.warn(
426
+ `Parent checkpoint ${parentCheckpointNumber} has invalid attestations, discarding pipelined work`,
427
+ { checkpointNumber: this.checkpointNumber, reason: validationStatus.reason },
428
+ );
429
+ this.emitPipelinedCheckpointDiscarded('parent-invalid-attestations');
430
+ await this.enqueueInvalidation(validationStatus);
431
+ return false;
432
+ }
433
+
434
+ // The pending chain is valid. But did the parent checkpoint land on L1 at all?
435
+ if (checkpointedNumber < parentCheckpointNumber) {
436
+ this.log.warn(`Parent checkpoint ${parentCheckpointNumber} did not land on L1, discarding pipelined work`, {
437
+ checkpointNumber: this.checkpointNumber,
438
+ checkpointedNumber,
439
+ });
440
+ this.emitPipelinedCheckpointDiscarded('parent-not-on-l1');
441
+ return false;
442
+ }
443
+
444
+ // It landed. But is it the one we were expecting?
445
+ const expectedHash = this.proposedCheckpointData.header.hash().toString();
446
+ if (tips.checkpointed.checkpoint.hash !== expectedHash) {
447
+ this.log.warn(`Parent checkpoint ${parentCheckpointNumber} hash mismatch on L1, discarding pipelined work`, {
448
+ checkpointNumber: this.checkpointNumber,
449
+ expectedHash,
450
+ actualHash: tips.checkpointed.checkpoint.hash,
451
+ });
452
+ this.emitPipelinedCheckpointDiscarded('parent-hash-mismatch');
453
+ return false;
454
+ }
455
+
456
+ return true;
457
+ } else {
458
+ // We didn't see a proposed checkpoint at build time, so we built on checkpointed parent from two slots ago.
459
+ // But if a new checkpoint for the previous slot appeared on L1 in the meantime, our checkpoint assumed the wrong parent,
460
+ // so we have to discard our work. This can happen if we're somehow cut off from p2p and fail to see the checkpoint
461
+ // proposal for the previous slot.
462
+ if (checkpointedNumber > parentCheckpointNumber) {
463
+ this.log.warn(
464
+ `Unexpected checkpoint ${checkpointedNumber} landed on L1 after we built on top of parent ${parentCheckpointNumber}, discarding pipelined work`,
465
+ { checkpointNumber: this.checkpointNumber, checkpointedNumber },
466
+ );
467
+ this.emitPipelinedCheckpointDiscarded('unexpected-parent-appeared');
468
+ return false;
469
+ }
470
+
471
+ return true;
472
+ }
473
+ }
474
+
475
+ /** Emits the pipelined-checkpoint-discarded event and records the metric. */
476
+ private emitPipelinedCheckpointDiscarded(reason: string): void {
477
+ this.metrics.recordPipelineParentCheckpointMismatch(reason);
478
+ this.eventEmitter.emit('pipelined-checkpoint-discarded', {
479
+ slot: this.targetSlot,
480
+ checkpointNumber: this.checkpointNumber,
481
+ reason,
482
+ });
483
+ }
484
+
485
+ /** Simulates and enqueues an invalidation request for the invalid parent checkpoint. */
486
+ private async enqueueInvalidation(validationStatus: ValidateCheckpointResult): Promise<void> {
487
+ if (this.config.skipInvalidateBlockAsProposer) {
488
+ this.log.warn(`Skipping checkpoint invalidation as proposer due to test configuration`);
489
+ return;
490
+ }
491
+ const invalidateRequest = await this.publisher.simulateInvalidateCheckpoint(validationStatus);
492
+ if (invalidateRequest) {
493
+ const submissionSlotStart = Number(getTimestampForSlot(this.targetSlot, this.l1Constants));
494
+ const txTimeoutAt = new Date((submissionSlotStart + this.l1Constants.slotDuration) * 1000);
495
+ this.publisher.enqueueInvalidateCheckpoint(invalidateRequest, { txTimeoutAt });
496
+ } else {
497
+ this.log.info(`Invalidation simulation returned undefined, checkpoint may have been removed already`, {
498
+ checkpointNumber: this.checkpointNumber,
499
+ });
155
500
  }
156
501
  }
157
502
 
@@ -159,42 +504,91 @@ export class CheckpointProposalJob implements Traceable {
159
504
  return {
160
505
  // nullish operator needed for tests
161
506
  [Attributes.COINBASE]: this.validatorClient.getCoinbaseForAttestor(this.attestorAddress)?.toString(),
162
- [Attributes.SLOT_NUMBER]: this.slot,
507
+ [Attributes.SLOT_NUMBER]: this.targetSlot,
163
508
  };
164
509
  })
165
- private async proposeCheckpoint(): Promise<Checkpoint | undefined> {
510
+ private async proposeCheckpoint(): Promise<CheckpointProposalBroadcast | undefined> {
166
511
  try {
512
+ const now = this.dateProvider.now();
513
+ if (this.epochCache.isProposerPipeliningEnabled() && this.proposedCheckpointData) {
514
+ // Measure against the wall-clock slot whose build window we are currently using.
515
+ // In pipelining mode `targetSlot` is intentionally one slot ahead, which makes the
516
+ // target-slot boundary a full slot away from the actual build start time.
517
+ const slotBoundaryMs = Number(getTimestampForSlot(this.slotNow, this.l1Constants)) * 1000;
518
+ this.checkpointMetrics.recordPipelinedCheckpointBuildStartOffsetFromSlotBoundary(now - slotBoundaryMs);
519
+ }
520
+ this.checkpointMetrics.startCheckpointTiming(now);
521
+
167
522
  // Get operator configured coinbase and fee recipient for this attestor
168
523
  const coinbase = this.validatorClient.getCoinbaseForAttestor(this.attestorAddress);
169
524
  const feeRecipient = this.validatorClient.getFeeRecipientForAttestor(this.attestorAddress);
170
525
 
171
526
  // Start the checkpoint
172
- this.setStateFn(SequencerState.INITIALIZING_CHECKPOINT, this.slot);
173
- this.metrics.incOpenSlot(this.slot, this.proposer?.toString() ?? 'unknown');
527
+ this.setStateFn(SequencerState.INITIALIZING_CHECKPOINT, this.targetSlot);
528
+ this.logCheckpointEvent('slot-started', `Starting checkpoint proposal for slot ${this.targetSlot}`, {
529
+ buildSlot: this.slotNow,
530
+ submissionSlot: this.targetSlot,
531
+ slot: this.targetSlot,
532
+ checkpointNumber: this.checkpointNumber,
533
+ pipelining: this.epochCache.isProposerPipeliningEnabled(),
534
+ proposer: this.proposer?.toString(),
535
+ attestorAddress: this.attestorAddress.toString(),
536
+ publisherAddress: this.publisher.getSenderAddress().toString(),
537
+ coinbase: coinbase.toString(),
538
+ });
539
+ this.metrics.incOpenSlot(this.targetSlot, this.proposer?.toString() ?? 'unknown');
174
540
 
175
541
  // Enqueues checkpoint invalidation (constant for the whole slot)
176
542
  if (this.invalidateCheckpoint && !this.config.skipInvalidateBlockAsProposer) {
177
543
  this.publisher.enqueueInvalidateCheckpoint(this.invalidateCheckpoint);
178
544
  }
179
545
 
180
- // Create checkpoint builder for the slot
546
+ // Build the simulation plan for this slot. When pipelining, this overrides L1's view of
547
+ // pending/archive/fee-header to "as if the proposed parent had landed", so both the
548
+ // mana-min-fee simulation (in the globals builder) and the pre-broadcast
549
+ // validateBlockHeader see the chain tip the eventual L1 send will see.
550
+ const isPipelining = this.epochCache.isProposerPipeliningEnabled();
551
+ this.checkpointSimulationOverridesPlan = await buildCheckpointSimulationOverridesPlan({
552
+ checkpointNumber: this.checkpointNumber,
553
+ proposedCheckpointData: isPipelining ? this.proposedCheckpointData : undefined,
554
+ invalidateToPendingCheckpointNumber: this.invalidateCheckpoint?.forcePendingCheckpointNumber,
555
+ checkpointedCheckpointNumber: this.checkpointedCheckpointNumber,
556
+ rollup: this.publisher.rollupContract,
557
+ signatureContext: this.signatureContext,
558
+ log: this.log,
559
+ });
560
+
181
561
  const checkpointGlobalVariables = await this.globalsBuilder.buildCheckpointGlobalVariables(
182
562
  coinbase,
183
563
  feeRecipient,
184
- this.slot,
564
+ this.targetSlot,
565
+ this.checkpointSimulationOverridesPlan,
185
566
  );
186
567
 
187
568
  // Collect L1 to L2 messages for the checkpoint and compute their hash
188
569
  const l1ToL2Messages = await this.l1ToL2MessageSource.getL1ToL2Messages(this.checkpointNumber);
189
570
  const inHash = computeInHashFromL1ToL2Messages(l1ToL2Messages);
190
571
 
191
- // Collect the out hashes of all the checkpoints before this one in the same epoch
192
- const previousCheckpointOutHashes = (await this.l2BlockSource.getCheckpointsDataForEpoch(this.epoch))
193
- .filter(c => c.checkpointNumber < this.checkpointNumber)
194
- .map(c => c.checkpointOutHash);
572
+ // Collect the out hashes of all the checkpoints before this one in the same epoch.
573
+ // Under pipelining the parent checkpoint may not be on L1 yet at build time, so the helper
574
+ // splices in the parent's checkpointOutHash from the locally-known proposed checkpoint so
575
+ // the resulting `epochOutHash` matches what validators (and L1) compute once the parent
576
+ // lands on L1.
577
+ const previousCheckpointOutHashes = await getPreviousCheckpointOutHashes({
578
+ blockSource: this.l2BlockSource,
579
+ epoch: this.targetEpoch,
580
+ checkpointNumber: this.checkpointNumber,
581
+ l1Constants: this.epochCache.getL1Constants(),
582
+ pipeliningEnabled: this.epochCache.isProposerPipeliningEnabled(),
583
+ proposedCheckpointData: this.proposedCheckpointData,
584
+ log: this.log,
585
+ });
195
586
 
196
- // Get the fee asset price modifier from the oracle
197
- const feeAssetPriceModifier = await this.publisher.getFeeAssetPriceModifier();
587
+ // Anchor the modifier to the predicted parent fee header: L1 will apply it against
588
+ // that, not against the latest published checkpoint (which lags by one under pipelining).
589
+ const predictedParentEthPerFeeAssetE12 =
590
+ this.checkpointSimulationOverridesPlan?.pendingCheckpointState?.feeHeader?.ethPerFeeAsset;
591
+ const feeAssetPriceModifier = await this.publisher.getFeeAssetPriceModifier(predictedParentEthPerFeeAssetE12);
198
592
 
199
593
  // Create a long-lived forked world state for the checkpoint builder
200
594
  await using fork = await this.worldState.fork(this.syncedToBlockNumber, { closeDelayMs: 12_000 });
@@ -218,11 +612,12 @@ export class CheckpointProposalJob implements Traceable {
218
612
 
219
613
  const checkpointProposalOptions: CheckpointProposalOptions = {
220
614
  publishFullTxs: !!this.config.publishTxsWithProposals,
221
- broadcastInvalidCheckpointProposal: this.config.broadcastInvalidBlockProposal,
615
+ broadcastInvalidCheckpointProposal:
616
+ this.config.broadcastInvalidCheckpointProposalOnly || this.config.broadcastInvalidBlockProposal,
222
617
  };
223
618
 
224
619
  let blocksInCheckpoint: L2Block[] = [];
225
- let blockPendingBroadcast: { block: L2Block; txs: Tx[] } | undefined = undefined;
620
+ let blockPendingBroadcast: BlockProposal | undefined = undefined;
226
621
  const checkpointBuildTimer = new Timer();
227
622
 
228
623
  try {
@@ -246,26 +641,49 @@ export class CheckpointProposalJob implements Traceable {
246
641
  }
247
642
 
248
643
  if (blocksInCheckpoint.length === 0) {
249
- this.log.warn(`No blocks were built for slot ${this.slot}`, { slot: this.slot });
250
- this.eventEmitter.emit('checkpoint-empty', { slot: this.slot });
644
+ this.logCheckpointEvent('build-failed', `Checkpoint build failed for slot ${this.targetSlot}`, {
645
+ slot: this.targetSlot,
646
+ checkpointNumber: this.checkpointNumber,
647
+ reason: 'no_blocks_built',
648
+ });
649
+ this.log.warn(`No blocks were built for slot ${this.targetSlot}`, {
650
+ slot: this.targetSlot,
651
+ checkpointNumber: this.checkpointNumber,
652
+ reason: 'no_blocks_built',
653
+ });
654
+ this.eventEmitter.emit('checkpoint-empty', { slot: this.targetSlot });
251
655
  return undefined;
252
656
  }
253
657
 
254
658
  const minBlocksForCheckpoint = this.config.minBlocksForCheckpoint;
255
659
  if (minBlocksForCheckpoint !== undefined && blocksInCheckpoint.length < minBlocksForCheckpoint) {
660
+ this.logCheckpointEvent('build-failed', `Checkpoint build failed for slot ${this.targetSlot}`, {
661
+ slot: this.targetSlot,
662
+ checkpointNumber: this.checkpointNumber,
663
+ blocksBuilt: blocksInCheckpoint.length,
664
+ minBlocksForCheckpoint,
665
+ reason: 'min_blocks_not_met',
666
+ });
256
667
  this.log.warn(
257
668
  `Checkpoint has fewer blocks than minimum (${blocksInCheckpoint.length} < ${minBlocksForCheckpoint}), skipping proposal`,
258
- { slot: this.slot, blocksBuilt: blocksInCheckpoint.length, minBlocksForCheckpoint },
669
+ {
670
+ slot: this.targetSlot,
671
+ checkpointNumber: this.checkpointNumber,
672
+ blocksBuilt: blocksInCheckpoint.length,
673
+ minBlocksForCheckpoint,
674
+ reason: 'min_blocks_not_met',
675
+ },
259
676
  );
260
677
  return undefined;
261
678
  }
262
679
 
263
680
  // Assemble and broadcast the checkpoint proposal, including the last block that was not
264
681
  // broadcasted yet, and wait to collect the committee attestations.
265
- this.setStateFn(SequencerState.ASSEMBLING_CHECKPOINT, this.slot);
682
+ this.setStateFn(SequencerState.ASSEMBLING_CHECKPOINT, this.targetSlot);
266
683
  const checkpoint = await checkpointBuilder.completeCheckpoint();
267
684
 
268
- // Final validation round for the checkpoint before we propose it, just for safety
685
+ // Final validation: per-block limits are only checked if the operator set them explicitly.
686
+ // Otherwise, checkpoint-level budgets were already enforced by the redistribution logic.
269
687
  try {
270
688
  validateCheckpoint(checkpoint, {
271
689
  rollupManaLimit: this.l1Constants.rollupManaLimit,
@@ -275,113 +693,118 @@ export class CheckpointProposalJob implements Traceable {
275
693
  maxTxsPerCheckpoint: this.config.maxTxsPerCheckpoint,
276
694
  });
277
695
  } catch (err) {
278
- this.log.error(`Built an invalid checkpoint at slot ${this.slot} (skipping proposal)`, err, {
696
+ this.logCheckpointEvent('build-failed', `Checkpoint build failed for slot ${this.targetSlot}`, {
697
+ slot: this.targetSlot,
698
+ checkpointNumber: this.checkpointNumber,
699
+ blocksBuilt: blocksInCheckpoint.length,
700
+ reason: 'invalid_checkpoint',
701
+ checkpoint: checkpoint.header.toInspect(),
702
+ });
703
+ this.log.error(`Built an invalid checkpoint at slot ${this.slotNow} (skipping proposal)`, err, {
704
+ slot: this.targetSlot,
705
+ checkpointNumber: this.checkpointNumber,
706
+ blocksBuilt: blocksInCheckpoint.length,
707
+ reason: 'invalid_checkpoint',
279
708
  checkpoint: checkpoint.header.toInspect(),
280
709
  });
281
710
  return undefined;
282
711
  }
283
712
 
284
713
  // Record checkpoint-level build metrics
285
- this.metrics.recordCheckpointBuild(
714
+ this.checkpointMetrics.recordCheckpointBuild(
286
715
  checkpointBuildTimer.ms(),
287
716
  blocksInCheckpoint.length,
288
717
  checkpoint.getStats().txCount,
289
718
  Number(checkpoint.header.totalManaUsed.toBigInt()),
290
719
  );
720
+ this.logCheckpointEvent('built', `Checkpoint built for slot ${this.targetSlot}`, {
721
+ slot: this.targetSlot,
722
+ buildSlot: this.slotNow,
723
+ checkpointNumber: this.checkpointNumber,
724
+ proposer: this.proposer?.toString(),
725
+ attestorAddress: this.attestorAddress.toString(),
726
+ publisherAddress: this.publisher.getSenderAddress().toString(),
727
+ blocksBuilt: blocksInCheckpoint.length,
728
+ txCount: checkpoint.getStats().txCount,
729
+ totalMana: Number(checkpoint.header.totalManaUsed.toBigInt()),
730
+ });
291
731
 
292
- // Do not collect attestations nor publish to L1 in fisherman mode
732
+ // In fisherman mode, return the checkpoint without broadcasting or collecting attestations
293
733
  if (this.config.fishermanMode) {
294
734
  this.log.info(
295
- `Built checkpoint for slot ${this.slot} with ${blocksInCheckpoint.length} blocks. ` +
735
+ `Built checkpoint for slot ${this.targetSlot} with ${blocksInCheckpoint.length} blocks. ` +
296
736
  `Skipping proposal in fisherman mode.`,
297
737
  {
298
- slot: this.slot,
738
+ slot: this.targetSlot,
299
739
  checkpoint: checkpoint.header.toInspect(),
300
740
  blocksBuilt: blocksInCheckpoint.length,
301
741
  },
302
742
  );
303
743
  this.metrics.recordCheckpointSuccess();
304
- return checkpoint;
744
+ // Return a broadcast result with a dummy proposal — fisherman mode skips attestation collection
745
+ return { checkpoint, proposal: undefined!, blockProposedAt: this.dateProvider.now() };
305
746
  }
306
747
 
307
- // Include the block pending broadcast in the checkpoint proposal if any
308
- const lastBlock = blockPendingBroadcast && {
309
- blockHeader: blockPendingBroadcast.block.header,
310
- indexWithinCheckpoint: blockPendingBroadcast.block.indexWithinCheckpoint,
311
- txs: blockPendingBroadcast.txs,
312
- };
748
+ // Validate the header against L1 state before broadcasting.
749
+ // If this fails the slot is aborted before any gossip work; state drift between here
750
+ // and the eventual L1 send is caught by the bundle simulate at send time.
751
+ try {
752
+ await this.publisher.validateBlockHeader(checkpoint.header, this.checkpointSimulationOverridesPlan);
753
+ } catch (err) {
754
+ this.log.error(`Pre-broadcast header validation failed for slot ${this.targetSlot}; aborting`, err, {
755
+ slot: this.targetSlot,
756
+ checkpointNumber: this.checkpointNumber,
757
+ });
758
+ this.metrics.recordCheckpointProposalFailed('header_validation_failed');
759
+ this.eventEmitter.emit('header-validation-failed', {
760
+ slot: this.targetSlot,
761
+ checkpointNumber: this.checkpointNumber,
762
+ reason: err instanceof Error ? err.message : String(err),
763
+ });
764
+ return undefined;
765
+ }
313
766
 
314
767
  // Create the checkpoint proposal and broadcast it
315
768
  const proposal = await this.validatorClient.createCheckpointProposal(
316
769
  checkpoint.header,
317
770
  checkpoint.archive.root,
771
+ this.checkpointNumber,
318
772
  feeAssetPriceModifier,
319
- lastBlock,
773
+ blockPendingBroadcast,
320
774
  this.proposer,
321
775
  checkpointProposalOptions,
322
776
  );
323
777
 
324
- const blockProposedAt = this.dateProvider.now();
325
- await this.p2pClient.broadcastCheckpointProposal(proposal);
326
-
327
- this.setStateFn(SequencerState.COLLECTING_ATTESTATIONS, this.slot);
328
- const attestations = await this.waitForAttestations(proposal);
329
- const blockAttestedAt = this.dateProvider.now();
778
+ // Advance our own optimistic proposed-checkpoint tip locally before gossiping. Gossipsub
779
+ // doesn't echo our own messages back, so this is how the proposer makes its own proposed
780
+ // checkpoint visible for pipelining the next slot. Built from local checkpoint data — never
781
+ // from the broadcast proposal archive, which may be deliberately corrupted under test flags.
782
+ // Fail closed: if this throws, the outer catch aborts the slot before gossiping.
783
+ await this.syncProposedCheckpointToArchiver(checkpoint, blocksInCheckpoint.length, feeAssetPriceModifier);
330
784
 
331
- this.metrics.recordCheckpointAttestationDelay(blockAttestedAt - blockProposedAt);
332
-
333
- // Proposer must sign over the attestations before pushing them to L1
334
- const signer = this.proposer ?? this.publisher.getSenderAddress();
335
- let attestationsSignature: Signature;
336
- try {
337
- attestationsSignature = await this.validatorClient.signAttestationsAndSigners(
338
- attestations,
339
- signer,
340
- this.slot,
341
- this.checkpointNumber,
342
- );
343
- } catch (err) {
344
- // We shouldn't really get here since we yield to another HA node
345
- // as soon as we see these errors when creating block or checkpoint proposals.
346
- if (this.handleHASigningError(err, 'Attestations signature')) {
347
- return undefined;
348
- }
349
- throw err;
350
- }
351
-
352
- // Enqueue publishing the checkpoint to L1
353
- this.setStateFn(SequencerState.PUBLISHING_CHECKPOINT, this.slot);
354
- const aztecSlotDuration = this.l1Constants.slotDuration;
355
- const slotStartBuildTimestamp = this.getSlotStartBuildTimestamp();
356
- const txTimeoutAt = new Date((slotStartBuildTimestamp + aztecSlotDuration) * 1000);
357
-
358
- // If we have been configured to potentially skip publishing checkpoint then roll the dice here
359
- if (
360
- this.config.skipPublishingCheckpointsPercent !== undefined &&
361
- this.config.skipPublishingCheckpointsPercent > 0
362
- ) {
363
- const result = Math.max(0, randomInt(100));
364
- if (result < this.config.skipPublishingCheckpointsPercent) {
365
- this.log.warn(
366
- `Skipping publishing proposal for checkpoint ${checkpoint.number}. Configured percentage: ${this.config.skipPublishingCheckpointsPercent}, generated value: ${result}`,
367
- );
368
- return checkpoint;
785
+ const blockProposedAt = this.dateProvider.now();
786
+ if (this.config.skipBroadcastCheckpointProposal) {
787
+ // Test-only: suppress the CheckpointProposal so peers never see a proposed checkpoint for
788
+ // this slot, but still broadcast the held last block standalone so peers' archivers ingest
789
+ // it as a proposed-but-uncheckpointed tip — the exact orphan-block state that
790
+ // pruneOrphanProposedBlocks / checkSync must handle.
791
+ if (blockPendingBroadcast && !this.config.skipBroadcastProposals) {
792
+ await this.p2pClient.broadcastProposal(blockPendingBroadcast);
369
793
  }
794
+ } else if (!this.config.skipBroadcastProposals) {
795
+ await this.p2pClient.broadcastCheckpointProposal(proposal);
796
+ this.checkpointMetrics.noteCheckpointBroadcast(this.dateProvider.now());
370
797
  }
371
798
 
372
- await this.publisher.enqueueProposeCheckpoint(checkpoint, attestations, attestationsSignature, {
373
- txTimeoutAt,
374
- forcePendingCheckpointNumber: this.invalidateCheckpoint?.forcePendingCheckpointNumber,
375
- });
376
-
377
- return checkpoint;
799
+ // Return immediately after broadcast — attestation collection happens in the background
800
+ return { checkpoint, proposal, blockProposedAt };
378
801
  } catch (err) {
379
802
  if (err && (err instanceof DutyAlreadySignedError || err instanceof SlashingProtectionError)) {
380
803
  // swallow this error. It's already been logged by a function deeper in the stack
381
804
  return undefined;
382
805
  }
383
806
 
384
- this.log.error(`Error building checkpoint at slot ${this.slot}`, err);
807
+ this.log.error(`Error building checkpoint at slot ${this.targetSlot}`, err);
385
808
  return undefined;
386
809
  }
387
810
  }
@@ -397,26 +820,35 @@ export class CheckpointProposalJob implements Traceable {
397
820
  blockProposalOptions: BlockProposalOptions,
398
821
  ): Promise<{
399
822
  blocksInCheckpoint: L2Block[];
400
- blockPendingBroadcast: { block: L2Block; txs: Tx[] } | undefined;
823
+ blockPendingBroadcast: BlockProposal | undefined;
401
824
  }> {
402
825
  const blocksInCheckpoint: L2Block[] = [];
403
826
  const txHashesAlreadyIncluded = new Set<string>();
404
827
  const initialBlockNumber = BlockNumber(this.syncedToBlockNumber + 1);
405
828
 
406
829
  // Last block in the checkpoint will usually be flagged as pending broadcast, so we send it along with the checkpoint proposal
407
- let blockPendingBroadcast: { block: L2Block; txs: Tx[] } | undefined = undefined;
830
+ let blockPendingBroadcast: BlockProposal | undefined = undefined;
408
831
 
409
832
  while (true) {
410
833
  const blocksBuilt = blocksInCheckpoint.length;
411
834
  const indexWithinCheckpoint = IndexWithinCheckpoint(blocksBuilt);
412
835
  const blockNumber = BlockNumber(initialBlockNumber + blocksBuilt);
413
836
 
837
+ if (blocksBuilt >= this.config.maxBlocksPerCheckpoint) {
838
+ this.log.debug(`Reached max blocks per checkpoint`, {
839
+ slot: this.targetSlot,
840
+ blocksBuilt,
841
+ maxBlocksPerCheckpoint: this.config.maxBlocksPerCheckpoint,
842
+ });
843
+ break;
844
+ }
845
+
414
846
  const secondsIntoSlot = this.getSecondsIntoSlot();
415
847
  const timingInfo = this.timetable.canStartNextBlock(secondsIntoSlot);
416
848
 
417
849
  if (!timingInfo.canStart) {
418
850
  this.log.debug(`Not enough time left in slot to start another block`, {
419
- slot: this.slot,
851
+ slot: this.targetSlot,
420
852
  blocksBuilt,
421
853
  secondsIntoSlot,
422
854
  });
@@ -437,22 +869,23 @@ export class CheckpointProposalJob implements Traceable {
437
869
  txHashesAlreadyIncluded,
438
870
  });
439
871
 
440
- // TODO(palla/mbps): Review these conditions. We may want to keep trying in some scenarios.
441
- if (!buildResult && timingInfo.isLastBlock) {
442
- // If no block was produced due to not enough txs and this was the last subslot, exit
443
- break;
444
- } else if (!buildResult && timingInfo.deadline !== undefined) {
445
- // But if there is still time for more blocks, wait until the next subslot and try again
872
+ // If we failed to build the block due to insufficient txs, we try again if there is still time left in the slot
873
+ if ('failure' in buildResult) {
874
+ // If this was the last subslot, or we're running with a single block per slot, we're done
875
+ if (timingInfo.isLastBlock || timingInfo.deadline === undefined) {
876
+ break;
877
+ }
878
+ // Otherwise, if there is still time for more blocks, we wait until the next subslot and try again
446
879
  await this.waitUntilNextSubslot(timingInfo.deadline);
447
880
  continue;
448
- } else if (!buildResult) {
449
- // Exit if there is no possibility of building more blocks
450
- break;
451
- } else if ('error' in buildResult) {
452
- // If there was an error building the block, just exit the loop and give up the rest of the slot
881
+ }
882
+
883
+ // If there was an error building the block, we just exit the loop and give up the rest of the slot.
884
+ // We don't want to risk building more blocks if something went wrong.
885
+ if ('error' in buildResult) {
453
886
  if (!(buildResult.error instanceof SequencerInterruptedError)) {
454
- this.log.warn(`Halting block building for slot ${this.slot}`, {
455
- slot: this.slot,
887
+ this.log.warn(`Halting block building for slot ${this.targetSlot}`, {
888
+ slot: this.targetSlot,
456
889
  blocksBuilt,
457
890
  error: buildResult.error,
458
891
  });
@@ -461,41 +894,42 @@ export class CheckpointProposalJob implements Traceable {
461
894
  }
462
895
 
463
896
  const { block, usedTxs } = buildResult;
897
+ this.checkpointMetrics.noteCheckpointBlockBuilt(this.dateProvider.now(), {
898
+ isFirstBlock: blocksBuilt === 0,
899
+ isLastBlock: timingInfo.isLastBlock,
900
+ });
901
+
464
902
  blocksInCheckpoint.push(block);
903
+ usedTxs.forEach(tx => txHashesAlreadyIncluded.add(tx.txHash.toString()));
465
904
 
466
- // Sync the proposed block to the archiver to make it available
467
- // Note that the checkpoint builder uses its own fork so it should not need to wait for this syncing
468
- // Eventually we should refactor the checkpoint builder to not need a separate long-lived fork
469
- // Fire and forget - don't block the critical path, but log errors
470
- this.syncProposedBlockToArchiver(block).catch(err => {
471
- this.log.error(`Failed to sync proposed block ${block.number} to archiver`, { blockNumber: block.number, err });
905
+ // Sign the block proposal. This will throw if HA signing fails.
906
+ const proposal = await this.createBlockProposal(block, inHash, usedTxs, {
907
+ ...blockProposalOptions,
908
+ broadcastInvalidBlockProposal:
909
+ blockProposalOptions.broadcastInvalidBlockProposal ||
910
+ block.indexWithinCheckpoint === this.config.invalidBlockProposalIndexWithinCheckpoint,
472
911
  });
473
912
 
474
- usedTxs.forEach(tx => txHashesAlreadyIncluded.add(tx.txHash.toString()));
913
+ // Sync the proposed block to the archiver to make it available, only after we've managed to sign the proposal,
914
+ // so we avoid polluting our archive with a block that would fail.
915
+ // We wait for the sync to succeed, as this helps catch consistency errors, even if it means we lose some time for block-building.
916
+ // If this throws, we abort the entire checkpoint.
917
+ await this.syncProposedBlockToArchiver(block);
475
918
 
476
- // If this is the last block, exit the loop now so we start collecting attestations
919
+ // If this is the last block, do not broadcast it, since it will be included in the checkpoint proposal.
477
920
  if (timingInfo.isLastBlock) {
478
- this.log.verbose(`Completed final block ${blockNumber} for slot ${this.slot}`, {
479
- slot: this.slot,
921
+ this.log.verbose(`Completed final block ${blockNumber} for slot ${this.targetSlot}`, {
922
+ slot: this.targetSlot,
480
923
  blockNumber,
481
924
  blocksBuilt,
482
925
  });
483
- blockPendingBroadcast = { block, txs: usedTxs };
926
+
927
+ blockPendingBroadcast = proposal;
484
928
  break;
485
929
  }
486
930
 
487
- // For non-last blocks, broadcast the block proposal (unless we're in fisherman mode)
488
- // If the block is the last one, we'll broadcast it along with the checkpoint at the end of the loop
489
- if (!this.config.fishermanMode) {
490
- const proposal = await this.validatorClient.createBlockProposal(
491
- block.header,
492
- block.indexWithinCheckpoint,
493
- inHash,
494
- block.archive.root,
495
- usedTxs,
496
- this.proposer,
497
- blockProposalOptions,
498
- );
931
+ // Once we have a signed proposal and the archiver agreed with our proposed block, then we broadcast it.
932
+ if (proposal && !this.config.skipBroadcastProposals) {
499
933
  await this.p2pClient.broadcastProposal(proposal);
500
934
  }
501
935
 
@@ -503,19 +937,44 @@ export class CheckpointProposalJob implements Traceable {
503
937
  await this.waitUntilNextSubslot(timingInfo.deadline);
504
938
  }
505
939
 
506
- this.log.verbose(`Block building loop completed for slot ${this.slot}`, {
507
- slot: this.slot,
940
+ this.log.verbose(`Block building loop completed for slot ${this.targetSlot}`, {
941
+ slot: this.targetSlot,
508
942
  blocksBuilt: blocksInCheckpoint.length,
509
943
  });
510
944
 
511
945
  return { blocksInCheckpoint, blockPendingBroadcast };
512
946
  }
513
947
 
948
+ /** Creates a block proposal for a given block via the validator client (unless in fisherman mode) */
949
+ private createBlockProposal(
950
+ block: L2Block,
951
+ inHash: Fr,
952
+ usedTxs: Tx[],
953
+ blockProposalOptions: BlockProposalOptions,
954
+ ): Promise<BlockProposal | undefined> {
955
+ if (this.config.fishermanMode) {
956
+ this.log.info(`Skipping block proposal for block ${block.number} in fisherman mode`);
957
+ return Promise.resolve(undefined);
958
+ }
959
+ return this.validatorClient.createBlockProposal(
960
+ block.header,
961
+ this.checkpointNumber,
962
+ block.indexWithinCheckpoint,
963
+ inHash,
964
+ block.archive.root,
965
+ usedTxs,
966
+ this.proposer,
967
+ blockProposalOptions,
968
+ );
969
+ }
970
+
514
971
  /** Sleeps until it is time to produce the next block in the slot */
515
972
  @trackSpan('CheckpointProposalJob.waitUntilNextSubslot')
516
973
  private async waitUntilNextSubslot(nextSubslotStart: number) {
517
- this.setStateFn(SequencerState.WAITING_UNTIL_NEXT_BLOCK, this.slot);
518
- this.log.verbose(`Waiting until time for the next block at ${nextSubslotStart}s into slot`, { slot: this.slot });
974
+ this.setStateFn(SequencerState.WAITING_UNTIL_NEXT_BLOCK, this.targetSlot);
975
+ this.log.verbose(`Waiting until time for the next block at ${nextSubslotStart}s into slot`, {
976
+ slot: this.targetSlot,
977
+ });
519
978
  await this.waitUntilTimeInSlot(nextSubslotStart);
520
979
  }
521
980
 
@@ -531,27 +990,45 @@ export class CheckpointProposalJob implements Traceable {
531
990
  buildDeadline: Date | undefined;
532
991
  txHashesAlreadyIncluded: Set<string>;
533
992
  },
534
- ): Promise<{ block: L2Block; usedTxs: Tx[] } | { error: Error } | undefined> {
993
+ ): Promise<
994
+ { block: L2Block; usedTxs: Tx[] } | { failure: 'insufficient-txs' | 'insufficient-valid-txs' } | { error: Error }
995
+ > {
535
996
  const { blockTimestamp, forceCreate, blockNumber, indexWithinCheckpoint, buildDeadline, txHashesAlreadyIncluded } =
536
997
  opts;
537
998
 
538
999
  this.log.verbose(
539
- `Preparing block ${blockNumber} index ${indexWithinCheckpoint} at checkpoint ${this.checkpointNumber} for slot ${this.slot}`,
1000
+ `Preparing block ${blockNumber} index ${indexWithinCheckpoint} at checkpoint ${this.checkpointNumber} for slot ${this.targetSlot}`,
540
1001
  { ...checkpointBuilder.getConstantData(), ...opts },
541
1002
  );
542
1003
 
543
1004
  try {
544
1005
  // Wait until we have enough txs to build the block
545
- const minTxs = this.config.minTxsPerBlock;
546
- const { availableTxs, canStartBuilding } = await this.waitForMinTxs(opts);
1006
+ const { availableTxs, canStartBuilding, minTxs } = await this.waitForMinTxs(opts);
547
1007
  if (!canStartBuilding) {
1008
+ this.logCheckpointEvent('block-build-failed', `Block build failed for slot ${this.targetSlot}`, {
1009
+ reason: 'insufficient_txs',
1010
+ blockNumber,
1011
+ slot: this.targetSlot,
1012
+ checkpointNumber: this.checkpointNumber,
1013
+ indexWithinCheckpoint,
1014
+ availableTxs,
1015
+ minTxs,
1016
+ });
548
1017
  this.log.warn(
549
- `Not enough txs to build block ${blockNumber} at index ${indexWithinCheckpoint} in slot ${this.slot} (got ${availableTxs} txs but needs ${minTxs})`,
550
- { blockNumber, slot: this.slot, indexWithinCheckpoint },
1018
+ `Not enough txs to build block ${blockNumber} at index ${indexWithinCheckpoint} in slot ${this.targetSlot} (got ${availableTxs} txs but needs ${minTxs})`,
1019
+ {
1020
+ reason: 'insufficient_txs',
1021
+ blockNumber,
1022
+ slot: this.targetSlot,
1023
+ checkpointNumber: this.checkpointNumber,
1024
+ indexWithinCheckpoint,
1025
+ availableTxs,
1026
+ minTxs,
1027
+ },
551
1028
  );
552
- this.eventEmitter.emit('block-tx-count-check-failed', { minTxs, availableTxs, slot: this.slot });
1029
+ this.eventEmitter.emit('block-tx-count-check-failed', { minTxs, availableTxs, slot: this.targetSlot });
553
1030
  this.metrics.recordBlockProposalFailed('insufficient_txs');
554
- return undefined;
1031
+ return { failure: 'insufficient-txs' };
555
1032
  }
556
1033
 
557
1034
  // Create iterator to pending txs. We filter out txs already included in previous blocks in the checkpoint
@@ -562,14 +1039,16 @@ export class CheckpointProposalJob implements Traceable {
562
1039
  );
563
1040
 
564
1041
  this.log.debug(
565
- `Building block ${blockNumber} at index ${indexWithinCheckpoint} for slot ${this.slot} with ${availableTxs} available txs`,
566
- { slot: this.slot, blockNumber, indexWithinCheckpoint },
1042
+ `Building block ${blockNumber} at index ${indexWithinCheckpoint} for slot ${this.targetSlot} with ${availableTxs} available txs`,
1043
+ { slot: this.targetSlot, blockNumber, indexWithinCheckpoint },
567
1044
  );
568
- this.setStateFn(SequencerState.CREATING_BLOCK, this.slot);
1045
+ this.setStateFn(SequencerState.CREATING_BLOCK, this.targetSlot);
569
1046
 
570
- // Per-block limits derived at startup by computeBlockLimits(), further capped
1047
+ // Per-block limits are operator overrides (from SEQ_MAX_L2_BLOCK_GAS etc.) further capped
571
1048
  // by remaining checkpoint-level budgets inside CheckpointBuilder before each block is built.
572
- const blockBuilderOptions: PublicProcessorLimits = {
1049
+ // minValidTxs is passed into the builder so it can reject the block *before* updating state.
1050
+ const minValidTxs = forceCreate ? 0 : (this.config.minValidTxsPerBlock ?? minTxs);
1051
+ const blockBuilderOptions: BlockBuilderOptions = {
573
1052
  maxTransactions: this.config.maxTxsPerBlock,
574
1053
  maxBlockGas:
575
1054
  this.config.maxL2BlockGas !== undefined || this.config.maxDABlockGas !== undefined
@@ -577,9 +1056,14 @@ export class CheckpointProposalJob implements Traceable {
577
1056
  : undefined,
578
1057
  deadline: buildDeadline,
579
1058
  isBuildingProposal: true,
1059
+ minValidTxs,
1060
+ maxBlocksPerCheckpoint: this.timetable.maxNumberOfBlocks,
1061
+ perBlockAllocationMultiplier: this.config.perBlockAllocationMultiplier,
580
1062
  };
581
1063
 
582
- // Actually build the block by executing txs
1064
+ // Actually build the block by executing txs. The builder throws InsufficientValidTxsError
1065
+ // if the number of successfully processed txs is below minValidTxs, ensuring state is not
1066
+ // updated for blocks that will be discarded.
583
1067
  const buildResult = await this.buildSingleBlockWithCheckpointBuilder(
584
1068
  checkpointBuilder,
585
1069
  pendingTxs,
@@ -591,22 +1075,38 @@ export class CheckpointProposalJob implements Traceable {
591
1075
  // If any txs failed during execution, drop them from the mempool so we don't pick them up again
592
1076
  await this.dropFailedTxsFromP2P(buildResult.failedTxs);
593
1077
 
594
- // Check if we have created a block with enough txs. If there were invalid txs in the pool, or if execution took
595
- // too long, then we may not get to minTxsPerBlock after executing public functions.
596
- const minValidTxs = this.config.minValidTxsPerBlock ?? minTxs;
597
- const numTxs = buildResult.status === 'no-valid-txs' ? 0 : buildResult.numTxs;
598
- if (buildResult.status === 'no-valid-txs' || (!forceCreate && numTxs < minValidTxs)) {
1078
+ if (buildResult.status === 'insufficient-valid-txs') {
1079
+ this.logCheckpointEvent('block-build-failed', `Block build failed for slot ${this.targetSlot}`, {
1080
+ reason: 'insufficient_valid_txs',
1081
+ slot: this.targetSlot,
1082
+ checkpointNumber: this.checkpointNumber,
1083
+ blockNumber,
1084
+ numTxs: buildResult.processedCount,
1085
+ indexWithinCheckpoint,
1086
+ minValidTxs,
1087
+ });
599
1088
  this.log.warn(
600
- `Block ${blockNumber} at index ${indexWithinCheckpoint} on slot ${this.slot} has too few valid txs to be proposed`,
601
- { slot: this.slot, blockNumber, numTxs, indexWithinCheckpoint, minValidTxs, buildResult: buildResult.status },
1089
+ `Block ${blockNumber} at index ${indexWithinCheckpoint} on slot ${this.targetSlot} has too few valid txs to be proposed`,
1090
+ {
1091
+ reason: 'insufficient_valid_txs',
1092
+ slot: this.targetSlot,
1093
+ checkpointNumber: this.checkpointNumber,
1094
+ blockNumber,
1095
+ numTxs: buildResult.processedCount,
1096
+ indexWithinCheckpoint,
1097
+ minValidTxs,
1098
+ },
602
1099
  );
603
- this.eventEmitter.emit('block-build-failed', { reason: `Insufficient valid txs`, slot: this.slot });
1100
+ this.eventEmitter.emit('block-build-failed', {
1101
+ reason: `Insufficient valid txs`,
1102
+ slot: this.targetSlot,
1103
+ });
604
1104
  this.metrics.recordBlockProposalFailed('insufficient_valid_txs');
605
- return undefined;
1105
+ return { failure: 'insufficient-valid-txs' };
606
1106
  }
607
1107
 
608
1108
  // Block creation succeeded, emit stats and metrics
609
- const { block, publicProcessorDuration, usedTxs, blockBuildDuration } = buildResult;
1109
+ const { block, publicProcessorDuration, usedTxs, blockBuildDuration, numTxs } = buildResult;
610
1110
 
611
1111
  const blockStats = {
612
1112
  eventName: 'l2-block-built',
@@ -620,30 +1120,53 @@ export class CheckpointProposalJob implements Traceable {
620
1120
  const manaPerSec = block.header.totalManaUsed.toNumberUnsafe() / (blockBuildDuration / 1000);
621
1121
 
622
1122
  this.log.info(
623
- `Built block ${block.number} at checkpoint ${this.checkpointNumber} for slot ${this.slot} with ${numTxs} txs`,
1123
+ `Built block ${block.number} at checkpoint ${this.checkpointNumber} for slot ${this.targetSlot} with ${numTxs} txs`,
624
1124
  { blockHash, txHashes, manaPerSec, ...blockStats },
625
1125
  );
626
1126
 
627
- this.eventEmitter.emit('block-proposed', { blockNumber: block.number, slot: this.slot });
628
- this.metrics.recordBuiltBlock(blockBuildDuration, block.header.totalManaUsed.toNumberUnsafe());
1127
+ // `slot` is the target/submission slot (may be one ahead when pipelining),
1128
+ // `buildSlot` is the wall-clock slot during which the block was actually built.
1129
+ this.eventEmitter.emit('block-proposed', {
1130
+ blockNumber: block.number,
1131
+ blockHash,
1132
+ checkpointNumber: this.checkpointNumber,
1133
+ indexWithinCheckpoint: block.indexWithinCheckpoint,
1134
+ slot: this.targetSlot,
1135
+ buildSlot: this.slotNow,
1136
+ });
1137
+ this.metrics.recordBuiltBlock(blockBuildDuration, block.header.totalManaUsed.toNumberUnsafe(), this.targetSlot);
629
1138
 
630
1139
  return { block, usedTxs };
631
1140
  } catch (err: any) {
632
- this.eventEmitter.emit('block-build-failed', { reason: err.message, slot: this.slot });
633
- this.log.error(`Error building block`, err, { blockNumber, slot: this.slot });
1141
+ this.eventEmitter.emit('block-build-failed', {
1142
+ reason: err.message,
1143
+ slot: this.targetSlot,
1144
+ });
1145
+ this.logCheckpointEvent('block-build-failed', `Block build failed for slot ${this.targetSlot}`, {
1146
+ reason: err instanceof Error ? err.message : String(err),
1147
+ slot: this.targetSlot,
1148
+ checkpointNumber: this.checkpointNumber,
1149
+ blockNumber,
1150
+ });
1151
+ this.log.error(`Error building block`, err, {
1152
+ reason: err instanceof Error ? err.message : String(err),
1153
+ slot: this.targetSlot,
1154
+ checkpointNumber: this.checkpointNumber,
1155
+ blockNumber,
1156
+ });
634
1157
  this.metrics.recordBlockProposalFailed(err.name || 'unknown_error');
635
1158
  this.metrics.recordFailedBlock();
636
1159
  return { error: err };
637
1160
  }
638
1161
  }
639
1162
 
640
- /** Uses the checkpoint builder to build a block, catching specific txs */
1163
+ /** Uses the checkpoint builder to build a block, catching InsufficientValidTxsError. */
641
1164
  private async buildSingleBlockWithCheckpointBuilder(
642
1165
  checkpointBuilder: CheckpointBuilder,
643
1166
  pendingTxs: AsyncIterable<Tx>,
644
1167
  blockNumber: BlockNumber,
645
1168
  blockTimestamp: bigint,
646
- blockBuilderOptions: PublicProcessorLimits,
1169
+ blockBuilderOptions: BlockBuilderOptions,
647
1170
  ) {
648
1171
  try {
649
1172
  const workTimer = new Timer();
@@ -651,8 +1174,12 @@ export class CheckpointProposalJob implements Traceable {
651
1174
  const blockBuildDuration = workTimer.ms();
652
1175
  return { ...result, blockBuildDuration, status: 'success' as const };
653
1176
  } catch (err: unknown) {
654
- if (isErrorClass(err, NoValidTxsError)) {
655
- return { failedTxs: err.failedTxs, status: 'no-valid-txs' as const };
1177
+ if (isErrorClass(err, InsufficientValidTxsError)) {
1178
+ return {
1179
+ failedTxs: err.failedTxs,
1180
+ processedCount: err.processedCount,
1181
+ status: 'insufficient-valid-txs' as const,
1182
+ };
656
1183
  }
657
1184
  throw err;
658
1185
  }
@@ -665,7 +1192,7 @@ export class CheckpointProposalJob implements Traceable {
665
1192
  blockNumber: BlockNumber;
666
1193
  indexWithinCheckpoint: IndexWithinCheckpoint;
667
1194
  buildDeadline: Date | undefined;
668
- }): Promise<{ canStartBuilding: boolean; availableTxs: number }> {
1195
+ }): Promise<{ canStartBuilding: boolean; availableTxs: number; minTxs: number }> {
669
1196
  const { indexWithinCheckpoint, blockNumber, buildDeadline, forceCreate } = opts;
670
1197
 
671
1198
  // We only allow a block with 0 txs in the first block of the checkpoint
@@ -682,20 +1209,50 @@ export class CheckpointProposalJob implements Traceable {
682
1209
  // If we're past deadline, or we have no deadline, give up
683
1210
  const now = this.dateProvider.nowAsDate();
684
1211
  if (startBuildingDeadline === undefined || now >= startBuildingDeadline) {
685
- return { canStartBuilding: false, availableTxs: availableTxs };
1212
+ return { canStartBuilding: false, availableTxs, minTxs };
686
1213
  }
687
1214
 
688
1215
  // Wait a bit before checking again
689
- this.setStateFn(SequencerState.WAITING_FOR_TXS, this.slot);
1216
+ this.setStateFn(SequencerState.WAITING_FOR_TXS, this.targetSlot);
690
1217
  this.log.verbose(
691
- `Waiting for enough txs to build block ${blockNumber} at index ${indexWithinCheckpoint} in slot ${this.slot} (have ${availableTxs} but need ${minTxs})`,
692
- { blockNumber, slot: this.slot, indexWithinCheckpoint },
1218
+ `Waiting for enough txs to build block ${blockNumber} at index ${indexWithinCheckpoint} in slot ${this.targetSlot} (have ${availableTxs} but need ${minTxs})`,
1219
+ { blockNumber, slot: this.targetSlot, indexWithinCheckpoint },
693
1220
  );
694
1221
  await this.waitForTxsPollingInterval();
695
1222
  availableTxs = await this.p2pClient.getPendingTxCount();
696
1223
  }
697
1224
 
698
- return { canStartBuilding: true, availableTxs };
1225
+ return { canStartBuilding: true, availableTxs, minTxs };
1226
+ }
1227
+
1228
+ private async getSignedCommitteeAttestations(
1229
+ broadcast: CheckpointProposalBroadcast,
1230
+ ): Promise<{ attestations: CommitteeAttestationsAndSigners; attestationsSignature: Signature } | undefined> {
1231
+ const { proposal, blockProposedAt } = broadcast;
1232
+ this.setStateFn(SequencerState.COLLECTING_ATTESTATIONS, this.targetSlot);
1233
+ const attestations = await this.waitForAttestations(proposal);
1234
+ if (!attestations) {
1235
+ return undefined;
1236
+ }
1237
+ this.checkpointMetrics.recordCheckpointAttestationDelay(this.dateProvider.now() - blockProposedAt);
1238
+
1239
+ // Proposer must sign over the attestations before pushing them to L1
1240
+ const signer = this.proposer ?? this.publisher.getSenderAddress();
1241
+ try {
1242
+ const attestationsSignature = await this.validatorClient.signAttestationsAndSigners(
1243
+ attestations,
1244
+ signer,
1245
+ this.targetSlot,
1246
+ this.checkpointNumber,
1247
+ );
1248
+ return { attestations, attestationsSignature };
1249
+ } catch (err) {
1250
+ if (this.handleHASigningError(err, 'Attestations signature')) {
1251
+ return;
1252
+ }
1253
+ this.log.error(`Error signing attestations for checkpoint proposal at slot ${proposal.slotNumber}`, err);
1254
+ return undefined;
1255
+ }
699
1256
  }
700
1257
 
701
1258
  /**
@@ -703,10 +1260,12 @@ export class CheckpointProposalJob implements Traceable {
703
1260
  * This is run after all blocks for the checkpoint have been built.
704
1261
  */
705
1262
  @trackSpan('CheckpointProposalJob.waitForAttestations')
706
- private async waitForAttestations(proposal: CheckpointProposal): Promise<CommitteeAttestationsAndSigners> {
1263
+ private async waitForAttestations(
1264
+ proposal: CheckpointProposal,
1265
+ ): Promise<CommitteeAttestationsAndSigners | undefined> {
707
1266
  if (this.config.fishermanMode) {
708
1267
  this.log.debug('Skipping attestation collection in fisherman mode');
709
- return CommitteeAttestationsAndSigners.empty();
1268
+ return CommitteeAttestationsAndSigners.empty(this.getSignatureContext());
710
1269
  }
711
1270
 
712
1271
  const slotNumber = proposal.slotNumber;
@@ -716,21 +1275,27 @@ export class CheckpointProposalJob implements Traceable {
716
1275
  throw new Error('No committee when collecting attestations');
717
1276
  } else if (committee.length === 0) {
718
1277
  this.log.verbose(`Attesting committee is empty`);
719
- return CommitteeAttestationsAndSigners.empty();
1278
+ return CommitteeAttestationsAndSigners.empty(this.getSignatureContext());
720
1279
  } else {
721
1280
  this.log.debug(`Attesting committee length is ${committee.length}`, { committee });
722
1281
  }
723
1282
 
724
- const numberOfRequiredAttestations = Math.floor((committee.length * 2) / 3) + 1;
1283
+ const numberOfRequiredAttestations = computeQuorum(committee.length);
725
1284
 
726
1285
  if (this.config.skipCollectingAttestations) {
727
1286
  this.log.warn('Skipping attestation collection as per config (attesting with own keys only)');
728
- const attestations = await this.validatorClient?.collectOwnAttestations(proposal);
729
- return new CommitteeAttestationsAndSigners(orderAttestations(attestations ?? [], committee));
1287
+ const attestations = await this.validatorClient?.collectOwnAttestations(proposal, this.checkpointNumber);
1288
+ this.logCheckpointAttestations('collected', committee, attestations ?? [], numberOfRequiredAttestations, {
1289
+ reason: 'collect_own_only',
1290
+ });
1291
+ return new CommitteeAttestationsAndSigners(
1292
+ orderAttestations(attestations ?? [], committee),
1293
+ this.getSignatureContext(),
1294
+ );
730
1295
  }
731
1296
 
732
1297
  const attestationTimeAllowed = this.config.enforceTimeTable
733
- ? this.timetable.getMaxAllowedTime(SequencerState.PUBLISHING_CHECKPOINT)!
1298
+ ? this.timetable.getCheckpointAttestationDeadline()
734
1299
  : this.l1Constants.slotDuration;
735
1300
  const attestationDeadline = new Date((this.getSlotStartBuildTimestamp() + attestationTimeAllowed) * 1000);
736
1301
 
@@ -743,6 +1308,7 @@ export class CheckpointProposalJob implements Traceable {
743
1308
  proposal,
744
1309
  numberOfRequiredAttestations,
745
1310
  attestationDeadline,
1311
+ this.checkpointNumber,
746
1312
  );
747
1313
 
748
1314
  collectedAttestationsCount = attestations.length;
@@ -761,6 +1327,9 @@ export class CheckpointProposalJob implements Traceable {
761
1327
 
762
1328
  // Rollup contract requires that the signatures are provided in the order of the committee
763
1329
  const sorted = orderAttestations(trimmed, committee);
1330
+ this.logCheckpointAttestations('collected', committee, attestations, numberOfRequiredAttestations, {
1331
+ submittedCount: trimmed.length,
1332
+ });
764
1333
 
765
1334
  // Manipulate the attestations if we've been configured to do so
766
1335
  if (
@@ -772,17 +1341,56 @@ export class CheckpointProposalJob implements Traceable {
772
1341
  return this.manipulateAttestations(proposal.slotNumber, epoch, seed, committee, sorted);
773
1342
  }
774
1343
 
775
- return new CommitteeAttestationsAndSigners(sorted);
1344
+ return new CommitteeAttestationsAndSigners(sorted, this.getSignatureContext());
776
1345
  } catch (err) {
777
1346
  if (err && err instanceof AttestationTimeoutError) {
778
1347
  collectedAttestationsCount = err.collectedCount;
1348
+ this.logCheckpointAttestations('failed', committee, undefined, numberOfRequiredAttestations, {
1349
+ collectedCount: collectedAttestationsCount,
1350
+ reason: 'timeout',
1351
+ });
1352
+ this.log.error(
1353
+ `Timeout while waiting for attestations for checkpoint proposal at slot ${proposal.slotNumber} (collected ${collectedAttestationsCount}/${numberOfRequiredAttestations})`,
1354
+ err,
1355
+ );
1356
+ } else {
1357
+ this.logCheckpointAttestations('failed', committee, undefined, numberOfRequiredAttestations, {
1358
+ collectedCount: collectedAttestationsCount,
1359
+ reason: err instanceof Error ? err.message : String(err),
1360
+ });
1361
+ this.log.error(`Error collecting attestations for checkpoint proposal at slot ${proposal.slotNumber}`, err);
779
1362
  }
780
- throw err;
1363
+ return undefined;
781
1364
  } finally {
782
1365
  this.metrics.recordCollectedAttestations(collectedAttestationsCount, collectAttestationsTimer.ms());
783
1366
  }
784
1367
  }
785
1368
 
1369
+ private logCheckpointAttestations(
1370
+ status: 'collected' | 'failed',
1371
+ committee: EthAddress[],
1372
+ attestations: CheckpointAttestation[] | undefined,
1373
+ requiredAttestations: number,
1374
+ opts: { collectedCount?: number; submittedCount?: number; reason?: string } = {},
1375
+ ) {
1376
+ const signedValidators =
1377
+ attestations
1378
+ ?.map(attestation => attestation.getSender()?.toString())
1379
+ .filter((address): address is `0x${string}` => address !== undefined) ?? [];
1380
+ const collectedCount = opts.collectedCount ?? new Set(signedValidators).size;
1381
+ const missingValidatorCount = status === 'failed' ? Math.max(0, requiredAttestations - collectedCount) : undefined;
1382
+ this.logCheckpointEvent(`attestations-${status}`, `Checkpoint attestations ${status} for slot ${this.targetSlot}`, {
1383
+ slot: this.targetSlot,
1384
+ checkpointNumber: this.checkpointNumber,
1385
+ committeeSize: committee.length,
1386
+ requiredAttestations,
1387
+ collectedAttestations: collectedCount,
1388
+ ...(opts.submittedCount !== undefined && { submittedAttestations: opts.submittedCount }),
1389
+ ...(missingValidatorCount !== undefined && { missingValidatorCount }),
1390
+ ...(opts.reason !== undefined && { reason: opts.reason }),
1391
+ });
1392
+ }
1393
+
786
1394
  /** Breaks the attestations before publishing based on attack configs */
787
1395
  private manipulateAttestations(
788
1396
  slotNumber: SlotNumber,
@@ -826,7 +1434,7 @@ export class CheckpointProposalJob implements Traceable {
826
1434
  unfreeze(attestations[targetIndex]).signature = generateRecoverableSignature();
827
1435
  }
828
1436
  }
829
- return new CommitteeAttestationsAndSigners(attestations);
1437
+ return new CommitteeAttestationsAndSigners(attestations, this.getSignatureContext());
830
1438
  }
831
1439
 
832
1440
  if (this.config.shuffleAttestationOrdering) {
@@ -848,11 +1456,11 @@ export class CheckpointProposalJob implements Traceable {
848
1456
  [shuffled[i], shuffled[j]] = [shuffled[j], shuffled[i]];
849
1457
  }
850
1458
 
851
- const signers = new CommitteeAttestationsAndSigners(attestations).getSigners();
852
- return new MaliciousCommitteeAttestationsAndSigners(shuffled, signers);
1459
+ const signers = new CommitteeAttestationsAndSigners(attestations, this.getSignatureContext()).getSigners();
1460
+ return new MaliciousCommitteeAttestationsAndSigners(shuffled, signers, this.getSignatureContext());
853
1461
  }
854
1462
 
855
- return new CommitteeAttestationsAndSigners(attestations);
1463
+ return new CommitteeAttestationsAndSigners(attestations, this.getSignatureContext());
856
1464
  }
857
1465
 
858
1466
  private async dropFailedTxsFromP2P(failedTxs: FailedTx[]) {
@@ -869,9 +1477,13 @@ export class CheckpointProposalJob implements Traceable {
869
1477
  * Adds the proposed block to the archiver so it's available via P2P.
870
1478
  * Gossip doesn't echo messages back to the sender, so the proposer's archiver/world-state
871
1479
  * would never receive its own block without this explicit sync.
1480
+ *
1481
+ * In fisherman mode we skip this push: the fisherman builds blocks locally for validation
1482
+ * and fee analysis only, and pushing them to the archiver causes spurious reorg cascades
1483
+ * whenever the real proposer's block arrives from L1.
872
1484
  */
873
1485
  private async syncProposedBlockToArchiver(block: L2Block): Promise<void> {
874
- if (this.config.skipPushProposedBlocksToArchiver !== false) {
1486
+ if (this.config.skipPushProposedBlocksToArchiver || this.config.fishermanMode) {
875
1487
  this.log.warn(`Skipping push of proposed block ${block.number} to archiver`, {
876
1488
  blockNumber: block.number,
877
1489
  slot: block.header.globalVariables.slotNumber,
@@ -885,23 +1497,62 @@ export class CheckpointProposalJob implements Traceable {
885
1497
  await this.blockSink.addBlock(block);
886
1498
  }
887
1499
 
1500
+ /**
1501
+ * Adds the proposed checkpoint to the archiver so the proposer's optimistic proposed-checkpoint
1502
+ * tip advances locally. Gossip doesn't echo our own messages back, so without this the proposer
1503
+ * would never see its own proposed checkpoint and couldn't pipeline the next slot.
1504
+ *
1505
+ * Only runs under proposer pipelining (the proposed tip is a pipelining-only concept) and is
1506
+ * skipped whenever proposed blocks aren't pushed (`skipPushProposedBlocksToArchiver`, fisherman
1507
+ * mode): the archiver derives the checkpoint archive from its stored blocks, so without them the
1508
+ * push would fail. All blocks were already added (and awaited) during block building, so this
1509
+ * needs no retry — they are guaranteed present by the time we get here.
1510
+ */
1511
+ private async syncProposedCheckpointToArchiver(
1512
+ checkpoint: Checkpoint,
1513
+ blockCount: number,
1514
+ feeAssetPriceModifier: bigint,
1515
+ ): Promise<void> {
1516
+ if (this.config.skipPushProposedBlocksToArchiver || this.config.fishermanMode) {
1517
+ return;
1518
+ }
1519
+ if (!this.epochCache.isProposerPipeliningEnabled()) {
1520
+ return;
1521
+ }
1522
+ const startBlock = BlockNumber(this.syncedToBlockNumber + 1);
1523
+ this.log.debug(`Syncing proposed checkpoint ${this.checkpointNumber} to archiver`, {
1524
+ checkpointNumber: this.checkpointNumber,
1525
+ slot: this.targetSlot,
1526
+ startBlock,
1527
+ blockCount,
1528
+ });
1529
+ await this.blockSink.addProposedCheckpoint({
1530
+ header: checkpoint.header,
1531
+ checkpointNumber: this.checkpointNumber,
1532
+ startBlock,
1533
+ blockCount,
1534
+ totalManaUsed: checkpoint.header.totalManaUsed.toBigInt(),
1535
+ feeAssetPriceModifier,
1536
+ });
1537
+ }
1538
+
888
1539
  /** Runs fee analysis and logs checkpoint outcome as fisherman */
889
1540
  private async handleCheckpointEndAsFisherman(checkpoint: Checkpoint | undefined) {
890
1541
  // Perform L1 fee analysis before clearing requests
891
1542
  // The callback is invoked asynchronously after the next block is mined
892
- const feeAnalysis = await this.publisher.analyzeL1Fees(this.slot, analysis =>
1543
+ const feeAnalysis = await this.publisher.analyzeL1Fees(this.targetSlot, analysis =>
893
1544
  this.metrics.recordFishermanFeeAnalysis(analysis),
894
1545
  );
895
1546
 
896
1547
  if (checkpoint) {
897
- this.log.info(`Validation checkpoint building SUCCEEDED for slot ${this.slot}`, {
1548
+ this.log.info(`Validation checkpoint building SUCCEEDED for slot ${this.targetSlot}`, {
898
1549
  ...checkpoint.toCheckpointInfo(),
899
1550
  ...checkpoint.getStats(),
900
1551
  feeAnalysisId: feeAnalysis?.id,
901
1552
  });
902
1553
  } else {
903
- this.log.warn(`Validation block building FAILED for slot ${this.slot}`, {
904
- slot: this.slot,
1554
+ this.log.warn(`Validation block building FAILED for slot ${this.targetSlot}`, {
1555
+ slot: this.targetSlot,
905
1556
  feeAnalysisId: feeAnalysis?.id,
906
1557
  });
907
1558
  this.metrics.recordCheckpointProposalFailed('block_build_failed');
@@ -915,15 +1566,15 @@ export class CheckpointProposalJob implements Traceable {
915
1566
  */
916
1567
  private handleHASigningError(err: any, errorContext: string): boolean {
917
1568
  if (err instanceof DutyAlreadySignedError) {
918
- this.log.info(`${errorContext} for slot ${this.slot} already signed by another HA node, yielding`, {
919
- slot: this.slot,
1569
+ this.log.info(`${errorContext} for slot ${this.targetSlot} already signed by another HA node, yielding`, {
1570
+ slot: this.targetSlot,
920
1571
  signedByNode: err.signedByNode,
921
1572
  });
922
1573
  return true;
923
1574
  }
924
1575
  if (err instanceof SlashingProtectionError) {
925
- this.log.info(`${errorContext} for slot ${this.slot} blocked by slashing protection, yielding`, {
926
- slot: this.slot,
1576
+ this.log.info(`${errorContext} for slot ${this.targetSlot} blocked by slashing protection, yielding`, {
1577
+ slot: this.targetSlot,
927
1578
  existingMessageHash: err.existingMessageHash,
928
1579
  attemptedMessageHash: err.attemptedMessageHash,
929
1580
  });
@@ -946,7 +1597,7 @@ export class CheckpointProposalJob implements Traceable {
946
1597
  }
947
1598
 
948
1599
  private getSlotStartBuildTimestamp(): number {
949
- return getSlotStartBuildTimestamp(this.slot, this.l1Constants);
1600
+ return getSlotStartBuildTimestamp(this.slotNow, this.l1Constants);
950
1601
  }
951
1602
 
952
1603
  private getSecondsIntoSlot(): number {