@aztec/sequencer-client 0.0.1-commit.001888fc → 0.0.1-commit.017a351

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (105) hide show
  1. package/README.md +281 -21
  2. package/dest/client/sequencer-client.d.ts +8 -15
  3. package/dest/client/sequencer-client.d.ts.map +1 -1
  4. package/dest/client/sequencer-client.js +35 -87
  5. package/dest/config.d.ts +4 -1
  6. package/dest/config.d.ts.map +1 -1
  7. package/dest/config.js +35 -17
  8. package/dest/global_variable_builder/fee_predictor.d.ts +37 -0
  9. package/dest/global_variable_builder/fee_predictor.d.ts.map +1 -0
  10. package/dest/global_variable_builder/fee_predictor.js +128 -0
  11. package/dest/global_variable_builder/fee_provider.d.ts +21 -0
  12. package/dest/global_variable_builder/fee_provider.d.ts.map +1 -0
  13. package/dest/global_variable_builder/fee_provider.js +58 -0
  14. package/dest/global_variable_builder/global_builder.d.ts +14 -14
  15. package/dest/global_variable_builder/global_builder.d.ts.map +1 -1
  16. package/dest/global_variable_builder/global_builder.js +16 -51
  17. package/dest/global_variable_builder/index.d.ts +4 -2
  18. package/dest/global_variable_builder/index.d.ts.map +1 -1
  19. package/dest/global_variable_builder/index.js +2 -0
  20. package/dest/index.d.ts +2 -2
  21. package/dest/index.d.ts.map +1 -1
  22. package/dest/index.js +1 -1
  23. package/dest/publisher/config.d.ts +15 -3
  24. package/dest/publisher/config.d.ts.map +1 -1
  25. package/dest/publisher/config.js +19 -4
  26. package/dest/publisher/l1_tx_failed_store/failed_tx_store.d.ts +3 -4
  27. package/dest/publisher/l1_tx_failed_store/failed_tx_store.d.ts.map +1 -1
  28. package/dest/publisher/sequencer-bundle-simulator.d.ts +96 -0
  29. package/dest/publisher/sequencer-bundle-simulator.d.ts.map +1 -0
  30. package/dest/publisher/sequencer-bundle-simulator.js +198 -0
  31. package/dest/publisher/sequencer-publisher-factory.d.ts +3 -5
  32. package/dest/publisher/sequencer-publisher-factory.d.ts.map +1 -1
  33. package/dest/publisher/sequencer-publisher-factory.js +2 -3
  34. package/dest/publisher/sequencer-publisher.d.ts +73 -65
  35. package/dest/publisher/sequencer-publisher.d.ts.map +1 -1
  36. package/dest/publisher/sequencer-publisher.js +314 -532
  37. package/dest/sequencer/automine/automine_factory.d.ts +54 -0
  38. package/dest/sequencer/automine/automine_factory.d.ts.map +1 -0
  39. package/dest/sequencer/automine/automine_factory.js +84 -0
  40. package/dest/sequencer/automine/automine_sequencer.d.ts +151 -0
  41. package/dest/sequencer/automine/automine_sequencer.d.ts.map +1 -0
  42. package/dest/sequencer/automine/automine_sequencer.js +509 -0
  43. package/dest/sequencer/chain_state_overrides.d.ts +61 -0
  44. package/dest/sequencer/chain_state_overrides.d.ts.map +1 -0
  45. package/dest/sequencer/chain_state_overrides.js +98 -0
  46. package/dest/sequencer/checkpoint_proposal_job.d.ts +60 -10
  47. package/dest/sequencer/checkpoint_proposal_job.d.ts.map +1 -1
  48. package/dest/sequencer/checkpoint_proposal_job.js +732 -180
  49. package/dest/sequencer/checkpoint_proposal_job_metrics.d.ts +34 -0
  50. package/dest/sequencer/checkpoint_proposal_job_metrics.d.ts.map +1 -0
  51. package/dest/sequencer/checkpoint_proposal_job_metrics.js +72 -0
  52. package/dest/sequencer/checkpoint_voter.d.ts +1 -2
  53. package/dest/sequencer/checkpoint_voter.d.ts.map +1 -1
  54. package/dest/sequencer/checkpoint_voter.js +2 -5
  55. package/dest/sequencer/events.d.ts +48 -2
  56. package/dest/sequencer/events.d.ts.map +1 -1
  57. package/dest/sequencer/index.d.ts +3 -1
  58. package/dest/sequencer/index.d.ts.map +1 -1
  59. package/dest/sequencer/index.js +2 -0
  60. package/dest/sequencer/metrics.d.ts +13 -10
  61. package/dest/sequencer/metrics.d.ts.map +1 -1
  62. package/dest/sequencer/metrics.js +45 -20
  63. package/dest/sequencer/sequencer.d.ts +62 -14
  64. package/dest/sequencer/sequencer.d.ts.map +1 -1
  65. package/dest/sequencer/sequencer.js +349 -107
  66. package/dest/sequencer/timetable.d.ts +11 -1
  67. package/dest/sequencer/timetable.d.ts.map +1 -1
  68. package/dest/sequencer/timetable.js +41 -36
  69. package/dest/sequencer/types.d.ts +2 -2
  70. package/dest/sequencer/types.d.ts.map +1 -1
  71. package/dest/test/mock_checkpoint_builder.d.ts +4 -8
  72. package/dest/test/mock_checkpoint_builder.d.ts.map +1 -1
  73. package/dest/test/utils.d.ts +1 -1
  74. package/dest/test/utils.d.ts.map +1 -1
  75. package/dest/test/utils.js +7 -6
  76. package/package.json +27 -27
  77. package/src/client/sequencer-client.ts +55 -116
  78. package/src/config.ts +40 -14
  79. package/src/global_variable_builder/README.md +44 -0
  80. package/src/global_variable_builder/fee_predictor.ts +172 -0
  81. package/src/global_variable_builder/fee_provider.ts +75 -0
  82. package/src/global_variable_builder/global_builder.ts +25 -63
  83. package/src/global_variable_builder/index.ts +3 -1
  84. package/src/index.ts +10 -1
  85. package/src/publisher/config.ts +40 -6
  86. package/src/publisher/l1_tx_failed_store/failed_tx_store.ts +3 -1
  87. package/src/publisher/sequencer-bundle-simulator.ts +254 -0
  88. package/src/publisher/sequencer-publisher-factory.ts +3 -6
  89. package/src/publisher/sequencer-publisher.ts +373 -577
  90. package/src/sequencer/README.md +162 -450
  91. package/src/sequencer/automine/README.md +46 -0
  92. package/src/sequencer/automine/automine_factory.ts +145 -0
  93. package/src/sequencer/automine/automine_sequencer.ts +595 -0
  94. package/src/sequencer/chain_state_overrides.ts +162 -0
  95. package/src/sequencer/checkpoint_proposal_job.ts +872 -207
  96. package/src/sequencer/checkpoint_proposal_job_metrics.ts +128 -0
  97. package/src/sequencer/checkpoint_voter.ts +1 -12
  98. package/src/sequencer/events.ts +51 -2
  99. package/src/sequencer/index.ts +2 -0
  100. package/src/sequencer/metrics.ts +57 -24
  101. package/src/sequencer/sequencer.ts +429 -121
  102. package/src/sequencer/timetable.ts +49 -44
  103. package/src/sequencer/types.ts +1 -1
  104. package/src/test/mock_checkpoint_builder.ts +3 -3
  105. package/src/test/utils.ts +28 -10
@@ -1,4 +1,5 @@
1
1
  import type { EpochCache } from '@aztec/epoch-cache';
2
+ import type { SimulationOverridesPlan } from '@aztec/ethereum/contracts';
2
3
  import {
3
4
  BlockNumber,
4
5
  CheckpointNumber,
@@ -13,11 +14,12 @@ import {
13
14
  generateUnrecoverableSignature,
14
15
  } from '@aztec/foundation/crypto/secp256k1-signer';
15
16
  import { Fr } from '@aztec/foundation/curves/bn254';
17
+ import { InterruptError, TimeoutError } from '@aztec/foundation/error';
16
18
  import { EthAddress } from '@aztec/foundation/eth-address';
17
19
  import { Signature } from '@aztec/foundation/eth-signature';
18
20
  import { filter } from '@aztec/foundation/iterator';
19
21
  import { type Logger, type LoggerBindings, createLogger } from '@aztec/foundation/log';
20
- import { sleep, sleepUntil } from '@aztec/foundation/sleep';
22
+ import { InterruptibleSleep } from '@aztec/foundation/sleep';
21
23
  import { type DateProvider, Timer } from '@aztec/foundation/timer';
22
24
  import { type TypedEventEmitter, isErrorClass, unfreeze } from '@aztec/foundation/types';
23
25
  import type { P2P } from '@aztec/p2p';
@@ -29,18 +31,32 @@ import {
29
31
  type L2BlockSink,
30
32
  type L2BlockSource,
31
33
  MaliciousCommitteeAttestationsAndSigners,
34
+ type ProposedCheckpointSink,
35
+ type ValidateCheckpointResult,
32
36
  } from '@aztec/stdlib/block';
33
- import { type Checkpoint, validateCheckpoint } from '@aztec/stdlib/checkpoint';
34
- import { getSlotStartBuildTimestamp } from '@aztec/stdlib/epoch-helpers';
37
+ import {
38
+ type Checkpoint,
39
+ type ProposedCheckpointData,
40
+ getPreviousCheckpointOutHashes,
41
+ validateCheckpoint,
42
+ } from '@aztec/stdlib/checkpoint';
43
+ import { computeQuorum, getSlotStartBuildTimestamp, getTimestampForSlot } from '@aztec/stdlib/epoch-helpers';
35
44
  import { Gas } from '@aztec/stdlib/gas';
36
45
  import {
46
+ type BlockBuilderOptions,
37
47
  InsufficientValidTxsError,
38
- type PublicProcessorLimits,
39
48
  type ResolvedSequencerConfig,
40
49
  type WorldStateSynchronizer,
41
50
  } from '@aztec/stdlib/interfaces/server';
42
51
  import { type L1ToL2MessageSource, computeInHashFromL1ToL2Messages } from '@aztec/stdlib/messaging';
43
- import type { BlockProposalOptions, CheckpointProposal, CheckpointProposalOptions } from '@aztec/stdlib/p2p';
52
+ import type {
53
+ BlockProposal,
54
+ BlockProposalOptions,
55
+ CheckpointAttestation,
56
+ CheckpointProposal,
57
+ CheckpointProposalOptions,
58
+ CoordinationSignatureContext,
59
+ } from '@aztec/stdlib/p2p';
44
60
  import { orderAttestations, trimAttestations } from '@aztec/stdlib/p2p';
45
61
  import type { L2BlockBuiltStats } from '@aztec/stdlib/stats';
46
62
  import { type FailedTx, Tx } from '@aztec/stdlib/tx';
@@ -51,6 +67,8 @@ import { DutyAlreadySignedError, SlashingProtectionError } from '@aztec/validato
51
67
 
52
68
  import type { GlobalVariableBuilder } from '../global_variable_builder/global_builder.js';
53
69
  import type { InvalidateCheckpointRequest, SequencerPublisher } from '../publisher/sequencer-publisher.js';
70
+ import { buildCheckpointSimulationOverridesPlan } from './chain_state_overrides.js';
71
+ import type { CheckpointProposalJobMetricsRecorder } from './checkpoint_proposal_job_metrics.js';
54
72
  import { CheckpointVoter } from './checkpoint_voter.js';
55
73
  import { SequencerInterruptedError } from './errors.js';
56
74
  import type { SequencerEvents } from './events.js';
@@ -61,6 +79,21 @@ import { SequencerState } from './utils.js';
61
79
 
62
80
  /** How much time to sleep while waiting for min transactions to accumulate for a block */
63
81
  const TXS_POLLING_MS = 500;
82
+ const ARCHIVER_SYNC_POLLING_MS = 200;
83
+
84
+ /** Result from proposeCheckpoint when a checkpoint was successfully built and broadcast. */
85
+ type CheckpointProposalBroadcast = {
86
+ checkpoint: Checkpoint;
87
+ proposal: CheckpointProposal;
88
+ blockProposedAt: number;
89
+ };
90
+
91
+ /** Result after attestation collection and signing, ready for L1 submission. */
92
+ type CheckpointProposalResult = {
93
+ checkpoint: Checkpoint;
94
+ attestations: CommitteeAttestationsAndSigners;
95
+ attestationsSignature: Signature;
96
+ };
64
97
 
65
98
  /**
66
99
  * Handles the execution of a checkpoint proposal after the initial preparation phase.
@@ -70,12 +103,32 @@ const TXS_POLLING_MS = 500;
70
103
  */
71
104
  export class CheckpointProposalJob implements Traceable {
72
105
  protected readonly log: Logger;
106
+ private readonly checkpointEventLog: Logger;
107
+
108
+ /** Tracks the fire-and-forget L1 submission promise so it can be awaited during shutdown. */
109
+ private pendingL1Submission: Promise<void> | undefined;
110
+ private readonly interruptibleSleep = new InterruptibleSleep();
111
+ private interrupted = false;
112
+
113
+ /**
114
+ * Chain state overrides built once per slot in proposeCheckpoint after the checkpoint is
115
+ * complete. Carries the pending parent override (archive + slot + fee header) for pipelining,
116
+ * or the invalidation pending override when rolling back. Consumed by
117
+ * publisher.validateBlockHeader before broadcast.
118
+ */
119
+ private checkpointSimulationOverridesPlan?: SimulationOverridesPlan;
120
+
121
+ private getSignatureContext(): CoordinationSignatureContext {
122
+ return this.signatureContext;
123
+ }
73
124
 
74
125
  constructor(
75
- private readonly epoch: EpochNumber,
76
- private readonly slot: SlotNumber,
126
+ private readonly slotNow: SlotNumber,
127
+ private readonly targetSlot: SlotNumber,
128
+ private readonly targetEpoch: EpochNumber,
77
129
  private readonly checkpointNumber: CheckpointNumber,
78
130
  private readonly syncedToBlockNumber: BlockNumber,
131
+ private readonly checkpointedCheckpointNumber: CheckpointNumber,
79
132
  // TODO(palla/mbps): Can we remove the proposer in favor of attestorAddress? Need to check fisherman-node flows.
80
133
  private readonly proposer: EthAddress | undefined,
81
134
  private readonly publisher: SequencerPublisher,
@@ -88,25 +141,74 @@ export class CheckpointProposalJob implements Traceable {
88
141
  private readonly l1ToL2MessageSource: L1ToL2MessageSource,
89
142
  private readonly l2BlockSource: L2BlockSource,
90
143
  private readonly checkpointsBuilder: FullNodeCheckpointsBuilder,
91
- private readonly blockSink: L2BlockSink,
144
+ private readonly blockSink: L2BlockSink & ProposedCheckpointSink,
92
145
  private readonly l1Constants: SequencerRollupConstants,
146
+ private readonly signatureContext: CoordinationSignatureContext,
93
147
  protected config: ResolvedSequencerConfig,
94
148
  protected timetable: SequencerTimetable,
95
149
  private readonly slasherClient: SlasherClientInterface | undefined,
96
150
  private readonly epochCache: EpochCache,
97
151
  private readonly dateProvider: DateProvider,
98
152
  private readonly metrics: SequencerMetrics,
99
- private readonly eventEmitter: TypedEventEmitter<SequencerEvents>,
100
- private readonly setStateFn: (state: SequencerState, slot?: SlotNumber) => void,
153
+ private readonly checkpointMetrics: CheckpointProposalJobMetricsRecorder,
154
+ protected readonly eventEmitter: TypedEventEmitter<SequencerEvents>,
155
+ private readonly setStateFn: (state: SequencerState, slot?: SlotNumber, timeReferenceSlot?: SlotNumber) => void,
101
156
  public readonly tracer: Tracer,
102
157
  bindings?: LoggerBindings,
158
+ private readonly proposedCheckpointData?: ProposedCheckpointData,
103
159
  ) {
104
- this.log = createLogger('sequencer:checkpoint-proposal', { ...bindings, instanceId: `slot-${slot}` });
160
+ this.log = createLogger('sequencer:checkpoint-proposal', {
161
+ ...bindings,
162
+ instanceId: `slot-${this.slotNow}`,
163
+ });
164
+ this.checkpointEventLog = createLogger('sequencer:checkpoint-events', {
165
+ ...bindings,
166
+ instanceId: `slot-${this.slotNow}`,
167
+ });
168
+ }
169
+
170
+ /** Awaits the pending L1 submission if one is in progress. Call during shutdown. */
171
+ public async awaitPendingSubmission(): Promise<void> {
172
+ this.log.info('Awaiting pending L1 payload submission');
173
+ await this.pendingL1Submission;
174
+ }
175
+
176
+ /** Interrupts job-owned waits so shutdown can finish. */
177
+ public interrupt(): void {
178
+ this.interrupted = true;
179
+ this.interruptibleSleep.interrupt(true);
180
+ }
181
+
182
+ private async awaitInterruptibleSleep(ms: number): Promise<void> {
183
+ if (this.interrupted) {
184
+ throw new SequencerInterruptedError();
185
+ }
186
+ if (ms <= 0) {
187
+ return;
188
+ }
189
+ try {
190
+ await this.interruptibleSleep.sleep(ms);
191
+ } catch (err) {
192
+ if (err instanceof InterruptError) {
193
+ throw new SequencerInterruptedError();
194
+ }
195
+ throw err;
196
+ }
197
+ }
198
+
199
+ private logCheckpointEvent(eventName: string, message: string, fields: Record<string, unknown>): void {
200
+ this.checkpointEventLog.debug(message, {
201
+ eventName: `sequencer-checkpoint-${eventName}`,
202
+ ...fields,
203
+ });
105
204
  }
106
205
 
107
206
  /**
108
207
  * Executes the checkpoint proposal job.
109
- * Returns the published checkpoint if successful, undefined otherwise.
208
+ * Builds blocks, assembles checkpoint, and broadcasts the proposal (blocking).
209
+ * Attestation collection, signing, and L1 submission are backgrounded so the
210
+ * work loop can return to IDLE immediately for consecutive slot proposals.
211
+ * Returns the built checkpoint if successful, undefined otherwise.
110
212
  */
111
213
  @trackSpan('CheckpointProposalJob.execute')
112
214
  public async execute(): Promise<Checkpoint | undefined> {
@@ -114,7 +216,7 @@ export class CheckpointProposalJob implements Traceable {
114
216
  // In fisherman mode, we simulate slashing but don't actually publish to L1
115
217
  // These are constant for the whole slot, so we only enqueue them once
116
218
  const votesPromises = new CheckpointVoter(
117
- this.slot,
219
+ this.targetSlot,
118
220
  this.publisher,
119
221
  this.attestorAddress,
120
222
  this.validatorClient,
@@ -125,33 +227,302 @@ export class CheckpointProposalJob implements Traceable {
125
227
  this.log,
126
228
  ).enqueueVotes();
127
229
 
128
- // Build and propose the checkpoint. This will enqueue the request on the publisher if a checkpoint is built.
129
- const checkpoint = await this.proposeCheckpoint();
130
-
131
- // Wait until the voting promises have resolved, so all requests are enqueued (not sent)
132
- await Promise.all(votesPromises);
133
-
134
- if (checkpoint) {
135
- this.metrics.recordCheckpointProposalSuccess();
230
+ // Build blocks, assemble checkpoint, and broadcast proposal (BLOCKING).
231
+ // Returns after broadcast — attestation collection is deferred.
232
+ const broadcast = await this.proposeCheckpoint();
233
+
234
+ if (!broadcast) {
235
+ await Promise.all(votesPromises);
236
+ // Still submit votes even without a checkpoint.
237
+ // Under proposer pipelining, vote-offenses signatures are EIP-712-bound to `targetSlot`
238
+ // (the pipelined slot in which the multicall is expected to mine). Submitting at the
239
+ // wall-clock time would let the multicall mine in a different L2 slot, causing
240
+ // signature verification to fail silently inside Multicall3. Delay submission to the
241
+ // start of `targetSlot` so the tx mines in the slot the vote was signed for.
242
+ if (!this.config.fishermanMode) {
243
+ this.pendingL1Submission = this.publisher.sendRequestsAt(this.targetSlot).then(() => {});
244
+ }
245
+ return undefined;
136
246
  }
137
247
 
248
+ const { checkpoint } = broadcast;
249
+ this.metrics.recordCheckpointProposalSuccess();
250
+
138
251
  // Do not post anything to L1 if we are fishermen, but do perform L1 fee analysis
139
252
  if (this.config.fishermanMode) {
140
253
  await this.handleCheckpointEndAsFisherman(checkpoint);
141
- return;
254
+ return checkpoint;
142
255
  }
143
256
 
144
- // Then send everything to L1
145
- const l1Response = await this.publisher.sendRequests();
146
- const proposedAction = l1Response?.successfulActions.find(a => a === 'propose');
147
- if (proposedAction) {
148
- this.eventEmitter.emit('checkpoint-published', { checkpoint: this.checkpointNumber, slot: this.slot });
149
- const coinbase = checkpoint?.header.coinbase;
150
- await this.metrics.incFilledSlot(this.publisher.getSenderAddress().toString(), coinbase);
151
- return checkpoint;
152
- } else if (checkpoint) {
153
- this.eventEmitter.emit('checkpoint-publish-failed', { ...l1Response, slot: this.slot });
154
- return undefined;
257
+ // Background the attestation signing → L1 pipeline so the work loop is unblocked
258
+ this.pendingL1Submission = this.waitForAttestationsAndEnqueueSubmissionAsync(broadcast, votesPromises);
259
+
260
+ // Return the built checkpoint immediately — the work loop is now unblocked
261
+ return checkpoint;
262
+ }
263
+
264
+ /**
265
+ * Background pipeline: collects attestations, signs them, enqueues the checkpoint, and submits to L1.
266
+ * Runs as a fire-and-forget task stored in `pendingL1Submission` so the work loop is unblocked.
267
+ */
268
+ private async waitForAttestationsAndEnqueueSubmissionAsync(
269
+ broadcast: CheckpointProposalBroadcast,
270
+ votesPromises: Promise<unknown>[],
271
+ ): Promise<void> {
272
+ const { checkpoint } = broadcast;
273
+
274
+ try {
275
+ // Wait for all votes actions, enqueued at the beginning, to resolve
276
+ await Promise.all(votesPromises);
277
+
278
+ // Try to collect attestations from the committee
279
+ const signedAttestations = await this.getSignedCommitteeAttestations(broadcast);
280
+
281
+ // Wait for the previous checkpoint to land on L1 before submitting, so we can check it
282
+ // matches the proposed checkpoint we used as parent, and has valid attestations.
283
+ if (signedAttestations && (await this.waitForValidParentCheckpointOnL1())) {
284
+ await this.enqueueCheckpointForSubmission({ checkpoint, ...signedAttestations });
285
+ }
286
+
287
+ // If we failed to collect attestations, at least check if we need to issue an invalidation
288
+ if (!signedAttestations && (await this.waitForSyncedL2SlotNumber(this.slotNow))) {
289
+ const validationStatus = await this.l2BlockSource.getPendingChainValidationStatus();
290
+ if (!validationStatus.valid) {
291
+ this.log.warn(
292
+ `Checkpoint ${validationStatus.checkpoint.checkpointNumber} has invalid attestations, enqueuing invalidation in spite of attestation collection failure`,
293
+ { checkpoint: validationStatus.checkpoint, reason: validationStatus.reason },
294
+ );
295
+ await this.enqueueInvalidation(validationStatus);
296
+ }
297
+ }
298
+
299
+ // Send whatever was enqueued: votes + (propose | invalidation | nothing).
300
+ const l1Response = await this.publisher.sendRequestsAt(this.targetSlot);
301
+ const proposedAction = l1Response?.successfulActions.find(a => a === 'propose');
302
+ if (proposedAction) {
303
+ this.logCheckpointEvent('published', `Checkpoint published for slot ${this.targetSlot}`, {
304
+ slot: this.targetSlot,
305
+ checkpointNumber: this.checkpointNumber,
306
+ successfulActions: l1Response?.successfulActions,
307
+ sentActions: l1Response?.sentActions,
308
+ });
309
+ this.eventEmitter.emit('checkpoint-published', { checkpoint: this.checkpointNumber, slot: this.targetSlot });
310
+ const coinbase = checkpoint.header.coinbase;
311
+ await this.metrics.incFilledSlot(this.publisher.getSenderAddress().toString(), coinbase);
312
+ } else {
313
+ this.logCheckpointEvent('publish-failed', `Checkpoint publish failed for slot ${this.targetSlot}`, {
314
+ slot: this.targetSlot,
315
+ checkpointNumber: this.checkpointNumber,
316
+ successfulActions: l1Response?.successfulActions,
317
+ failedActions: l1Response?.failedActions,
318
+ sentActions: l1Response?.sentActions,
319
+ expiredActions: l1Response?.expiredActions,
320
+ reason: 'propose_action_not_successful',
321
+ });
322
+ this.log.warn(`Checkpoint publish failed for slot ${this.targetSlot}`, {
323
+ slot: this.targetSlot,
324
+ checkpointNumber: this.checkpointNumber,
325
+ successfulActions: l1Response?.successfulActions,
326
+ failedActions: l1Response?.failedActions,
327
+ sentActions: l1Response?.sentActions,
328
+ expiredActions: l1Response?.expiredActions,
329
+ reason: 'propose_action_not_successful',
330
+ });
331
+ this.eventEmitter.emit('checkpoint-publish-failed', { ...l1Response, slot: this.targetSlot });
332
+ this.metrics.recordPipelineDiscard();
333
+ }
334
+ } catch (err) {
335
+ if (err instanceof SequencerInterruptedError) {
336
+ return;
337
+ }
338
+ this.logCheckpointEvent('publish-failed', `Checkpoint publish failed for slot ${this.targetSlot}`, {
339
+ slot: this.targetSlot,
340
+ checkpointNumber: this.checkpointNumber,
341
+ reason: err instanceof Error ? err.message : String(err),
342
+ });
343
+ this.log.error(`Background attestation/L1 pipeline failed for slot ${this.targetSlot}`, err, {
344
+ slot: this.targetSlot,
345
+ checkpointNumber: this.checkpointNumber,
346
+ reason: err instanceof Error ? err.message : String(err),
347
+ });
348
+ this.eventEmitter.emit('checkpoint-publish-failed', { slot: this.targetSlot });
349
+ this.metrics.recordPipelineDiscard();
350
+ }
351
+ }
352
+
353
+ /** Enqueues the checkpoint for L1 submission. Called after pipeline sleep in execute(). */
354
+ private async enqueueCheckpointForSubmission(result: CheckpointProposalResult): Promise<void> {
355
+ const { checkpoint, attestations, attestationsSignature } = result;
356
+
357
+ // Build-frame deadlines are measured against `slotNow`; observers still see `targetSlot`.
358
+ this.setStateFn(SequencerState.PUBLISHING_CHECKPOINT, this.targetSlot, this.slotNow);
359
+ const aztecSlotDuration = this.l1Constants.slotDuration;
360
+ const submissionSlotStart = Number(getTimestampForSlot(this.targetSlot, this.l1Constants));
361
+ const txTimeoutAt = new Date((submissionSlotStart + aztecSlotDuration) * 1000);
362
+
363
+ // If we have been configured to potentially skip publishing checkpoint then roll the dice here
364
+ if (
365
+ this.config.skipPublishingCheckpointsPercent !== undefined &&
366
+ this.config.skipPublishingCheckpointsPercent > 0
367
+ ) {
368
+ const roll = Math.max(0, randomInt(100));
369
+ if (roll < this.config.skipPublishingCheckpointsPercent) {
370
+ this.log.warn(
371
+ `Skipping publishing proposal for checkpoint ${checkpoint.number}. Configured percentage: ${this.config.skipPublishingCheckpointsPercent}, generated value: ${roll}`,
372
+ );
373
+ return;
374
+ }
375
+ }
376
+
377
+ await this.publisher.enqueueProposeCheckpoint(checkpoint, attestations, attestationsSignature, {
378
+ txTimeoutAt,
379
+ });
380
+ }
381
+
382
+ /**
383
+ * Wait until the archiver syncs past the given L2 slot number.
384
+ * The deadline is the end of `this.targetSlot`, beyond which any pipelined work would miss its
385
+ * L1 submission window and is no longer useful.
386
+ */
387
+ private async waitForSyncedL2SlotNumber(waitForSlot: SlotNumber): Promise<boolean> {
388
+ const targetSlotStart = Number(getTimestampForSlot(this.targetSlot, this.l1Constants));
389
+ const targetSlotEndMs = (targetSlotStart + this.l1Constants.slotDuration) * 1000;
390
+ const syncDelayTolerance = this.l1Constants.ethereumSlotDuration * 2 * 1000;
391
+ const timeoutSeconds = Math.max(0.1, (targetSlotEndMs + syncDelayTolerance - this.dateProvider.now()) / 1000);
392
+
393
+ try {
394
+ const timer = new Timer();
395
+ while (true) {
396
+ const syncedSlot = await this.l2BlockSource.getSyncedL2SlotNumber();
397
+ if (syncedSlot !== undefined && syncedSlot >= waitForSlot) {
398
+ return true;
399
+ }
400
+ if (timeoutSeconds && timer.s() > timeoutSeconds) {
401
+ throw new TimeoutError(`Timeout awaiting archiver sync past slot ${waitForSlot}`);
402
+ }
403
+ await this.awaitInterruptibleSleep(ARCHIVER_SYNC_POLLING_MS);
404
+ }
405
+ } catch (err) {
406
+ if (err instanceof SequencerInterruptedError) {
407
+ throw err;
408
+ }
409
+ this.log.warn(
410
+ `Archiver did not sync L1 past slot ${waitForSlot} before slot ${this.targetSlot} expired, discarding pipelined work`,
411
+ { checkpointNumber: this.checkpointNumber },
412
+ );
413
+ this.emitPipelinedCheckpointDiscarded('archiver-sync-timeout');
414
+ return false;
415
+ }
416
+ }
417
+
418
+ /**
419
+ * Waits for the parent checkpoint to land on L1 before submitting a pipelined checkpoint.
420
+ * Polls until the archiver has synced L1 past the parent's slot, then verifies:
421
+ * - If we built on a proposed parent: it must have landed on L1 with matching hash and valid attestations.
422
+ * - If we built without a proposed parent: no new checkpoint must have appeared for that slot.
423
+ * If the parent has invalid attestations, enqueues an invalidation. Returns whether to proceed with the proposal.
424
+ */
425
+ protected async waitForValidParentCheckpointOnL1(): Promise<boolean> {
426
+ if (this.config.skipWaitForValidParentCheckpointOnL1) {
427
+ this.log.warn(`Skipping waitForValidParentCheckpointOnL1 due to test configuration`, {
428
+ checkpointNumber: this.checkpointNumber,
429
+ });
430
+ return true;
431
+ }
432
+
433
+ const parentCheckpointNumber = CheckpointNumber(this.checkpointNumber - 1);
434
+
435
+ // Wait until archiver has synced L1 past the parent's slot (slotNow)
436
+ if (!(await this.waitForSyncedL2SlotNumber(this.slotNow))) {
437
+ return false;
438
+ }
439
+
440
+ const tips = await this.l2BlockSource.getL2Tips();
441
+ const checkpointedNumber = tips.checkpointed.checkpoint.number;
442
+
443
+ // We built on top of a proposed checkpoint. Verify it landed on L1 as expected.
444
+ if (this.proposedCheckpointData) {
445
+ // After syncing from L1 we see the chain tip has invalid attestations. This means the parent checkpoint was posted
446
+ // with invalid attestations, or it built on top of something with invalid attestations and didnt invalidate them.
447
+ // Either way, we thought our parent would be valid, so we have to throw away our work. But at least we'll try and
448
+ // invalidate on L1 so we clean up the chain for the next proposer. And we'll slash them, but that's handled elsewhere.
449
+ const validationStatus = await this.l2BlockSource.getPendingChainValidationStatus();
450
+ if (!validationStatus.valid) {
451
+ this.log.warn(
452
+ `Parent checkpoint ${parentCheckpointNumber} has invalid attestations, discarding pipelined work`,
453
+ { checkpointNumber: this.checkpointNumber, reason: validationStatus.reason },
454
+ );
455
+ this.emitPipelinedCheckpointDiscarded('parent-invalid-attestations');
456
+ await this.enqueueInvalidation(validationStatus);
457
+ return false;
458
+ }
459
+
460
+ // The pending chain is valid. But did the parent checkpoint land on L1 at all?
461
+ if (checkpointedNumber < parentCheckpointNumber) {
462
+ this.log.warn(`Parent checkpoint ${parentCheckpointNumber} did not land on L1, discarding pipelined work`, {
463
+ checkpointNumber: this.checkpointNumber,
464
+ checkpointedNumber,
465
+ });
466
+ this.emitPipelinedCheckpointDiscarded('parent-not-on-l1');
467
+ return false;
468
+ }
469
+
470
+ // It landed. But is it the one we were expecting?
471
+ const expectedHash = this.proposedCheckpointData.header.hash().toString();
472
+ if (tips.checkpointed.checkpoint.hash !== expectedHash) {
473
+ this.log.warn(`Parent checkpoint ${parentCheckpointNumber} hash mismatch on L1, discarding pipelined work`, {
474
+ checkpointNumber: this.checkpointNumber,
475
+ expectedHash,
476
+ actualHash: tips.checkpointed.checkpoint.hash,
477
+ });
478
+ this.emitPipelinedCheckpointDiscarded('parent-hash-mismatch');
479
+ return false;
480
+ }
481
+
482
+ return true;
483
+ } else {
484
+ // We didn't see a proposed checkpoint at build time, so we built on checkpointed parent from two slots ago.
485
+ // But if a new checkpoint for the previous slot appeared on L1 in the meantime, our checkpoint assumed the wrong parent,
486
+ // so we have to discard our work. This can happen if we're somehow cut off from p2p and fail to see the checkpoint
487
+ // proposal for the previous slot.
488
+ if (checkpointedNumber > parentCheckpointNumber) {
489
+ this.log.warn(
490
+ `Unexpected checkpoint ${checkpointedNumber} landed on L1 after we built on top of parent ${parentCheckpointNumber}, discarding pipelined work`,
491
+ { checkpointNumber: this.checkpointNumber, checkpointedNumber },
492
+ );
493
+ this.emitPipelinedCheckpointDiscarded('unexpected-parent-appeared');
494
+ return false;
495
+ }
496
+
497
+ return true;
498
+ }
499
+ }
500
+
501
+ /** Emits the pipelined-checkpoint-discarded event and records the metric. */
502
+ private emitPipelinedCheckpointDiscarded(reason: string): void {
503
+ this.metrics.recordPipelineParentCheckpointMismatch(reason);
504
+ this.eventEmitter.emit('pipelined-checkpoint-discarded', {
505
+ slot: this.targetSlot,
506
+ checkpointNumber: this.checkpointNumber,
507
+ reason,
508
+ });
509
+ }
510
+
511
+ /** Simulates and enqueues an invalidation request for the invalid parent checkpoint. */
512
+ private async enqueueInvalidation(validationStatus: ValidateCheckpointResult): Promise<void> {
513
+ if (this.config.skipInvalidateBlockAsProposer) {
514
+ this.log.warn(`Skipping checkpoint invalidation as proposer due to test configuration`);
515
+ return;
516
+ }
517
+ const invalidateRequest = await this.publisher.simulateInvalidateCheckpoint(validationStatus);
518
+ if (invalidateRequest) {
519
+ const submissionSlotStart = Number(getTimestampForSlot(this.targetSlot, this.l1Constants));
520
+ const txTimeoutAt = new Date((submissionSlotStart + this.l1Constants.slotDuration) * 1000);
521
+ this.publisher.enqueueInvalidateCheckpoint(invalidateRequest, { txTimeoutAt });
522
+ } else {
523
+ this.log.info(`Invalidation simulation returned undefined, checkpoint may have been removed already`, {
524
+ checkpointNumber: this.checkpointNumber,
525
+ });
155
526
  }
156
527
  }
157
528
 
@@ -159,42 +530,89 @@ export class CheckpointProposalJob implements Traceable {
159
530
  return {
160
531
  // nullish operator needed for tests
161
532
  [Attributes.COINBASE]: this.validatorClient.getCoinbaseForAttestor(this.attestorAddress)?.toString(),
162
- [Attributes.SLOT_NUMBER]: this.slot,
533
+ [Attributes.SLOT_NUMBER]: this.targetSlot,
163
534
  };
164
535
  })
165
- private async proposeCheckpoint(): Promise<Checkpoint | undefined> {
536
+ private async proposeCheckpoint(): Promise<CheckpointProposalBroadcast | undefined> {
166
537
  try {
538
+ const now = this.dateProvider.now();
539
+ if (this.proposedCheckpointData) {
540
+ // Measure against the wall-clock slot whose build window we are currently using.
541
+ // In pipelining mode `targetSlot` is intentionally one slot ahead, which makes the
542
+ // target-slot boundary a full slot away from the actual build start time.
543
+ const slotBoundaryMs = Number(getTimestampForSlot(this.slotNow, this.l1Constants)) * 1000;
544
+ this.checkpointMetrics.recordPipelinedCheckpointBuildStartOffsetFromSlotBoundary(now - slotBoundaryMs);
545
+ }
546
+ this.checkpointMetrics.startCheckpointTiming(now);
547
+
167
548
  // Get operator configured coinbase and fee recipient for this attestor
168
549
  const coinbase = this.validatorClient.getCoinbaseForAttestor(this.attestorAddress);
169
550
  const feeRecipient = this.validatorClient.getFeeRecipientForAttestor(this.attestorAddress);
170
551
 
171
552
  // Start the checkpoint
172
- this.setStateFn(SequencerState.INITIALIZING_CHECKPOINT, this.slot);
173
- this.metrics.incOpenSlot(this.slot, this.proposer?.toString() ?? 'unknown');
553
+ this.setStateFn(SequencerState.INITIALIZING_CHECKPOINT, this.targetSlot, this.slotNow);
554
+ this.logCheckpointEvent('slot-started', `Starting checkpoint proposal for slot ${this.targetSlot}`, {
555
+ buildSlot: this.slotNow,
556
+ submissionSlot: this.targetSlot,
557
+ slot: this.targetSlot,
558
+ checkpointNumber: this.checkpointNumber,
559
+ proposer: this.proposer?.toString(),
560
+ attestorAddress: this.attestorAddress.toString(),
561
+ publisherAddress: this.publisher.getSenderAddress().toString(),
562
+ coinbase: coinbase.toString(),
563
+ });
564
+ this.metrics.incOpenSlot(this.targetSlot, this.proposer?.toString() ?? 'unknown');
174
565
 
175
566
  // Enqueues checkpoint invalidation (constant for the whole slot)
176
567
  if (this.invalidateCheckpoint && !this.config.skipInvalidateBlockAsProposer) {
177
568
  this.publisher.enqueueInvalidateCheckpoint(this.invalidateCheckpoint);
178
569
  }
179
570
 
180
- // Create checkpoint builder for the slot
571
+ // Build the simulation plan for this slot. When pipelining, this overrides L1's view of
572
+ // pending/archive/fee-header to "as if the proposed parent had landed", so both the
573
+ // mana-min-fee simulation (in the globals builder) and the pre-broadcast
574
+ // validateBlockHeader see the chain tip the eventual L1 send will see.
575
+ this.checkpointSimulationOverridesPlan = await buildCheckpointSimulationOverridesPlan({
576
+ checkpointNumber: this.checkpointNumber,
577
+ proposedCheckpointData: this.proposedCheckpointData,
578
+ invalidateToPendingCheckpointNumber: this.invalidateCheckpoint?.forcePendingCheckpointNumber,
579
+ checkpointedCheckpointNumber: this.checkpointedCheckpointNumber,
580
+ rollup: this.publisher.rollupContract,
581
+ signatureContext: this.signatureContext,
582
+ log: this.log,
583
+ });
584
+
181
585
  const checkpointGlobalVariables = await this.globalsBuilder.buildCheckpointGlobalVariables(
182
586
  coinbase,
183
587
  feeRecipient,
184
- this.slot,
588
+ this.targetSlot,
589
+ this.checkpointSimulationOverridesPlan,
185
590
  );
186
591
 
187
592
  // Collect L1 to L2 messages for the checkpoint and compute their hash
188
593
  const l1ToL2Messages = await this.l1ToL2MessageSource.getL1ToL2Messages(this.checkpointNumber);
189
594
  const inHash = computeInHashFromL1ToL2Messages(l1ToL2Messages);
190
595
 
191
- // Collect the out hashes of all the checkpoints before this one in the same epoch
192
- const previousCheckpointOutHashes = (await this.l2BlockSource.getCheckpointsDataForEpoch(this.epoch))
193
- .filter(c => c.checkpointNumber < this.checkpointNumber)
194
- .map(c => c.checkpointOutHash);
596
+ // Collect the out hashes of all the checkpoints before this one in the same epoch.
597
+ // Under pipelining the parent checkpoint may not be on L1 yet at build time, so the helper
598
+ // splices in the parent's checkpointOutHash from the locally-known proposed checkpoint so
599
+ // the resulting `epochOutHash` matches what validators (and L1) compute once the parent
600
+ // lands on L1.
601
+ const previousCheckpointOutHashes = await getPreviousCheckpointOutHashes({
602
+ blockSource: this.l2BlockSource,
603
+ epoch: this.targetEpoch,
604
+ checkpointNumber: this.checkpointNumber,
605
+ l1Constants: this.epochCache.getL1Constants(),
606
+ pipeliningEnabled: true,
607
+ proposedCheckpointData: this.proposedCheckpointData,
608
+ log: this.log,
609
+ });
195
610
 
196
- // Get the fee asset price modifier from the oracle
197
- const feeAssetPriceModifier = await this.publisher.getFeeAssetPriceModifier();
611
+ // Anchor the modifier to the predicted parent fee header: L1 will apply it against
612
+ // that, not against the latest published checkpoint (which lags by one under pipelining).
613
+ const predictedParentEthPerFeeAssetE12 =
614
+ this.checkpointSimulationOverridesPlan?.pendingCheckpointState?.feeHeader?.ethPerFeeAsset;
615
+ const feeAssetPriceModifier = await this.publisher.getFeeAssetPriceModifier(predictedParentEthPerFeeAssetE12);
198
616
 
199
617
  // Create a long-lived forked world state for the checkpoint builder
200
618
  await using fork = await this.worldState.fork(this.syncedToBlockNumber, { closeDelayMs: 12_000 });
@@ -218,11 +636,12 @@ export class CheckpointProposalJob implements Traceable {
218
636
 
219
637
  const checkpointProposalOptions: CheckpointProposalOptions = {
220
638
  publishFullTxs: !!this.config.publishTxsWithProposals,
221
- broadcastInvalidCheckpointProposal: this.config.broadcastInvalidBlockProposal,
639
+ broadcastInvalidCheckpointProposal:
640
+ this.config.broadcastInvalidCheckpointProposalOnly || this.config.broadcastInvalidBlockProposal,
222
641
  };
223
642
 
224
643
  let blocksInCheckpoint: L2Block[] = [];
225
- let blockPendingBroadcast: { block: L2Block; txs: Tx[] } | undefined = undefined;
644
+ let blockPendingBroadcast: BlockProposal | undefined = undefined;
226
645
  const checkpointBuildTimer = new Timer();
227
646
 
228
647
  try {
@@ -246,26 +665,49 @@ export class CheckpointProposalJob implements Traceable {
246
665
  }
247
666
 
248
667
  if (blocksInCheckpoint.length === 0) {
249
- this.log.warn(`No blocks were built for slot ${this.slot}`, { slot: this.slot });
250
- this.eventEmitter.emit('checkpoint-empty', { slot: this.slot });
668
+ this.logCheckpointEvent('build-failed', `Checkpoint build failed for slot ${this.targetSlot}`, {
669
+ slot: this.targetSlot,
670
+ checkpointNumber: this.checkpointNumber,
671
+ reason: 'no_blocks_built',
672
+ });
673
+ this.log.warn(`No blocks were built for slot ${this.targetSlot}`, {
674
+ slot: this.targetSlot,
675
+ checkpointNumber: this.checkpointNumber,
676
+ reason: 'no_blocks_built',
677
+ });
678
+ this.eventEmitter.emit('checkpoint-empty', { slot: this.targetSlot });
251
679
  return undefined;
252
680
  }
253
681
 
254
682
  const minBlocksForCheckpoint = this.config.minBlocksForCheckpoint;
255
683
  if (minBlocksForCheckpoint !== undefined && blocksInCheckpoint.length < minBlocksForCheckpoint) {
684
+ this.logCheckpointEvent('build-failed', `Checkpoint build failed for slot ${this.targetSlot}`, {
685
+ slot: this.targetSlot,
686
+ checkpointNumber: this.checkpointNumber,
687
+ blocksBuilt: blocksInCheckpoint.length,
688
+ minBlocksForCheckpoint,
689
+ reason: 'min_blocks_not_met',
690
+ });
256
691
  this.log.warn(
257
692
  `Checkpoint has fewer blocks than minimum (${blocksInCheckpoint.length} < ${minBlocksForCheckpoint}), skipping proposal`,
258
- { slot: this.slot, blocksBuilt: blocksInCheckpoint.length, minBlocksForCheckpoint },
693
+ {
694
+ slot: this.targetSlot,
695
+ checkpointNumber: this.checkpointNumber,
696
+ blocksBuilt: blocksInCheckpoint.length,
697
+ minBlocksForCheckpoint,
698
+ reason: 'min_blocks_not_met',
699
+ },
259
700
  );
260
701
  return undefined;
261
702
  }
262
703
 
263
704
  // Assemble and broadcast the checkpoint proposal, including the last block that was not
264
705
  // broadcasted yet, and wait to collect the committee attestations.
265
- this.setStateFn(SequencerState.ASSEMBLING_CHECKPOINT, this.slot);
706
+ this.setStateFn(SequencerState.ASSEMBLING_CHECKPOINT, this.targetSlot, this.slotNow);
266
707
  const checkpoint = await checkpointBuilder.completeCheckpoint();
267
708
 
268
- // Final validation round for the checkpoint before we propose it, just for safety
709
+ // Final validation: per-block limits are only checked if the operator set them explicitly.
710
+ // Otherwise, checkpoint-level budgets were already enforced by the redistribution logic.
269
711
  try {
270
712
  validateCheckpoint(checkpoint, {
271
713
  rollupManaLimit: this.l1Constants.rollupManaLimit,
@@ -275,113 +717,118 @@ export class CheckpointProposalJob implements Traceable {
275
717
  maxTxsPerCheckpoint: this.config.maxTxsPerCheckpoint,
276
718
  });
277
719
  } catch (err) {
278
- this.log.error(`Built an invalid checkpoint at slot ${this.slot} (skipping proposal)`, err, {
720
+ this.logCheckpointEvent('build-failed', `Checkpoint build failed for slot ${this.targetSlot}`, {
721
+ slot: this.targetSlot,
722
+ checkpointNumber: this.checkpointNumber,
723
+ blocksBuilt: blocksInCheckpoint.length,
724
+ reason: 'invalid_checkpoint',
725
+ checkpoint: checkpoint.header.toInspect(),
726
+ });
727
+ this.log.error(`Built an invalid checkpoint at slot ${this.slotNow} (skipping proposal)`, err, {
728
+ slot: this.targetSlot,
729
+ checkpointNumber: this.checkpointNumber,
730
+ blocksBuilt: blocksInCheckpoint.length,
731
+ reason: 'invalid_checkpoint',
279
732
  checkpoint: checkpoint.header.toInspect(),
280
733
  });
281
734
  return undefined;
282
735
  }
283
736
 
284
737
  // Record checkpoint-level build metrics
285
- this.metrics.recordCheckpointBuild(
738
+ this.checkpointMetrics.recordCheckpointBuild(
286
739
  checkpointBuildTimer.ms(),
287
740
  blocksInCheckpoint.length,
288
741
  checkpoint.getStats().txCount,
289
742
  Number(checkpoint.header.totalManaUsed.toBigInt()),
290
743
  );
744
+ this.logCheckpointEvent('built', `Checkpoint built for slot ${this.targetSlot}`, {
745
+ slot: this.targetSlot,
746
+ buildSlot: this.slotNow,
747
+ checkpointNumber: this.checkpointNumber,
748
+ proposer: this.proposer?.toString(),
749
+ attestorAddress: this.attestorAddress.toString(),
750
+ publisherAddress: this.publisher.getSenderAddress().toString(),
751
+ blocksBuilt: blocksInCheckpoint.length,
752
+ txCount: checkpoint.getStats().txCount,
753
+ totalMana: Number(checkpoint.header.totalManaUsed.toBigInt()),
754
+ });
291
755
 
292
- // Do not collect attestations nor publish to L1 in fisherman mode
756
+ // In fisherman mode, return the checkpoint without broadcasting or collecting attestations
293
757
  if (this.config.fishermanMode) {
294
758
  this.log.info(
295
- `Built checkpoint for slot ${this.slot} with ${blocksInCheckpoint.length} blocks. ` +
759
+ `Built checkpoint for slot ${this.targetSlot} with ${blocksInCheckpoint.length} blocks. ` +
296
760
  `Skipping proposal in fisherman mode.`,
297
761
  {
298
- slot: this.slot,
762
+ slot: this.targetSlot,
299
763
  checkpoint: checkpoint.header.toInspect(),
300
764
  blocksBuilt: blocksInCheckpoint.length,
301
765
  },
302
766
  );
303
767
  this.metrics.recordCheckpointSuccess();
304
- return checkpoint;
768
+ // Return a broadcast result with a dummy proposal — fisherman mode skips attestation collection
769
+ return { checkpoint, proposal: undefined!, blockProposedAt: this.dateProvider.now() };
305
770
  }
306
771
 
307
- // Include the block pending broadcast in the checkpoint proposal if any
308
- const lastBlock = blockPendingBroadcast && {
309
- blockHeader: blockPendingBroadcast.block.header,
310
- indexWithinCheckpoint: blockPendingBroadcast.block.indexWithinCheckpoint,
311
- txs: blockPendingBroadcast.txs,
312
- };
772
+ // Validate the header against L1 state before broadcasting.
773
+ // If this fails the slot is aborted before any gossip work; state drift between here
774
+ // and the eventual L1 send is caught by the bundle simulate at send time.
775
+ try {
776
+ await this.publisher.validateBlockHeader(checkpoint.header, this.checkpointSimulationOverridesPlan);
777
+ } catch (err) {
778
+ this.log.error(`Pre-broadcast header validation failed for slot ${this.targetSlot}; aborting`, err, {
779
+ slot: this.targetSlot,
780
+ checkpointNumber: this.checkpointNumber,
781
+ });
782
+ this.metrics.recordCheckpointProposalFailed('header_validation_failed');
783
+ this.eventEmitter.emit('header-validation-failed', {
784
+ slot: this.targetSlot,
785
+ checkpointNumber: this.checkpointNumber,
786
+ reason: err instanceof Error ? err.message : String(err),
787
+ });
788
+ return undefined;
789
+ }
313
790
 
314
791
  // Create the checkpoint proposal and broadcast it
315
792
  const proposal = await this.validatorClient.createCheckpointProposal(
316
793
  checkpoint.header,
317
794
  checkpoint.archive.root,
795
+ this.checkpointNumber,
318
796
  feeAssetPriceModifier,
319
- lastBlock,
797
+ blockPendingBroadcast,
320
798
  this.proposer,
321
799
  checkpointProposalOptions,
322
800
  );
323
801
 
324
- const blockProposedAt = this.dateProvider.now();
325
- await this.p2pClient.broadcastCheckpointProposal(proposal);
326
-
327
- this.setStateFn(SequencerState.COLLECTING_ATTESTATIONS, this.slot);
328
- const attestations = await this.waitForAttestations(proposal);
329
- const blockAttestedAt = this.dateProvider.now();
330
-
331
- this.metrics.recordCheckpointAttestationDelay(blockAttestedAt - blockProposedAt);
802
+ // Advance our own optimistic proposed-checkpoint tip locally before gossiping. Gossipsub
803
+ // doesn't echo our own messages back, so this is how the proposer makes its own proposed
804
+ // checkpoint visible for pipelining the next slot. Built from local checkpoint data — never
805
+ // from the broadcast proposal archive, which may be deliberately corrupted under test flags.
806
+ // Fail closed: if this throws, the outer catch aborts the slot before gossiping.
807
+ await this.syncProposedCheckpointToArchiver(checkpoint, blocksInCheckpoint.length, feeAssetPriceModifier);
332
808
 
333
- // Proposer must sign over the attestations before pushing them to L1
334
- const signer = this.proposer ?? this.publisher.getSenderAddress();
335
- let attestationsSignature: Signature;
336
- try {
337
- attestationsSignature = await this.validatorClient.signAttestationsAndSigners(
338
- attestations,
339
- signer,
340
- this.slot,
341
- this.checkpointNumber,
342
- );
343
- } catch (err) {
344
- // We shouldn't really get here since we yield to another HA node
345
- // as soon as we see these errors when creating block or checkpoint proposals.
346
- if (this.handleHASigningError(err, 'Attestations signature')) {
347
- return undefined;
348
- }
349
- throw err;
350
- }
351
-
352
- // Enqueue publishing the checkpoint to L1
353
- this.setStateFn(SequencerState.PUBLISHING_CHECKPOINT, this.slot);
354
- const aztecSlotDuration = this.l1Constants.slotDuration;
355
- const slotStartBuildTimestamp = this.getSlotStartBuildTimestamp();
356
- const txTimeoutAt = new Date((slotStartBuildTimestamp + aztecSlotDuration) * 1000);
357
-
358
- // If we have been configured to potentially skip publishing checkpoint then roll the dice here
359
- if (
360
- this.config.skipPublishingCheckpointsPercent !== undefined &&
361
- this.config.skipPublishingCheckpointsPercent > 0
362
- ) {
363
- const result = Math.max(0, randomInt(100));
364
- if (result < this.config.skipPublishingCheckpointsPercent) {
365
- this.log.warn(
366
- `Skipping publishing proposal for checkpoint ${checkpoint.number}. Configured percentage: ${this.config.skipPublishingCheckpointsPercent}, generated value: ${result}`,
367
- );
368
- return checkpoint;
809
+ const blockProposedAt = this.dateProvider.now();
810
+ if (this.config.skipBroadcastCheckpointProposal) {
811
+ // Test-only: suppress the CheckpointProposal so peers never see a proposed checkpoint for
812
+ // this slot, but still broadcast the held last block standalone so peers' archivers ingest
813
+ // it as a proposed-but-uncheckpointed tip — the exact orphan-block state that
814
+ // pruneOrphanProposedBlocks / checkSync must handle.
815
+ if (blockPendingBroadcast && !this.config.skipBroadcastProposals) {
816
+ await this.p2pClient.broadcastProposal(blockPendingBroadcast);
369
817
  }
818
+ } else if (!this.config.skipBroadcastProposals) {
819
+ await this.p2pClient.broadcastCheckpointProposal(proposal);
820
+ this.checkpointMetrics.noteCheckpointBroadcast(this.dateProvider.now());
370
821
  }
371
822
 
372
- await this.publisher.enqueueProposeCheckpoint(checkpoint, attestations, attestationsSignature, {
373
- txTimeoutAt,
374
- forcePendingCheckpointNumber: this.invalidateCheckpoint?.forcePendingCheckpointNumber,
375
- });
376
-
377
- return checkpoint;
823
+ // Return immediately after broadcast — attestation collection happens in the background
824
+ return { checkpoint, proposal, blockProposedAt };
378
825
  } catch (err) {
379
826
  if (err && (err instanceof DutyAlreadySignedError || err instanceof SlashingProtectionError)) {
380
827
  // swallow this error. It's already been logged by a function deeper in the stack
381
828
  return undefined;
382
829
  }
383
830
 
384
- this.log.error(`Error building checkpoint at slot ${this.slot}`, err);
831
+ this.log.error(`Error building checkpoint at slot ${this.targetSlot}`, err);
385
832
  return undefined;
386
833
  }
387
834
  }
@@ -397,26 +844,35 @@ export class CheckpointProposalJob implements Traceable {
397
844
  blockProposalOptions: BlockProposalOptions,
398
845
  ): Promise<{
399
846
  blocksInCheckpoint: L2Block[];
400
- blockPendingBroadcast: { block: L2Block; txs: Tx[] } | undefined;
847
+ blockPendingBroadcast: BlockProposal | undefined;
401
848
  }> {
402
849
  const blocksInCheckpoint: L2Block[] = [];
403
850
  const txHashesAlreadyIncluded = new Set<string>();
404
851
  const initialBlockNumber = BlockNumber(this.syncedToBlockNumber + 1);
405
852
 
406
853
  // Last block in the checkpoint will usually be flagged as pending broadcast, so we send it along with the checkpoint proposal
407
- let blockPendingBroadcast: { block: L2Block; txs: Tx[] } | undefined = undefined;
854
+ let blockPendingBroadcast: BlockProposal | undefined = undefined;
408
855
 
409
856
  while (true) {
410
857
  const blocksBuilt = blocksInCheckpoint.length;
411
858
  const indexWithinCheckpoint = IndexWithinCheckpoint(blocksBuilt);
412
859
  const blockNumber = BlockNumber(initialBlockNumber + blocksBuilt);
413
860
 
861
+ if (blocksBuilt >= this.config.maxBlocksPerCheckpoint) {
862
+ this.log.debug(`Reached max blocks per checkpoint`, {
863
+ slot: this.targetSlot,
864
+ blocksBuilt,
865
+ maxBlocksPerCheckpoint: this.config.maxBlocksPerCheckpoint,
866
+ });
867
+ break;
868
+ }
869
+
414
870
  const secondsIntoSlot = this.getSecondsIntoSlot();
415
871
  const timingInfo = this.timetable.canStartNextBlock(secondsIntoSlot);
416
872
 
417
873
  if (!timingInfo.canStart) {
418
874
  this.log.debug(`Not enough time left in slot to start another block`, {
419
- slot: this.slot,
875
+ slot: this.targetSlot,
420
876
  blocksBuilt,
421
877
  secondsIntoSlot,
422
878
  });
@@ -437,22 +893,23 @@ export class CheckpointProposalJob implements Traceable {
437
893
  txHashesAlreadyIncluded,
438
894
  });
439
895
 
440
- // TODO(palla/mbps): Review these conditions. We may want to keep trying in some scenarios.
441
- if (!buildResult && timingInfo.isLastBlock) {
442
- // If no block was produced due to not enough txs and this was the last subslot, exit
443
- break;
444
- } else if (!buildResult && timingInfo.deadline !== undefined) {
445
- // But if there is still time for more blocks, wait until the next subslot and try again
896
+ // If we failed to build the block due to insufficient txs, we try again if there is still time left in the slot
897
+ if ('failure' in buildResult) {
898
+ // If this was the last subslot, or we're running with a single block per slot, we're done
899
+ if (timingInfo.isLastBlock || timingInfo.deadline === undefined) {
900
+ break;
901
+ }
902
+ // Otherwise, if there is still time for more blocks, we wait until the next subslot and try again
446
903
  await this.waitUntilNextSubslot(timingInfo.deadline);
447
904
  continue;
448
- } else if (!buildResult) {
449
- // Exit if there is no possibility of building more blocks
450
- break;
451
- } else if ('error' in buildResult) {
452
- // If there was an error building the block, just exit the loop and give up the rest of the slot
905
+ }
906
+
907
+ // If there was an error building the block, we just exit the loop and give up the rest of the slot.
908
+ // We don't want to risk building more blocks if something went wrong.
909
+ if ('error' in buildResult) {
453
910
  if (!(buildResult.error instanceof SequencerInterruptedError)) {
454
- this.log.warn(`Halting block building for slot ${this.slot}`, {
455
- slot: this.slot,
911
+ this.log.warn(`Halting block building for slot ${this.targetSlot}`, {
912
+ slot: this.targetSlot,
456
913
  blocksBuilt,
457
914
  error: buildResult.error,
458
915
  });
@@ -461,38 +918,42 @@ export class CheckpointProposalJob implements Traceable {
461
918
  }
462
919
 
463
920
  const { block, usedTxs } = buildResult;
921
+ this.checkpointMetrics.noteCheckpointBlockBuilt(this.dateProvider.now(), {
922
+ isFirstBlock: blocksBuilt === 0,
923
+ isLastBlock: timingInfo.isLastBlock,
924
+ });
925
+
464
926
  blocksInCheckpoint.push(block);
927
+ usedTxs.forEach(tx => txHashesAlreadyIncluded.add(tx.txHash.toString()));
465
928
 
466
- // Sync the proposed block to the archiver to make it available
467
- // We wait for the sync to succeed, as this helps catch consistency errors, even if it means we lose some time for block-building
468
- // If this throws, we abort the entire checkpoint
469
- await this.syncProposedBlockToArchiver(block);
929
+ // Sign the block proposal. This will throw if HA signing fails.
930
+ const proposal = await this.createBlockProposal(block, inHash, usedTxs, {
931
+ ...blockProposalOptions,
932
+ broadcastInvalidBlockProposal:
933
+ blockProposalOptions.broadcastInvalidBlockProposal ||
934
+ block.indexWithinCheckpoint === this.config.invalidBlockProposalIndexWithinCheckpoint,
935
+ });
470
936
 
471
- usedTxs.forEach(tx => txHashesAlreadyIncluded.add(tx.txHash.toString()));
937
+ // Sync the proposed block to the archiver to make it available, only after we've managed to sign the proposal,
938
+ // so we avoid polluting our archive with a block that would fail.
939
+ // We wait for the sync to succeed, as this helps catch consistency errors, even if it means we lose some time for block-building.
940
+ // If this throws, we abort the entire checkpoint.
941
+ await this.syncProposedBlockToArchiver(block);
472
942
 
473
- // If this is the last block, exit the loop now so we start collecting attestations
943
+ // If this is the last block, do not broadcast it, since it will be included in the checkpoint proposal.
474
944
  if (timingInfo.isLastBlock) {
475
- this.log.verbose(`Completed final block ${blockNumber} for slot ${this.slot}`, {
476
- slot: this.slot,
945
+ this.log.verbose(`Completed final block ${blockNumber} for slot ${this.targetSlot}`, {
946
+ slot: this.targetSlot,
477
947
  blockNumber,
478
948
  blocksBuilt,
479
949
  });
480
- blockPendingBroadcast = { block, txs: usedTxs };
950
+
951
+ blockPendingBroadcast = proposal;
481
952
  break;
482
953
  }
483
954
 
484
- // For non-last blocks, broadcast the block proposal (unless we're in fisherman mode)
485
- // If the block is the last one, we'll broadcast it along with the checkpoint at the end of the loop
486
- if (!this.config.fishermanMode) {
487
- const proposal = await this.validatorClient.createBlockProposal(
488
- block.header,
489
- block.indexWithinCheckpoint,
490
- inHash,
491
- block.archive.root,
492
- usedTxs,
493
- this.proposer,
494
- blockProposalOptions,
495
- );
955
+ // Once we have a signed proposal and the archiver agreed with our proposed block, then we broadcast it.
956
+ if (proposal && !this.config.skipBroadcastProposals) {
496
957
  await this.p2pClient.broadcastProposal(proposal);
497
958
  }
498
959
 
@@ -500,19 +961,44 @@ export class CheckpointProposalJob implements Traceable {
500
961
  await this.waitUntilNextSubslot(timingInfo.deadline);
501
962
  }
502
963
 
503
- this.log.verbose(`Block building loop completed for slot ${this.slot}`, {
504
- slot: this.slot,
964
+ this.log.verbose(`Block building loop completed for slot ${this.targetSlot}`, {
965
+ slot: this.targetSlot,
505
966
  blocksBuilt: blocksInCheckpoint.length,
506
967
  });
507
968
 
508
969
  return { blocksInCheckpoint, blockPendingBroadcast };
509
970
  }
510
971
 
972
+ /** Creates a block proposal for a given block via the validator client (unless in fisherman mode) */
973
+ private createBlockProposal(
974
+ block: L2Block,
975
+ inHash: Fr,
976
+ usedTxs: Tx[],
977
+ blockProposalOptions: BlockProposalOptions,
978
+ ): Promise<BlockProposal | undefined> {
979
+ if (this.config.fishermanMode) {
980
+ this.log.info(`Skipping block proposal for block ${block.number} in fisherman mode`);
981
+ return Promise.resolve(undefined);
982
+ }
983
+ return this.validatorClient.createBlockProposal(
984
+ block.header,
985
+ this.checkpointNumber,
986
+ block.indexWithinCheckpoint,
987
+ inHash,
988
+ block.archive.root,
989
+ usedTxs,
990
+ this.proposer,
991
+ blockProposalOptions,
992
+ );
993
+ }
994
+
511
995
  /** Sleeps until it is time to produce the next block in the slot */
512
996
  @trackSpan('CheckpointProposalJob.waitUntilNextSubslot')
513
997
  private async waitUntilNextSubslot(nextSubslotStart: number) {
514
- this.setStateFn(SequencerState.WAITING_UNTIL_NEXT_BLOCK, this.slot);
515
- this.log.verbose(`Waiting until time for the next block at ${nextSubslotStart}s into slot`, { slot: this.slot });
998
+ this.setStateFn(SequencerState.WAITING_UNTIL_NEXT_BLOCK, this.targetSlot, this.slotNow);
999
+ this.log.verbose(`Waiting until time for the next block at ${nextSubslotStart}s into slot`, {
1000
+ slot: this.targetSlot,
1001
+ });
516
1002
  await this.waitUntilTimeInSlot(nextSubslotStart);
517
1003
  }
518
1004
 
@@ -528,12 +1014,14 @@ export class CheckpointProposalJob implements Traceable {
528
1014
  buildDeadline: Date | undefined;
529
1015
  txHashesAlreadyIncluded: Set<string>;
530
1016
  },
531
- ): Promise<{ block: L2Block; usedTxs: Tx[] } | { error: Error } | undefined> {
1017
+ ): Promise<
1018
+ { block: L2Block; usedTxs: Tx[] } | { failure: 'insufficient-txs' | 'insufficient-valid-txs' } | { error: Error }
1019
+ > {
532
1020
  const { blockTimestamp, forceCreate, blockNumber, indexWithinCheckpoint, buildDeadline, txHashesAlreadyIncluded } =
533
1021
  opts;
534
1022
 
535
1023
  this.log.verbose(
536
- `Preparing block ${blockNumber} index ${indexWithinCheckpoint} at checkpoint ${this.checkpointNumber} for slot ${this.slot}`,
1024
+ `Preparing block ${blockNumber} index ${indexWithinCheckpoint} at checkpoint ${this.checkpointNumber} for slot ${this.targetSlot}`,
537
1025
  { ...checkpointBuilder.getConstantData(), ...opts },
538
1026
  );
539
1027
 
@@ -541,13 +1029,30 @@ export class CheckpointProposalJob implements Traceable {
541
1029
  // Wait until we have enough txs to build the block
542
1030
  const { availableTxs, canStartBuilding, minTxs } = await this.waitForMinTxs(opts);
543
1031
  if (!canStartBuilding) {
1032
+ this.logCheckpointEvent('block-build-failed', `Block build failed for slot ${this.targetSlot}`, {
1033
+ reason: 'insufficient_txs',
1034
+ blockNumber,
1035
+ slot: this.targetSlot,
1036
+ checkpointNumber: this.checkpointNumber,
1037
+ indexWithinCheckpoint,
1038
+ availableTxs,
1039
+ minTxs,
1040
+ });
544
1041
  this.log.warn(
545
- `Not enough txs to build block ${blockNumber} at index ${indexWithinCheckpoint} in slot ${this.slot} (got ${availableTxs} txs but needs ${minTxs})`,
546
- { blockNumber, slot: this.slot, indexWithinCheckpoint },
1042
+ `Not enough txs to build block ${blockNumber} at index ${indexWithinCheckpoint} in slot ${this.targetSlot} (got ${availableTxs} txs but needs ${minTxs})`,
1043
+ {
1044
+ reason: 'insufficient_txs',
1045
+ blockNumber,
1046
+ slot: this.targetSlot,
1047
+ checkpointNumber: this.checkpointNumber,
1048
+ indexWithinCheckpoint,
1049
+ availableTxs,
1050
+ minTxs,
1051
+ },
547
1052
  );
548
- this.eventEmitter.emit('block-tx-count-check-failed', { minTxs, availableTxs, slot: this.slot });
1053
+ this.eventEmitter.emit('block-tx-count-check-failed', { minTxs, availableTxs, slot: this.targetSlot });
549
1054
  this.metrics.recordBlockProposalFailed('insufficient_txs');
550
- return undefined;
1055
+ return { failure: 'insufficient-txs' };
551
1056
  }
552
1057
 
553
1058
  // Create iterator to pending txs. We filter out txs already included in previous blocks in the checkpoint
@@ -558,16 +1063,16 @@ export class CheckpointProposalJob implements Traceable {
558
1063
  );
559
1064
 
560
1065
  this.log.debug(
561
- `Building block ${blockNumber} at index ${indexWithinCheckpoint} for slot ${this.slot} with ${availableTxs} available txs`,
562
- { slot: this.slot, blockNumber, indexWithinCheckpoint },
1066
+ `Building block ${blockNumber} at index ${indexWithinCheckpoint} for slot ${this.targetSlot} with ${availableTxs} available txs`,
1067
+ { slot: this.targetSlot, blockNumber, indexWithinCheckpoint },
563
1068
  );
564
- this.setStateFn(SequencerState.CREATING_BLOCK, this.slot);
1069
+ this.setStateFn(SequencerState.CREATING_BLOCK, this.targetSlot, this.slotNow);
565
1070
 
566
- // Per-block limits derived at startup by computeBlockLimits(), further capped
1071
+ // Per-block limits are operator overrides (from SEQ_MAX_L2_BLOCK_GAS etc.) further capped
567
1072
  // by remaining checkpoint-level budgets inside CheckpointBuilder before each block is built.
568
1073
  // minValidTxs is passed into the builder so it can reject the block *before* updating state.
569
1074
  const minValidTxs = forceCreate ? 0 : (this.config.minValidTxsPerBlock ?? minTxs);
570
- const blockBuilderOptions: PublicProcessorLimits & { minValidTxs?: number } = {
1075
+ const blockBuilderOptions: BlockBuilderOptions = {
571
1076
  maxTransactions: this.config.maxTxsPerBlock,
572
1077
  maxBlockGas:
573
1078
  this.config.maxL2BlockGas !== undefined || this.config.maxDABlockGas !== undefined
@@ -576,6 +1081,8 @@ export class CheckpointProposalJob implements Traceable {
576
1081
  deadline: buildDeadline,
577
1082
  isBuildingProposal: true,
578
1083
  minValidTxs,
1084
+ maxBlocksPerCheckpoint: this.timetable.maxNumberOfBlocks,
1085
+ perBlockAllocationMultiplier: this.config.perBlockAllocationMultiplier,
579
1086
  };
580
1087
 
581
1088
  // Actually build the block by executing txs. The builder throws InsufficientValidTxsError
@@ -593,19 +1100,33 @@ export class CheckpointProposalJob implements Traceable {
593
1100
  await this.dropFailedTxsFromP2P(buildResult.failedTxs);
594
1101
 
595
1102
  if (buildResult.status === 'insufficient-valid-txs') {
1103
+ this.logCheckpointEvent('block-build-failed', `Block build failed for slot ${this.targetSlot}`, {
1104
+ reason: 'insufficient_valid_txs',
1105
+ slot: this.targetSlot,
1106
+ checkpointNumber: this.checkpointNumber,
1107
+ blockNumber,
1108
+ numTxs: buildResult.processedCount,
1109
+ indexWithinCheckpoint,
1110
+ minValidTxs,
1111
+ });
596
1112
  this.log.warn(
597
- `Block ${blockNumber} at index ${indexWithinCheckpoint} on slot ${this.slot} has too few valid txs to be proposed`,
1113
+ `Block ${blockNumber} at index ${indexWithinCheckpoint} on slot ${this.targetSlot} has too few valid txs to be proposed`,
598
1114
  {
599
- slot: this.slot,
1115
+ reason: 'insufficient_valid_txs',
1116
+ slot: this.targetSlot,
1117
+ checkpointNumber: this.checkpointNumber,
600
1118
  blockNumber,
601
1119
  numTxs: buildResult.processedCount,
602
1120
  indexWithinCheckpoint,
603
1121
  minValidTxs,
604
1122
  },
605
1123
  );
606
- this.eventEmitter.emit('block-build-failed', { reason: `Insufficient valid txs`, slot: this.slot });
1124
+ this.eventEmitter.emit('block-build-failed', {
1125
+ reason: `Insufficient valid txs`,
1126
+ slot: this.targetSlot,
1127
+ });
607
1128
  this.metrics.recordBlockProposalFailed('insufficient_valid_txs');
608
- return undefined;
1129
+ return { failure: 'insufficient-valid-txs' };
609
1130
  }
610
1131
 
611
1132
  // Block creation succeeded, emit stats and metrics
@@ -623,17 +1144,40 @@ export class CheckpointProposalJob implements Traceable {
623
1144
  const manaPerSec = block.header.totalManaUsed.toNumberUnsafe() / (blockBuildDuration / 1000);
624
1145
 
625
1146
  this.log.info(
626
- `Built block ${block.number} at checkpoint ${this.checkpointNumber} for slot ${this.slot} with ${numTxs} txs`,
1147
+ `Built block ${block.number} at checkpoint ${this.checkpointNumber} for slot ${this.targetSlot} with ${numTxs} txs`,
627
1148
  { blockHash, txHashes, manaPerSec, ...blockStats },
628
1149
  );
629
1150
 
630
- this.eventEmitter.emit('block-proposed', { blockNumber: block.number, slot: this.slot });
631
- this.metrics.recordBuiltBlock(blockBuildDuration, block.header.totalManaUsed.toNumberUnsafe());
1151
+ // `slot` is the target/submission slot (may be one ahead when pipelining),
1152
+ // `buildSlot` is the wall-clock slot during which the block was actually built.
1153
+ this.eventEmitter.emit('block-proposed', {
1154
+ blockNumber: block.number,
1155
+ blockHash,
1156
+ checkpointNumber: this.checkpointNumber,
1157
+ indexWithinCheckpoint: block.indexWithinCheckpoint,
1158
+ slot: this.targetSlot,
1159
+ buildSlot: this.slotNow,
1160
+ });
1161
+ this.metrics.recordBuiltBlock(blockBuildDuration, block.header.totalManaUsed.toNumberUnsafe(), this.targetSlot);
632
1162
 
633
1163
  return { block, usedTxs };
634
1164
  } catch (err: any) {
635
- this.eventEmitter.emit('block-build-failed', { reason: err.message, slot: this.slot });
636
- this.log.error(`Error building block`, err, { blockNumber, slot: this.slot });
1165
+ this.eventEmitter.emit('block-build-failed', {
1166
+ reason: err.message,
1167
+ slot: this.targetSlot,
1168
+ });
1169
+ this.logCheckpointEvent('block-build-failed', `Block build failed for slot ${this.targetSlot}`, {
1170
+ reason: err instanceof Error ? err.message : String(err),
1171
+ slot: this.targetSlot,
1172
+ checkpointNumber: this.checkpointNumber,
1173
+ blockNumber,
1174
+ });
1175
+ this.log.error(`Error building block`, err, {
1176
+ reason: err instanceof Error ? err.message : String(err),
1177
+ slot: this.targetSlot,
1178
+ checkpointNumber: this.checkpointNumber,
1179
+ blockNumber,
1180
+ });
637
1181
  this.metrics.recordBlockProposalFailed(err.name || 'unknown_error');
638
1182
  this.metrics.recordFailedBlock();
639
1183
  return { error: err };
@@ -646,7 +1190,7 @@ export class CheckpointProposalJob implements Traceable {
646
1190
  pendingTxs: AsyncIterable<Tx>,
647
1191
  blockNumber: BlockNumber,
648
1192
  blockTimestamp: bigint,
649
- blockBuilderOptions: PublicProcessorLimits & { minValidTxs?: number },
1193
+ blockBuilderOptions: BlockBuilderOptions,
650
1194
  ) {
651
1195
  try {
652
1196
  const workTimer = new Timer();
@@ -693,10 +1237,10 @@ export class CheckpointProposalJob implements Traceable {
693
1237
  }
694
1238
 
695
1239
  // Wait a bit before checking again
696
- this.setStateFn(SequencerState.WAITING_FOR_TXS, this.slot);
1240
+ this.setStateFn(SequencerState.WAITING_FOR_TXS, this.targetSlot, this.slotNow);
697
1241
  this.log.verbose(
698
- `Waiting for enough txs to build block ${blockNumber} at index ${indexWithinCheckpoint} in slot ${this.slot} (have ${availableTxs} but need ${minTxs})`,
699
- { blockNumber, slot: this.slot, indexWithinCheckpoint },
1242
+ `Waiting for enough txs to build block ${blockNumber} at index ${indexWithinCheckpoint} in slot ${this.targetSlot} (have ${availableTxs} but need ${minTxs})`,
1243
+ { blockNumber, slot: this.targetSlot, indexWithinCheckpoint },
700
1244
  );
701
1245
  await this.waitForTxsPollingInterval();
702
1246
  availableTxs = await this.p2pClient.getPendingTxCount();
@@ -705,15 +1249,47 @@ export class CheckpointProposalJob implements Traceable {
705
1249
  return { canStartBuilding: true, availableTxs, minTxs };
706
1250
  }
707
1251
 
1252
+ private async getSignedCommitteeAttestations(
1253
+ broadcast: CheckpointProposalBroadcast,
1254
+ ): Promise<{ attestations: CommitteeAttestationsAndSigners; attestationsSignature: Signature } | undefined> {
1255
+ const { proposal, blockProposedAt } = broadcast;
1256
+ this.setStateFn(SequencerState.COLLECTING_ATTESTATIONS, this.targetSlot, this.slotNow);
1257
+ const attestations = await this.waitForAttestations(proposal);
1258
+ if (!attestations) {
1259
+ return undefined;
1260
+ }
1261
+ this.checkpointMetrics.recordCheckpointAttestationDelay(this.dateProvider.now() - blockProposedAt);
1262
+
1263
+ // Proposer must sign over the attestations before pushing them to L1
1264
+ const signer = this.proposer ?? this.publisher.getSenderAddress();
1265
+ try {
1266
+ const attestationsSignature = await this.validatorClient.signAttestationsAndSigners(
1267
+ attestations,
1268
+ signer,
1269
+ this.targetSlot,
1270
+ this.checkpointNumber,
1271
+ );
1272
+ return { attestations, attestationsSignature };
1273
+ } catch (err) {
1274
+ if (this.handleHASigningError(err, 'Attestations signature')) {
1275
+ return;
1276
+ }
1277
+ this.log.error(`Error signing attestations for checkpoint proposal at slot ${proposal.slotNumber}`, err);
1278
+ return undefined;
1279
+ }
1280
+ }
1281
+
708
1282
  /**
709
1283
  * Waits for enough attestations to be collected via p2p.
710
1284
  * This is run after all blocks for the checkpoint have been built.
711
1285
  */
712
1286
  @trackSpan('CheckpointProposalJob.waitForAttestations')
713
- private async waitForAttestations(proposal: CheckpointProposal): Promise<CommitteeAttestationsAndSigners> {
1287
+ private async waitForAttestations(
1288
+ proposal: CheckpointProposal,
1289
+ ): Promise<CommitteeAttestationsAndSigners | undefined> {
714
1290
  if (this.config.fishermanMode) {
715
1291
  this.log.debug('Skipping attestation collection in fisherman mode');
716
- return CommitteeAttestationsAndSigners.empty();
1292
+ return CommitteeAttestationsAndSigners.empty(this.getSignatureContext());
717
1293
  }
718
1294
 
719
1295
  const slotNumber = proposal.slotNumber;
@@ -723,21 +1299,27 @@ export class CheckpointProposalJob implements Traceable {
723
1299
  throw new Error('No committee when collecting attestations');
724
1300
  } else if (committee.length === 0) {
725
1301
  this.log.verbose(`Attesting committee is empty`);
726
- return CommitteeAttestationsAndSigners.empty();
1302
+ return CommitteeAttestationsAndSigners.empty(this.getSignatureContext());
727
1303
  } else {
728
1304
  this.log.debug(`Attesting committee length is ${committee.length}`, { committee });
729
1305
  }
730
1306
 
731
- const numberOfRequiredAttestations = Math.floor((committee.length * 2) / 3) + 1;
1307
+ const numberOfRequiredAttestations = computeQuorum(committee.length);
732
1308
 
733
1309
  if (this.config.skipCollectingAttestations) {
734
1310
  this.log.warn('Skipping attestation collection as per config (attesting with own keys only)');
735
- const attestations = await this.validatorClient?.collectOwnAttestations(proposal);
736
- return new CommitteeAttestationsAndSigners(orderAttestations(attestations ?? [], committee));
1311
+ const attestations = await this.validatorClient?.collectOwnAttestations(proposal, this.checkpointNumber);
1312
+ this.logCheckpointAttestations('collected', committee, attestations ?? [], numberOfRequiredAttestations, {
1313
+ reason: 'collect_own_only',
1314
+ });
1315
+ return new CommitteeAttestationsAndSigners(
1316
+ orderAttestations(attestations ?? [], committee),
1317
+ this.getSignatureContext(),
1318
+ );
737
1319
  }
738
1320
 
739
1321
  const attestationTimeAllowed = this.config.enforceTimeTable
740
- ? this.timetable.getMaxAllowedTime(SequencerState.PUBLISHING_CHECKPOINT)!
1322
+ ? this.timetable.getCheckpointAttestationDeadline()
741
1323
  : this.l1Constants.slotDuration;
742
1324
  const attestationDeadline = new Date((this.getSlotStartBuildTimestamp() + attestationTimeAllowed) * 1000);
743
1325
 
@@ -750,6 +1332,7 @@ export class CheckpointProposalJob implements Traceable {
750
1332
  proposal,
751
1333
  numberOfRequiredAttestations,
752
1334
  attestationDeadline,
1335
+ this.checkpointNumber,
753
1336
  );
754
1337
 
755
1338
  collectedAttestationsCount = attestations.length;
@@ -768,6 +1351,9 @@ export class CheckpointProposalJob implements Traceable {
768
1351
 
769
1352
  // Rollup contract requires that the signatures are provided in the order of the committee
770
1353
  const sorted = orderAttestations(trimmed, committee);
1354
+ this.logCheckpointAttestations('collected', committee, attestations, numberOfRequiredAttestations, {
1355
+ submittedCount: trimmed.length,
1356
+ });
771
1357
 
772
1358
  // Manipulate the attestations if we've been configured to do so
773
1359
  if (
@@ -779,17 +1365,56 @@ export class CheckpointProposalJob implements Traceable {
779
1365
  return this.manipulateAttestations(proposal.slotNumber, epoch, seed, committee, sorted);
780
1366
  }
781
1367
 
782
- return new CommitteeAttestationsAndSigners(sorted);
1368
+ return new CommitteeAttestationsAndSigners(sorted, this.getSignatureContext());
783
1369
  } catch (err) {
784
1370
  if (err && err instanceof AttestationTimeoutError) {
785
1371
  collectedAttestationsCount = err.collectedCount;
1372
+ this.logCheckpointAttestations('failed', committee, undefined, numberOfRequiredAttestations, {
1373
+ collectedCount: collectedAttestationsCount,
1374
+ reason: 'timeout',
1375
+ });
1376
+ this.log.error(
1377
+ `Timeout while waiting for attestations for checkpoint proposal at slot ${proposal.slotNumber} (collected ${collectedAttestationsCount}/${numberOfRequiredAttestations})`,
1378
+ err,
1379
+ );
1380
+ } else {
1381
+ this.logCheckpointAttestations('failed', committee, undefined, numberOfRequiredAttestations, {
1382
+ collectedCount: collectedAttestationsCount,
1383
+ reason: err instanceof Error ? err.message : String(err),
1384
+ });
1385
+ this.log.error(`Error collecting attestations for checkpoint proposal at slot ${proposal.slotNumber}`, err);
786
1386
  }
787
- throw err;
1387
+ return undefined;
788
1388
  } finally {
789
1389
  this.metrics.recordCollectedAttestations(collectedAttestationsCount, collectAttestationsTimer.ms());
790
1390
  }
791
1391
  }
792
1392
 
1393
+ private logCheckpointAttestations(
1394
+ status: 'collected' | 'failed',
1395
+ committee: EthAddress[],
1396
+ attestations: CheckpointAttestation[] | undefined,
1397
+ requiredAttestations: number,
1398
+ opts: { collectedCount?: number; submittedCount?: number; reason?: string } = {},
1399
+ ) {
1400
+ const signedValidators =
1401
+ attestations
1402
+ ?.map(attestation => attestation.getSender()?.toString())
1403
+ .filter((address): address is `0x${string}` => address !== undefined) ?? [];
1404
+ const collectedCount = opts.collectedCount ?? new Set(signedValidators).size;
1405
+ const missingValidatorCount = status === 'failed' ? Math.max(0, requiredAttestations - collectedCount) : undefined;
1406
+ this.logCheckpointEvent(`attestations-${status}`, `Checkpoint attestations ${status} for slot ${this.targetSlot}`, {
1407
+ slot: this.targetSlot,
1408
+ checkpointNumber: this.checkpointNumber,
1409
+ committeeSize: committee.length,
1410
+ requiredAttestations,
1411
+ collectedAttestations: collectedCount,
1412
+ ...(opts.submittedCount !== undefined && { submittedAttestations: opts.submittedCount }),
1413
+ ...(missingValidatorCount !== undefined && { missingValidatorCount }),
1414
+ ...(opts.reason !== undefined && { reason: opts.reason }),
1415
+ });
1416
+ }
1417
+
793
1418
  /** Breaks the attestations before publishing based on attack configs */
794
1419
  private manipulateAttestations(
795
1420
  slotNumber: SlotNumber,
@@ -833,7 +1458,7 @@ export class CheckpointProposalJob implements Traceable {
833
1458
  unfreeze(attestations[targetIndex]).signature = generateRecoverableSignature();
834
1459
  }
835
1460
  }
836
- return new CommitteeAttestationsAndSigners(attestations);
1461
+ return new CommitteeAttestationsAndSigners(attestations, this.getSignatureContext());
837
1462
  }
838
1463
 
839
1464
  if (this.config.shuffleAttestationOrdering) {
@@ -855,11 +1480,11 @@ export class CheckpointProposalJob implements Traceable {
855
1480
  [shuffled[i], shuffled[j]] = [shuffled[j], shuffled[i]];
856
1481
  }
857
1482
 
858
- const signers = new CommitteeAttestationsAndSigners(attestations).getSigners();
859
- return new MaliciousCommitteeAttestationsAndSigners(shuffled, signers);
1483
+ const signers = new CommitteeAttestationsAndSigners(attestations, this.getSignatureContext()).getSigners();
1484
+ return new MaliciousCommitteeAttestationsAndSigners(shuffled, signers, this.getSignatureContext());
860
1485
  }
861
1486
 
862
- return new CommitteeAttestationsAndSigners(attestations);
1487
+ return new CommitteeAttestationsAndSigners(attestations, this.getSignatureContext());
863
1488
  }
864
1489
 
865
1490
  private async dropFailedTxsFromP2P(failedTxs: FailedTx[]) {
@@ -876,9 +1501,13 @@ export class CheckpointProposalJob implements Traceable {
876
1501
  * Adds the proposed block to the archiver so it's available via P2P.
877
1502
  * Gossip doesn't echo messages back to the sender, so the proposer's archiver/world-state
878
1503
  * would never receive its own block without this explicit sync.
1504
+ *
1505
+ * In fisherman mode we skip this push: the fisherman builds blocks locally for validation
1506
+ * and fee analysis only, and pushing them to the archiver causes spurious reorg cascades
1507
+ * whenever the real proposer's block arrives from L1.
879
1508
  */
880
1509
  private async syncProposedBlockToArchiver(block: L2Block): Promise<void> {
881
- if (this.config.skipPushProposedBlocksToArchiver !== false) {
1510
+ if (this.config.skipPushProposedBlocksToArchiver || this.config.fishermanMode) {
882
1511
  this.log.warn(`Skipping push of proposed block ${block.number} to archiver`, {
883
1512
  blockNumber: block.number,
884
1513
  slot: block.header.globalVariables.slotNumber,
@@ -892,23 +1521,58 @@ export class CheckpointProposalJob implements Traceable {
892
1521
  await this.blockSink.addBlock(block);
893
1522
  }
894
1523
 
1524
+ /**
1525
+ * Adds the proposed checkpoint to the archiver so the proposer's optimistic proposed-checkpoint
1526
+ * tip advances locally. Gossip doesn't echo our own messages back, so without this the proposer
1527
+ * would never see its own proposed checkpoint and couldn't pipeline the next slot.
1528
+ *
1529
+ * Skipped whenever proposed blocks aren't pushed (`skipPushProposedBlocksToArchiver`, fisherman
1530
+ * mode): the archiver derives the checkpoint archive from its stored blocks, so without them the
1531
+ * push would fail. All blocks were already added (and awaited) during block building, so this
1532
+ * needs no retry — they are guaranteed present by the time we get here.
1533
+ */
1534
+ private async syncProposedCheckpointToArchiver(
1535
+ checkpoint: Checkpoint,
1536
+ blockCount: number,
1537
+ feeAssetPriceModifier: bigint,
1538
+ ): Promise<void> {
1539
+ if (this.config.skipPushProposedBlocksToArchiver || this.config.fishermanMode) {
1540
+ return;
1541
+ }
1542
+ const startBlock = BlockNumber(this.syncedToBlockNumber + 1);
1543
+ this.log.debug(`Syncing proposed checkpoint ${this.checkpointNumber} to archiver`, {
1544
+ checkpointNumber: this.checkpointNumber,
1545
+ slot: this.targetSlot,
1546
+ startBlock,
1547
+ blockCount,
1548
+ });
1549
+ await this.blockSink.addProposedCheckpoint({
1550
+ header: checkpoint.header,
1551
+ checkpointNumber: this.checkpointNumber,
1552
+ startBlock,
1553
+ blockCount,
1554
+ totalManaUsed: checkpoint.header.totalManaUsed.toBigInt(),
1555
+ feeAssetPriceModifier,
1556
+ });
1557
+ }
1558
+
895
1559
  /** Runs fee analysis and logs checkpoint outcome as fisherman */
896
1560
  private async handleCheckpointEndAsFisherman(checkpoint: Checkpoint | undefined) {
897
1561
  // Perform L1 fee analysis before clearing requests
898
1562
  // The callback is invoked asynchronously after the next block is mined
899
- const feeAnalysis = await this.publisher.analyzeL1Fees(this.slot, analysis =>
1563
+ const feeAnalysis = await this.publisher.analyzeL1Fees(this.targetSlot, analysis =>
900
1564
  this.metrics.recordFishermanFeeAnalysis(analysis),
901
1565
  );
902
1566
 
903
1567
  if (checkpoint) {
904
- this.log.info(`Validation checkpoint building SUCCEEDED for slot ${this.slot}`, {
1568
+ this.log.info(`Validation checkpoint building SUCCEEDED for slot ${this.targetSlot}`, {
905
1569
  ...checkpoint.toCheckpointInfo(),
906
1570
  ...checkpoint.getStats(),
907
1571
  feeAnalysisId: feeAnalysis?.id,
908
1572
  });
909
1573
  } else {
910
- this.log.warn(`Validation block building FAILED for slot ${this.slot}`, {
911
- slot: this.slot,
1574
+ this.log.warn(`Validation block building FAILED for slot ${this.targetSlot}`, {
1575
+ slot: this.targetSlot,
912
1576
  feeAnalysisId: feeAnalysis?.id,
913
1577
  });
914
1578
  this.metrics.recordCheckpointProposalFailed('block_build_failed');
@@ -922,15 +1586,15 @@ export class CheckpointProposalJob implements Traceable {
922
1586
  */
923
1587
  private handleHASigningError(err: any, errorContext: string): boolean {
924
1588
  if (err instanceof DutyAlreadySignedError) {
925
- this.log.info(`${errorContext} for slot ${this.slot} already signed by another HA node, yielding`, {
926
- slot: this.slot,
1589
+ this.log.info(`${errorContext} for slot ${this.targetSlot} already signed by another HA node, yielding`, {
1590
+ slot: this.targetSlot,
927
1591
  signedByNode: err.signedByNode,
928
1592
  });
929
1593
  return true;
930
1594
  }
931
1595
  if (err instanceof SlashingProtectionError) {
932
- this.log.info(`${errorContext} for slot ${this.slot} blocked by slashing protection, yielding`, {
933
- slot: this.slot,
1596
+ this.log.info(`${errorContext} for slot ${this.targetSlot} blocked by slashing protection, yielding`, {
1597
+ slot: this.targetSlot,
934
1598
  existingMessageHash: err.existingMessageHash,
935
1599
  attemptedMessageHash: err.attemptedMessageHash,
936
1600
  });
@@ -944,16 +1608,17 @@ export class CheckpointProposalJob implements Traceable {
944
1608
  protected async waitUntilTimeInSlot(targetSecondsIntoSlot: number): Promise<void> {
945
1609
  const slotStartTimestamp = this.getSlotStartBuildTimestamp();
946
1610
  const targetTimestamp = slotStartTimestamp + targetSecondsIntoSlot;
947
- await sleepUntil(new Date(targetTimestamp * 1000), this.dateProvider.nowAsDate());
1611
+ const delayMs = targetTimestamp * 1000 - this.dateProvider.nowAsDate().getTime();
1612
+ await this.awaitInterruptibleSleep(delayMs);
948
1613
  }
949
1614
 
950
- /** Waits the polling interval for transactions. Extracted for test overriding. */
1615
+ /** Pause between mempool polls in waitForMinTxs. Extracted for test overriding. */
951
1616
  protected async waitForTxsPollingInterval(): Promise<void> {
952
- await sleep(TXS_POLLING_MS);
1617
+ await this.awaitInterruptibleSleep(TXS_POLLING_MS);
953
1618
  }
954
1619
 
955
1620
  private getSlotStartBuildTimestamp(): number {
956
- return getSlotStartBuildTimestamp(this.slot, this.l1Constants);
1621
+ return getSlotStartBuildTimestamp(this.slotNow, this.l1Constants);
957
1622
  }
958
1623
 
959
1624
  private getSecondsIntoSlot(): number {