@aztec/sequencer-client 0.0.1-commit.5de5ca79e → 0.0.1-commit.6201a7b05

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. package/dest/client/sequencer-client.d.ts +6 -1
  2. package/dest/client/sequencer-client.d.ts.map +1 -1
  3. package/dest/client/sequencer-client.js +11 -17
  4. package/dest/config.d.ts +1 -1
  5. package/dest/config.d.ts.map +1 -1
  6. package/dest/config.js +13 -12
  7. package/dest/global_variable_builder/fee_predictor.d.ts +37 -0
  8. package/dest/global_variable_builder/fee_predictor.d.ts.map +1 -0
  9. package/dest/global_variable_builder/fee_predictor.js +128 -0
  10. package/dest/global_variable_builder/fee_provider.d.ts +21 -0
  11. package/dest/global_variable_builder/fee_provider.d.ts.map +1 -0
  12. package/dest/global_variable_builder/fee_provider.js +58 -0
  13. package/dest/global_variable_builder/global_builder.d.ts +15 -14
  14. package/dest/global_variable_builder/global_builder.d.ts.map +1 -1
  15. package/dest/global_variable_builder/global_builder.js +16 -51
  16. package/dest/global_variable_builder/index.d.ts +4 -2
  17. package/dest/global_variable_builder/index.d.ts.map +1 -1
  18. package/dest/global_variable_builder/index.js +2 -0
  19. package/dest/publisher/config.d.ts +13 -1
  20. package/dest/publisher/config.d.ts.map +1 -1
  21. package/dest/publisher/config.js +19 -4
  22. package/dest/publisher/l1_tx_failed_store/failed_tx_store.d.ts +3 -4
  23. package/dest/publisher/l1_tx_failed_store/failed_tx_store.d.ts.map +1 -1
  24. package/dest/publisher/sequencer-publisher-factory.d.ts +3 -5
  25. package/dest/publisher/sequencer-publisher-factory.d.ts.map +1 -1
  26. package/dest/publisher/sequencer-publisher-factory.js +2 -3
  27. package/dest/publisher/sequencer-publisher.d.ts +48 -42
  28. package/dest/publisher/sequencer-publisher.d.ts.map +1 -1
  29. package/dest/publisher/sequencer-publisher.js +119 -111
  30. package/dest/sequencer/chain_state_overrides.d.ts +25 -0
  31. package/dest/sequencer/chain_state_overrides.d.ts.map +1 -0
  32. package/dest/sequencer/chain_state_overrides.js +39 -0
  33. package/dest/sequencer/checkpoint_proposal_job.d.ts +41 -11
  34. package/dest/sequencer/checkpoint_proposal_job.d.ts.map +1 -1
  35. package/dest/sequencer/checkpoint_proposal_job.js +525 -149
  36. package/dest/sequencer/checkpoint_proposal_job_metrics.d.ts +34 -0
  37. package/dest/sequencer/checkpoint_proposal_job_metrics.d.ts.map +1 -0
  38. package/dest/sequencer/checkpoint_proposal_job_metrics.js +72 -0
  39. package/dest/sequencer/checkpoint_voter.d.ts +1 -2
  40. package/dest/sequencer/checkpoint_voter.d.ts.map +1 -1
  41. package/dest/sequencer/checkpoint_voter.js +2 -5
  42. package/dest/sequencer/events.d.ts +6 -1
  43. package/dest/sequencer/events.d.ts.map +1 -1
  44. package/dest/sequencer/metrics.d.ts +9 -10
  45. package/dest/sequencer/metrics.d.ts.map +1 -1
  46. package/dest/sequencer/metrics.js +34 -20
  47. package/dest/sequencer/sequencer.d.ts +27 -7
  48. package/dest/sequencer/sequencer.d.ts.map +1 -1
  49. package/dest/sequencer/sequencer.js +106 -28
  50. package/dest/sequencer/timetable.d.ts +14 -1
  51. package/dest/sequencer/timetable.d.ts.map +1 -1
  52. package/dest/sequencer/timetable.js +45 -36
  53. package/dest/test/utils.d.ts +1 -1
  54. package/dest/test/utils.d.ts.map +1 -1
  55. package/dest/test/utils.js +7 -6
  56. package/package.json +27 -27
  57. package/src/client/sequencer-client.ts +16 -24
  58. package/src/config.ts +13 -11
  59. package/src/global_variable_builder/README.md +44 -0
  60. package/src/global_variable_builder/fee_predictor.ts +172 -0
  61. package/src/global_variable_builder/fee_provider.ts +75 -0
  62. package/src/global_variable_builder/global_builder.ts +26 -63
  63. package/src/global_variable_builder/index.ts +3 -1
  64. package/src/publisher/config.ts +38 -4
  65. package/src/publisher/l1_tx_failed_store/failed_tx_store.ts +3 -1
  66. package/src/publisher/sequencer-publisher-factory.ts +3 -6
  67. package/src/publisher/sequencer-publisher.ts +183 -159
  68. package/src/sequencer/README.md +82 -13
  69. package/src/sequencer/chain_state_overrides.ts +87 -0
  70. package/src/sequencer/checkpoint_proposal_job.ts +616 -167
  71. package/src/sequencer/checkpoint_proposal_job_metrics.ts +128 -0
  72. package/src/sequencer/checkpoint_voter.ts +1 -12
  73. package/src/sequencer/events.ts +5 -0
  74. package/src/sequencer/metrics.ts +43 -24
  75. package/src/sequencer/sequencer.ts +148 -32
  76. package/src/sequencer/timetable.ts +57 -45
  77. package/src/test/utils.ts +28 -10
@@ -1,4 +1,5 @@
1
1
  import type { EpochCache } from '@aztec/epoch-cache';
2
+ import type { SimulationOverridesPlan } from '@aztec/ethereum/contracts';
2
3
  import {
3
4
  BlockNumber,
4
5
  CheckpointNumber,
@@ -17,6 +18,7 @@ import { EthAddress } from '@aztec/foundation/eth-address';
17
18
  import { Signature } from '@aztec/foundation/eth-signature';
18
19
  import { filter } from '@aztec/foundation/iterator';
19
20
  import { type Logger, type LoggerBindings, createLogger } from '@aztec/foundation/log';
21
+ import { retryUntil } from '@aztec/foundation/retry';
20
22
  import { sleep, sleepUntil } from '@aztec/foundation/sleep';
21
23
  import { type DateProvider, Timer } from '@aztec/foundation/timer';
22
24
  import { type TypedEventEmitter, isErrorClass, unfreeze } from '@aztec/foundation/types';
@@ -29,9 +31,10 @@ import {
29
31
  type L2BlockSink,
30
32
  type L2BlockSource,
31
33
  MaliciousCommitteeAttestationsAndSigners,
34
+ type ValidateCheckpointResult,
32
35
  } from '@aztec/stdlib/block';
33
- import { type Checkpoint, validateCheckpoint } from '@aztec/stdlib/checkpoint';
34
- import { getSlotStartBuildTimestamp, getTimestampForSlot } from '@aztec/stdlib/epoch-helpers';
36
+ import { type Checkpoint, type ProposedCheckpointData, validateCheckpoint } from '@aztec/stdlib/checkpoint';
37
+ import { computeQuorum, getSlotStartBuildTimestamp, getTimestampForSlot } from '@aztec/stdlib/epoch-helpers';
35
38
  import { Gas } from '@aztec/stdlib/gas';
36
39
  import {
37
40
  type BlockBuilderOptions,
@@ -43,8 +46,10 @@ import { type L1ToL2MessageSource, computeInHashFromL1ToL2Messages } from '@azte
43
46
  import type {
44
47
  BlockProposal,
45
48
  BlockProposalOptions,
49
+ CheckpointAttestation,
46
50
  CheckpointProposal,
47
51
  CheckpointProposalOptions,
52
+ CoordinationSignatureContext,
48
53
  } from '@aztec/stdlib/p2p';
49
54
  import { orderAttestations, trimAttestations } from '@aztec/stdlib/p2p';
50
55
  import type { L2BlockBuiltStats } from '@aztec/stdlib/stats';
@@ -56,6 +61,11 @@ import { DutyAlreadySignedError, SlashingProtectionError } from '@aztec/validato
56
61
 
57
62
  import type { GlobalVariableBuilder } from '../global_variable_builder/global_builder.js';
58
63
  import type { InvalidateCheckpointRequest, SequencerPublisher } from '../publisher/sequencer-publisher.js';
64
+ import {
65
+ buildPipelinedParentSimulationOverridesPlan,
66
+ buildSubmissionSimulationOverridesPlan,
67
+ } from './chain_state_overrides.js';
68
+ import type { CheckpointProposalJobMetricsRecorder } from './checkpoint_proposal_job_metrics.js';
59
69
  import { CheckpointVoter } from './checkpoint_voter.js';
60
70
  import { SequencerInterruptedError } from './errors.js';
61
71
  import type { SequencerEvents } from './events.js';
@@ -67,6 +77,20 @@ import { SequencerState } from './utils.js';
67
77
  /** How much time to sleep while waiting for min transactions to accumulate for a block */
68
78
  const TXS_POLLING_MS = 500;
69
79
 
80
+ /** Result from proposeCheckpoint when a checkpoint was successfully built and broadcast. */
81
+ type CheckpointProposalBroadcast = {
82
+ checkpoint: Checkpoint;
83
+ proposal: CheckpointProposal;
84
+ blockProposedAt: number;
85
+ };
86
+
87
+ /** Result after attestation collection and signing, ready for L1 submission. */
88
+ type CheckpointProposalResult = {
89
+ checkpoint: Checkpoint;
90
+ attestations: CommitteeAttestationsAndSigners;
91
+ attestationsSignature: Signature;
92
+ };
93
+
70
94
  /**
71
95
  * Handles the execution of a checkpoint proposal after the initial preparation phase.
72
96
  * This includes building blocks, collecting attestations, and publishing the checkpoint to L1,
@@ -75,11 +99,21 @@ const TXS_POLLING_MS = 500;
75
99
  */
76
100
  export class CheckpointProposalJob implements Traceable {
77
101
  protected readonly log: Logger;
102
+ private readonly checkpointEventLog: Logger;
103
+
104
+ /** Tracks the fire-and-forget L1 submission promise so it can be awaited during shutdown. */
105
+ private pendingL1Submission: Promise<void> | undefined;
106
+
107
+ /** Pipelined parent chain state used while building and later submitting this checkpoint. */
108
+ private pipelinedParentSimulationOverridesPlan?: SimulationOverridesPlan;
109
+
110
+ private getSignatureContext(): CoordinationSignatureContext {
111
+ return this.signatureContext;
112
+ }
78
113
 
79
114
  constructor(
80
115
  private readonly slotNow: SlotNumber,
81
116
  private readonly targetSlot: SlotNumber,
82
- private readonly epochNow: EpochNumber,
83
117
  private readonly targetEpoch: EpochNumber,
84
118
  private readonly checkpointNumber: CheckpointNumber,
85
119
  private readonly syncedToBlockNumber: BlockNumber,
@@ -97,36 +131,49 @@ export class CheckpointProposalJob implements Traceable {
97
131
  private readonly checkpointsBuilder: FullNodeCheckpointsBuilder,
98
132
  private readonly blockSink: L2BlockSink,
99
133
  private readonly l1Constants: SequencerRollupConstants,
134
+ private readonly signatureContext: CoordinationSignatureContext,
100
135
  protected config: ResolvedSequencerConfig,
101
136
  protected timetable: SequencerTimetable,
102
137
  private readonly slasherClient: SlasherClientInterface | undefined,
103
138
  private readonly epochCache: EpochCache,
104
139
  private readonly dateProvider: DateProvider,
105
140
  private readonly metrics: SequencerMetrics,
106
- private readonly eventEmitter: TypedEventEmitter<SequencerEvents>,
141
+ private readonly checkpointMetrics: CheckpointProposalJobMetricsRecorder,
142
+ protected readonly eventEmitter: TypedEventEmitter<SequencerEvents>,
107
143
  private readonly setStateFn: (state: SequencerState, slot?: SlotNumber) => void,
108
144
  public readonly tracer: Tracer,
109
145
  bindings?: LoggerBindings,
146
+ private readonly proposedCheckpointData?: ProposedCheckpointData,
110
147
  ) {
111
148
  this.log = createLogger('sequencer:checkpoint-proposal', {
112
149
  ...bindings,
113
150
  instanceId: `slot-${this.slotNow}`,
114
151
  });
152
+ this.checkpointEventLog = createLogger('sequencer:checkpoint-events', {
153
+ ...bindings,
154
+ instanceId: `slot-${this.slotNow}`,
155
+ });
115
156
  }
116
157
 
117
- /** The wall-clock slot during which the proposer builds. */
118
- private get slot(): SlotNumber {
119
- return this.slotNow;
158
+ /** Awaits the pending L1 submission if one is in progress. Call during shutdown. */
159
+ public async awaitPendingSubmission(): Promise<void> {
160
+ this.log.info('Awaiting pending L1 payload submission');
161
+ await this.pendingL1Submission;
120
162
  }
121
163
 
122
- /** The wall-clock epoch. */
123
- private get epoch(): EpochNumber {
124
- return this.epochNow;
164
+ private logCheckpointEvent(eventName: string, message: string, fields: Record<string, unknown>): void {
165
+ this.checkpointEventLog.debug(message, {
166
+ eventName: `sequencer-checkpoint-${eventName}`,
167
+ ...fields,
168
+ });
125
169
  }
126
170
 
127
171
  /**
128
172
  * Executes the checkpoint proposal job.
129
- * Returns the published checkpoint if successful, undefined otherwise.
173
+ * Builds blocks, assembles checkpoint, and broadcasts the proposal (blocking).
174
+ * Attestation collection, signing, and L1 submission are backgrounded so the
175
+ * work loop can return to IDLE immediately for consecutive slot proposals.
176
+ * Returns the built checkpoint if successful, undefined otherwise.
130
177
  */
131
178
  @trackSpan('CheckpointProposalJob.execute')
132
179
  public async execute(): Promise<Checkpoint | undefined> {
@@ -145,56 +192,304 @@ export class CheckpointProposalJob implements Traceable {
145
192
  this.log,
146
193
  ).enqueueVotes();
147
194
 
148
- // Build and propose the checkpoint. This will enqueue the request on the publisher if a checkpoint is built.
149
- const checkpoint = await this.proposeCheckpoint();
195
+ // Build blocks, assemble checkpoint, and broadcast proposal (BLOCKING).
196
+ // Returns after broadcast — attestation collection is deferred.
197
+ const broadcast = await this.proposeCheckpoint();
150
198
 
151
- // Wait until the voting promises have resolved, so all requests are enqueued (not sent)
152
- await Promise.all(votesPromises);
153
-
154
- if (checkpoint) {
155
- this.metrics.recordCheckpointProposalSuccess();
199
+ if (!broadcast) {
200
+ await Promise.all(votesPromises);
201
+ // Still submit votes even without a checkpoint
202
+ if (!this.config.fishermanMode) {
203
+ this.pendingL1Submission = this.publisher.sendRequestsAt(this.dateProvider.nowAsDate()).then(() => {});
204
+ }
205
+ return undefined;
156
206
  }
157
207
 
208
+ const { checkpoint } = broadcast;
209
+ this.metrics.recordCheckpointProposalSuccess();
210
+
158
211
  // Do not post anything to L1 if we are fishermen, but do perform L1 fee analysis
159
212
  if (this.config.fishermanMode) {
160
213
  await this.handleCheckpointEndAsFisherman(checkpoint);
161
- return;
214
+ return checkpoint;
162
215
  }
163
216
 
164
- // If pipelining, wait until the submission slot so L1 recognizes the pipelined proposer
165
- if (this.epochCache.isProposerPipeliningEnabled()) {
166
- const submissionSlotTimestamp =
167
- getTimestampForSlot(this.targetSlot, this.l1Constants) - BigInt(this.l1Constants.ethereumSlotDuration);
168
- this.log.info(`Waiting until submission slot ${this.targetSlot} for L1 submission`, {
169
- slot: this.slot,
170
- submissionSlot: this.targetSlot,
171
- submissionSlotTimestamp,
217
+ // Background the attestation signing L1 pipeline so the work loop is unblocked
218
+ this.pendingL1Submission = this.waitForAttestationsAndEnqueueSubmissionAsync(broadcast, votesPromises);
219
+
220
+ // Return the built checkpoint immediately — the work loop is now unblocked
221
+ return checkpoint;
222
+ }
223
+
224
+ /**
225
+ * Background pipeline: collects attestations, signs them, enqueues the checkpoint, and submits to L1.
226
+ * Runs as a fire-and-forget task stored in `pendingL1Submission` so the work loop is unblocked.
227
+ */
228
+ private async waitForAttestationsAndEnqueueSubmissionAsync(
229
+ broadcast: CheckpointProposalBroadcast,
230
+ votesPromises: Promise<unknown>[],
231
+ ): Promise<void> {
232
+ const { checkpoint } = broadcast;
233
+ const isPipelining = this.epochCache.isProposerPipeliningEnabled();
234
+
235
+ try {
236
+ // Wait for all votes actions, enqueued at the beginning, to resolve
237
+ await Promise.all(votesPromises);
238
+
239
+ // Try to collect attestations from the committee
240
+ const signedAttestations = await this.getSignedCommitteeAttestations(broadcast);
241
+
242
+ // If pipelining, wait for the previous checkpoint to land on L1 before submitting,
243
+ // so we can check it matches the proposed checkpoint we used as parent, and has valid attestations.
244
+ if (signedAttestations && (!isPipelining || (await this.waitForValidParentCheckpointOnL1()))) {
245
+ await this.enqueueCheckpointForSubmission({ checkpoint, ...signedAttestations });
246
+ }
247
+
248
+ // If we failed to collect attestations, at least check if we need to issue an invalidation
249
+ // Note that if we are not pipelining, we enqueued the invalidation at the beginning
250
+ if (!signedAttestations && isPipelining && (await this.waitForSyncedL2SlotNumber(this.slotNow))) {
251
+ const validationStatus = await this.l2BlockSource.getPendingChainValidationStatus();
252
+ if (!validationStatus.valid) {
253
+ this.log.warn(
254
+ `Checkpoint ${validationStatus.checkpoint.checkpointNumber} has invalid attestations, enqueuing invalidation in spite of attestation collection failure`,
255
+ { checkpoint: validationStatus.checkpoint, reason: validationStatus.reason },
256
+ );
257
+ await this.enqueueInvalidation(validationStatus);
258
+ }
259
+ }
260
+
261
+ // Send whatever was enqueued: votes + (propose | invalidation | nothing).
262
+ // Compute the earliest time to submit: pipeline slot start when pipelining, now otherwise.
263
+ const submitAfter = isPipelining
264
+ ? new Date(Number(getTimestampForSlot(this.targetSlot, this.l1Constants)) * 1000)
265
+ : new Date(this.dateProvider.now());
266
+
267
+ const l1Response = await this.publisher.sendRequestsAt(submitAfter);
268
+ const proposedAction = l1Response?.successfulActions.find(a => a === 'propose');
269
+ if (proposedAction) {
270
+ this.logCheckpointEvent('published', `Checkpoint published for slot ${this.targetSlot}`, {
271
+ slot: this.targetSlot,
272
+ checkpointNumber: this.checkpointNumber,
273
+ successfulActions: l1Response?.successfulActions,
274
+ sentActions: l1Response?.sentActions,
275
+ });
276
+ this.eventEmitter.emit('checkpoint-published', { checkpoint: this.checkpointNumber, slot: this.targetSlot });
277
+ const coinbase = checkpoint.header.coinbase;
278
+ await this.metrics.incFilledSlot(this.publisher.getSenderAddress().toString(), coinbase);
279
+ } else {
280
+ this.logCheckpointEvent('publish-failed', `Checkpoint publish failed for slot ${this.targetSlot}`, {
281
+ slot: this.targetSlot,
282
+ checkpointNumber: this.checkpointNumber,
283
+ successfulActions: l1Response?.successfulActions,
284
+ failedActions: l1Response?.failedActions,
285
+ sentActions: l1Response?.sentActions,
286
+ expiredActions: l1Response?.expiredActions,
287
+ reason: 'propose_action_not_successful',
288
+ });
289
+ this.log.warn(`Checkpoint publish failed for slot ${this.targetSlot}`, {
290
+ slot: this.targetSlot,
291
+ checkpointNumber: this.checkpointNumber,
292
+ successfulActions: l1Response?.successfulActions,
293
+ failedActions: l1Response?.failedActions,
294
+ sentActions: l1Response?.sentActions,
295
+ expiredActions: l1Response?.expiredActions,
296
+ reason: 'propose_action_not_successful',
297
+ });
298
+ this.eventEmitter.emit('checkpoint-publish-failed', { ...l1Response, slot: this.targetSlot });
299
+ if (isPipelining) {
300
+ this.metrics.recordPipelineDiscard();
301
+ }
302
+ }
303
+ } catch (err) {
304
+ if (err instanceof SequencerInterruptedError) {
305
+ return;
306
+ }
307
+ this.logCheckpointEvent('publish-failed', `Checkpoint publish failed for slot ${this.targetSlot}`, {
308
+ slot: this.targetSlot,
309
+ checkpointNumber: this.checkpointNumber,
310
+ reason: err instanceof Error ? err.message : String(err),
311
+ });
312
+ this.log.error(`Background attestation/L1 pipeline failed for slot ${this.targetSlot}`, err, {
313
+ slot: this.targetSlot,
314
+ checkpointNumber: this.checkpointNumber,
315
+ reason: err instanceof Error ? err.message : String(err),
172
316
  });
173
- await sleepUntil(new Date(Number(submissionSlotTimestamp) * 1000), this.dateProvider.nowAsDate());
317
+ this.eventEmitter.emit('checkpoint-publish-failed', { slot: this.targetSlot });
318
+ if (isPipelining) {
319
+ this.metrics.recordPipelineDiscard();
320
+ }
321
+ }
322
+ }
323
+
324
+ /** Enqueues the checkpoint for L1 submission. Called after pipeline sleep in execute(). */
325
+ private async enqueueCheckpointForSubmission(result: CheckpointProposalResult): Promise<void> {
326
+ const { checkpoint, attestations, attestationsSignature } = result;
327
+
328
+ this.setStateFn(SequencerState.PUBLISHING_CHECKPOINT, this.targetSlot);
329
+ const aztecSlotDuration = this.l1Constants.slotDuration;
330
+ const submissionSlotStart = Number(getTimestampForSlot(this.targetSlot, this.l1Constants));
331
+ const txTimeoutAt = new Date((submissionSlotStart + aztecSlotDuration) * 1000);
332
+
333
+ // If we have been configured to potentially skip publishing checkpoint then roll the dice here
334
+ if (
335
+ this.config.skipPublishingCheckpointsPercent !== undefined &&
336
+ this.config.skipPublishingCheckpointsPercent > 0
337
+ ) {
338
+ const roll = Math.max(0, randomInt(100));
339
+ if (roll < this.config.skipPublishingCheckpointsPercent) {
340
+ this.log.warn(
341
+ `Skipping publishing proposal for checkpoint ${checkpoint.number}. Configured percentage: ${this.config.skipPublishingCheckpointsPercent}, generated value: ${roll}`,
342
+ );
343
+ return;
344
+ }
345
+ }
346
+
347
+ const isPipelining = this.epochCache.isProposerPipeliningEnabled();
348
+ const submissionSimulationOverridesPlan = buildSubmissionSimulationOverridesPlan({
349
+ pipelinedParentPlan: this.pipelinedParentSimulationOverridesPlan,
350
+ invalidateToPendingCheckpointNumber: this.invalidateCheckpoint?.forcePendingCheckpointNumber,
351
+ lastArchiveRoot: checkpoint.header.lastArchiveRoot,
352
+ pipeliningEnabled: isPipelining,
353
+ });
354
+
355
+ await this.publisher.enqueueProposeCheckpoint(checkpoint, attestations, attestationsSignature, {
356
+ txTimeoutAt,
357
+ ...(submissionSimulationOverridesPlan ? { simulationOverridesPlan: submissionSimulationOverridesPlan } : {}),
358
+ });
359
+ }
360
+
361
+ /**
362
+ * Wait until the archiver syncs past the given L2 slot number.
363
+ * The deadline is the end of `this.targetSlot`, beyond which any pipelined work would miss its
364
+ * L1 submission window and is no longer useful.
365
+ */
366
+ private async waitForSyncedL2SlotNumber(waitForSlot: SlotNumber): Promise<boolean> {
367
+ const targetSlotStart = Number(getTimestampForSlot(this.targetSlot, this.l1Constants));
368
+ const targetSlotEndMs = (targetSlotStart + this.l1Constants.slotDuration) * 1000;
369
+ const syncDelayTolerance = this.l1Constants.ethereumSlotDuration * 2 * 1000;
370
+ const timeoutSeconds = Math.max(0.1, (targetSlotEndMs + syncDelayTolerance - this.dateProvider.now()) / 1000);
371
+
372
+ try {
373
+ return await retryUntil(
374
+ async () => {
375
+ const syncedSlot = await this.l2BlockSource.getSyncedL2SlotNumber();
376
+ return syncedSlot !== undefined && syncedSlot >= waitForSlot;
377
+ },
378
+ `archiver sync past slot ${waitForSlot}`,
379
+ timeoutSeconds,
380
+ 0.2,
381
+ );
382
+ } catch {
383
+ this.log.warn(
384
+ `Archiver did not sync L1 past slot ${waitForSlot} before slot ${this.targetSlot} expired, discarding pipelined work`,
385
+ { checkpointNumber: this.checkpointNumber },
386
+ );
387
+ this.emitPipelinedCheckpointDiscarded('archiver-sync-timeout');
388
+ return false;
389
+ }
390
+ }
391
+
392
+ /**
393
+ * Waits for the parent checkpoint to land on L1 before submitting a pipelined checkpoint.
394
+ * Polls until the archiver has synced L1 past the parent's slot, then verifies:
395
+ * - If we built on a proposed parent: it must have landed on L1 with matching hash and valid attestations.
396
+ * - If we built without a proposed parent: no new checkpoint must have appeared for that slot.
397
+ * If the parent has invalid attestations, enqueues an invalidation. Returns whether to proceed with the proposal.
398
+ */
399
+ protected async waitForValidParentCheckpointOnL1(): Promise<boolean> {
400
+ const parentCheckpointNumber = CheckpointNumber(this.checkpointNumber - 1);
401
+
402
+ // Wait until archiver has synced L1 past the parent's slot (slotNow)
403
+ if (!(await this.waitForSyncedL2SlotNumber(this.slotNow))) {
404
+ return false;
405
+ }
174
406
 
175
- // After waking, verify the parent checkpoint wasn't pruned during the sleep.
176
- // We check L1's pending tip directly instead of canProposeAt, which also validates the proposer
177
- // identity and would fail because the timestamp resolves to a different slot's proposer.
178
- const l1Tips = await this.publisher.rollupContract.getTips();
179
- if (l1Tips.pending < this.checkpointNumber - 1) {
407
+ const tips = await this.l2BlockSource.getL2Tips();
408
+ const checkpointedNumber = tips.checkpointed.checkpoint.number;
409
+
410
+ // We built on top of a proposed checkpoint. Verify it landed on L1 as expected.
411
+ if (this.proposedCheckpointData) {
412
+ // After syncing from L1 we see the chain tip has invalid attestations. This means the parent checkpoint was posted
413
+ // with invalid attestations, or it built on top of something with invalid attestations and didnt invalidate them.
414
+ // Either way, we thought our parent would be valid, so we have to throw away our work. But at least we'll try and
415
+ // invalidate on L1 so we clean up the chain for the next proposer. And we'll slash them, but that's handled elsewhere.
416
+ const validationStatus = await this.l2BlockSource.getPendingChainValidationStatus();
417
+ if (!validationStatus.valid) {
180
418
  this.log.warn(
181
- `Parent checkpoint was pruned during pipelining sleep (L1 pending=${l1Tips.pending}, expected>=${this.checkpointNumber - 1}), skipping L1 submission for checkpoint ${this.checkpointNumber}`,
419
+ `Parent checkpoint ${parentCheckpointNumber} has invalid attestations, discarding pipelined work`,
420
+ { checkpointNumber: this.checkpointNumber, reason: validationStatus.reason },
182
421
  );
183
- return undefined;
422
+ this.emitPipelinedCheckpointDiscarded('parent-invalid-attestations');
423
+ await this.enqueueInvalidation(validationStatus);
424
+ return false;
425
+ }
426
+
427
+ // The pending chain is valid. But did the parent checkpoint land on L1 at all?
428
+ if (checkpointedNumber < parentCheckpointNumber) {
429
+ this.log.warn(`Parent checkpoint ${parentCheckpointNumber} did not land on L1, discarding pipelined work`, {
430
+ checkpointNumber: this.checkpointNumber,
431
+ checkpointedNumber,
432
+ });
433
+ this.emitPipelinedCheckpointDiscarded('parent-not-on-l1');
434
+ return false;
184
435
  }
436
+
437
+ // It landed. But is it the one we were expecting?
438
+ const expectedHash = this.proposedCheckpointData.header.hash().toString();
439
+ if (tips.checkpointed.checkpoint.hash !== expectedHash) {
440
+ this.log.warn(`Parent checkpoint ${parentCheckpointNumber} hash mismatch on L1, discarding pipelined work`, {
441
+ checkpointNumber: this.checkpointNumber,
442
+ expectedHash,
443
+ actualHash: tips.checkpointed.checkpoint.hash,
444
+ });
445
+ this.emitPipelinedCheckpointDiscarded('parent-hash-mismatch');
446
+ return false;
447
+ }
448
+
449
+ return true;
450
+ } else {
451
+ // We didn't see a proposed checkpoint at build time, so we built on checkpointed parent from two slots ago.
452
+ // But if a new checkpoint for the previous slot appeared on L1 in the meantime, our checkpoint assumed the wrong parent,
453
+ // so we have to discard our work. This can happen if we're somehow cut off from p2p and fail to see the checkpoint
454
+ // proposal for the previous slot.
455
+ if (checkpointedNumber > parentCheckpointNumber) {
456
+ this.log.warn(
457
+ `Unexpected checkpoint ${checkpointedNumber} landed on L1 after we built on top of parent ${parentCheckpointNumber}, discarding pipelined work`,
458
+ { checkpointNumber: this.checkpointNumber, checkpointedNumber },
459
+ );
460
+ this.emitPipelinedCheckpointDiscarded('unexpected-parent-appeared');
461
+ return false;
462
+ }
463
+
464
+ return true;
185
465
  }
466
+ }
186
467
 
187
- // Then send everything to L1
188
- const l1Response = await this.publisher.sendRequests();
189
- const proposedAction = l1Response?.successfulActions.find(a => a === 'propose');
190
- if (proposedAction) {
191
- this.eventEmitter.emit('checkpoint-published', { checkpoint: this.checkpointNumber, slot: this.slot });
192
- const coinbase = checkpoint?.header.coinbase;
193
- await this.metrics.incFilledSlot(this.publisher.getSenderAddress().toString(), coinbase);
194
- return checkpoint;
195
- } else if (checkpoint) {
196
- this.eventEmitter.emit('checkpoint-publish-failed', { ...l1Response, slot: this.slot });
197
- return undefined;
468
+ /** Emits the pipelined-checkpoint-discarded event and records the metric. */
469
+ private emitPipelinedCheckpointDiscarded(reason: string): void {
470
+ this.metrics.recordPipelineParentCheckpointMismatch(reason);
471
+ this.eventEmitter.emit('pipelined-checkpoint-discarded', {
472
+ slot: this.targetSlot,
473
+ checkpointNumber: this.checkpointNumber,
474
+ reason,
475
+ });
476
+ }
477
+
478
+ /** Simulates and enqueues an invalidation request for the invalid parent checkpoint. */
479
+ private async enqueueInvalidation(validationStatus: ValidateCheckpointResult): Promise<void> {
480
+ if (this.config.skipInvalidateBlockAsProposer) {
481
+ this.log.warn(`Skipping checkpoint invalidation as proposer due to test configuration`);
482
+ return;
483
+ }
484
+ const invalidateRequest = await this.publisher.simulateInvalidateCheckpoint(validationStatus);
485
+ if (invalidateRequest) {
486
+ const submissionSlotStart = Number(getTimestampForSlot(this.targetSlot, this.l1Constants));
487
+ const txTimeoutAt = new Date((submissionSlotStart + this.l1Constants.slotDuration) * 1000);
488
+ this.publisher.enqueueInvalidateCheckpoint(invalidateRequest, { txTimeoutAt });
489
+ } else {
490
+ this.log.info(`Invalidation simulation returned undefined, checkpoint may have been removed already`, {
491
+ checkpointNumber: this.checkpointNumber,
492
+ });
198
493
  }
199
494
  }
200
495
 
@@ -205,19 +500,33 @@ export class CheckpointProposalJob implements Traceable {
205
500
  [Attributes.SLOT_NUMBER]: this.targetSlot,
206
501
  };
207
502
  })
208
- private async proposeCheckpoint(): Promise<Checkpoint | undefined> {
503
+ private async proposeCheckpoint(): Promise<CheckpointProposalBroadcast | undefined> {
209
504
  try {
505
+ const now = this.dateProvider.now();
506
+ if (this.epochCache.isProposerPipeliningEnabled() && this.proposedCheckpointData) {
507
+ // Measure against the wall-clock slot whose build window we are currently using.
508
+ // In pipelining mode `targetSlot` is intentionally one slot ahead, which makes the
509
+ // target-slot boundary a full slot away from the actual build start time.
510
+ const slotBoundaryMs = Number(getTimestampForSlot(this.slotNow, this.l1Constants)) * 1000;
511
+ this.checkpointMetrics.recordPipelinedCheckpointBuildStartOffsetFromSlotBoundary(now - slotBoundaryMs);
512
+ }
513
+ this.checkpointMetrics.startCheckpointTiming(now);
514
+
210
515
  // Get operator configured coinbase and fee recipient for this attestor
211
516
  const coinbase = this.validatorClient.getCoinbaseForAttestor(this.attestorAddress);
212
517
  const feeRecipient = this.validatorClient.getFeeRecipientForAttestor(this.attestorAddress);
213
518
 
214
519
  // Start the checkpoint
215
520
  this.setStateFn(SequencerState.INITIALIZING_CHECKPOINT, this.targetSlot);
216
- this.log.info(`Starting checkpoint proposal`, {
217
- buildSlot: this.slot,
521
+ this.logCheckpointEvent('slot-started', `Starting checkpoint proposal for slot ${this.targetSlot}`, {
522
+ buildSlot: this.slotNow,
218
523
  submissionSlot: this.targetSlot,
524
+ slot: this.targetSlot,
525
+ checkpointNumber: this.checkpointNumber,
219
526
  pipelining: this.epochCache.isProposerPipeliningEnabled(),
220
527
  proposer: this.proposer?.toString(),
528
+ attestorAddress: this.attestorAddress.toString(),
529
+ publisherAddress: this.publisher.getSenderAddress().toString(),
221
530
  coinbase: coinbase.toString(),
222
531
  });
223
532
  this.metrics.incOpenSlot(this.targetSlot, this.proposer?.toString() ?? 'unknown');
@@ -227,11 +536,24 @@ export class CheckpointProposalJob implements Traceable {
227
536
  this.publisher.enqueueInvalidateCheckpoint(this.invalidateCheckpoint);
228
537
  }
229
538
 
230
- // Create checkpoint builder for the slot
539
+ // Create checkpoint builder for the slot.
540
+ // When pipelining, force the proposed checkpoint number and fee header to our parent so the
541
+ // fee computation sees the same chain tip that L1 will see once the previous pipelined checkpoint lands.
542
+ const isPipelining = this.epochCache.isProposerPipeliningEnabled();
543
+ this.pipelinedParentSimulationOverridesPlan = isPipelining
544
+ ? await buildPipelinedParentSimulationOverridesPlan({
545
+ checkpointNumber: this.checkpointNumber,
546
+ proposedCheckpointData: this.proposedCheckpointData,
547
+ rollup: this.publisher.rollupContract,
548
+ log: this.log,
549
+ })
550
+ : undefined;
551
+
231
552
  const checkpointGlobalVariables = await this.globalsBuilder.buildCheckpointGlobalVariables(
232
553
  coinbase,
233
554
  feeRecipient,
234
555
  this.targetSlot,
556
+ this.pipelinedParentSimulationOverridesPlan,
235
557
  );
236
558
 
237
559
  // Collect L1 to L2 messages for the checkpoint and compute their hash
@@ -272,7 +594,7 @@ export class CheckpointProposalJob implements Traceable {
272
594
  };
273
595
 
274
596
  let blocksInCheckpoint: L2Block[] = [];
275
- let blockPendingBroadcast: { block: L2Block; txs: Tx[] } | undefined = undefined;
597
+ let blockPendingBroadcast: BlockProposal | undefined = undefined;
276
598
  const checkpointBuildTimer = new Timer();
277
599
 
278
600
  try {
@@ -296,16 +618,38 @@ export class CheckpointProposalJob implements Traceable {
296
618
  }
297
619
 
298
620
  if (blocksInCheckpoint.length === 0) {
299
- this.log.warn(`No blocks were built for slot ${this.targetSlot}`, { slot: this.targetSlot });
621
+ this.logCheckpointEvent('build-failed', `Checkpoint build failed for slot ${this.targetSlot}`, {
622
+ slot: this.targetSlot,
623
+ checkpointNumber: this.checkpointNumber,
624
+ reason: 'no_blocks_built',
625
+ });
626
+ this.log.warn(`No blocks were built for slot ${this.targetSlot}`, {
627
+ slot: this.targetSlot,
628
+ checkpointNumber: this.checkpointNumber,
629
+ reason: 'no_blocks_built',
630
+ });
300
631
  this.eventEmitter.emit('checkpoint-empty', { slot: this.targetSlot });
301
632
  return undefined;
302
633
  }
303
634
 
304
635
  const minBlocksForCheckpoint = this.config.minBlocksForCheckpoint;
305
636
  if (minBlocksForCheckpoint !== undefined && blocksInCheckpoint.length < minBlocksForCheckpoint) {
637
+ this.logCheckpointEvent('build-failed', `Checkpoint build failed for slot ${this.targetSlot}`, {
638
+ slot: this.targetSlot,
639
+ checkpointNumber: this.checkpointNumber,
640
+ blocksBuilt: blocksInCheckpoint.length,
641
+ minBlocksForCheckpoint,
642
+ reason: 'min_blocks_not_met',
643
+ });
306
644
  this.log.warn(
307
645
  `Checkpoint has fewer blocks than minimum (${blocksInCheckpoint.length} < ${minBlocksForCheckpoint}), skipping proposal`,
308
- { slot: this.targetSlot, blocksBuilt: blocksInCheckpoint.length, minBlocksForCheckpoint },
646
+ {
647
+ slot: this.targetSlot,
648
+ checkpointNumber: this.checkpointNumber,
649
+ blocksBuilt: blocksInCheckpoint.length,
650
+ minBlocksForCheckpoint,
651
+ reason: 'min_blocks_not_met',
652
+ },
309
653
  );
310
654
  return undefined;
311
655
  }
@@ -326,21 +670,43 @@ export class CheckpointProposalJob implements Traceable {
326
670
  maxTxsPerCheckpoint: this.config.maxTxsPerCheckpoint,
327
671
  });
328
672
  } catch (err) {
329
- this.log.error(`Built an invalid checkpoint at slot ${this.slot} (skipping proposal)`, err, {
673
+ this.logCheckpointEvent('build-failed', `Checkpoint build failed for slot ${this.targetSlot}`, {
674
+ slot: this.targetSlot,
675
+ checkpointNumber: this.checkpointNumber,
676
+ blocksBuilt: blocksInCheckpoint.length,
677
+ reason: 'invalid_checkpoint',
678
+ checkpoint: checkpoint.header.toInspect(),
679
+ });
680
+ this.log.error(`Built an invalid checkpoint at slot ${this.slotNow} (skipping proposal)`, err, {
681
+ slot: this.targetSlot,
682
+ checkpointNumber: this.checkpointNumber,
683
+ blocksBuilt: blocksInCheckpoint.length,
684
+ reason: 'invalid_checkpoint',
330
685
  checkpoint: checkpoint.header.toInspect(),
331
686
  });
332
687
  return undefined;
333
688
  }
334
689
 
335
690
  // Record checkpoint-level build metrics
336
- this.metrics.recordCheckpointBuild(
691
+ this.checkpointMetrics.recordCheckpointBuild(
337
692
  checkpointBuildTimer.ms(),
338
693
  blocksInCheckpoint.length,
339
694
  checkpoint.getStats().txCount,
340
695
  Number(checkpoint.header.totalManaUsed.toBigInt()),
341
696
  );
697
+ this.logCheckpointEvent('built', `Checkpoint built for slot ${this.targetSlot}`, {
698
+ slot: this.targetSlot,
699
+ buildSlot: this.slotNow,
700
+ checkpointNumber: this.checkpointNumber,
701
+ proposer: this.proposer?.toString(),
702
+ attestorAddress: this.attestorAddress.toString(),
703
+ publisherAddress: this.publisher.getSenderAddress().toString(),
704
+ blocksBuilt: blocksInCheckpoint.length,
705
+ txCount: checkpoint.getStats().txCount,
706
+ totalMana: Number(checkpoint.header.totalManaUsed.toBigInt()),
707
+ });
342
708
 
343
- // Do not collect attestations nor publish to L1 in fisherman mode
709
+ // In fisherman mode, return the checkpoint without broadcasting or collecting attestations
344
710
  if (this.config.fishermanMode) {
345
711
  this.log.info(
346
712
  `Built checkpoint for slot ${this.targetSlot} with ${blocksInCheckpoint.length} blocks. ` +
@@ -352,87 +718,36 @@ export class CheckpointProposalJob implements Traceable {
352
718
  },
353
719
  );
354
720
  this.metrics.recordCheckpointSuccess();
355
- return checkpoint;
721
+ // Return a broadcast result with a dummy proposal — fisherman mode skips attestation collection
722
+ return { checkpoint, proposal: undefined!, blockProposedAt: this.dateProvider.now() };
356
723
  }
357
724
 
358
- // Include the block pending broadcast in the checkpoint proposal if any
359
- const lastBlock = blockPendingBroadcast && {
360
- blockHeader: blockPendingBroadcast.block.header,
361
- indexWithinCheckpoint: blockPendingBroadcast.block.indexWithinCheckpoint,
362
- txs: blockPendingBroadcast.txs,
363
- };
364
-
365
725
  // Create the checkpoint proposal and broadcast it
366
726
  const proposal = await this.validatorClient.createCheckpointProposal(
367
727
  checkpoint.header,
368
728
  checkpoint.archive.root,
729
+ this.checkpointNumber,
369
730
  feeAssetPriceModifier,
370
- lastBlock,
731
+ blockPendingBroadcast,
371
732
  this.proposer,
372
733
  checkpointProposalOptions,
373
734
  );
374
735
 
375
736
  const blockProposedAt = this.dateProvider.now();
376
- await this.p2pClient.broadcastCheckpointProposal(proposal);
377
-
378
- this.setStateFn(SequencerState.COLLECTING_ATTESTATIONS, this.targetSlot);
379
- const attestations = await this.waitForAttestations(proposal);
380
- const blockAttestedAt = this.dateProvider.now();
381
-
382
- this.metrics.recordCheckpointAttestationDelay(blockAttestedAt - blockProposedAt);
383
-
384
- // Proposer must sign over the attestations before pushing them to L1
385
- const signer = this.proposer ?? this.publisher.getSenderAddress();
386
- let attestationsSignature: Signature;
387
- try {
388
- attestationsSignature = await this.validatorClient.signAttestationsAndSigners(
389
- attestations,
390
- signer,
391
- this.targetSlot,
392
- this.checkpointNumber,
393
- );
394
- } catch (err) {
395
- // We shouldn't really get here since we yield to another HA node
396
- // as soon as we see these errors when creating block or checkpoint proposals.
397
- if (this.handleHASigningError(err, 'Attestations signature')) {
398
- return undefined;
399
- }
400
- throw err;
401
- }
402
-
403
- // Enqueue publishing the checkpoint to L1
404
- this.setStateFn(SequencerState.PUBLISHING_CHECKPOINT, this.targetSlot);
405
- const aztecSlotDuration = this.l1Constants.slotDuration;
406
- const submissionSlotStart = Number(getTimestampForSlot(this.targetSlot, this.l1Constants));
407
- const txTimeoutAt = new Date((submissionSlotStart + aztecSlotDuration) * 1000);
408
-
409
- // If we have been configured to potentially skip publishing checkpoint then roll the dice here
410
- if (
411
- this.config.skipPublishingCheckpointsPercent !== undefined &&
412
- this.config.skipPublishingCheckpointsPercent > 0
413
- ) {
414
- const result = Math.max(0, randomInt(100));
415
- if (result < this.config.skipPublishingCheckpointsPercent) {
416
- this.log.warn(
417
- `Skipping publishing proposal for checkpoint ${checkpoint.number}. Configured percentage: ${this.config.skipPublishingCheckpointsPercent}, generated value: ${result}`,
418
- );
419
- return checkpoint;
420
- }
737
+ if (!this.config.skipBroadcastProposals) {
738
+ await this.p2pClient.broadcastCheckpointProposal(proposal);
739
+ this.checkpointMetrics.noteCheckpointBroadcast(this.dateProvider.now());
421
740
  }
422
741
 
423
- await this.publisher.enqueueProposeCheckpoint(checkpoint, attestations, attestationsSignature, {
424
- txTimeoutAt,
425
- forcePendingCheckpointNumber: this.invalidateCheckpoint?.forcePendingCheckpointNumber,
426
- });
427
-
428
- return checkpoint;
742
+ // Return immediately after broadcast — attestation collection happens in the background
743
+ return { checkpoint, proposal, blockProposedAt };
429
744
  } catch (err) {
430
745
  if (err && (err instanceof DutyAlreadySignedError || err instanceof SlashingProtectionError)) {
431
746
  // swallow this error. It's already been logged by a function deeper in the stack
432
747
  return undefined;
433
748
  }
434
749
 
435
- this.log.error(`Error building checkpoint at slot ${this.slot}`, err);
750
+ this.log.error(`Error building checkpoint at slot ${this.targetSlot}`, err);
436
751
  return undefined;
437
752
  }
438
753
  }
@@ -448,14 +763,14 @@ export class CheckpointProposalJob implements Traceable {
448
763
  blockProposalOptions: BlockProposalOptions,
449
764
  ): Promise<{
450
765
  blocksInCheckpoint: L2Block[];
451
- blockPendingBroadcast: { block: L2Block; txs: Tx[] } | undefined;
766
+ blockPendingBroadcast: BlockProposal | undefined;
452
767
  }> {
453
768
  const blocksInCheckpoint: L2Block[] = [];
454
769
  const txHashesAlreadyIncluded = new Set<string>();
455
770
  const initialBlockNumber = BlockNumber(this.syncedToBlockNumber + 1);
456
771
 
457
772
  // Last block in the checkpoint will usually be flagged as pending broadcast, so we send it along with the checkpoint proposal
458
- let blockPendingBroadcast: { block: L2Block; txs: Tx[] } | undefined = undefined;
773
+ let blockPendingBroadcast: BlockProposal | undefined = undefined;
459
774
 
460
775
  while (true) {
461
776
  const blocksBuilt = blocksInCheckpoint.length;
@@ -488,19 +803,20 @@ export class CheckpointProposalJob implements Traceable {
488
803
  txHashesAlreadyIncluded,
489
804
  });
490
805
 
491
- // TODO(palla/mbps): Review these conditions. We may want to keep trying in some scenarios.
492
- if (!buildResult && timingInfo.isLastBlock) {
493
- // If no block was produced due to not enough txs and this was the last subslot, exit
494
- break;
495
- } else if (!buildResult && timingInfo.deadline !== undefined) {
496
- // But if there is still time for more blocks, wait until the next subslot and try again
806
+ // If we failed to build the block due to insufficient txs, we try again if there is still time left in the slot
807
+ if ('failure' in buildResult) {
808
+ // If this was the last subslot, or we're running with a single block per slot, we're done
809
+ if (timingInfo.isLastBlock || timingInfo.deadline === undefined) {
810
+ break;
811
+ }
812
+ // Otherwise, if there is still time for more blocks, we wait until the next subslot and try again
497
813
  await this.waitUntilNextSubslot(timingInfo.deadline);
498
814
  continue;
499
- } else if (!buildResult) {
500
- // Exit if there is no possibility of building more blocks
501
- break;
502
- } else if ('error' in buildResult) {
503
- // If there was an error building the block, just exit the loop and give up the rest of the slot
815
+ }
816
+
817
+ // If there was an error building the block, we just exit the loop and give up the rest of the slot.
818
+ // We don't want to risk building more blocks if something went wrong.
819
+ if ('error' in buildResult) {
504
820
  if (!(buildResult.error instanceof SequencerInterruptedError)) {
505
821
  this.log.warn(`Halting block building for slot ${this.targetSlot}`, {
506
822
  slot: this.targetSlot,
@@ -512,35 +828,39 @@ export class CheckpointProposalJob implements Traceable {
512
828
  }
513
829
 
514
830
  const { block, usedTxs } = buildResult;
831
+ this.checkpointMetrics.noteCheckpointBlockBuilt(this.dateProvider.now(), {
832
+ isFirstBlock: blocksBuilt === 0,
833
+ isLastBlock: timingInfo.isLastBlock,
834
+ });
835
+
515
836
  blocksInCheckpoint.push(block);
516
837
  usedTxs.forEach(tx => txHashesAlreadyIncluded.add(tx.txHash.toString()));
517
838
 
518
- // If this is the last block, sync it to the archiver and exit the loop
519
- // so we can build the checkpoint and start collecting attestations.
839
+ // Sign the block proposal. This will throw if HA signing fails.
840
+ const proposal = await this.createBlockProposal(block, inHash, usedTxs, blockProposalOptions);
841
+
842
+ // Sync the proposed block to the archiver to make it available, only after we've managed to sign the proposal,
843
+ // so we avoid polluting our archive with a block that would fail.
844
+ // We wait for the sync to succeed, as this helps catch consistency errors, even if it means we lose some time for block-building.
845
+ // If this throws, we abort the entire checkpoint.
846
+ await this.syncProposedBlockToArchiver(block);
847
+
848
+ // If this is the last block, do not broadcast it, since it will be included in the checkpoint proposal.
520
849
  if (timingInfo.isLastBlock) {
521
- await this.syncProposedBlockToArchiver(block);
522
850
  this.log.verbose(`Completed final block ${blockNumber} for slot ${this.targetSlot}`, {
523
851
  slot: this.targetSlot,
524
852
  blockNumber,
525
853
  blocksBuilt,
526
854
  });
527
- blockPendingBroadcast = { block, txs: usedTxs };
855
+
856
+ blockPendingBroadcast = proposal;
528
857
  break;
529
858
  }
530
859
 
531
- // Broadcast the block proposal (unless we're in fisherman mode) unless the block is the last one,
532
- // in which case we'll broadcast it along with the checkpoint at the end of the loop.
533
- // Note that we only send the block to the archiver if we manage to create the proposal, so if there's
534
- // a HA error we don't pollute our archiver with a block that won't make it to the chain.
535
- const proposal = await this.createBlockProposal(block, inHash, usedTxs, blockProposalOptions);
536
-
537
- // Sync the proposed block to the archiver to make it available, only after we've managed to sign the proposal.
538
- // We wait for the sync to succeed, as this helps catch consistency errors, even if it means we lose some time for block-building.
539
- // If this throws, we abort the entire checkpoint.
540
- await this.syncProposedBlockToArchiver(block);
541
-
542
860
  // Once we have a signed proposal and the archiver agreed with our proposed block, then we broadcast it.
543
- proposal && (await this.p2pClient.broadcastProposal(proposal));
861
+ if (proposal && !this.config.skipBroadcastProposals) {
862
+ await this.p2pClient.broadcastProposal(proposal);
863
+ }
544
864
 
545
865
  // Wait until the next block's start time
546
866
  await this.waitUntilNextSubslot(timingInfo.deadline);
@@ -567,6 +887,7 @@ export class CheckpointProposalJob implements Traceable {
567
887
  }
568
888
  return this.validatorClient.createBlockProposal(
569
889
  block.header,
890
+ this.checkpointNumber,
570
891
  block.indexWithinCheckpoint,
571
892
  inHash,
572
893
  block.archive.root,
@@ -598,7 +919,9 @@ export class CheckpointProposalJob implements Traceable {
598
919
  buildDeadline: Date | undefined;
599
920
  txHashesAlreadyIncluded: Set<string>;
600
921
  },
601
- ): Promise<{ block: L2Block; usedTxs: Tx[] } | { error: Error } | undefined> {
922
+ ): Promise<
923
+ { block: L2Block; usedTxs: Tx[] } | { failure: 'insufficient-txs' | 'insufficient-valid-txs' } | { error: Error }
924
+ > {
602
925
  const { blockTimestamp, forceCreate, blockNumber, indexWithinCheckpoint, buildDeadline, txHashesAlreadyIncluded } =
603
926
  opts;
604
927
 
@@ -611,13 +934,30 @@ export class CheckpointProposalJob implements Traceable {
611
934
  // Wait until we have enough txs to build the block
612
935
  const { availableTxs, canStartBuilding, minTxs } = await this.waitForMinTxs(opts);
613
936
  if (!canStartBuilding) {
937
+ this.logCheckpointEvent('block-build-failed', `Block build failed for slot ${this.targetSlot}`, {
938
+ reason: 'insufficient_txs',
939
+ blockNumber,
940
+ slot: this.targetSlot,
941
+ checkpointNumber: this.checkpointNumber,
942
+ indexWithinCheckpoint,
943
+ availableTxs,
944
+ minTxs,
945
+ });
614
946
  this.log.warn(
615
947
  `Not enough txs to build block ${blockNumber} at index ${indexWithinCheckpoint} in slot ${this.targetSlot} (got ${availableTxs} txs but needs ${minTxs})`,
616
- { blockNumber, slot: this.targetSlot, indexWithinCheckpoint },
948
+ {
949
+ reason: 'insufficient_txs',
950
+ blockNumber,
951
+ slot: this.targetSlot,
952
+ checkpointNumber: this.checkpointNumber,
953
+ indexWithinCheckpoint,
954
+ availableTxs,
955
+ minTxs,
956
+ },
617
957
  );
618
958
  this.eventEmitter.emit('block-tx-count-check-failed', { minTxs, availableTxs, slot: this.targetSlot });
619
959
  this.metrics.recordBlockProposalFailed('insufficient_txs');
620
- return undefined;
960
+ return { failure: 'insufficient-txs' };
621
961
  }
622
962
 
623
963
  // Create iterator to pending txs. We filter out txs already included in previous blocks in the checkpoint
@@ -665,10 +1005,21 @@ export class CheckpointProposalJob implements Traceable {
665
1005
  await this.dropFailedTxsFromP2P(buildResult.failedTxs);
666
1006
 
667
1007
  if (buildResult.status === 'insufficient-valid-txs') {
1008
+ this.logCheckpointEvent('block-build-failed', `Block build failed for slot ${this.targetSlot}`, {
1009
+ reason: 'insufficient_valid_txs',
1010
+ slot: this.targetSlot,
1011
+ checkpointNumber: this.checkpointNumber,
1012
+ blockNumber,
1013
+ numTxs: buildResult.processedCount,
1014
+ indexWithinCheckpoint,
1015
+ minValidTxs,
1016
+ });
668
1017
  this.log.warn(
669
1018
  `Block ${blockNumber} at index ${indexWithinCheckpoint} on slot ${this.targetSlot} has too few valid txs to be proposed`,
670
1019
  {
1020
+ reason: 'insufficient_valid_txs',
671
1021
  slot: this.targetSlot,
1022
+ checkpointNumber: this.checkpointNumber,
672
1023
  blockNumber,
673
1024
  numTxs: buildResult.processedCount,
674
1025
  indexWithinCheckpoint,
@@ -680,7 +1031,7 @@ export class CheckpointProposalJob implements Traceable {
680
1031
  slot: this.targetSlot,
681
1032
  });
682
1033
  this.metrics.recordBlockProposalFailed('insufficient_valid_txs');
683
- return undefined;
1034
+ return { failure: 'insufficient-valid-txs' };
684
1035
  }
685
1036
 
686
1037
  // Block creation succeeded, emit stats and metrics
@@ -702,12 +1053,14 @@ export class CheckpointProposalJob implements Traceable {
702
1053
  { blockHash, txHashes, manaPerSec, ...blockStats },
703
1054
  );
704
1055
 
1056
+ // `slot` is the target/submission slot (may be one ahead when pipelining),
1057
+ // `buildSlot` is the wall-clock slot during which the block was actually built.
705
1058
  this.eventEmitter.emit('block-proposed', {
706
1059
  blockNumber: block.number,
707
1060
  slot: this.targetSlot,
708
1061
  buildSlot: this.slotNow,
709
1062
  });
710
- this.metrics.recordBuiltBlock(blockBuildDuration, block.header.totalManaUsed.toNumberUnsafe());
1063
+ this.metrics.recordBuiltBlock(blockBuildDuration, block.header.totalManaUsed.toNumberUnsafe(), this.targetSlot);
711
1064
 
712
1065
  return { block, usedTxs };
713
1066
  } catch (err: any) {
@@ -715,7 +1068,18 @@ export class CheckpointProposalJob implements Traceable {
715
1068
  reason: err.message,
716
1069
  slot: this.targetSlot,
717
1070
  });
718
- this.log.error(`Error building block`, err, { blockNumber, slot: this.targetSlot });
1071
+ this.logCheckpointEvent('block-build-failed', `Block build failed for slot ${this.targetSlot}`, {
1072
+ reason: err instanceof Error ? err.message : String(err),
1073
+ slot: this.targetSlot,
1074
+ checkpointNumber: this.checkpointNumber,
1075
+ blockNumber,
1076
+ });
1077
+ this.log.error(`Error building block`, err, {
1078
+ reason: err instanceof Error ? err.message : String(err),
1079
+ slot: this.targetSlot,
1080
+ checkpointNumber: this.checkpointNumber,
1081
+ blockNumber,
1082
+ });
719
1083
  this.metrics.recordBlockProposalFailed(err.name || 'unknown_error');
720
1084
  this.metrics.recordFailedBlock();
721
1085
  return { error: err };
@@ -787,15 +1151,47 @@ export class CheckpointProposalJob implements Traceable {
787
1151
  return { canStartBuilding: true, availableTxs, minTxs };
788
1152
  }
789
1153
 
1154
+ private async getSignedCommitteeAttestations(
1155
+ broadcast: CheckpointProposalBroadcast,
1156
+ ): Promise<{ attestations: CommitteeAttestationsAndSigners; attestationsSignature: Signature } | undefined> {
1157
+ const { proposal, blockProposedAt } = broadcast;
1158
+ this.setStateFn(SequencerState.COLLECTING_ATTESTATIONS, this.targetSlot);
1159
+ const attestations = await this.waitForAttestations(proposal);
1160
+ if (!attestations) {
1161
+ return undefined;
1162
+ }
1163
+ this.checkpointMetrics.recordCheckpointAttestationDelay(this.dateProvider.now() - blockProposedAt);
1164
+
1165
+ // Proposer must sign over the attestations before pushing them to L1
1166
+ const signer = this.proposer ?? this.publisher.getSenderAddress();
1167
+ try {
1168
+ const attestationsSignature = await this.validatorClient.signAttestationsAndSigners(
1169
+ attestations,
1170
+ signer,
1171
+ this.targetSlot,
1172
+ this.checkpointNumber,
1173
+ );
1174
+ return { attestations, attestationsSignature };
1175
+ } catch (err) {
1176
+ if (this.handleHASigningError(err, 'Attestations signature')) {
1177
+ return;
1178
+ }
1179
+ this.log.error(`Error signing attestations for checkpoint proposal at slot ${proposal.slotNumber}`, err);
1180
+ return undefined;
1181
+ }
1182
+ }
1183
+
790
1184
  /**
791
1185
  * Waits for enough attestations to be collected via p2p.
792
1186
  * This is run after all blocks for the checkpoint have been built.
793
1187
  */
794
1188
  @trackSpan('CheckpointProposalJob.waitForAttestations')
795
- private async waitForAttestations(proposal: CheckpointProposal): Promise<CommitteeAttestationsAndSigners> {
1189
+ private async waitForAttestations(
1190
+ proposal: CheckpointProposal,
1191
+ ): Promise<CommitteeAttestationsAndSigners | undefined> {
796
1192
  if (this.config.fishermanMode) {
797
1193
  this.log.debug('Skipping attestation collection in fisherman mode');
798
- return CommitteeAttestationsAndSigners.empty();
1194
+ return CommitteeAttestationsAndSigners.empty(this.getSignatureContext());
799
1195
  }
800
1196
 
801
1197
  const slotNumber = proposal.slotNumber;
@@ -805,21 +1201,27 @@ export class CheckpointProposalJob implements Traceable {
805
1201
  throw new Error('No committee when collecting attestations');
806
1202
  } else if (committee.length === 0) {
807
1203
  this.log.verbose(`Attesting committee is empty`);
808
- return CommitteeAttestationsAndSigners.empty();
1204
+ return CommitteeAttestationsAndSigners.empty(this.getSignatureContext());
809
1205
  } else {
810
1206
  this.log.debug(`Attesting committee length is ${committee.length}`, { committee });
811
1207
  }
812
1208
 
813
- const numberOfRequiredAttestations = Math.floor((committee.length * 2) / 3) + 1;
1209
+ const numberOfRequiredAttestations = computeQuorum(committee.length);
814
1210
 
815
1211
  if (this.config.skipCollectingAttestations) {
816
1212
  this.log.warn('Skipping attestation collection as per config (attesting with own keys only)');
817
- const attestations = await this.validatorClient?.collectOwnAttestations(proposal);
818
- return new CommitteeAttestationsAndSigners(orderAttestations(attestations ?? [], committee));
1213
+ const attestations = await this.validatorClient?.collectOwnAttestations(proposal, this.checkpointNumber);
1214
+ this.logCheckpointAttestations('collected', committee, attestations ?? [], numberOfRequiredAttestations, {
1215
+ reason: 'collect_own_only',
1216
+ });
1217
+ return new CommitteeAttestationsAndSigners(
1218
+ orderAttestations(attestations ?? [], committee),
1219
+ this.getSignatureContext(),
1220
+ );
819
1221
  }
820
1222
 
821
1223
  const attestationTimeAllowed = this.config.enforceTimeTable
822
- ? this.timetable.getMaxAllowedTime(SequencerState.PUBLISHING_CHECKPOINT)!
1224
+ ? this.timetable.getCheckpointAttestationDeadline()
823
1225
  : this.l1Constants.slotDuration;
824
1226
  const attestationDeadline = new Date((this.getSlotStartBuildTimestamp() + attestationTimeAllowed) * 1000);
825
1227
 
@@ -832,6 +1234,7 @@ export class CheckpointProposalJob implements Traceable {
832
1234
  proposal,
833
1235
  numberOfRequiredAttestations,
834
1236
  attestationDeadline,
1237
+ this.checkpointNumber,
835
1238
  );
836
1239
 
837
1240
  collectedAttestationsCount = attestations.length;
@@ -850,6 +1253,9 @@ export class CheckpointProposalJob implements Traceable {
850
1253
 
851
1254
  // Rollup contract requires that the signatures are provided in the order of the committee
852
1255
  const sorted = orderAttestations(trimmed, committee);
1256
+ this.logCheckpointAttestations('collected', committee, attestations, numberOfRequiredAttestations, {
1257
+ submittedCount: trimmed.length,
1258
+ });
853
1259
 
854
1260
  // Manipulate the attestations if we've been configured to do so
855
1261
  if (
@@ -861,17 +1267,56 @@ export class CheckpointProposalJob implements Traceable {
861
1267
  return this.manipulateAttestations(proposal.slotNumber, epoch, seed, committee, sorted);
862
1268
  }
863
1269
 
864
- return new CommitteeAttestationsAndSigners(sorted);
1270
+ return new CommitteeAttestationsAndSigners(sorted, this.getSignatureContext());
865
1271
  } catch (err) {
866
1272
  if (err && err instanceof AttestationTimeoutError) {
867
1273
  collectedAttestationsCount = err.collectedCount;
1274
+ this.logCheckpointAttestations('failed', committee, undefined, numberOfRequiredAttestations, {
1275
+ collectedCount: collectedAttestationsCount,
1276
+ reason: 'timeout',
1277
+ });
1278
+ this.log.error(
1279
+ `Timeout while waiting for attestations for checkpoint proposal at slot ${proposal.slotNumber} (collected ${collectedAttestationsCount}/${numberOfRequiredAttestations})`,
1280
+ err,
1281
+ );
1282
+ } else {
1283
+ this.logCheckpointAttestations('failed', committee, undefined, numberOfRequiredAttestations, {
1284
+ collectedCount: collectedAttestationsCount,
1285
+ reason: err instanceof Error ? err.message : String(err),
1286
+ });
1287
+ this.log.error(`Error collecting attestations for checkpoint proposal at slot ${proposal.slotNumber}`, err);
868
1288
  }
869
- throw err;
1289
+ return undefined;
870
1290
  } finally {
871
1291
  this.metrics.recordCollectedAttestations(collectedAttestationsCount, collectAttestationsTimer.ms());
872
1292
  }
873
1293
  }
874
1294
 
1295
+ private logCheckpointAttestations(
1296
+ status: 'collected' | 'failed',
1297
+ committee: EthAddress[],
1298
+ attestations: CheckpointAttestation[] | undefined,
1299
+ requiredAttestations: number,
1300
+ opts: { collectedCount?: number; submittedCount?: number; reason?: string } = {},
1301
+ ) {
1302
+ const signedValidators =
1303
+ attestations
1304
+ ?.map(attestation => attestation.getSender()?.toString())
1305
+ .filter((address): address is `0x${string}` => address !== undefined) ?? [];
1306
+ const collectedCount = opts.collectedCount ?? new Set(signedValidators).size;
1307
+ const missingValidatorCount = status === 'failed' ? Math.max(0, requiredAttestations - collectedCount) : undefined;
1308
+ this.logCheckpointEvent(`attestations-${status}`, `Checkpoint attestations ${status} for slot ${this.targetSlot}`, {
1309
+ slot: this.targetSlot,
1310
+ checkpointNumber: this.checkpointNumber,
1311
+ committeeSize: committee.length,
1312
+ requiredAttestations,
1313
+ collectedAttestations: collectedCount,
1314
+ ...(opts.submittedCount !== undefined && { submittedAttestations: opts.submittedCount }),
1315
+ ...(missingValidatorCount !== undefined && { missingValidatorCount }),
1316
+ ...(opts.reason !== undefined && { reason: opts.reason }),
1317
+ });
1318
+ }
1319
+
875
1320
  /** Breaks the attestations before publishing based on attack configs */
876
1321
  private manipulateAttestations(
877
1322
  slotNumber: SlotNumber,
@@ -915,7 +1360,7 @@ export class CheckpointProposalJob implements Traceable {
915
1360
  unfreeze(attestations[targetIndex]).signature = generateRecoverableSignature();
916
1361
  }
917
1362
  }
918
- return new CommitteeAttestationsAndSigners(attestations);
1363
+ return new CommitteeAttestationsAndSigners(attestations, this.getSignatureContext());
919
1364
  }
920
1365
 
921
1366
  if (this.config.shuffleAttestationOrdering) {
@@ -937,11 +1382,11 @@ export class CheckpointProposalJob implements Traceable {
937
1382
  [shuffled[i], shuffled[j]] = [shuffled[j], shuffled[i]];
938
1383
  }
939
1384
 
940
- const signers = new CommitteeAttestationsAndSigners(attestations).getSigners();
941
- return new MaliciousCommitteeAttestationsAndSigners(shuffled, signers);
1385
+ const signers = new CommitteeAttestationsAndSigners(attestations, this.getSignatureContext()).getSigners();
1386
+ return new MaliciousCommitteeAttestationsAndSigners(shuffled, signers, this.getSignatureContext());
942
1387
  }
943
1388
 
944
- return new CommitteeAttestationsAndSigners(attestations);
1389
+ return new CommitteeAttestationsAndSigners(attestations, this.getSignatureContext());
945
1390
  }
946
1391
 
947
1392
  private async dropFailedTxsFromP2P(failedTxs: FailedTx[]) {
@@ -958,9 +1403,13 @@ export class CheckpointProposalJob implements Traceable {
958
1403
  * Adds the proposed block to the archiver so it's available via P2P.
959
1404
  * Gossip doesn't echo messages back to the sender, so the proposer's archiver/world-state
960
1405
  * would never receive its own block without this explicit sync.
1406
+ *
1407
+ * In fisherman mode we skip this push: the fisherman builds blocks locally for validation
1408
+ * and fee analysis only, and pushing them to the archiver causes spurious reorg cascades
1409
+ * whenever the real proposer's block arrives from L1.
961
1410
  */
962
1411
  private async syncProposedBlockToArchiver(block: L2Block): Promise<void> {
963
- if (this.config.skipPushProposedBlocksToArchiver !== false) {
1412
+ if (this.config.skipPushProposedBlocksToArchiver || this.config.fishermanMode) {
964
1413
  this.log.warn(`Skipping push of proposed block ${block.number} to archiver`, {
965
1414
  blockNumber: block.number,
966
1415
  slot: block.header.globalVariables.slotNumber,
@@ -1035,7 +1484,7 @@ export class CheckpointProposalJob implements Traceable {
1035
1484
  }
1036
1485
 
1037
1486
  private getSlotStartBuildTimestamp(): number {
1038
- return getSlotStartBuildTimestamp(this.slot, this.l1Constants);
1487
+ return getSlotStartBuildTimestamp(this.slotNow, this.l1Constants);
1039
1488
  }
1040
1489
 
1041
1490
  private getSecondsIntoSlot(): number {