@aztec/sequencer-client 0.0.1-commit.e588bc7e5 → 0.0.1-commit.e5a3663dd

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. package/dest/client/sequencer-client.d.ts +3 -1
  2. package/dest/client/sequencer-client.d.ts.map +1 -1
  3. package/dest/client/sequencer-client.js +3 -4
  4. package/dest/config.d.ts +2 -1
  5. package/dest/config.d.ts.map +1 -1
  6. package/dest/config.js +16 -14
  7. package/dest/global_variable_builder/fee_predictor.d.ts +37 -0
  8. package/dest/global_variable_builder/fee_predictor.d.ts.map +1 -0
  9. package/dest/global_variable_builder/fee_predictor.js +128 -0
  10. package/dest/global_variable_builder/fee_provider.d.ts +21 -0
  11. package/dest/global_variable_builder/fee_provider.d.ts.map +1 -0
  12. package/dest/global_variable_builder/fee_provider.js +58 -0
  13. package/dest/global_variable_builder/global_builder.d.ts +4 -9
  14. package/dest/global_variable_builder/global_builder.d.ts.map +1 -1
  15. package/dest/global_variable_builder/global_builder.js +3 -41
  16. package/dest/global_variable_builder/index.d.ts +3 -1
  17. package/dest/global_variable_builder/index.d.ts.map +1 -1
  18. package/dest/global_variable_builder/index.js +2 -0
  19. package/dest/publisher/config.d.ts +1 -1
  20. package/dest/publisher/config.d.ts.map +1 -1
  21. package/dest/publisher/config.js +2 -2
  22. package/dest/publisher/l1_tx_failed_store/failed_tx_store.d.ts +3 -4
  23. package/dest/publisher/l1_tx_failed_store/failed_tx_store.d.ts.map +1 -1
  24. package/dest/publisher/sequencer-publisher-factory.d.ts +1 -3
  25. package/dest/publisher/sequencer-publisher-factory.d.ts.map +1 -1
  26. package/dest/publisher/sequencer-publisher-factory.js +0 -1
  27. package/dest/publisher/sequencer-publisher.d.ts +16 -35
  28. package/dest/publisher/sequencer-publisher.d.ts.map +1 -1
  29. package/dest/publisher/sequencer-publisher.js +62 -94
  30. package/dest/sequencer/chain_state_overrides.d.ts +25 -0
  31. package/dest/sequencer/chain_state_overrides.d.ts.map +1 -0
  32. package/dest/sequencer/chain_state_overrides.js +39 -0
  33. package/dest/sequencer/checkpoint_proposal_job.d.ts +33 -22
  34. package/dest/sequencer/checkpoint_proposal_job.d.ts.map +1 -1
  35. package/dest/sequencer/checkpoint_proposal_job.js +479 -170
  36. package/dest/sequencer/checkpoint_proposal_job_metrics.d.ts +34 -0
  37. package/dest/sequencer/checkpoint_proposal_job_metrics.d.ts.map +1 -0
  38. package/dest/sequencer/checkpoint_proposal_job_metrics.js +72 -0
  39. package/dest/sequencer/events.d.ts +6 -1
  40. package/dest/sequencer/events.d.ts.map +1 -1
  41. package/dest/sequencer/metrics.d.ts +9 -10
  42. package/dest/sequencer/metrics.d.ts.map +1 -1
  43. package/dest/sequencer/metrics.js +34 -20
  44. package/dest/sequencer/sequencer.d.ts +16 -5
  45. package/dest/sequencer/sequencer.d.ts.map +1 -1
  46. package/dest/sequencer/sequencer.js +63 -30
  47. package/dest/sequencer/timetable.d.ts +14 -1
  48. package/dest/sequencer/timetable.d.ts.map +1 -1
  49. package/dest/sequencer/timetable.js +45 -36
  50. package/dest/test/utils.d.ts +1 -1
  51. package/dest/test/utils.d.ts.map +1 -1
  52. package/dest/test/utils.js +7 -6
  53. package/package.json +27 -27
  54. package/src/client/sequencer-client.ts +5 -7
  55. package/src/config.ts +15 -11
  56. package/src/global_variable_builder/README.md +44 -0
  57. package/src/global_variable_builder/fee_predictor.ts +172 -0
  58. package/src/global_variable_builder/fee_provider.ts +75 -0
  59. package/src/global_variable_builder/global_builder.ts +7 -54
  60. package/src/global_variable_builder/index.ts +2 -0
  61. package/src/publisher/config.ts +6 -4
  62. package/src/publisher/l1_tx_failed_store/failed_tx_store.ts +3 -1
  63. package/src/publisher/sequencer-publisher-factory.ts +0 -3
  64. package/src/publisher/sequencer-publisher.ts +108 -156
  65. package/src/sequencer/README.md +82 -13
  66. package/src/sequencer/chain_state_overrides.ts +87 -0
  67. package/src/sequencer/checkpoint_proposal_job.ts +572 -218
  68. package/src/sequencer/checkpoint_proposal_job_metrics.ts +128 -0
  69. package/src/sequencer/events.ts +5 -0
  70. package/src/sequencer/metrics.ts +43 -24
  71. package/src/sequencer/sequencer.ts +81 -35
  72. package/src/sequencer/timetable.ts +57 -45
  73. package/src/test/utils.ts +28 -10
@@ -1,5 +1,5 @@
1
1
  import type { EpochCache } from '@aztec/epoch-cache';
2
- import { type FeeHeader, RollupContract } from '@aztec/ethereum/contracts';
2
+ import type { SimulationOverridesPlan } from '@aztec/ethereum/contracts';
3
3
  import {
4
4
  BlockNumber,
5
5
  CheckpointNumber,
@@ -18,6 +18,7 @@ import { EthAddress } from '@aztec/foundation/eth-address';
18
18
  import { Signature } from '@aztec/foundation/eth-signature';
19
19
  import { filter } from '@aztec/foundation/iterator';
20
20
  import { type Logger, type LoggerBindings, createLogger } from '@aztec/foundation/log';
21
+ import { retryUntil } from '@aztec/foundation/retry';
21
22
  import { sleep, sleepUntil } from '@aztec/foundation/sleep';
22
23
  import { type DateProvider, Timer } from '@aztec/foundation/timer';
23
24
  import { type TypedEventEmitter, isErrorClass, unfreeze } from '@aztec/foundation/types';
@@ -30,6 +31,7 @@ import {
30
31
  type L2BlockSink,
31
32
  type L2BlockSource,
32
33
  MaliciousCommitteeAttestationsAndSigners,
34
+ type ValidateCheckpointResult,
33
35
  } from '@aztec/stdlib/block';
34
36
  import { type Checkpoint, type ProposedCheckpointData, validateCheckpoint } from '@aztec/stdlib/checkpoint';
35
37
  import { computeQuorum, getSlotStartBuildTimestamp, getTimestampForSlot } from '@aztec/stdlib/epoch-helpers';
@@ -44,8 +46,10 @@ import { type L1ToL2MessageSource, computeInHashFromL1ToL2Messages } from '@azte
44
46
  import type {
45
47
  BlockProposal,
46
48
  BlockProposalOptions,
49
+ CheckpointAttestation,
47
50
  CheckpointProposal,
48
51
  CheckpointProposalOptions,
52
+ CoordinationSignatureContext,
49
53
  } from '@aztec/stdlib/p2p';
50
54
  import { orderAttestations, trimAttestations } from '@aztec/stdlib/p2p';
51
55
  import type { L2BlockBuiltStats } from '@aztec/stdlib/stats';
@@ -57,6 +61,11 @@ import { DutyAlreadySignedError, SlashingProtectionError } from '@aztec/validato
57
61
 
58
62
  import type { GlobalVariableBuilder } from '../global_variable_builder/global_builder.js';
59
63
  import type { InvalidateCheckpointRequest, SequencerPublisher } from '../publisher/sequencer-publisher.js';
64
+ import {
65
+ buildPipelinedParentSimulationOverridesPlan,
66
+ buildSubmissionSimulationOverridesPlan,
67
+ } from './chain_state_overrides.js';
68
+ import type { CheckpointProposalJobMetricsRecorder } from './checkpoint_proposal_job_metrics.js';
60
69
  import { CheckpointVoter } from './checkpoint_voter.js';
61
70
  import { SequencerInterruptedError } from './errors.js';
62
71
  import type { SequencerEvents } from './events.js';
@@ -68,7 +77,14 @@ import { SequencerState } from './utils.js';
68
77
  /** How much time to sleep while waiting for min transactions to accumulate for a block */
69
78
  const TXS_POLLING_MS = 500;
70
79
 
71
- /** Result from proposeCheckpoint when a checkpoint was successfully built and attested. */
80
+ /** Result from proposeCheckpoint when a checkpoint was successfully built and broadcast. */
81
+ type CheckpointProposalBroadcast = {
82
+ checkpoint: Checkpoint;
83
+ proposal: CheckpointProposal;
84
+ blockProposedAt: number;
85
+ };
86
+
87
+ /** Result after attestation collection and signing, ready for L1 submission. */
72
88
  type CheckpointProposalResult = {
73
89
  checkpoint: Checkpoint;
74
90
  attestations: CommitteeAttestationsAndSigners;
@@ -83,12 +99,17 @@ type CheckpointProposalResult = {
83
99
  */
84
100
  export class CheckpointProposalJob implements Traceable {
85
101
  protected readonly log: Logger;
102
+ private readonly checkpointEventLog: Logger;
86
103
 
87
104
  /** Tracks the fire-and-forget L1 submission promise so it can be awaited during shutdown. */
88
105
  private pendingL1Submission: Promise<void> | undefined;
89
106
 
90
- /** Fee header override computed during proposeCheckpoint, reused in enqueueCheckpointForSubmission. */
91
- private computedForceProposedFeeHeader?: { checkpointNumber: CheckpointNumber; feeHeader: FeeHeader };
107
+ /** Pipelined parent chain state used while building and later submitting this checkpoint. */
108
+ private pipelinedParentSimulationOverridesPlan?: SimulationOverridesPlan;
109
+
110
+ private getSignatureContext(): CoordinationSignatureContext {
111
+ return this.signatureContext;
112
+ }
92
113
 
93
114
  constructor(
94
115
  private readonly slotNow: SlotNumber,
@@ -110,13 +131,15 @@ export class CheckpointProposalJob implements Traceable {
110
131
  private readonly checkpointsBuilder: FullNodeCheckpointsBuilder,
111
132
  private readonly blockSink: L2BlockSink,
112
133
  private readonly l1Constants: SequencerRollupConstants,
134
+ private readonly signatureContext: CoordinationSignatureContext,
113
135
  protected config: ResolvedSequencerConfig,
114
136
  protected timetable: SequencerTimetable,
115
137
  private readonly slasherClient: SlasherClientInterface | undefined,
116
138
  private readonly epochCache: EpochCache,
117
139
  private readonly dateProvider: DateProvider,
118
140
  private readonly metrics: SequencerMetrics,
119
- private readonly eventEmitter: TypedEventEmitter<SequencerEvents>,
141
+ private readonly checkpointMetrics: CheckpointProposalJobMetricsRecorder,
142
+ protected readonly eventEmitter: TypedEventEmitter<SequencerEvents>,
120
143
  private readonly setStateFn: (state: SequencerState, slot?: SlotNumber) => void,
121
144
  public readonly tracer: Tracer,
122
145
  bindings?: LoggerBindings,
@@ -126,6 +149,10 @@ export class CheckpointProposalJob implements Traceable {
126
149
  ...bindings,
127
150
  instanceId: `slot-${this.slotNow}`,
128
151
  });
152
+ this.checkpointEventLog = createLogger('sequencer:checkpoint-events', {
153
+ ...bindings,
154
+ instanceId: `slot-${this.slotNow}`,
155
+ });
129
156
  }
130
157
 
131
158
  /** Awaits the pending L1 submission if one is in progress. Call during shutdown. */
@@ -134,10 +161,18 @@ export class CheckpointProposalJob implements Traceable {
134
161
  await this.pendingL1Submission;
135
162
  }
136
163
 
164
+ private logCheckpointEvent(eventName: string, message: string, fields: Record<string, unknown>): void {
165
+ this.checkpointEventLog.debug(message, {
166
+ eventName: `sequencer-checkpoint-${eventName}`,
167
+ ...fields,
168
+ });
169
+ }
170
+
137
171
  /**
138
172
  * Executes the checkpoint proposal job.
139
- * Builds blocks, collects attestations, enqueues requests, and schedules L1 submission as a
140
- * background task so the work loop can return to IDLE immediately.
173
+ * Builds blocks, assembles checkpoint, and broadcasts the proposal (blocking).
174
+ * Attestation collection, signing, and L1 submission are backgrounded so the
175
+ * work loop can return to IDLE immediately for consecutive slot proposals.
141
176
  * Returns the built checkpoint if successful, undefined otherwise.
142
177
  */
143
178
  @trackSpan('CheckpointProposalJob.execute')
@@ -157,73 +192,133 @@ export class CheckpointProposalJob implements Traceable {
157
192
  this.log,
158
193
  ).enqueueVotes();
159
194
 
160
- // Build and propose the checkpoint. Builds blocks, broadcasts, collects attestations, and signs.
161
- // Does NOT enqueue to L1 yet that happens after the pipeline sleep.
162
- const proposalResult = await this.proposeCheckpoint();
163
- const checkpoint = proposalResult?.checkpoint;
195
+ // Build blocks, assemble checkpoint, and broadcast proposal (BLOCKING).
196
+ // Returns after broadcastattestation collection is deferred.
197
+ const broadcast = await this.proposeCheckpoint();
164
198
 
165
- // Wait until the voting promises have resolved, so all requests are enqueued (not sent)
166
- await Promise.all(votesPromises);
167
-
168
- if (checkpoint) {
169
- this.metrics.recordCheckpointProposalSuccess();
199
+ if (!broadcast) {
200
+ await Promise.all(votesPromises);
201
+ // Still submit votes even without a checkpoint
202
+ if (!this.config.fishermanMode) {
203
+ this.pendingL1Submission = this.publisher.sendRequestsAt(this.dateProvider.nowAsDate()).then(() => {});
204
+ }
205
+ return undefined;
170
206
  }
171
207
 
208
+ const { checkpoint } = broadcast;
209
+ this.metrics.recordCheckpointProposalSuccess();
210
+
172
211
  // Do not post anything to L1 if we are fishermen, but do perform L1 fee analysis
173
212
  if (this.config.fishermanMode) {
174
213
  await this.handleCheckpointEndAsFisherman(checkpoint);
175
- return;
214
+ return checkpoint;
176
215
  }
177
216
 
178
- // Enqueue the checkpoint for L1 submission
179
- if (proposalResult) {
180
- try {
181
- await this.enqueueCheckpointForSubmission(proposalResult);
182
- } catch (err) {
183
- this.log.error(`Failed to enqueue checkpoint for L1 submission at slot ${this.targetSlot}`, err);
184
- // Continue to sendRequestsAt so votes are still sent
217
+ // Background the attestation signing → L1 pipeline so the work loop is unblocked
218
+ this.pendingL1Submission = this.waitForAttestationsAndEnqueueSubmissionAsync(broadcast, votesPromises);
219
+
220
+ // Return the built checkpoint immediately — the work loop is now unblocked
221
+ return checkpoint;
222
+ }
223
+
224
+ /**
225
+ * Background pipeline: collects attestations, signs them, enqueues the checkpoint, and submits to L1.
226
+ * Runs as a fire-and-forget task stored in `pendingL1Submission` so the work loop is unblocked.
227
+ */
228
+ private async waitForAttestationsAndEnqueueSubmissionAsync(
229
+ broadcast: CheckpointProposalBroadcast,
230
+ votesPromises: Promise<unknown>[],
231
+ ): Promise<void> {
232
+ const { checkpoint } = broadcast;
233
+ const isPipelining = this.epochCache.isProposerPipeliningEnabled();
234
+
235
+ try {
236
+ // Wait for all votes actions, enqueued at the beginning, to resolve
237
+ await Promise.all(votesPromises);
238
+
239
+ // Try to collect attestations from the committee
240
+ const signedAttestations = await this.getSignedCommitteeAttestations(broadcast);
241
+
242
+ // If pipelining, wait for the previous checkpoint to land on L1 before submitting,
243
+ // so we can check it matches the proposed checkpoint we used as parent, and has valid attestations.
244
+ if (signedAttestations && (!isPipelining || (await this.waitForValidParentCheckpointOnL1()))) {
245
+ await this.enqueueCheckpointForSubmission({ checkpoint, ...signedAttestations });
185
246
  }
186
- }
187
247
 
188
- // Compute the earliest time to submit: pipeline slot start when pipelining, now otherwise.
189
- const submitAfter = this.epochCache.isProposerPipeliningEnabled()
190
- ? new Date(Number(getTimestampForSlot(this.targetSlot, this.l1Constants)) * 1000)
191
- : new Date(this.dateProvider.now());
192
-
193
- // TODO(https://github.com/AztecProtocol/aztec-packages/pull/21250): should discard the pending submission if a reorg occurs underneath
194
-
195
- // Schedule L1 submission in the background so the work loop returns immediately.
196
- // The publisher will sleep until submitAfter, then send the bundled requests.
197
- // The promise is stored so it can be awaited during shutdown.
198
- this.pendingL1Submission = this.publisher
199
- .sendRequestsAt(submitAfter)
200
- .then(async l1Response => {
201
- const proposedAction = l1Response?.successfulActions.find(a => a === 'propose');
202
- if (proposedAction) {
203
- this.eventEmitter.emit('checkpoint-published', { checkpoint: this.checkpointNumber, slot: this.targetSlot });
204
- const coinbase = checkpoint?.header.coinbase;
205
- await this.metrics.incFilledSlot(this.publisher.getSenderAddress().toString(), coinbase);
206
- } else if (checkpoint) {
207
- this.eventEmitter.emit('checkpoint-publish-failed', { ...l1Response, slot: this.targetSlot });
208
-
209
- if (this.epochCache.isProposerPipeliningEnabled()) {
210
- this.metrics.recordPipelineDiscard();
211
- }
248
+ // If we failed to collect attestations, at least check if we need to issue an invalidation
249
+ // Note that if we are not pipelining, we enqueued the invalidation at the beginning
250
+ if (!signedAttestations && isPipelining && (await this.waitForSyncedL2SlotNumber(this.slotNow))) {
251
+ const validationStatus = await this.l2BlockSource.getPendingChainValidationStatus();
252
+ if (!validationStatus.valid) {
253
+ this.log.warn(
254
+ `Checkpoint ${validationStatus.checkpoint.checkpointNumber} has invalid attestations, enqueuing invalidation in spite of attestation collection failure`,
255
+ { checkpoint: validationStatus.checkpoint, reason: validationStatus.reason },
256
+ );
257
+ await this.enqueueInvalidation(validationStatus);
212
258
  }
213
- })
214
- .catch(err => {
215
- this.log.error(`Background L1 submission failed for slot ${this.targetSlot}`, err);
216
- if (checkpoint) {
217
- this.eventEmitter.emit('checkpoint-publish-failed', { slot: this.targetSlot });
218
-
219
- if (this.epochCache.isProposerPipeliningEnabled()) {
220
- this.metrics.recordPipelineDiscard();
221
- }
259
+ }
260
+
261
+ // Send whatever was enqueued: votes + (propose | invalidation | nothing).
262
+ // Compute the earliest time to submit: pipeline slot start when pipelining, now otherwise.
263
+ const submitAfter = isPipelining
264
+ ? new Date(Number(getTimestampForSlot(this.targetSlot, this.l1Constants)) * 1000)
265
+ : new Date(this.dateProvider.now());
266
+
267
+ const l1Response = await this.publisher.sendRequestsAt(submitAfter);
268
+ const proposedAction = l1Response?.successfulActions.find(a => a === 'propose');
269
+ if (proposedAction) {
270
+ this.logCheckpointEvent('published', `Checkpoint published for slot ${this.targetSlot}`, {
271
+ slot: this.targetSlot,
272
+ checkpointNumber: this.checkpointNumber,
273
+ successfulActions: l1Response?.successfulActions,
274
+ sentActions: l1Response?.sentActions,
275
+ });
276
+ this.eventEmitter.emit('checkpoint-published', { checkpoint: this.checkpointNumber, slot: this.targetSlot });
277
+ const coinbase = checkpoint.header.coinbase;
278
+ await this.metrics.incFilledSlot(this.publisher.getSenderAddress().toString(), coinbase);
279
+ } else {
280
+ this.logCheckpointEvent('publish-failed', `Checkpoint publish failed for slot ${this.targetSlot}`, {
281
+ slot: this.targetSlot,
282
+ checkpointNumber: this.checkpointNumber,
283
+ successfulActions: l1Response?.successfulActions,
284
+ failedActions: l1Response?.failedActions,
285
+ sentActions: l1Response?.sentActions,
286
+ expiredActions: l1Response?.expiredActions,
287
+ reason: 'propose_action_not_successful',
288
+ });
289
+ this.log.warn(`Checkpoint publish failed for slot ${this.targetSlot}`, {
290
+ slot: this.targetSlot,
291
+ checkpointNumber: this.checkpointNumber,
292
+ successfulActions: l1Response?.successfulActions,
293
+ failedActions: l1Response?.failedActions,
294
+ sentActions: l1Response?.sentActions,
295
+ expiredActions: l1Response?.expiredActions,
296
+ reason: 'propose_action_not_successful',
297
+ });
298
+ this.eventEmitter.emit('checkpoint-publish-failed', { ...l1Response, slot: this.targetSlot });
299
+ if (isPipelining) {
300
+ this.metrics.recordPipelineDiscard();
222
301
  }
302
+ }
303
+ } catch (err) {
304
+ if (err instanceof SequencerInterruptedError) {
305
+ return;
306
+ }
307
+ this.logCheckpointEvent('publish-failed', `Checkpoint publish failed for slot ${this.targetSlot}`, {
308
+ slot: this.targetSlot,
309
+ checkpointNumber: this.checkpointNumber,
310
+ reason: err instanceof Error ? err.message : String(err),
223
311
  });
224
-
225
- // Return the built checkpoint immediately — the work loop is now unblocked
226
- return checkpoint;
312
+ this.log.error(`Background attestation/L1 pipeline failed for slot ${this.targetSlot}`, err, {
313
+ slot: this.targetSlot,
314
+ checkpointNumber: this.checkpointNumber,
315
+ reason: err instanceof Error ? err.message : String(err),
316
+ });
317
+ this.eventEmitter.emit('checkpoint-publish-failed', { slot: this.targetSlot });
318
+ if (isPipelining) {
319
+ this.metrics.recordPipelineDiscard();
320
+ }
321
+ }
227
322
  }
228
323
 
229
324
  /** Enqueues the checkpoint for L1 submission. Called after pipeline sleep in execute(). */
@@ -249,13 +344,155 @@ export class CheckpointProposalJob implements Traceable {
249
344
  }
250
345
  }
251
346
 
347
+ const isPipelining = this.epochCache.isProposerPipeliningEnabled();
348
+ const submissionSimulationOverridesPlan = buildSubmissionSimulationOverridesPlan({
349
+ pipelinedParentPlan: this.pipelinedParentSimulationOverridesPlan,
350
+ invalidateToPendingCheckpointNumber: this.invalidateCheckpoint?.forcePendingCheckpointNumber,
351
+ lastArchiveRoot: checkpoint.header.lastArchiveRoot,
352
+ pipeliningEnabled: isPipelining,
353
+ });
354
+
252
355
  await this.publisher.enqueueProposeCheckpoint(checkpoint, attestations, attestationsSignature, {
253
356
  txTimeoutAt,
254
- forcePendingCheckpointNumber: this.invalidateCheckpoint?.forcePendingCheckpointNumber,
255
- forceProposedFeeHeader: this.computedForceProposedFeeHeader,
357
+ ...(submissionSimulationOverridesPlan ? { simulationOverridesPlan: submissionSimulationOverridesPlan } : {}),
358
+ });
359
+ }
360
+
361
+ /**
362
+ * Wait until the archiver syncs past the given L2 slot number.
363
+ * The deadline is the end of `this.targetSlot`, beyond which any pipelined work would miss its
364
+ * L1 submission window and is no longer useful.
365
+ */
366
+ private async waitForSyncedL2SlotNumber(waitForSlot: SlotNumber): Promise<boolean> {
367
+ const targetSlotStart = Number(getTimestampForSlot(this.targetSlot, this.l1Constants));
368
+ const targetSlotEndMs = (targetSlotStart + this.l1Constants.slotDuration) * 1000;
369
+ const syncDelayTolerance = this.l1Constants.ethereumSlotDuration * 2 * 1000;
370
+ const timeoutSeconds = Math.max(0.1, (targetSlotEndMs + syncDelayTolerance - this.dateProvider.now()) / 1000);
371
+
372
+ try {
373
+ return await retryUntil(
374
+ async () => {
375
+ const syncedSlot = await this.l2BlockSource.getSyncedL2SlotNumber();
376
+ return syncedSlot !== undefined && syncedSlot >= waitForSlot;
377
+ },
378
+ `archiver sync past slot ${waitForSlot}`,
379
+ timeoutSeconds,
380
+ 0.2,
381
+ );
382
+ } catch {
383
+ this.log.warn(
384
+ `Archiver did not sync L1 past slot ${waitForSlot} before slot ${this.targetSlot} expired, discarding pipelined work`,
385
+ { checkpointNumber: this.checkpointNumber },
386
+ );
387
+ this.emitPipelinedCheckpointDiscarded('archiver-sync-timeout');
388
+ return false;
389
+ }
390
+ }
391
+
392
+ /**
393
+ * Waits for the parent checkpoint to land on L1 before submitting a pipelined checkpoint.
394
+ * Polls until the archiver has synced L1 past the parent's slot, then verifies:
395
+ * - If we built on a proposed parent: it must have landed on L1 with matching hash and valid attestations.
396
+ * - If we built without a proposed parent: no new checkpoint must have appeared for that slot.
397
+ * If the parent has invalid attestations, enqueues an invalidation. Returns whether to proceed with the proposal.
398
+ */
399
+ protected async waitForValidParentCheckpointOnL1(): Promise<boolean> {
400
+ const parentCheckpointNumber = CheckpointNumber(this.checkpointNumber - 1);
401
+
402
+ // Wait until archiver has synced L1 past the parent's slot (slotNow)
403
+ if (!(await this.waitForSyncedL2SlotNumber(this.slotNow))) {
404
+ return false;
405
+ }
406
+
407
+ const tips = await this.l2BlockSource.getL2Tips();
408
+ const checkpointedNumber = tips.checkpointed.checkpoint.number;
409
+
410
+ // We built on top of a proposed checkpoint. Verify it landed on L1 as expected.
411
+ if (this.proposedCheckpointData) {
412
+ // After syncing from L1 we see the chain tip has invalid attestations. This means the parent checkpoint was posted
413
+ // with invalid attestations, or it built on top of something with invalid attestations and didnt invalidate them.
414
+ // Either way, we thought our parent would be valid, so we have to throw away our work. But at least we'll try and
415
+ // invalidate on L1 so we clean up the chain for the next proposer. And we'll slash them, but that's handled elsewhere.
416
+ const validationStatus = await this.l2BlockSource.getPendingChainValidationStatus();
417
+ if (!validationStatus.valid) {
418
+ this.log.warn(
419
+ `Parent checkpoint ${parentCheckpointNumber} has invalid attestations, discarding pipelined work`,
420
+ { checkpointNumber: this.checkpointNumber, reason: validationStatus.reason },
421
+ );
422
+ this.emitPipelinedCheckpointDiscarded('parent-invalid-attestations');
423
+ await this.enqueueInvalidation(validationStatus);
424
+ return false;
425
+ }
426
+
427
+ // The pending chain is valid. But did the parent checkpoint land on L1 at all?
428
+ if (checkpointedNumber < parentCheckpointNumber) {
429
+ this.log.warn(`Parent checkpoint ${parentCheckpointNumber} did not land on L1, discarding pipelined work`, {
430
+ checkpointNumber: this.checkpointNumber,
431
+ checkpointedNumber,
432
+ });
433
+ this.emitPipelinedCheckpointDiscarded('parent-not-on-l1');
434
+ return false;
435
+ }
436
+
437
+ // It landed. But is it the one we were expecting?
438
+ const expectedHash = this.proposedCheckpointData.header.hash().toString();
439
+ if (tips.checkpointed.checkpoint.hash !== expectedHash) {
440
+ this.log.warn(`Parent checkpoint ${parentCheckpointNumber} hash mismatch on L1, discarding pipelined work`, {
441
+ checkpointNumber: this.checkpointNumber,
442
+ expectedHash,
443
+ actualHash: tips.checkpointed.checkpoint.hash,
444
+ });
445
+ this.emitPipelinedCheckpointDiscarded('parent-hash-mismatch');
446
+ return false;
447
+ }
448
+
449
+ return true;
450
+ } else {
451
+ // We didn't see a proposed checkpoint at build time, so we built on checkpointed parent from two slots ago.
452
+ // But if a new checkpoint for the previous slot appeared on L1 in the meantime, our checkpoint assumed the wrong parent,
453
+ // so we have to discard our work. This can happen if we're somehow cut off from p2p and fail to see the checkpoint
454
+ // proposal for the previous slot.
455
+ if (checkpointedNumber > parentCheckpointNumber) {
456
+ this.log.warn(
457
+ `Unexpected checkpoint ${checkpointedNumber} landed on L1 after we built on top of parent ${parentCheckpointNumber}, discarding pipelined work`,
458
+ { checkpointNumber: this.checkpointNumber, checkpointedNumber },
459
+ );
460
+ this.emitPipelinedCheckpointDiscarded('unexpected-parent-appeared');
461
+ return false;
462
+ }
463
+
464
+ return true;
465
+ }
466
+ }
467
+
468
+ /** Emits the pipelined-checkpoint-discarded event and records the metric. */
469
+ private emitPipelinedCheckpointDiscarded(reason: string): void {
470
+ this.metrics.recordPipelineParentCheckpointMismatch(reason);
471
+ this.eventEmitter.emit('pipelined-checkpoint-discarded', {
472
+ slot: this.targetSlot,
473
+ checkpointNumber: this.checkpointNumber,
474
+ reason,
256
475
  });
257
476
  }
258
477
 
478
+ /** Simulates and enqueues an invalidation request for the invalid parent checkpoint. */
479
+ private async enqueueInvalidation(validationStatus: ValidateCheckpointResult): Promise<void> {
480
+ if (this.config.skipInvalidateBlockAsProposer) {
481
+ this.log.warn(`Skipping checkpoint invalidation as proposer due to test configuration`);
482
+ return;
483
+ }
484
+ const invalidateRequest = await this.publisher.simulateInvalidateCheckpoint(validationStatus);
485
+ if (invalidateRequest) {
486
+ const submissionSlotStart = Number(getTimestampForSlot(this.targetSlot, this.l1Constants));
487
+ const txTimeoutAt = new Date((submissionSlotStart + this.l1Constants.slotDuration) * 1000);
488
+ this.publisher.enqueueInvalidateCheckpoint(invalidateRequest, { txTimeoutAt });
489
+ } else {
490
+ this.log.info(`Invalidation simulation returned undefined, checkpoint may have been removed already`, {
491
+ checkpointNumber: this.checkpointNumber,
492
+ });
493
+ }
494
+ }
495
+
259
496
  @trackSpan('CheckpointProposalJob.proposeCheckpoint', function () {
260
497
  return {
261
498
  // nullish operator needed for tests
@@ -263,19 +500,33 @@ export class CheckpointProposalJob implements Traceable {
263
500
  [Attributes.SLOT_NUMBER]: this.targetSlot,
264
501
  };
265
502
  })
266
- private async proposeCheckpoint(): Promise<CheckpointProposalResult | undefined> {
503
+ private async proposeCheckpoint(): Promise<CheckpointProposalBroadcast | undefined> {
267
504
  try {
505
+ const now = this.dateProvider.now();
506
+ if (this.epochCache.isProposerPipeliningEnabled() && this.proposedCheckpointData) {
507
+ // Measure against the wall-clock slot whose build window we are currently using.
508
+ // In pipelining mode `targetSlot` is intentionally one slot ahead, which makes the
509
+ // target-slot boundary a full slot away from the actual build start time.
510
+ const slotBoundaryMs = Number(getTimestampForSlot(this.slotNow, this.l1Constants)) * 1000;
511
+ this.checkpointMetrics.recordPipelinedCheckpointBuildStartOffsetFromSlotBoundary(now - slotBoundaryMs);
512
+ }
513
+ this.checkpointMetrics.startCheckpointTiming(now);
514
+
268
515
  // Get operator configured coinbase and fee recipient for this attestor
269
516
  const coinbase = this.validatorClient.getCoinbaseForAttestor(this.attestorAddress);
270
517
  const feeRecipient = this.validatorClient.getFeeRecipientForAttestor(this.attestorAddress);
271
518
 
272
519
  // Start the checkpoint
273
520
  this.setStateFn(SequencerState.INITIALIZING_CHECKPOINT, this.targetSlot);
274
- this.log.info(`Starting checkpoint proposal`, {
521
+ this.logCheckpointEvent('slot-started', `Starting checkpoint proposal for slot ${this.targetSlot}`, {
275
522
  buildSlot: this.slotNow,
276
523
  submissionSlot: this.targetSlot,
524
+ slot: this.targetSlot,
525
+ checkpointNumber: this.checkpointNumber,
277
526
  pipelining: this.epochCache.isProposerPipeliningEnabled(),
278
527
  proposer: this.proposer?.toString(),
528
+ attestorAddress: this.attestorAddress.toString(),
529
+ publisherAddress: this.publisher.getSenderAddress().toString(),
279
530
  coinbase: coinbase.toString(),
280
531
  });
281
532
  this.metrics.incOpenSlot(this.targetSlot, this.proposer?.toString() ?? 'unknown');
@@ -289,21 +540,20 @@ export class CheckpointProposalJob implements Traceable {
289
540
  // When pipelining, force the proposed checkpoint number and fee header to our parent so the
290
541
  // fee computation sees the same chain tip that L1 will see once the previous pipelined checkpoint lands.
291
542
  const isPipelining = this.epochCache.isProposerPipeliningEnabled();
292
- const parentCheckpointNumber = isPipelining ? CheckpointNumber(this.checkpointNumber - 1) : undefined;
293
-
294
- // Compute the parent's fee header override when pipelining
295
- if (isPipelining && this.proposedCheckpointData) {
296
- this.computedForceProposedFeeHeader = await this.computeForceProposedFeeHeader(parentCheckpointNumber!);
297
- }
543
+ this.pipelinedParentSimulationOverridesPlan = isPipelining
544
+ ? await buildPipelinedParentSimulationOverridesPlan({
545
+ checkpointNumber: this.checkpointNumber,
546
+ proposedCheckpointData: this.proposedCheckpointData,
547
+ rollup: this.publisher.rollupContract,
548
+ log: this.log,
549
+ })
550
+ : undefined;
298
551
 
299
552
  const checkpointGlobalVariables = await this.globalsBuilder.buildCheckpointGlobalVariables(
300
553
  coinbase,
301
554
  feeRecipient,
302
555
  this.targetSlot,
303
- {
304
- forcePendingCheckpointNumber: parentCheckpointNumber,
305
- forceProposedFeeHeader: this.computedForceProposedFeeHeader,
306
- },
556
+ this.pipelinedParentSimulationOverridesPlan,
307
557
  );
308
558
 
309
559
  // Collect L1 to L2 messages for the checkpoint and compute their hash
@@ -344,7 +594,7 @@ export class CheckpointProposalJob implements Traceable {
344
594
  };
345
595
 
346
596
  let blocksInCheckpoint: L2Block[] = [];
347
- let blockPendingBroadcast: { block: L2Block; txs: Tx[] } | undefined = undefined;
597
+ let blockPendingBroadcast: BlockProposal | undefined = undefined;
348
598
  const checkpointBuildTimer = new Timer();
349
599
 
350
600
  try {
@@ -368,16 +618,38 @@ export class CheckpointProposalJob implements Traceable {
368
618
  }
369
619
 
370
620
  if (blocksInCheckpoint.length === 0) {
371
- this.log.warn(`No blocks were built for slot ${this.targetSlot}`, { slot: this.targetSlot });
621
+ this.logCheckpointEvent('build-failed', `Checkpoint build failed for slot ${this.targetSlot}`, {
622
+ slot: this.targetSlot,
623
+ checkpointNumber: this.checkpointNumber,
624
+ reason: 'no_blocks_built',
625
+ });
626
+ this.log.warn(`No blocks were built for slot ${this.targetSlot}`, {
627
+ slot: this.targetSlot,
628
+ checkpointNumber: this.checkpointNumber,
629
+ reason: 'no_blocks_built',
630
+ });
372
631
  this.eventEmitter.emit('checkpoint-empty', { slot: this.targetSlot });
373
632
  return undefined;
374
633
  }
375
634
 
376
635
  const minBlocksForCheckpoint = this.config.minBlocksForCheckpoint;
377
636
  if (minBlocksForCheckpoint !== undefined && blocksInCheckpoint.length < minBlocksForCheckpoint) {
637
+ this.logCheckpointEvent('build-failed', `Checkpoint build failed for slot ${this.targetSlot}`, {
638
+ slot: this.targetSlot,
639
+ checkpointNumber: this.checkpointNumber,
640
+ blocksBuilt: blocksInCheckpoint.length,
641
+ minBlocksForCheckpoint,
642
+ reason: 'min_blocks_not_met',
643
+ });
378
644
  this.log.warn(
379
645
  `Checkpoint has fewer blocks than minimum (${blocksInCheckpoint.length} < ${minBlocksForCheckpoint}), skipping proposal`,
380
- { slot: this.targetSlot, blocksBuilt: blocksInCheckpoint.length, minBlocksForCheckpoint },
646
+ {
647
+ slot: this.targetSlot,
648
+ checkpointNumber: this.checkpointNumber,
649
+ blocksBuilt: blocksInCheckpoint.length,
650
+ minBlocksForCheckpoint,
651
+ reason: 'min_blocks_not_met',
652
+ },
381
653
  );
382
654
  return undefined;
383
655
  }
@@ -398,21 +670,43 @@ export class CheckpointProposalJob implements Traceable {
398
670
  maxTxsPerCheckpoint: this.config.maxTxsPerCheckpoint,
399
671
  });
400
672
  } catch (err) {
673
+ this.logCheckpointEvent('build-failed', `Checkpoint build failed for slot ${this.targetSlot}`, {
674
+ slot: this.targetSlot,
675
+ checkpointNumber: this.checkpointNumber,
676
+ blocksBuilt: blocksInCheckpoint.length,
677
+ reason: 'invalid_checkpoint',
678
+ checkpoint: checkpoint.header.toInspect(),
679
+ });
401
680
  this.log.error(`Built an invalid checkpoint at slot ${this.slotNow} (skipping proposal)`, err, {
681
+ slot: this.targetSlot,
682
+ checkpointNumber: this.checkpointNumber,
683
+ blocksBuilt: blocksInCheckpoint.length,
684
+ reason: 'invalid_checkpoint',
402
685
  checkpoint: checkpoint.header.toInspect(),
403
686
  });
404
687
  return undefined;
405
688
  }
406
689
 
407
690
  // Record checkpoint-level build metrics
408
- this.metrics.recordCheckpointBuild(
691
+ this.checkpointMetrics.recordCheckpointBuild(
409
692
  checkpointBuildTimer.ms(),
410
693
  blocksInCheckpoint.length,
411
694
  checkpoint.getStats().txCount,
412
695
  Number(checkpoint.header.totalManaUsed.toBigInt()),
413
696
  );
697
+ this.logCheckpointEvent('built', `Checkpoint built for slot ${this.targetSlot}`, {
698
+ slot: this.targetSlot,
699
+ buildSlot: this.slotNow,
700
+ checkpointNumber: this.checkpointNumber,
701
+ proposer: this.proposer?.toString(),
702
+ attestorAddress: this.attestorAddress.toString(),
703
+ publisherAddress: this.publisher.getSenderAddress().toString(),
704
+ blocksBuilt: blocksInCheckpoint.length,
705
+ txCount: checkpoint.getStats().txCount,
706
+ totalMana: Number(checkpoint.header.totalManaUsed.toBigInt()),
707
+ });
414
708
 
415
- // Do not collect attestations nor publish to L1 in fisherman mode
709
+ // In fisherman mode, return the checkpoint without broadcasting or collecting attestations
416
710
  if (this.config.fishermanMode) {
417
711
  this.log.info(
418
712
  `Built checkpoint for slot ${this.targetSlot} with ${blocksInCheckpoint.length} blocks. ` +
@@ -424,60 +718,29 @@ export class CheckpointProposalJob implements Traceable {
424
718
  },
425
719
  );
426
720
  this.metrics.recordCheckpointSuccess();
427
- return {
428
- checkpoint,
429
- attestations: CommitteeAttestationsAndSigners.empty(),
430
- attestationsSignature: Signature.empty(),
431
- };
721
+ // Return a broadcast result with a dummy proposal — fisherman mode skips attestation collection
722
+ return { checkpoint, proposal: undefined!, blockProposedAt: this.dateProvider.now() };
432
723
  }
433
724
 
434
- // Include the block pending broadcast in the checkpoint proposal if any
435
- const lastBlock = blockPendingBroadcast && {
436
- blockHeader: blockPendingBroadcast.block.header,
437
- indexWithinCheckpoint: blockPendingBroadcast.block.indexWithinCheckpoint,
438
- txs: blockPendingBroadcast.txs,
439
- };
440
-
441
725
  // Create the checkpoint proposal and broadcast it
442
726
  const proposal = await this.validatorClient.createCheckpointProposal(
443
727
  checkpoint.header,
444
728
  checkpoint.archive.root,
729
+ this.checkpointNumber,
445
730
  feeAssetPriceModifier,
446
- lastBlock,
731
+ blockPendingBroadcast,
447
732
  this.proposer,
448
733
  checkpointProposalOptions,
449
734
  );
450
735
 
451
736
  const blockProposedAt = this.dateProvider.now();
452
- await this.p2pClient.broadcastCheckpointProposal(proposal);
453
-
454
- this.setStateFn(SequencerState.COLLECTING_ATTESTATIONS, this.targetSlot);
455
- const attestations = await this.waitForAttestations(proposal);
456
- const blockAttestedAt = this.dateProvider.now();
457
-
458
- this.metrics.recordCheckpointAttestationDelay(blockAttestedAt - blockProposedAt);
459
-
460
- // Proposer must sign over the attestations before pushing them to L1
461
- const signer = this.proposer ?? this.publisher.getSenderAddress();
462
- let attestationsSignature: Signature;
463
- try {
464
- attestationsSignature = await this.validatorClient.signAttestationsAndSigners(
465
- attestations,
466
- signer,
467
- this.targetSlot,
468
- this.checkpointNumber,
469
- );
470
- } catch (err) {
471
- // We shouldn't really get here since we yield to another HA node
472
- // as soon as we see these errors when creating block or checkpoint proposals.
473
- if (this.handleHASigningError(err, 'Attestations signature')) {
474
- return undefined;
475
- }
476
- throw err;
737
+ if (!this.config.skipBroadcastProposals) {
738
+ await this.p2pClient.broadcastCheckpointProposal(proposal);
739
+ this.checkpointMetrics.noteCheckpointBroadcast(this.dateProvider.now());
477
740
  }
478
741
 
479
- // Return the result for the caller to enqueue after the pipeline sleep
480
- return { checkpoint, attestations, attestationsSignature };
742
+ // Return immediately after broadcast attestation collection happens in the background
743
+ return { checkpoint, proposal, blockProposedAt };
481
744
  } catch (err) {
482
745
  if (err && (err instanceof DutyAlreadySignedError || err instanceof SlashingProtectionError)) {
483
746
  // swallow this error. It's already been logged by a function deeper in the stack
@@ -500,20 +763,29 @@ export class CheckpointProposalJob implements Traceable {
500
763
  blockProposalOptions: BlockProposalOptions,
501
764
  ): Promise<{
502
765
  blocksInCheckpoint: L2Block[];
503
- blockPendingBroadcast: { block: L2Block; txs: Tx[] } | undefined;
766
+ blockPendingBroadcast: BlockProposal | undefined;
504
767
  }> {
505
768
  const blocksInCheckpoint: L2Block[] = [];
506
769
  const txHashesAlreadyIncluded = new Set<string>();
507
770
  const initialBlockNumber = BlockNumber(this.syncedToBlockNumber + 1);
508
771
 
509
772
  // Last block in the checkpoint will usually be flagged as pending broadcast, so we send it along with the checkpoint proposal
510
- let blockPendingBroadcast: { block: L2Block; txs: Tx[] } | undefined = undefined;
773
+ let blockPendingBroadcast: BlockProposal | undefined = undefined;
511
774
 
512
775
  while (true) {
513
776
  const blocksBuilt = blocksInCheckpoint.length;
514
777
  const indexWithinCheckpoint = IndexWithinCheckpoint(blocksBuilt);
515
778
  const blockNumber = BlockNumber(initialBlockNumber + blocksBuilt);
516
779
 
780
+ if (blocksBuilt >= this.config.maxBlocksPerCheckpoint) {
781
+ this.log.debug(`Reached max blocks per checkpoint`, {
782
+ slot: this.targetSlot,
783
+ blocksBuilt,
784
+ maxBlocksPerCheckpoint: this.config.maxBlocksPerCheckpoint,
785
+ });
786
+ break;
787
+ }
788
+
517
789
  const secondsIntoSlot = this.getSecondsIntoSlot();
518
790
  const timingInfo = this.timetable.canStartNextBlock(secondsIntoSlot);
519
791
 
@@ -540,19 +812,20 @@ export class CheckpointProposalJob implements Traceable {
540
812
  txHashesAlreadyIncluded,
541
813
  });
542
814
 
543
- // TODO(palla/mbps): Review these conditions. We may want to keep trying in some scenarios.
544
- if (!buildResult && timingInfo.isLastBlock) {
545
- // If no block was produced due to not enough txs and this was the last subslot, exit
546
- break;
547
- } else if (!buildResult && timingInfo.deadline !== undefined) {
548
- // But if there is still time for more blocks, wait until the next subslot and try again
815
+ // If we failed to build the block due to insufficient txs, we try again if there is still time left in the slot
816
+ if ('failure' in buildResult) {
817
+ // If this was the last subslot, or we're running with a single block per slot, we're done
818
+ if (timingInfo.isLastBlock || timingInfo.deadline === undefined) {
819
+ break;
820
+ }
821
+ // Otherwise, if there is still time for more blocks, we wait until the next subslot and try again
549
822
  await this.waitUntilNextSubslot(timingInfo.deadline);
550
823
  continue;
551
- } else if (!buildResult) {
552
- // Exit if there is no possibility of building more blocks
553
- break;
554
- } else if ('error' in buildResult) {
555
- // If there was an error building the block, just exit the loop and give up the rest of the slot
824
+ }
825
+
826
+ // If there was an error building the block, we just exit the loop and give up the rest of the slot.
827
+ // We don't want to risk building more blocks if something went wrong.
828
+ if ('error' in buildResult) {
556
829
  if (!(buildResult.error instanceof SequencerInterruptedError)) {
557
830
  this.log.warn(`Halting block building for slot ${this.targetSlot}`, {
558
831
  slot: this.targetSlot,
@@ -564,35 +837,39 @@ export class CheckpointProposalJob implements Traceable {
564
837
  }
565
838
 
566
839
  const { block, usedTxs } = buildResult;
840
+ this.checkpointMetrics.noteCheckpointBlockBuilt(this.dateProvider.now(), {
841
+ isFirstBlock: blocksBuilt === 0,
842
+ isLastBlock: timingInfo.isLastBlock,
843
+ });
844
+
567
845
  blocksInCheckpoint.push(block);
568
846
  usedTxs.forEach(tx => txHashesAlreadyIncluded.add(tx.txHash.toString()));
569
847
 
570
- // If this is the last block, sync it to the archiver and exit the loop
571
- // so we can build the checkpoint and start collecting attestations.
848
+ // Sign the block proposal. This will throw if HA signing fails.
849
+ const proposal = await this.createBlockProposal(block, inHash, usedTxs, blockProposalOptions);
850
+
851
+ // Sync the proposed block to the archiver to make it available, only after we've managed to sign the proposal,
852
+ // so we avoid polluting our archive with a block that would fail.
853
+ // We wait for the sync to succeed, as this helps catch consistency errors, even if it means we lose some time for block-building.
854
+ // If this throws, we abort the entire checkpoint.
855
+ await this.syncProposedBlockToArchiver(block);
856
+
857
+ // If this is the last block, do not broadcast it, since it will be included in the checkpoint proposal.
572
858
  if (timingInfo.isLastBlock) {
573
- await this.syncProposedBlockToArchiver(block);
574
859
  this.log.verbose(`Completed final block ${blockNumber} for slot ${this.targetSlot}`, {
575
860
  slot: this.targetSlot,
576
861
  blockNumber,
577
862
  blocksBuilt,
578
863
  });
579
- blockPendingBroadcast = { block, txs: usedTxs };
864
+
865
+ blockPendingBroadcast = proposal;
580
866
  break;
581
867
  }
582
868
 
583
- // Broadcast the block proposal (unless we're in fisherman mode) unless the block is the last one,
584
- // in which case we'll broadcast it along with the checkpoint at the end of the loop.
585
- // Note that we only send the block to the archiver if we manage to create the proposal, so if there's
586
- // a HA error we don't pollute our archiver with a block that won't make it to the chain.
587
- const proposal = await this.createBlockProposal(block, inHash, usedTxs, blockProposalOptions);
588
-
589
- // Sync the proposed block to the archiver to make it available, only after we've managed to sign the proposal.
590
- // We wait for the sync to succeed, as this helps catch consistency errors, even if it means we lose some time for block-building.
591
- // If this throws, we abort the entire checkpoint.
592
- await this.syncProposedBlockToArchiver(block);
593
-
594
869
  // Once we have a signed proposal and the archiver agreed with our proposed block, then we broadcast it.
595
- proposal && (await this.p2pClient.broadcastProposal(proposal));
870
+ if (proposal && !this.config.skipBroadcastProposals) {
871
+ await this.p2pClient.broadcastProposal(proposal);
872
+ }
596
873
 
597
874
  // Wait until the next block's start time
598
875
  await this.waitUntilNextSubslot(timingInfo.deadline);
@@ -619,6 +896,7 @@ export class CheckpointProposalJob implements Traceable {
619
896
  }
620
897
  return this.validatorClient.createBlockProposal(
621
898
  block.header,
899
+ this.checkpointNumber,
622
900
  block.indexWithinCheckpoint,
623
901
  inHash,
624
902
  block.archive.root,
@@ -650,7 +928,9 @@ export class CheckpointProposalJob implements Traceable {
650
928
  buildDeadline: Date | undefined;
651
929
  txHashesAlreadyIncluded: Set<string>;
652
930
  },
653
- ): Promise<{ block: L2Block; usedTxs: Tx[] } | { error: Error } | undefined> {
931
+ ): Promise<
932
+ { block: L2Block; usedTxs: Tx[] } | { failure: 'insufficient-txs' | 'insufficient-valid-txs' } | { error: Error }
933
+ > {
654
934
  const { blockTimestamp, forceCreate, blockNumber, indexWithinCheckpoint, buildDeadline, txHashesAlreadyIncluded } =
655
935
  opts;
656
936
 
@@ -663,13 +943,30 @@ export class CheckpointProposalJob implements Traceable {
663
943
  // Wait until we have enough txs to build the block
664
944
  const { availableTxs, canStartBuilding, minTxs } = await this.waitForMinTxs(opts);
665
945
  if (!canStartBuilding) {
946
+ this.logCheckpointEvent('block-build-failed', `Block build failed for slot ${this.targetSlot}`, {
947
+ reason: 'insufficient_txs',
948
+ blockNumber,
949
+ slot: this.targetSlot,
950
+ checkpointNumber: this.checkpointNumber,
951
+ indexWithinCheckpoint,
952
+ availableTxs,
953
+ minTxs,
954
+ });
666
955
  this.log.warn(
667
956
  `Not enough txs to build block ${blockNumber} at index ${indexWithinCheckpoint} in slot ${this.targetSlot} (got ${availableTxs} txs but needs ${minTxs})`,
668
- { blockNumber, slot: this.targetSlot, indexWithinCheckpoint },
957
+ {
958
+ reason: 'insufficient_txs',
959
+ blockNumber,
960
+ slot: this.targetSlot,
961
+ checkpointNumber: this.checkpointNumber,
962
+ indexWithinCheckpoint,
963
+ availableTxs,
964
+ minTxs,
965
+ },
669
966
  );
670
967
  this.eventEmitter.emit('block-tx-count-check-failed', { minTxs, availableTxs, slot: this.targetSlot });
671
968
  this.metrics.recordBlockProposalFailed('insufficient_txs');
672
- return undefined;
969
+ return { failure: 'insufficient-txs' };
673
970
  }
674
971
 
675
972
  // Create iterator to pending txs. We filter out txs already included in previous blocks in the checkpoint
@@ -717,10 +1014,21 @@ export class CheckpointProposalJob implements Traceable {
717
1014
  await this.dropFailedTxsFromP2P(buildResult.failedTxs);
718
1015
 
719
1016
  if (buildResult.status === 'insufficient-valid-txs') {
1017
+ this.logCheckpointEvent('block-build-failed', `Block build failed for slot ${this.targetSlot}`, {
1018
+ reason: 'insufficient_valid_txs',
1019
+ slot: this.targetSlot,
1020
+ checkpointNumber: this.checkpointNumber,
1021
+ blockNumber,
1022
+ numTxs: buildResult.processedCount,
1023
+ indexWithinCheckpoint,
1024
+ minValidTxs,
1025
+ });
720
1026
  this.log.warn(
721
1027
  `Block ${blockNumber} at index ${indexWithinCheckpoint} on slot ${this.targetSlot} has too few valid txs to be proposed`,
722
1028
  {
1029
+ reason: 'insufficient_valid_txs',
723
1030
  slot: this.targetSlot,
1031
+ checkpointNumber: this.checkpointNumber,
724
1032
  blockNumber,
725
1033
  numTxs: buildResult.processedCount,
726
1034
  indexWithinCheckpoint,
@@ -732,7 +1040,7 @@ export class CheckpointProposalJob implements Traceable {
732
1040
  slot: this.targetSlot,
733
1041
  });
734
1042
  this.metrics.recordBlockProposalFailed('insufficient_valid_txs');
735
- return undefined;
1043
+ return { failure: 'insufficient-valid-txs' };
736
1044
  }
737
1045
 
738
1046
  // Block creation succeeded, emit stats and metrics
@@ -761,7 +1069,7 @@ export class CheckpointProposalJob implements Traceable {
761
1069
  slot: this.targetSlot,
762
1070
  buildSlot: this.slotNow,
763
1071
  });
764
- this.metrics.recordBuiltBlock(blockBuildDuration, block.header.totalManaUsed.toNumberUnsafe());
1072
+ this.metrics.recordBuiltBlock(blockBuildDuration, block.header.totalManaUsed.toNumberUnsafe(), this.targetSlot);
765
1073
 
766
1074
  return { block, usedTxs };
767
1075
  } catch (err: any) {
@@ -769,7 +1077,18 @@ export class CheckpointProposalJob implements Traceable {
769
1077
  reason: err.message,
770
1078
  slot: this.targetSlot,
771
1079
  });
772
- this.log.error(`Error building block`, err, { blockNumber, slot: this.targetSlot });
1080
+ this.logCheckpointEvent('block-build-failed', `Block build failed for slot ${this.targetSlot}`, {
1081
+ reason: err instanceof Error ? err.message : String(err),
1082
+ slot: this.targetSlot,
1083
+ checkpointNumber: this.checkpointNumber,
1084
+ blockNumber,
1085
+ });
1086
+ this.log.error(`Error building block`, err, {
1087
+ reason: err instanceof Error ? err.message : String(err),
1088
+ slot: this.targetSlot,
1089
+ checkpointNumber: this.checkpointNumber,
1090
+ blockNumber,
1091
+ });
773
1092
  this.metrics.recordBlockProposalFailed(err.name || 'unknown_error');
774
1093
  this.metrics.recordFailedBlock();
775
1094
  return { error: err };
@@ -841,15 +1160,47 @@ export class CheckpointProposalJob implements Traceable {
841
1160
  return { canStartBuilding: true, availableTxs, minTxs };
842
1161
  }
843
1162
 
1163
+ private async getSignedCommitteeAttestations(
1164
+ broadcast: CheckpointProposalBroadcast,
1165
+ ): Promise<{ attestations: CommitteeAttestationsAndSigners; attestationsSignature: Signature } | undefined> {
1166
+ const { proposal, blockProposedAt } = broadcast;
1167
+ this.setStateFn(SequencerState.COLLECTING_ATTESTATIONS, this.targetSlot);
1168
+ const attestations = await this.waitForAttestations(proposal);
1169
+ if (!attestations) {
1170
+ return undefined;
1171
+ }
1172
+ this.checkpointMetrics.recordCheckpointAttestationDelay(this.dateProvider.now() - blockProposedAt);
1173
+
1174
+ // Proposer must sign over the attestations before pushing them to L1
1175
+ const signer = this.proposer ?? this.publisher.getSenderAddress();
1176
+ try {
1177
+ const attestationsSignature = await this.validatorClient.signAttestationsAndSigners(
1178
+ attestations,
1179
+ signer,
1180
+ this.targetSlot,
1181
+ this.checkpointNumber,
1182
+ );
1183
+ return { attestations, attestationsSignature };
1184
+ } catch (err) {
1185
+ if (this.handleHASigningError(err, 'Attestations signature')) {
1186
+ return;
1187
+ }
1188
+ this.log.error(`Error signing attestations for checkpoint proposal at slot ${proposal.slotNumber}`, err);
1189
+ return undefined;
1190
+ }
1191
+ }
1192
+
844
1193
  /**
845
1194
  * Waits for enough attestations to be collected via p2p.
846
1195
  * This is run after all blocks for the checkpoint have been built.
847
1196
  */
848
1197
  @trackSpan('CheckpointProposalJob.waitForAttestations')
849
- private async waitForAttestations(proposal: CheckpointProposal): Promise<CommitteeAttestationsAndSigners> {
1198
+ private async waitForAttestations(
1199
+ proposal: CheckpointProposal,
1200
+ ): Promise<CommitteeAttestationsAndSigners | undefined> {
850
1201
  if (this.config.fishermanMode) {
851
1202
  this.log.debug('Skipping attestation collection in fisherman mode');
852
- return CommitteeAttestationsAndSigners.empty();
1203
+ return CommitteeAttestationsAndSigners.empty(this.getSignatureContext());
853
1204
  }
854
1205
 
855
1206
  const slotNumber = proposal.slotNumber;
@@ -859,7 +1210,7 @@ export class CheckpointProposalJob implements Traceable {
859
1210
  throw new Error('No committee when collecting attestations');
860
1211
  } else if (committee.length === 0) {
861
1212
  this.log.verbose(`Attesting committee is empty`);
862
- return CommitteeAttestationsAndSigners.empty();
1213
+ return CommitteeAttestationsAndSigners.empty(this.getSignatureContext());
863
1214
  } else {
864
1215
  this.log.debug(`Attesting committee length is ${committee.length}`, { committee });
865
1216
  }
@@ -868,12 +1219,18 @@ export class CheckpointProposalJob implements Traceable {
868
1219
 
869
1220
  if (this.config.skipCollectingAttestations) {
870
1221
  this.log.warn('Skipping attestation collection as per config (attesting with own keys only)');
871
- const attestations = await this.validatorClient?.collectOwnAttestations(proposal);
872
- return new CommitteeAttestationsAndSigners(orderAttestations(attestations ?? [], committee));
1222
+ const attestations = await this.validatorClient?.collectOwnAttestations(proposal, this.checkpointNumber);
1223
+ this.logCheckpointAttestations('collected', committee, attestations ?? [], numberOfRequiredAttestations, {
1224
+ reason: 'collect_own_only',
1225
+ });
1226
+ return new CommitteeAttestationsAndSigners(
1227
+ orderAttestations(attestations ?? [], committee),
1228
+ this.getSignatureContext(),
1229
+ );
873
1230
  }
874
1231
 
875
1232
  const attestationTimeAllowed = this.config.enforceTimeTable
876
- ? this.timetable.getMaxAllowedTime(SequencerState.PUBLISHING_CHECKPOINT)!
1233
+ ? this.timetable.getCheckpointAttestationDeadline()
877
1234
  : this.l1Constants.slotDuration;
878
1235
  const attestationDeadline = new Date((this.getSlotStartBuildTimestamp() + attestationTimeAllowed) * 1000);
879
1236
 
@@ -886,6 +1243,7 @@ export class CheckpointProposalJob implements Traceable {
886
1243
  proposal,
887
1244
  numberOfRequiredAttestations,
888
1245
  attestationDeadline,
1246
+ this.checkpointNumber,
889
1247
  );
890
1248
 
891
1249
  collectedAttestationsCount = attestations.length;
@@ -904,6 +1262,9 @@ export class CheckpointProposalJob implements Traceable {
904
1262
 
905
1263
  // Rollup contract requires that the signatures are provided in the order of the committee
906
1264
  const sorted = orderAttestations(trimmed, committee);
1265
+ this.logCheckpointAttestations('collected', committee, attestations, numberOfRequiredAttestations, {
1266
+ submittedCount: trimmed.length,
1267
+ });
907
1268
 
908
1269
  // Manipulate the attestations if we've been configured to do so
909
1270
  if (
@@ -915,17 +1276,56 @@ export class CheckpointProposalJob implements Traceable {
915
1276
  return this.manipulateAttestations(proposal.slotNumber, epoch, seed, committee, sorted);
916
1277
  }
917
1278
 
918
- return new CommitteeAttestationsAndSigners(sorted);
1279
+ return new CommitteeAttestationsAndSigners(sorted, this.getSignatureContext());
919
1280
  } catch (err) {
920
1281
  if (err && err instanceof AttestationTimeoutError) {
921
1282
  collectedAttestationsCount = err.collectedCount;
1283
+ this.logCheckpointAttestations('failed', committee, undefined, numberOfRequiredAttestations, {
1284
+ collectedCount: collectedAttestationsCount,
1285
+ reason: 'timeout',
1286
+ });
1287
+ this.log.error(
1288
+ `Timeout while waiting for attestations for checkpoint proposal at slot ${proposal.slotNumber} (collected ${collectedAttestationsCount}/${numberOfRequiredAttestations})`,
1289
+ err,
1290
+ );
1291
+ } else {
1292
+ this.logCheckpointAttestations('failed', committee, undefined, numberOfRequiredAttestations, {
1293
+ collectedCount: collectedAttestationsCount,
1294
+ reason: err instanceof Error ? err.message : String(err),
1295
+ });
1296
+ this.log.error(`Error collecting attestations for checkpoint proposal at slot ${proposal.slotNumber}`, err);
922
1297
  }
923
- throw err;
1298
+ return undefined;
924
1299
  } finally {
925
1300
  this.metrics.recordCollectedAttestations(collectedAttestationsCount, collectAttestationsTimer.ms());
926
1301
  }
927
1302
  }
928
1303
 
1304
+ private logCheckpointAttestations(
1305
+ status: 'collected' | 'failed',
1306
+ committee: EthAddress[],
1307
+ attestations: CheckpointAttestation[] | undefined,
1308
+ requiredAttestations: number,
1309
+ opts: { collectedCount?: number; submittedCount?: number; reason?: string } = {},
1310
+ ) {
1311
+ const signedValidators =
1312
+ attestations
1313
+ ?.map(attestation => attestation.getSender()?.toString())
1314
+ .filter((address): address is `0x${string}` => address !== undefined) ?? [];
1315
+ const collectedCount = opts.collectedCount ?? new Set(signedValidators).size;
1316
+ const missingValidatorCount = status === 'failed' ? Math.max(0, requiredAttestations - collectedCount) : undefined;
1317
+ this.logCheckpointEvent(`attestations-${status}`, `Checkpoint attestations ${status} for slot ${this.targetSlot}`, {
1318
+ slot: this.targetSlot,
1319
+ checkpointNumber: this.checkpointNumber,
1320
+ committeeSize: committee.length,
1321
+ requiredAttestations,
1322
+ collectedAttestations: collectedCount,
1323
+ ...(opts.submittedCount !== undefined && { submittedAttestations: opts.submittedCount }),
1324
+ ...(missingValidatorCount !== undefined && { missingValidatorCount }),
1325
+ ...(opts.reason !== undefined && { reason: opts.reason }),
1326
+ });
1327
+ }
1328
+
929
1329
  /** Breaks the attestations before publishing based on attack configs */
930
1330
  private manipulateAttestations(
931
1331
  slotNumber: SlotNumber,
@@ -969,7 +1369,7 @@ export class CheckpointProposalJob implements Traceable {
969
1369
  unfreeze(attestations[targetIndex]).signature = generateRecoverableSignature();
970
1370
  }
971
1371
  }
972
- return new CommitteeAttestationsAndSigners(attestations);
1372
+ return new CommitteeAttestationsAndSigners(attestations, this.getSignatureContext());
973
1373
  }
974
1374
 
975
1375
  if (this.config.shuffleAttestationOrdering) {
@@ -991,11 +1391,11 @@ export class CheckpointProposalJob implements Traceable {
991
1391
  [shuffled[i], shuffled[j]] = [shuffled[j], shuffled[i]];
992
1392
  }
993
1393
 
994
- const signers = new CommitteeAttestationsAndSigners(attestations).getSigners();
995
- return new MaliciousCommitteeAttestationsAndSigners(shuffled, signers);
1394
+ const signers = new CommitteeAttestationsAndSigners(attestations, this.getSignatureContext()).getSigners();
1395
+ return new MaliciousCommitteeAttestationsAndSigners(shuffled, signers, this.getSignatureContext());
996
1396
  }
997
1397
 
998
- return new CommitteeAttestationsAndSigners(attestations);
1398
+ return new CommitteeAttestationsAndSigners(attestations, this.getSignatureContext());
999
1399
  }
1000
1400
 
1001
1401
  private async dropFailedTxsFromP2P(failedTxs: FailedTx[]) {
@@ -1012,9 +1412,13 @@ export class CheckpointProposalJob implements Traceable {
1012
1412
  * Adds the proposed block to the archiver so it's available via P2P.
1013
1413
  * Gossip doesn't echo messages back to the sender, so the proposer's archiver/world-state
1014
1414
  * would never receive its own block without this explicit sync.
1415
+ *
1416
+ * In fisherman mode we skip this push: the fisherman builds blocks locally for validation
1417
+ * and fee analysis only, and pushing them to the archiver causes spurious reorg cascades
1418
+ * whenever the real proposer's block arrives from L1.
1015
1419
  */
1016
1420
  private async syncProposedBlockToArchiver(block: L2Block): Promise<void> {
1017
- if (this.config.skipPushProposedBlocksToArchiver) {
1421
+ if (this.config.skipPushProposedBlocksToArchiver || this.config.fishermanMode) {
1018
1422
  this.log.warn(`Skipping push of proposed block ${block.number} to archiver`, {
1019
1423
  blockNumber: block.number,
1020
1424
  slot: block.header.globalVariables.slotNumber,
@@ -1075,56 +1479,6 @@ export class CheckpointProposalJob implements Traceable {
1075
1479
  return false;
1076
1480
  }
1077
1481
 
1078
- /**
1079
- * In times of congestion we need to simulate using the correct fee header override for the previous block
1080
- * We calculate the correct fee header values.
1081
- *
1082
- * If we are in block 1, or the checkpoint we are querying does not exist, we return undefined. However
1083
- * If we are pipelining - where this function is called, the grandparentCheckpointNumber should always exist
1084
- * @param parentCheckpointNumber
1085
- * @returns
1086
- */
1087
- protected async computeForceProposedFeeHeader(parentCheckpointNumber: CheckpointNumber): Promise<
1088
- | {
1089
- checkpointNumber: CheckpointNumber;
1090
- feeHeader: FeeHeader;
1091
- }
1092
- | undefined
1093
- > {
1094
- if (!this.proposedCheckpointData) {
1095
- return undefined;
1096
- }
1097
-
1098
- const rollup = this.publisher.rollupContract;
1099
- const grandparentCheckpointNumber = CheckpointNumber(this.checkpointNumber - 2);
1100
- try {
1101
- const [grandparentCheckpoint, manaTarget] = await Promise.all([
1102
- rollup.getCheckpoint(grandparentCheckpointNumber),
1103
- rollup.getManaTarget(),
1104
- ]);
1105
-
1106
- if (!grandparentCheckpoint || !grandparentCheckpoint.feeHeader) {
1107
- this.log.error(
1108
- `Grandparent checkpoint or its feeHeader is undefined for checkpointNumber=${grandparentCheckpointNumber.toString()}`,
1109
- );
1110
- return undefined;
1111
- } else {
1112
- const parentFeeHeader = RollupContract.computeChildFeeHeader(
1113
- grandparentCheckpoint.feeHeader,
1114
- this.proposedCheckpointData.totalManaUsed,
1115
- this.proposedCheckpointData.feeAssetPriceModifier,
1116
- manaTarget,
1117
- );
1118
- return { checkpointNumber: parentCheckpointNumber, feeHeader: parentFeeHeader };
1119
- }
1120
- } catch (err) {
1121
- this.log.error(
1122
- `Failed to fetch grandparent checkpoint or mana target for checkpointNumber=${grandparentCheckpointNumber.toString()}: ${err}`,
1123
- );
1124
- return undefined;
1125
- }
1126
- }
1127
-
1128
1482
  /** Waits until a specific time within the current slot */
1129
1483
  @trackSpan('CheckpointProposalJob.waitUntilTimeInSlot')
1130
1484
  protected async waitUntilTimeInSlot(targetSecondsIntoSlot: number): Promise<void> {