@aztec/sequencer-client 0.0.1-commit.5de5ca79e → 0.0.1-commit.6201a7b05

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. package/dest/client/sequencer-client.d.ts +6 -1
  2. package/dest/client/sequencer-client.d.ts.map +1 -1
  3. package/dest/client/sequencer-client.js +11 -17
  4. package/dest/config.d.ts +1 -1
  5. package/dest/config.d.ts.map +1 -1
  6. package/dest/config.js +13 -12
  7. package/dest/global_variable_builder/fee_predictor.d.ts +37 -0
  8. package/dest/global_variable_builder/fee_predictor.d.ts.map +1 -0
  9. package/dest/global_variable_builder/fee_predictor.js +128 -0
  10. package/dest/global_variable_builder/fee_provider.d.ts +21 -0
  11. package/dest/global_variable_builder/fee_provider.d.ts.map +1 -0
  12. package/dest/global_variable_builder/fee_provider.js +58 -0
  13. package/dest/global_variable_builder/global_builder.d.ts +15 -14
  14. package/dest/global_variable_builder/global_builder.d.ts.map +1 -1
  15. package/dest/global_variable_builder/global_builder.js +16 -51
  16. package/dest/global_variable_builder/index.d.ts +4 -2
  17. package/dest/global_variable_builder/index.d.ts.map +1 -1
  18. package/dest/global_variable_builder/index.js +2 -0
  19. package/dest/publisher/config.d.ts +13 -1
  20. package/dest/publisher/config.d.ts.map +1 -1
  21. package/dest/publisher/config.js +19 -4
  22. package/dest/publisher/l1_tx_failed_store/failed_tx_store.d.ts +3 -4
  23. package/dest/publisher/l1_tx_failed_store/failed_tx_store.d.ts.map +1 -1
  24. package/dest/publisher/sequencer-publisher-factory.d.ts +3 -5
  25. package/dest/publisher/sequencer-publisher-factory.d.ts.map +1 -1
  26. package/dest/publisher/sequencer-publisher-factory.js +2 -3
  27. package/dest/publisher/sequencer-publisher.d.ts +48 -42
  28. package/dest/publisher/sequencer-publisher.d.ts.map +1 -1
  29. package/dest/publisher/sequencer-publisher.js +119 -111
  30. package/dest/sequencer/chain_state_overrides.d.ts +25 -0
  31. package/dest/sequencer/chain_state_overrides.d.ts.map +1 -0
  32. package/dest/sequencer/chain_state_overrides.js +39 -0
  33. package/dest/sequencer/checkpoint_proposal_job.d.ts +41 -11
  34. package/dest/sequencer/checkpoint_proposal_job.d.ts.map +1 -1
  35. package/dest/sequencer/checkpoint_proposal_job.js +525 -149
  36. package/dest/sequencer/checkpoint_proposal_job_metrics.d.ts +34 -0
  37. package/dest/sequencer/checkpoint_proposal_job_metrics.d.ts.map +1 -0
  38. package/dest/sequencer/checkpoint_proposal_job_metrics.js +72 -0
  39. package/dest/sequencer/checkpoint_voter.d.ts +1 -2
  40. package/dest/sequencer/checkpoint_voter.d.ts.map +1 -1
  41. package/dest/sequencer/checkpoint_voter.js +2 -5
  42. package/dest/sequencer/events.d.ts +6 -1
  43. package/dest/sequencer/events.d.ts.map +1 -1
  44. package/dest/sequencer/metrics.d.ts +9 -10
  45. package/dest/sequencer/metrics.d.ts.map +1 -1
  46. package/dest/sequencer/metrics.js +34 -20
  47. package/dest/sequencer/sequencer.d.ts +27 -7
  48. package/dest/sequencer/sequencer.d.ts.map +1 -1
  49. package/dest/sequencer/sequencer.js +106 -28
  50. package/dest/sequencer/timetable.d.ts +14 -1
  51. package/dest/sequencer/timetable.d.ts.map +1 -1
  52. package/dest/sequencer/timetable.js +45 -36
  53. package/dest/test/utils.d.ts +1 -1
  54. package/dest/test/utils.d.ts.map +1 -1
  55. package/dest/test/utils.js +7 -6
  56. package/package.json +27 -27
  57. package/src/client/sequencer-client.ts +16 -24
  58. package/src/config.ts +13 -11
  59. package/src/global_variable_builder/README.md +44 -0
  60. package/src/global_variable_builder/fee_predictor.ts +172 -0
  61. package/src/global_variable_builder/fee_provider.ts +75 -0
  62. package/src/global_variable_builder/global_builder.ts +26 -63
  63. package/src/global_variable_builder/index.ts +3 -1
  64. package/src/publisher/config.ts +38 -4
  65. package/src/publisher/l1_tx_failed_store/failed_tx_store.ts +3 -1
  66. package/src/publisher/sequencer-publisher-factory.ts +3 -6
  67. package/src/publisher/sequencer-publisher.ts +183 -159
  68. package/src/sequencer/README.md +82 -13
  69. package/src/sequencer/chain_state_overrides.ts +87 -0
  70. package/src/sequencer/checkpoint_proposal_job.ts +616 -167
  71. package/src/sequencer/checkpoint_proposal_job_metrics.ts +128 -0
  72. package/src/sequencer/checkpoint_voter.ts +1 -12
  73. package/src/sequencer/events.ts +5 -0
  74. package/src/sequencer/metrics.ts +43 -24
  75. package/src/sequencer/sequencer.ts +148 -32
  76. package/src/sequencer/timetable.ts +57 -45
  77. package/src/test/utils.ts +28 -10
@@ -436,17 +436,18 @@ function _apply_decs_2203_r(targetClass, memberDecs, classDecs, parentClass) {
436
436
  return (_apply_decs_2203_r = applyDecs2203RFactory())(targetClass, memberDecs, classDecs, parentClass);
437
437
  }
438
438
  var _dec, _dec1, _dec2, _dec3, _dec4, _dec5, _dec6, _dec7, _initProto;
439
- import { BlockNumber, IndexWithinCheckpoint } from '@aztec/foundation/branded-types';
439
+ import { BlockNumber, CheckpointNumber, IndexWithinCheckpoint } from '@aztec/foundation/branded-types';
440
440
  import { randomInt } from '@aztec/foundation/crypto/random';
441
441
  import { flipSignature, generateRecoverableSignature, generateUnrecoverableSignature } from '@aztec/foundation/crypto/secp256k1-signer';
442
442
  import { filter } from '@aztec/foundation/iterator';
443
443
  import { createLogger } from '@aztec/foundation/log';
444
+ import { retryUntil } from '@aztec/foundation/retry';
444
445
  import { sleep, sleepUntil } from '@aztec/foundation/sleep';
445
446
  import { Timer } from '@aztec/foundation/timer';
446
447
  import { isErrorClass, unfreeze } from '@aztec/foundation/types';
447
448
  import { CommitteeAttestationsAndSigners, MaliciousCommitteeAttestationsAndSigners } from '@aztec/stdlib/block';
448
449
  import { validateCheckpoint } from '@aztec/stdlib/checkpoint';
449
- import { getSlotStartBuildTimestamp, getTimestampForSlot } from '@aztec/stdlib/epoch-helpers';
450
+ import { computeQuorum, getSlotStartBuildTimestamp, getTimestampForSlot } from '@aztec/stdlib/epoch-helpers';
450
451
  import { Gas } from '@aztec/stdlib/gas';
451
452
  import { InsufficientValidTxsError } from '@aztec/stdlib/interfaces/server';
452
453
  import { computeInHashFromL1ToL2Messages } from '@aztec/stdlib/messaging';
@@ -454,6 +455,7 @@ import { orderAttestations, trimAttestations } from '@aztec/stdlib/p2p';
454
455
  import { AttestationTimeoutError } from '@aztec/stdlib/validators';
455
456
  import { Attributes, trackSpan } from '@aztec/telemetry-client';
456
457
  import { DutyAlreadySignedError, SlashingProtectionError } from '@aztec/validator-ha-signer/errors';
458
+ import { buildPipelinedParentSimulationOverridesPlan, buildSubmissionSimulationOverridesPlan } from './chain_state_overrides.js';
457
459
  import { CheckpointVoter } from './checkpoint_voter.js';
458
460
  import { SequencerInterruptedError } from './errors.js';
459
461
  import { SequencerState } from './utils.js';
@@ -473,7 +475,6 @@ _dec = trackSpan('CheckpointProposalJob.execute'), _dec1 = trackSpan('Checkpoint
473
475
  */ export class CheckpointProposalJob {
474
476
  slotNow;
475
477
  targetSlot;
476
- epochNow;
477
478
  targetEpoch;
478
479
  checkpointNumber;
479
480
  syncedToBlockNumber;
@@ -490,15 +491,18 @@ _dec = trackSpan('CheckpointProposalJob.execute'), _dec1 = trackSpan('Checkpoint
490
491
  checkpointsBuilder;
491
492
  blockSink;
492
493
  l1Constants;
494
+ signatureContext;
493
495
  config;
494
496
  timetable;
495
497
  slasherClient;
496
498
  epochCache;
497
499
  dateProvider;
498
500
  metrics;
501
+ checkpointMetrics;
499
502
  eventEmitter;
500
503
  setStateFn;
501
504
  tracer;
505
+ proposedCheckpointData;
502
506
  static{
503
507
  ({ e: [_initProto] } = _apply_decs_2203_r(this, [
504
508
  [
@@ -544,11 +548,16 @@ _dec = trackSpan('CheckpointProposalJob.execute'), _dec1 = trackSpan('Checkpoint
544
548
  ], []));
545
549
  }
546
550
  log;
547
- constructor(slotNow, targetSlot, epochNow, targetEpoch, checkpointNumber, syncedToBlockNumber, // TODO(palla/mbps): Can we remove the proposer in favor of attestorAddress? Need to check fisherman-node flows.
548
- proposer, publisher, attestorAddress, invalidateCheckpoint, validatorClient, globalsBuilder, p2pClient, worldState, l1ToL2MessageSource, l2BlockSource, checkpointsBuilder, blockSink, l1Constants, config, timetable, slasherClient, epochCache, dateProvider, metrics, eventEmitter, setStateFn, tracer, bindings){
551
+ checkpointEventLog;
552
+ /** Tracks the fire-and-forget L1 submission promise so it can be awaited during shutdown. */ pendingL1Submission;
553
+ /** Pipelined parent chain state used while building and later submitting this checkpoint. */ pipelinedParentSimulationOverridesPlan;
554
+ getSignatureContext() {
555
+ return this.signatureContext;
556
+ }
557
+ constructor(slotNow, targetSlot, targetEpoch, checkpointNumber, syncedToBlockNumber, // TODO(palla/mbps): Can we remove the proposer in favor of attestorAddress? Need to check fisherman-node flows.
558
+ proposer, publisher, attestorAddress, invalidateCheckpoint, validatorClient, globalsBuilder, p2pClient, worldState, l1ToL2MessageSource, l2BlockSource, checkpointsBuilder, blockSink, l1Constants, signatureContext, config, timetable, slasherClient, epochCache, dateProvider, metrics, checkpointMetrics, eventEmitter, setStateFn, tracer, bindings, proposedCheckpointData){
549
559
  this.slotNow = slotNow;
550
560
  this.targetSlot = targetSlot;
551
- this.epochNow = epochNow;
552
561
  this.targetEpoch = targetEpoch;
553
562
  this.checkpointNumber = checkpointNumber;
554
563
  this.syncedToBlockNumber = syncedToBlockNumber;
@@ -565,82 +574,311 @@ _dec = trackSpan('CheckpointProposalJob.execute'), _dec1 = trackSpan('Checkpoint
565
574
  this.checkpointsBuilder = checkpointsBuilder;
566
575
  this.blockSink = blockSink;
567
576
  this.l1Constants = l1Constants;
577
+ this.signatureContext = signatureContext;
568
578
  this.config = config;
569
579
  this.timetable = timetable;
570
580
  this.slasherClient = slasherClient;
571
581
  this.epochCache = epochCache;
572
582
  this.dateProvider = dateProvider;
573
583
  this.metrics = metrics;
584
+ this.checkpointMetrics = checkpointMetrics;
574
585
  this.eventEmitter = eventEmitter;
575
586
  this.setStateFn = setStateFn;
576
587
  this.tracer = tracer;
588
+ this.proposedCheckpointData = proposedCheckpointData;
577
589
  _initProto(this);
578
590
  this.log = createLogger('sequencer:checkpoint-proposal', {
579
591
  ...bindings,
580
592
  instanceId: `slot-${this.slotNow}`
581
593
  });
594
+ this.checkpointEventLog = createLogger('sequencer:checkpoint-events', {
595
+ ...bindings,
596
+ instanceId: `slot-${this.slotNow}`
597
+ });
582
598
  }
583
- /** The wall-clock slot during which the proposer builds. */ get slot() {
584
- return this.slotNow;
599
+ /** Awaits the pending L1 submission if one is in progress. Call during shutdown. */ async awaitPendingSubmission() {
600
+ this.log.info('Awaiting pending L1 payload submission');
601
+ await this.pendingL1Submission;
585
602
  }
586
- /** The wall-clock epoch. */ get epoch() {
587
- return this.epochNow;
603
+ logCheckpointEvent(eventName, message, fields) {
604
+ this.checkpointEventLog.debug(message, {
605
+ eventName: `sequencer-checkpoint-${eventName}`,
606
+ ...fields
607
+ });
588
608
  }
589
609
  /**
590
610
  * Executes the checkpoint proposal job.
591
- * Returns the published checkpoint if successful, undefined otherwise.
611
+ * Builds blocks, assembles checkpoint, and broadcasts the proposal (blocking).
612
+ * Attestation collection, signing, and L1 submission are backgrounded so the
613
+ * work loop can return to IDLE immediately for consecutive slot proposals.
614
+ * Returns the built checkpoint if successful, undefined otherwise.
592
615
  */ async execute() {
593
616
  // Enqueue governance and slashing votes (returns promises that will be awaited later)
594
617
  // In fisherman mode, we simulate slashing but don't actually publish to L1
595
618
  // These are constant for the whole slot, so we only enqueue them once
596
619
  const votesPromises = new CheckpointVoter(this.targetSlot, this.publisher, this.attestorAddress, this.validatorClient, this.slasherClient, this.l1Constants, this.config, this.metrics, this.log).enqueueVotes();
597
- // Build and propose the checkpoint. This will enqueue the request on the publisher if a checkpoint is built.
598
- const checkpoint = await this.proposeCheckpoint();
599
- // Wait until the voting promises have resolved, so all requests are enqueued (not sent)
600
- await Promise.all(votesPromises);
601
- if (checkpoint) {
602
- this.metrics.recordCheckpointProposalSuccess();
620
+ // Build blocks, assemble checkpoint, and broadcast proposal (BLOCKING).
621
+ // Returns after broadcast — attestation collection is deferred.
622
+ const broadcast = await this.proposeCheckpoint();
623
+ if (!broadcast) {
624
+ await Promise.all(votesPromises);
625
+ // Still submit votes even without a checkpoint
626
+ if (!this.config.fishermanMode) {
627
+ this.pendingL1Submission = this.publisher.sendRequestsAt(this.dateProvider.nowAsDate()).then(()=>{});
628
+ }
629
+ return undefined;
603
630
  }
631
+ const { checkpoint } = broadcast;
632
+ this.metrics.recordCheckpointProposalSuccess();
604
633
  // Do not post anything to L1 if we are fishermen, but do perform L1 fee analysis
605
634
  if (this.config.fishermanMode) {
606
635
  await this.handleCheckpointEndAsFisherman(checkpoint);
607
- return;
636
+ return checkpoint;
608
637
  }
609
- // If pipelining, wait until the submission slot so L1 recognizes the pipelined proposer
610
- if (this.epochCache.isProposerPipeliningEnabled()) {
611
- const submissionSlotTimestamp = getTimestampForSlot(this.targetSlot, this.l1Constants) - BigInt(this.l1Constants.ethereumSlotDuration);
612
- this.log.info(`Waiting until submission slot ${this.targetSlot} for L1 submission`, {
613
- slot: this.slot,
614
- submissionSlot: this.targetSlot,
615
- submissionSlotTimestamp
638
+ // Background the attestation signing L1 pipeline so the work loop is unblocked
639
+ this.pendingL1Submission = this.waitForAttestationsAndEnqueueSubmissionAsync(broadcast, votesPromises);
640
+ // Return the built checkpoint immediately — the work loop is now unblocked
641
+ return checkpoint;
642
+ }
643
+ /**
644
+ * Background pipeline: collects attestations, signs them, enqueues the checkpoint, and submits to L1.
645
+ * Runs as a fire-and-forget task stored in `pendingL1Submission` so the work loop is unblocked.
646
+ */ async waitForAttestationsAndEnqueueSubmissionAsync(broadcast, votesPromises) {
647
+ const { checkpoint } = broadcast;
648
+ const isPipelining = this.epochCache.isProposerPipeliningEnabled();
649
+ try {
650
+ // Wait for all votes actions, enqueued at the beginning, to resolve
651
+ await Promise.all(votesPromises);
652
+ // Try to collect attestations from the committee
653
+ const signedAttestations = await this.getSignedCommitteeAttestations(broadcast);
654
+ // If pipelining, wait for the previous checkpoint to land on L1 before submitting,
655
+ // so we can check it matches the proposed checkpoint we used as parent, and has valid attestations.
656
+ if (signedAttestations && (!isPipelining || await this.waitForValidParentCheckpointOnL1())) {
657
+ await this.enqueueCheckpointForSubmission({
658
+ checkpoint,
659
+ ...signedAttestations
660
+ });
661
+ }
662
+ // If we failed to collect attestations, at least check if we need to issue an invalidation
663
+ // Note that if we are not pipelining, we enqueued the invalidation at the beginning
664
+ if (!signedAttestations && isPipelining && await this.waitForSyncedL2SlotNumber(this.slotNow)) {
665
+ const validationStatus = await this.l2BlockSource.getPendingChainValidationStatus();
666
+ if (!validationStatus.valid) {
667
+ this.log.warn(`Checkpoint ${validationStatus.checkpoint.checkpointNumber} has invalid attestations, enqueuing invalidation in spite of attestation collection failure`, {
668
+ checkpoint: validationStatus.checkpoint,
669
+ reason: validationStatus.reason
670
+ });
671
+ await this.enqueueInvalidation(validationStatus);
672
+ }
673
+ }
674
+ // Send whatever was enqueued: votes + (propose | invalidation | nothing).
675
+ // Compute the earliest time to submit: pipeline slot start when pipelining, now otherwise.
676
+ const submitAfter = isPipelining ? new Date(Number(getTimestampForSlot(this.targetSlot, this.l1Constants)) * 1000) : new Date(this.dateProvider.now());
677
+ const l1Response = await this.publisher.sendRequestsAt(submitAfter);
678
+ const proposedAction = l1Response?.successfulActions.find((a)=>a === 'propose');
679
+ if (proposedAction) {
680
+ this.logCheckpointEvent('published', `Checkpoint published for slot ${this.targetSlot}`, {
681
+ slot: this.targetSlot,
682
+ checkpointNumber: this.checkpointNumber,
683
+ successfulActions: l1Response?.successfulActions,
684
+ sentActions: l1Response?.sentActions
685
+ });
686
+ this.eventEmitter.emit('checkpoint-published', {
687
+ checkpoint: this.checkpointNumber,
688
+ slot: this.targetSlot
689
+ });
690
+ const coinbase = checkpoint.header.coinbase;
691
+ await this.metrics.incFilledSlot(this.publisher.getSenderAddress().toString(), coinbase);
692
+ } else {
693
+ this.logCheckpointEvent('publish-failed', `Checkpoint publish failed for slot ${this.targetSlot}`, {
694
+ slot: this.targetSlot,
695
+ checkpointNumber: this.checkpointNumber,
696
+ successfulActions: l1Response?.successfulActions,
697
+ failedActions: l1Response?.failedActions,
698
+ sentActions: l1Response?.sentActions,
699
+ expiredActions: l1Response?.expiredActions,
700
+ reason: 'propose_action_not_successful'
701
+ });
702
+ this.log.warn(`Checkpoint publish failed for slot ${this.targetSlot}`, {
703
+ slot: this.targetSlot,
704
+ checkpointNumber: this.checkpointNumber,
705
+ successfulActions: l1Response?.successfulActions,
706
+ failedActions: l1Response?.failedActions,
707
+ sentActions: l1Response?.sentActions,
708
+ expiredActions: l1Response?.expiredActions,
709
+ reason: 'propose_action_not_successful'
710
+ });
711
+ this.eventEmitter.emit('checkpoint-publish-failed', {
712
+ ...l1Response,
713
+ slot: this.targetSlot
714
+ });
715
+ if (isPipelining) {
716
+ this.metrics.recordPipelineDiscard();
717
+ }
718
+ }
719
+ } catch (err) {
720
+ if (err instanceof SequencerInterruptedError) {
721
+ return;
722
+ }
723
+ this.logCheckpointEvent('publish-failed', `Checkpoint publish failed for slot ${this.targetSlot}`, {
724
+ slot: this.targetSlot,
725
+ checkpointNumber: this.checkpointNumber,
726
+ reason: err instanceof Error ? err.message : String(err)
616
727
  });
617
- await sleepUntil(new Date(Number(submissionSlotTimestamp) * 1000), this.dateProvider.nowAsDate());
618
- // After waking, verify the parent checkpoint wasn't pruned during the sleep.
619
- // We check L1's pending tip directly instead of canProposeAt, which also validates the proposer
620
- // identity and would fail because the timestamp resolves to a different slot's proposer.
621
- const l1Tips = await this.publisher.rollupContract.getTips();
622
- if (l1Tips.pending < this.checkpointNumber - 1) {
623
- this.log.warn(`Parent checkpoint was pruned during pipelining sleep (L1 pending=${l1Tips.pending}, expected>=${this.checkpointNumber - 1}), skipping L1 submission for checkpoint ${this.checkpointNumber}`);
624
- return undefined;
728
+ this.log.error(`Background attestation/L1 pipeline failed for slot ${this.targetSlot}`, err, {
729
+ slot: this.targetSlot,
730
+ checkpointNumber: this.checkpointNumber,
731
+ reason: err instanceof Error ? err.message : String(err)
732
+ });
733
+ this.eventEmitter.emit('checkpoint-publish-failed', {
734
+ slot: this.targetSlot
735
+ });
736
+ if (isPipelining) {
737
+ this.metrics.recordPipelineDiscard();
738
+ }
739
+ }
740
+ }
741
+ /** Enqueues the checkpoint for L1 submission. Called after pipeline sleep in execute(). */ async enqueueCheckpointForSubmission(result) {
742
+ const { checkpoint, attestations, attestationsSignature } = result;
743
+ this.setStateFn(SequencerState.PUBLISHING_CHECKPOINT, this.targetSlot);
744
+ const aztecSlotDuration = this.l1Constants.slotDuration;
745
+ const submissionSlotStart = Number(getTimestampForSlot(this.targetSlot, this.l1Constants));
746
+ const txTimeoutAt = new Date((submissionSlotStart + aztecSlotDuration) * 1000);
747
+ // If we have been configured to potentially skip publishing checkpoint then roll the dice here
748
+ if (this.config.skipPublishingCheckpointsPercent !== undefined && this.config.skipPublishingCheckpointsPercent > 0) {
749
+ const roll = Math.max(0, randomInt(100));
750
+ if (roll < this.config.skipPublishingCheckpointsPercent) {
751
+ this.log.warn(`Skipping publishing proposal for checkpoint ${checkpoint.number}. Configured percentage: ${this.config.skipPublishingCheckpointsPercent}, generated value: ${roll}`);
752
+ return;
625
753
  }
626
754
  }
627
- // Then send everything to L1
628
- const l1Response = await this.publisher.sendRequests();
629
- const proposedAction = l1Response?.successfulActions.find((a)=>a === 'propose');
630
- if (proposedAction) {
631
- this.eventEmitter.emit('checkpoint-published', {
632
- checkpoint: this.checkpointNumber,
633
- slot: this.slot
755
+ const isPipelining = this.epochCache.isProposerPipeliningEnabled();
756
+ const submissionSimulationOverridesPlan = buildSubmissionSimulationOverridesPlan({
757
+ pipelinedParentPlan: this.pipelinedParentSimulationOverridesPlan,
758
+ invalidateToPendingCheckpointNumber: this.invalidateCheckpoint?.forcePendingCheckpointNumber,
759
+ lastArchiveRoot: checkpoint.header.lastArchiveRoot,
760
+ pipeliningEnabled: isPipelining
761
+ });
762
+ await this.publisher.enqueueProposeCheckpoint(checkpoint, attestations, attestationsSignature, {
763
+ txTimeoutAt,
764
+ ...submissionSimulationOverridesPlan ? {
765
+ simulationOverridesPlan: submissionSimulationOverridesPlan
766
+ } : {}
767
+ });
768
+ }
769
+ /**
770
+ * Wait until the archiver syncs past the given L2 slot number.
771
+ * The deadline is the end of `this.targetSlot`, beyond which any pipelined work would miss its
772
+ * L1 submission window and is no longer useful.
773
+ */ async waitForSyncedL2SlotNumber(waitForSlot) {
774
+ const targetSlotStart = Number(getTimestampForSlot(this.targetSlot, this.l1Constants));
775
+ const targetSlotEndMs = (targetSlotStart + this.l1Constants.slotDuration) * 1000;
776
+ const syncDelayTolerance = this.l1Constants.ethereumSlotDuration * 2 * 1000;
777
+ const timeoutSeconds = Math.max(0.1, (targetSlotEndMs + syncDelayTolerance - this.dateProvider.now()) / 1000);
778
+ try {
779
+ return await retryUntil(async ()=>{
780
+ const syncedSlot = await this.l2BlockSource.getSyncedL2SlotNumber();
781
+ return syncedSlot !== undefined && syncedSlot >= waitForSlot;
782
+ }, `archiver sync past slot ${waitForSlot}`, timeoutSeconds, 0.2);
783
+ } catch {
784
+ this.log.warn(`Archiver did not sync L1 past slot ${waitForSlot} before slot ${this.targetSlot} expired, discarding pipelined work`, {
785
+ checkpointNumber: this.checkpointNumber
634
786
  });
635
- const coinbase = checkpoint?.header.coinbase;
636
- await this.metrics.incFilledSlot(this.publisher.getSenderAddress().toString(), coinbase);
637
- return checkpoint;
638
- } else if (checkpoint) {
639
- this.eventEmitter.emit('checkpoint-publish-failed', {
640
- ...l1Response,
641
- slot: this.slot
787
+ this.emitPipelinedCheckpointDiscarded('archiver-sync-timeout');
788
+ return false;
789
+ }
790
+ }
791
+ /**
792
+ * Waits for the parent checkpoint to land on L1 before submitting a pipelined checkpoint.
793
+ * Polls until the archiver has synced L1 past the parent's slot, then verifies:
794
+ * - If we built on a proposed parent: it must have landed on L1 with matching hash and valid attestations.
795
+ * - If we built without a proposed parent: no new checkpoint must have appeared for that slot.
796
+ * If the parent has invalid attestations, enqueues an invalidation. Returns whether to proceed with the proposal.
797
+ */ async waitForValidParentCheckpointOnL1() {
798
+ const parentCheckpointNumber = CheckpointNumber(this.checkpointNumber - 1);
799
+ // Wait until archiver has synced L1 past the parent's slot (slotNow)
800
+ if (!await this.waitForSyncedL2SlotNumber(this.slotNow)) {
801
+ return false;
802
+ }
803
+ const tips = await this.l2BlockSource.getL2Tips();
804
+ const checkpointedNumber = tips.checkpointed.checkpoint.number;
805
+ // We built on top of a proposed checkpoint. Verify it landed on L1 as expected.
806
+ if (this.proposedCheckpointData) {
807
+ // After syncing from L1 we see the chain tip has invalid attestations. This means the parent checkpoint was posted
808
+ // with invalid attestations, or it built on top of something with invalid attestations and didnt invalidate them.
809
+ // Either way, we thought our parent would be valid, so we have to throw away our work. But at least we'll try and
810
+ // invalidate on L1 so we clean up the chain for the next proposer. And we'll slash them, but that's handled elsewhere.
811
+ const validationStatus = await this.l2BlockSource.getPendingChainValidationStatus();
812
+ if (!validationStatus.valid) {
813
+ this.log.warn(`Parent checkpoint ${parentCheckpointNumber} has invalid attestations, discarding pipelined work`, {
814
+ checkpointNumber: this.checkpointNumber,
815
+ reason: validationStatus.reason
816
+ });
817
+ this.emitPipelinedCheckpointDiscarded('parent-invalid-attestations');
818
+ await this.enqueueInvalidation(validationStatus);
819
+ return false;
820
+ }
821
+ // The pending chain is valid. But did the parent checkpoint land on L1 at all?
822
+ if (checkpointedNumber < parentCheckpointNumber) {
823
+ this.log.warn(`Parent checkpoint ${parentCheckpointNumber} did not land on L1, discarding pipelined work`, {
824
+ checkpointNumber: this.checkpointNumber,
825
+ checkpointedNumber
826
+ });
827
+ this.emitPipelinedCheckpointDiscarded('parent-not-on-l1');
828
+ return false;
829
+ }
830
+ // It landed. But is it the one we were expecting?
831
+ const expectedHash = this.proposedCheckpointData.header.hash().toString();
832
+ if (tips.checkpointed.checkpoint.hash !== expectedHash) {
833
+ this.log.warn(`Parent checkpoint ${parentCheckpointNumber} hash mismatch on L1, discarding pipelined work`, {
834
+ checkpointNumber: this.checkpointNumber,
835
+ expectedHash,
836
+ actualHash: tips.checkpointed.checkpoint.hash
837
+ });
838
+ this.emitPipelinedCheckpointDiscarded('parent-hash-mismatch');
839
+ return false;
840
+ }
841
+ return true;
842
+ } else {
843
+ // We didn't see a proposed checkpoint at build time, so we built on checkpointed parent from two slots ago.
844
+ // But if a new checkpoint for the previous slot appeared on L1 in the meantime, our checkpoint assumed the wrong parent,
845
+ // so we have to discard our work. This can happen if we're somehow cut off from p2p and fail to see the checkpoint
846
+ // proposal for the previous slot.
847
+ if (checkpointedNumber > parentCheckpointNumber) {
848
+ this.log.warn(`Unexpected checkpoint ${checkpointedNumber} landed on L1 after we built on top of parent ${parentCheckpointNumber}, discarding pipelined work`, {
849
+ checkpointNumber: this.checkpointNumber,
850
+ checkpointedNumber
851
+ });
852
+ this.emitPipelinedCheckpointDiscarded('unexpected-parent-appeared');
853
+ return false;
854
+ }
855
+ return true;
856
+ }
857
+ }
858
+ /** Emits the pipelined-checkpoint-discarded event and records the metric. */ emitPipelinedCheckpointDiscarded(reason) {
859
+ this.metrics.recordPipelineParentCheckpointMismatch(reason);
860
+ this.eventEmitter.emit('pipelined-checkpoint-discarded', {
861
+ slot: this.targetSlot,
862
+ checkpointNumber: this.checkpointNumber,
863
+ reason
864
+ });
865
+ }
866
+ /** Simulates and enqueues an invalidation request for the invalid parent checkpoint. */ async enqueueInvalidation(validationStatus) {
867
+ if (this.config.skipInvalidateBlockAsProposer) {
868
+ this.log.warn(`Skipping checkpoint invalidation as proposer due to test configuration`);
869
+ return;
870
+ }
871
+ const invalidateRequest = await this.publisher.simulateInvalidateCheckpoint(validationStatus);
872
+ if (invalidateRequest) {
873
+ const submissionSlotStart = Number(getTimestampForSlot(this.targetSlot, this.l1Constants));
874
+ const txTimeoutAt = new Date((submissionSlotStart + this.l1Constants.slotDuration) * 1000);
875
+ this.publisher.enqueueInvalidateCheckpoint(invalidateRequest, {
876
+ txTimeoutAt
877
+ });
878
+ } else {
879
+ this.log.info(`Invalidation simulation returned undefined, checkpoint may have been removed already`, {
880
+ checkpointNumber: this.checkpointNumber
642
881
  });
643
- return undefined;
644
882
  }
645
883
  }
646
884
  async proposeCheckpoint() {
@@ -651,16 +889,29 @@ _dec = trackSpan('CheckpointProposalJob.execute'), _dec1 = trackSpan('Checkpoint
651
889
  hasError: false
652
890
  };
653
891
  try {
892
+ const now = this.dateProvider.now();
893
+ if (this.epochCache.isProposerPipeliningEnabled() && this.proposedCheckpointData) {
894
+ // Measure against the wall-clock slot whose build window we are currently using.
895
+ // In pipelining mode `targetSlot` is intentionally one slot ahead, which makes the
896
+ // target-slot boundary a full slot away from the actual build start time.
897
+ const slotBoundaryMs = Number(getTimestampForSlot(this.slotNow, this.l1Constants)) * 1000;
898
+ this.checkpointMetrics.recordPipelinedCheckpointBuildStartOffsetFromSlotBoundary(now - slotBoundaryMs);
899
+ }
900
+ this.checkpointMetrics.startCheckpointTiming(now);
654
901
  // Get operator configured coinbase and fee recipient for this attestor
655
902
  const coinbase = this.validatorClient.getCoinbaseForAttestor(this.attestorAddress);
656
903
  const feeRecipient = this.validatorClient.getFeeRecipientForAttestor(this.attestorAddress);
657
904
  // Start the checkpoint
658
905
  this.setStateFn(SequencerState.INITIALIZING_CHECKPOINT, this.targetSlot);
659
- this.log.info(`Starting checkpoint proposal`, {
660
- buildSlot: this.slot,
906
+ this.logCheckpointEvent('slot-started', `Starting checkpoint proposal for slot ${this.targetSlot}`, {
907
+ buildSlot: this.slotNow,
661
908
  submissionSlot: this.targetSlot,
909
+ slot: this.targetSlot,
910
+ checkpointNumber: this.checkpointNumber,
662
911
  pipelining: this.epochCache.isProposerPipeliningEnabled(),
663
912
  proposer: this.proposer?.toString(),
913
+ attestorAddress: this.attestorAddress.toString(),
914
+ publisherAddress: this.publisher.getSenderAddress().toString(),
664
915
  coinbase: coinbase.toString()
665
916
  });
666
917
  this.metrics.incOpenSlot(this.targetSlot, this.proposer?.toString() ?? 'unknown');
@@ -668,8 +919,17 @@ _dec = trackSpan('CheckpointProposalJob.execute'), _dec1 = trackSpan('Checkpoint
668
919
  if (this.invalidateCheckpoint && !this.config.skipInvalidateBlockAsProposer) {
669
920
  this.publisher.enqueueInvalidateCheckpoint(this.invalidateCheckpoint);
670
921
  }
671
- // Create checkpoint builder for the slot
672
- const checkpointGlobalVariables = await this.globalsBuilder.buildCheckpointGlobalVariables(coinbase, feeRecipient, this.targetSlot);
922
+ // Create checkpoint builder for the slot.
923
+ // When pipelining, force the proposed checkpoint number and fee header to our parent so the
924
+ // fee computation sees the same chain tip that L1 will see once the previous pipelined checkpoint lands.
925
+ const isPipelining = this.epochCache.isProposerPipeliningEnabled();
926
+ this.pipelinedParentSimulationOverridesPlan = isPipelining ? await buildPipelinedParentSimulationOverridesPlan({
927
+ checkpointNumber: this.checkpointNumber,
928
+ proposedCheckpointData: this.proposedCheckpointData,
929
+ rollup: this.publisher.rollupContract,
930
+ log: this.log
931
+ }) : undefined;
932
+ const checkpointGlobalVariables = await this.globalsBuilder.buildCheckpointGlobalVariables(coinbase, feeRecipient, this.targetSlot, this.pipelinedParentSimulationOverridesPlan);
673
933
  // Collect L1 to L2 messages for the checkpoint and compute their hash
674
934
  const l1ToL2Messages = await this.l1ToL2MessageSource.getL1ToL2Messages(this.checkpointNumber);
675
935
  const inHash = computeInHashFromL1ToL2Messages(l1ToL2Messages);
@@ -709,8 +969,15 @@ _dec = trackSpan('CheckpointProposalJob.execute'), _dec1 = trackSpan('Checkpoint
709
969
  throw err;
710
970
  }
711
971
  if (blocksInCheckpoint.length === 0) {
972
+ this.logCheckpointEvent('build-failed', `Checkpoint build failed for slot ${this.targetSlot}`, {
973
+ slot: this.targetSlot,
974
+ checkpointNumber: this.checkpointNumber,
975
+ reason: 'no_blocks_built'
976
+ });
712
977
  this.log.warn(`No blocks were built for slot ${this.targetSlot}`, {
713
- slot: this.targetSlot
978
+ slot: this.targetSlot,
979
+ checkpointNumber: this.checkpointNumber,
980
+ reason: 'no_blocks_built'
714
981
  });
715
982
  this.eventEmitter.emit('checkpoint-empty', {
716
983
  slot: this.targetSlot
@@ -719,10 +986,19 @@ _dec = trackSpan('CheckpointProposalJob.execute'), _dec1 = trackSpan('Checkpoint
719
986
  }
720
987
  const minBlocksForCheckpoint = this.config.minBlocksForCheckpoint;
721
988
  if (minBlocksForCheckpoint !== undefined && blocksInCheckpoint.length < minBlocksForCheckpoint) {
989
+ this.logCheckpointEvent('build-failed', `Checkpoint build failed for slot ${this.targetSlot}`, {
990
+ slot: this.targetSlot,
991
+ checkpointNumber: this.checkpointNumber,
992
+ blocksBuilt: blocksInCheckpoint.length,
993
+ minBlocksForCheckpoint,
994
+ reason: 'min_blocks_not_met'
995
+ });
722
996
  this.log.warn(`Checkpoint has fewer blocks than minimum (${blocksInCheckpoint.length} < ${minBlocksForCheckpoint}), skipping proposal`, {
723
997
  slot: this.targetSlot,
998
+ checkpointNumber: this.checkpointNumber,
724
999
  blocksBuilt: blocksInCheckpoint.length,
725
- minBlocksForCheckpoint
1000
+ minBlocksForCheckpoint,
1001
+ reason: 'min_blocks_not_met'
726
1002
  });
727
1003
  return undefined;
728
1004
  }
@@ -741,14 +1017,36 @@ _dec = trackSpan('CheckpointProposalJob.execute'), _dec1 = trackSpan('Checkpoint
741
1017
  maxTxsPerCheckpoint: this.config.maxTxsPerCheckpoint
742
1018
  });
743
1019
  } catch (err) {
744
- this.log.error(`Built an invalid checkpoint at slot ${this.slot} (skipping proposal)`, err, {
1020
+ this.logCheckpointEvent('build-failed', `Checkpoint build failed for slot ${this.targetSlot}`, {
1021
+ slot: this.targetSlot,
1022
+ checkpointNumber: this.checkpointNumber,
1023
+ blocksBuilt: blocksInCheckpoint.length,
1024
+ reason: 'invalid_checkpoint',
1025
+ checkpoint: checkpoint.header.toInspect()
1026
+ });
1027
+ this.log.error(`Built an invalid checkpoint at slot ${this.slotNow} (skipping proposal)`, err, {
1028
+ slot: this.targetSlot,
1029
+ checkpointNumber: this.checkpointNumber,
1030
+ blocksBuilt: blocksInCheckpoint.length,
1031
+ reason: 'invalid_checkpoint',
745
1032
  checkpoint: checkpoint.header.toInspect()
746
1033
  });
747
1034
  return undefined;
748
1035
  }
749
1036
  // Record checkpoint-level build metrics
750
- this.metrics.recordCheckpointBuild(checkpointBuildTimer.ms(), blocksInCheckpoint.length, checkpoint.getStats().txCount, Number(checkpoint.header.totalManaUsed.toBigInt()));
751
- // Do not collect attestations nor publish to L1 in fisherman mode
1037
+ this.checkpointMetrics.recordCheckpointBuild(checkpointBuildTimer.ms(), blocksInCheckpoint.length, checkpoint.getStats().txCount, Number(checkpoint.header.totalManaUsed.toBigInt()));
1038
+ this.logCheckpointEvent('built', `Checkpoint built for slot ${this.targetSlot}`, {
1039
+ slot: this.targetSlot,
1040
+ buildSlot: this.slotNow,
1041
+ checkpointNumber: this.checkpointNumber,
1042
+ proposer: this.proposer?.toString(),
1043
+ attestorAddress: this.attestorAddress.toString(),
1044
+ publisherAddress: this.publisher.getSenderAddress().toString(),
1045
+ blocksBuilt: blocksInCheckpoint.length,
1046
+ txCount: checkpoint.getStats().txCount,
1047
+ totalMana: Number(checkpoint.header.totalManaUsed.toBigInt())
1048
+ });
1049
+ // In fisherman mode, return the checkpoint without broadcasting or collecting attestations
752
1050
  if (this.config.fishermanMode) {
753
1051
  this.log.info(`Built checkpoint for slot ${this.targetSlot} with ${blocksInCheckpoint.length} blocks. ` + `Skipping proposal in fisherman mode.`, {
754
1052
  slot: this.targetSlot,
@@ -756,53 +1054,26 @@ _dec = trackSpan('CheckpointProposalJob.execute'), _dec1 = trackSpan('Checkpoint
756
1054
  blocksBuilt: blocksInCheckpoint.length
757
1055
  });
758
1056
  this.metrics.recordCheckpointSuccess();
759
- return checkpoint;
1057
+ // Return a broadcast result with a dummy proposal — fisherman mode skips attestation collection
1058
+ return {
1059
+ checkpoint,
1060
+ proposal: undefined,
1061
+ blockProposedAt: this.dateProvider.now()
1062
+ };
760
1063
  }
761
- // Include the block pending broadcast in the checkpoint proposal if any
762
- const lastBlock = blockPendingBroadcast && {
763
- blockHeader: blockPendingBroadcast.block.header,
764
- indexWithinCheckpoint: blockPendingBroadcast.block.indexWithinCheckpoint,
765
- txs: blockPendingBroadcast.txs
766
- };
767
1064
  // Create the checkpoint proposal and broadcast it
768
- const proposal = await this.validatorClient.createCheckpointProposal(checkpoint.header, checkpoint.archive.root, feeAssetPriceModifier, lastBlock, this.proposer, checkpointProposalOptions);
1065
+ const proposal = await this.validatorClient.createCheckpointProposal(checkpoint.header, checkpoint.archive.root, this.checkpointNumber, feeAssetPriceModifier, blockPendingBroadcast, this.proposer, checkpointProposalOptions);
769
1066
  const blockProposedAt = this.dateProvider.now();
770
- await this.p2pClient.broadcastCheckpointProposal(proposal);
771
- this.setStateFn(SequencerState.COLLECTING_ATTESTATIONS, this.targetSlot);
772
- const attestations = await this.waitForAttestations(proposal);
773
- const blockAttestedAt = this.dateProvider.now();
774
- this.metrics.recordCheckpointAttestationDelay(blockAttestedAt - blockProposedAt);
775
- // Proposer must sign over the attestations before pushing them to L1
776
- const signer = this.proposer ?? this.publisher.getSenderAddress();
777
- let attestationsSignature;
778
- try {
779
- attestationsSignature = await this.validatorClient.signAttestationsAndSigners(attestations, signer, this.targetSlot, this.checkpointNumber);
780
- } catch (err) {
781
- // We shouldn't really get here since we yield to another HA node
782
- // as soon as we see these errors when creating block or checkpoint proposals.
783
- if (this.handleHASigningError(err, 'Attestations signature')) {
784
- return undefined;
785
- }
786
- throw err;
1067
+ if (!this.config.skipBroadcastProposals) {
1068
+ await this.p2pClient.broadcastCheckpointProposal(proposal);
1069
+ this.checkpointMetrics.noteCheckpointBroadcast(this.dateProvider.now());
787
1070
  }
788
- // Enqueue publishing the checkpoint to L1
789
- this.setStateFn(SequencerState.PUBLISHING_CHECKPOINT, this.targetSlot);
790
- const aztecSlotDuration = this.l1Constants.slotDuration;
791
- const submissionSlotStart = Number(getTimestampForSlot(this.targetSlot, this.l1Constants));
792
- const txTimeoutAt = new Date((submissionSlotStart + aztecSlotDuration) * 1000);
793
- // If we have been configured to potentially skip publishing checkpoint then roll the dice here
794
- if (this.config.skipPublishingCheckpointsPercent !== undefined && this.config.skipPublishingCheckpointsPercent > 0) {
795
- const result = Math.max(0, randomInt(100));
796
- if (result < this.config.skipPublishingCheckpointsPercent) {
797
- this.log.warn(`Skipping publishing proposal for checkpoint ${checkpoint.number}. Configured percentage: ${this.config.skipPublishingCheckpointsPercent}, generated value: ${result}`);
798
- return checkpoint;
799
- }
800
- }
801
- await this.publisher.enqueueProposeCheckpoint(checkpoint, attestations, attestationsSignature, {
802
- txTimeoutAt,
803
- forcePendingCheckpointNumber: this.invalidateCheckpoint?.forcePendingCheckpointNumber
804
- });
805
- return checkpoint;
1071
+ // Return immediately after broadcast attestation collection happens in the background
1072
+ return {
1073
+ checkpoint,
1074
+ proposal,
1075
+ blockProposedAt
1076
+ };
806
1077
  } catch (e) {
807
1078
  env.error = e;
808
1079
  env.hasError = true;
@@ -815,7 +1086,7 @@ _dec = trackSpan('CheckpointProposalJob.execute'), _dec1 = trackSpan('Checkpoint
815
1086
  // swallow this error. It's already been logged by a function deeper in the stack
816
1087
  return undefined;
817
1088
  }
818
- this.log.error(`Error building checkpoint at slot ${this.slot}`, err);
1089
+ this.log.error(`Error building checkpoint at slot ${this.targetSlot}`, err);
819
1090
  return undefined;
820
1091
  }
821
1092
  }
@@ -852,17 +1123,19 @@ _dec = trackSpan('CheckpointProposalJob.execute'), _dec1 = trackSpan('Checkpoint
852
1123
  indexWithinCheckpoint,
853
1124
  txHashesAlreadyIncluded
854
1125
  });
855
- // TODO(palla/mbps): Review these conditions. We may want to keep trying in some scenarios.
856
- if (!buildResult && timingInfo.isLastBlock) {
857
- break;
858
- } else if (!buildResult && timingInfo.deadline !== undefined) {
859
- // But if there is still time for more blocks, wait until the next subslot and try again
1126
+ // If we failed to build the block due to insufficient txs, we try again if there is still time left in the slot
1127
+ if ('failure' in buildResult) {
1128
+ // If this was the last subslot, or we're running with a single block per slot, we're done
1129
+ if (timingInfo.isLastBlock || timingInfo.deadline === undefined) {
1130
+ break;
1131
+ }
1132
+ // Otherwise, if there is still time for more blocks, we wait until the next subslot and try again
860
1133
  await this.waitUntilNextSubslot(timingInfo.deadline);
861
1134
  continue;
862
- } else if (!buildResult) {
863
- break;
864
- } else if ('error' in buildResult) {
865
- // If there was an error building the block, just exit the loop and give up the rest of the slot
1135
+ }
1136
+ // If there was an error building the block, we just exit the loop and give up the rest of the slot.
1137
+ // We don't want to risk building more blocks if something went wrong.
1138
+ if ('error' in buildResult) {
866
1139
  if (!(buildResult.error instanceof SequencerInterruptedError)) {
867
1140
  this.log.warn(`Halting block building for slot ${this.targetSlot}`, {
868
1141
  slot: this.targetSlot,
@@ -873,34 +1146,33 @@ _dec = trackSpan('CheckpointProposalJob.execute'), _dec1 = trackSpan('Checkpoint
873
1146
  break;
874
1147
  }
875
1148
  const { block, usedTxs } = buildResult;
1149
+ this.checkpointMetrics.noteCheckpointBlockBuilt(this.dateProvider.now(), {
1150
+ isFirstBlock: blocksBuilt === 0,
1151
+ isLastBlock: timingInfo.isLastBlock
1152
+ });
876
1153
  blocksInCheckpoint.push(block);
877
1154
  usedTxs.forEach((tx)=>txHashesAlreadyIncluded.add(tx.txHash.toString()));
878
- // If this is the last block, sync it to the archiver and exit the loop
879
- // so we can build the checkpoint and start collecting attestations.
1155
+ // Sign the block proposal. This will throw if HA signing fails.
1156
+ const proposal = await this.createBlockProposal(block, inHash, usedTxs, blockProposalOptions);
1157
+ // Sync the proposed block to the archiver to make it available, only after we've managed to sign the proposal,
1158
+ // so we avoid polluting our archive with a block that would fail.
1159
+ // We wait for the sync to succeed, as this helps catch consistency errors, even if it means we lose some time for block-building.
1160
+ // If this throws, we abort the entire checkpoint.
1161
+ await this.syncProposedBlockToArchiver(block);
1162
+ // If this is the last block, do not broadcast it, since it will be included in the checkpoint proposal.
880
1163
  if (timingInfo.isLastBlock) {
881
- await this.syncProposedBlockToArchiver(block);
882
1164
  this.log.verbose(`Completed final block ${blockNumber} for slot ${this.targetSlot}`, {
883
1165
  slot: this.targetSlot,
884
1166
  blockNumber,
885
1167
  blocksBuilt
886
1168
  });
887
- blockPendingBroadcast = {
888
- block,
889
- txs: usedTxs
890
- };
1169
+ blockPendingBroadcast = proposal;
891
1170
  break;
892
1171
  }
893
- // Broadcast the block proposal (unless we're in fisherman mode) unless the block is the last one,
894
- // in which case we'll broadcast it along with the checkpoint at the end of the loop.
895
- // Note that we only send the block to the archiver if we manage to create the proposal, so if there's
896
- // a HA error we don't pollute our archiver with a block that won't make it to the chain.
897
- const proposal = await this.createBlockProposal(block, inHash, usedTxs, blockProposalOptions);
898
- // Sync the proposed block to the archiver to make it available, only after we've managed to sign the proposal.
899
- // We wait for the sync to succeed, as this helps catch consistency errors, even if it means we lose some time for block-building.
900
- // If this throws, we abort the entire checkpoint.
901
- await this.syncProposedBlockToArchiver(block);
902
1172
  // Once we have a signed proposal and the archiver agreed with our proposed block, then we broadcast it.
903
- proposal && await this.p2pClient.broadcastProposal(proposal);
1173
+ if (proposal && !this.config.skipBroadcastProposals) {
1174
+ await this.p2pClient.broadcastProposal(proposal);
1175
+ }
904
1176
  // Wait until the next block's start time
905
1177
  await this.waitUntilNextSubslot(timingInfo.deadline);
906
1178
  }
@@ -918,7 +1190,7 @@ _dec = trackSpan('CheckpointProposalJob.execute'), _dec1 = trackSpan('Checkpoint
918
1190
  this.log.info(`Skipping block proposal for block ${block.number} in fisherman mode`);
919
1191
  return Promise.resolve(undefined);
920
1192
  }
921
- return this.validatorClient.createBlockProposal(block.header, block.indexWithinCheckpoint, inHash, block.archive.root, usedTxs, this.proposer, blockProposalOptions);
1193
+ return this.validatorClient.createBlockProposal(block.header, this.checkpointNumber, block.indexWithinCheckpoint, inHash, block.archive.root, usedTxs, this.proposer, blockProposalOptions);
922
1194
  }
923
1195
  /** Sleeps until it is time to produce the next block in the slot */ async waitUntilNextSubslot(nextSubslotStart) {
924
1196
  this.setStateFn(SequencerState.WAITING_UNTIL_NEXT_BLOCK, this.targetSlot);
@@ -937,10 +1209,23 @@ _dec = trackSpan('CheckpointProposalJob.execute'), _dec1 = trackSpan('Checkpoint
937
1209
  // Wait until we have enough txs to build the block
938
1210
  const { availableTxs, canStartBuilding, minTxs } = await this.waitForMinTxs(opts);
939
1211
  if (!canStartBuilding) {
1212
+ this.logCheckpointEvent('block-build-failed', `Block build failed for slot ${this.targetSlot}`, {
1213
+ reason: 'insufficient_txs',
1214
+ blockNumber,
1215
+ slot: this.targetSlot,
1216
+ checkpointNumber: this.checkpointNumber,
1217
+ indexWithinCheckpoint,
1218
+ availableTxs,
1219
+ minTxs
1220
+ });
940
1221
  this.log.warn(`Not enough txs to build block ${blockNumber} at index ${indexWithinCheckpoint} in slot ${this.targetSlot} (got ${availableTxs} txs but needs ${minTxs})`, {
1222
+ reason: 'insufficient_txs',
941
1223
  blockNumber,
942
1224
  slot: this.targetSlot,
943
- indexWithinCheckpoint
1225
+ checkpointNumber: this.checkpointNumber,
1226
+ indexWithinCheckpoint,
1227
+ availableTxs,
1228
+ minTxs
944
1229
  });
945
1230
  this.eventEmitter.emit('block-tx-count-check-failed', {
946
1231
  minTxs,
@@ -948,7 +1233,9 @@ _dec = trackSpan('CheckpointProposalJob.execute'), _dec1 = trackSpan('Checkpoint
948
1233
  slot: this.targetSlot
949
1234
  });
950
1235
  this.metrics.recordBlockProposalFailed('insufficient_txs');
951
- return undefined;
1236
+ return {
1237
+ failure: 'insufficient-txs'
1238
+ };
952
1239
  }
953
1240
  // Create iterator to pending txs. We filter out txs already included in previous blocks in the checkpoint
954
1241
  // just in case p2p failed to sync the provisional block and didn't get to remove those txs from the mempool yet.
@@ -979,8 +1266,19 @@ _dec = trackSpan('CheckpointProposalJob.execute'), _dec1 = trackSpan('Checkpoint
979
1266
  // If any txs failed during execution, drop them from the mempool so we don't pick them up again
980
1267
  await this.dropFailedTxsFromP2P(buildResult.failedTxs);
981
1268
  if (buildResult.status === 'insufficient-valid-txs') {
1269
+ this.logCheckpointEvent('block-build-failed', `Block build failed for slot ${this.targetSlot}`, {
1270
+ reason: 'insufficient_valid_txs',
1271
+ slot: this.targetSlot,
1272
+ checkpointNumber: this.checkpointNumber,
1273
+ blockNumber,
1274
+ numTxs: buildResult.processedCount,
1275
+ indexWithinCheckpoint,
1276
+ minValidTxs
1277
+ });
982
1278
  this.log.warn(`Block ${blockNumber} at index ${indexWithinCheckpoint} on slot ${this.targetSlot} has too few valid txs to be proposed`, {
1279
+ reason: 'insufficient_valid_txs',
983
1280
  slot: this.targetSlot,
1281
+ checkpointNumber: this.checkpointNumber,
984
1282
  blockNumber,
985
1283
  numTxs: buildResult.processedCount,
986
1284
  indexWithinCheckpoint,
@@ -991,7 +1289,9 @@ _dec = trackSpan('CheckpointProposalJob.execute'), _dec1 = trackSpan('Checkpoint
991
1289
  slot: this.targetSlot
992
1290
  });
993
1291
  this.metrics.recordBlockProposalFailed('insufficient_valid_txs');
994
- return undefined;
1292
+ return {
1293
+ failure: 'insufficient-valid-txs'
1294
+ };
995
1295
  }
996
1296
  // Block creation succeeded, emit stats and metrics
997
1297
  const { block, publicProcessorDuration, usedTxs, blockBuildDuration, numTxs } = buildResult;
@@ -1010,12 +1310,14 @@ _dec = trackSpan('CheckpointProposalJob.execute'), _dec1 = trackSpan('Checkpoint
1010
1310
  manaPerSec,
1011
1311
  ...blockStats
1012
1312
  });
1313
+ // `slot` is the target/submission slot (may be one ahead when pipelining),
1314
+ // `buildSlot` is the wall-clock slot during which the block was actually built.
1013
1315
  this.eventEmitter.emit('block-proposed', {
1014
1316
  blockNumber: block.number,
1015
1317
  slot: this.targetSlot,
1016
1318
  buildSlot: this.slotNow
1017
1319
  });
1018
- this.metrics.recordBuiltBlock(blockBuildDuration, block.header.totalManaUsed.toNumberUnsafe());
1320
+ this.metrics.recordBuiltBlock(blockBuildDuration, block.header.totalManaUsed.toNumberUnsafe(), this.targetSlot);
1019
1321
  return {
1020
1322
  block,
1021
1323
  usedTxs
@@ -1025,9 +1327,17 @@ _dec = trackSpan('CheckpointProposalJob.execute'), _dec1 = trackSpan('Checkpoint
1025
1327
  reason: err.message,
1026
1328
  slot: this.targetSlot
1027
1329
  });
1330
+ this.logCheckpointEvent('block-build-failed', `Block build failed for slot ${this.targetSlot}`, {
1331
+ reason: err instanceof Error ? err.message : String(err),
1332
+ slot: this.targetSlot,
1333
+ checkpointNumber: this.checkpointNumber,
1334
+ blockNumber
1335
+ });
1028
1336
  this.log.error(`Error building block`, err, {
1029
- blockNumber,
1030
- slot: this.targetSlot
1337
+ reason: err instanceof Error ? err.message : String(err),
1338
+ slot: this.targetSlot,
1339
+ checkpointNumber: this.checkpointNumber,
1340
+ blockNumber
1031
1341
  });
1032
1342
  this.metrics.recordBlockProposalFailed(err.name || 'unknown_error');
1033
1343
  this.metrics.recordFailedBlock();
@@ -1090,13 +1400,37 @@ _dec = trackSpan('CheckpointProposalJob.execute'), _dec1 = trackSpan('Checkpoint
1090
1400
  minTxs
1091
1401
  };
1092
1402
  }
1403
+ async getSignedCommitteeAttestations(broadcast) {
1404
+ const { proposal, blockProposedAt } = broadcast;
1405
+ this.setStateFn(SequencerState.COLLECTING_ATTESTATIONS, this.targetSlot);
1406
+ const attestations = await this.waitForAttestations(proposal);
1407
+ if (!attestations) {
1408
+ return undefined;
1409
+ }
1410
+ this.checkpointMetrics.recordCheckpointAttestationDelay(this.dateProvider.now() - blockProposedAt);
1411
+ // Proposer must sign over the attestations before pushing them to L1
1412
+ const signer = this.proposer ?? this.publisher.getSenderAddress();
1413
+ try {
1414
+ const attestationsSignature = await this.validatorClient.signAttestationsAndSigners(attestations, signer, this.targetSlot, this.checkpointNumber);
1415
+ return {
1416
+ attestations,
1417
+ attestationsSignature
1418
+ };
1419
+ } catch (err) {
1420
+ if (this.handleHASigningError(err, 'Attestations signature')) {
1421
+ return;
1422
+ }
1423
+ this.log.error(`Error signing attestations for checkpoint proposal at slot ${proposal.slotNumber}`, err);
1424
+ return undefined;
1425
+ }
1426
+ }
1093
1427
  /**
1094
1428
  * Waits for enough attestations to be collected via p2p.
1095
1429
  * This is run after all blocks for the checkpoint have been built.
1096
1430
  */ async waitForAttestations(proposal) {
1097
1431
  if (this.config.fishermanMode) {
1098
1432
  this.log.debug('Skipping attestation collection in fisherman mode');
1099
- return CommitteeAttestationsAndSigners.empty();
1433
+ return CommitteeAttestationsAndSigners.empty(this.getSignatureContext());
1100
1434
  }
1101
1435
  const slotNumber = proposal.slotNumber;
1102
1436
  const { committee, seed, epoch } = await this.epochCache.getCommittee(slotNumber);
@@ -1104,25 +1438,28 @@ _dec = trackSpan('CheckpointProposalJob.execute'), _dec1 = trackSpan('Checkpoint
1104
1438
  throw new Error('No committee when collecting attestations');
1105
1439
  } else if (committee.length === 0) {
1106
1440
  this.log.verbose(`Attesting committee is empty`);
1107
- return CommitteeAttestationsAndSigners.empty();
1441
+ return CommitteeAttestationsAndSigners.empty(this.getSignatureContext());
1108
1442
  } else {
1109
1443
  this.log.debug(`Attesting committee length is ${committee.length}`, {
1110
1444
  committee
1111
1445
  });
1112
1446
  }
1113
- const numberOfRequiredAttestations = Math.floor(committee.length * 2 / 3) + 1;
1447
+ const numberOfRequiredAttestations = computeQuorum(committee.length);
1114
1448
  if (this.config.skipCollectingAttestations) {
1115
1449
  this.log.warn('Skipping attestation collection as per config (attesting with own keys only)');
1116
- const attestations = await this.validatorClient?.collectOwnAttestations(proposal);
1117
- return new CommitteeAttestationsAndSigners(orderAttestations(attestations ?? [], committee));
1450
+ const attestations = await this.validatorClient?.collectOwnAttestations(proposal, this.checkpointNumber);
1451
+ this.logCheckpointAttestations('collected', committee, attestations ?? [], numberOfRequiredAttestations, {
1452
+ reason: 'collect_own_only'
1453
+ });
1454
+ return new CommitteeAttestationsAndSigners(orderAttestations(attestations ?? [], committee), this.getSignatureContext());
1118
1455
  }
1119
- const attestationTimeAllowed = this.config.enforceTimeTable ? this.timetable.getMaxAllowedTime(SequencerState.PUBLISHING_CHECKPOINT) : this.l1Constants.slotDuration;
1456
+ const attestationTimeAllowed = this.config.enforceTimeTable ? this.timetable.getCheckpointAttestationDeadline() : this.l1Constants.slotDuration;
1120
1457
  const attestationDeadline = new Date((this.getSlotStartBuildTimestamp() + attestationTimeAllowed) * 1000);
1121
1458
  this.metrics.recordRequiredAttestations(numberOfRequiredAttestations, attestationTimeAllowed);
1122
1459
  const collectAttestationsTimer = new Timer();
1123
1460
  let collectedAttestationsCount = 0;
1124
1461
  try {
1125
- const attestations = await this.validatorClient.collectAttestations(proposal, numberOfRequiredAttestations, attestationDeadline);
1462
+ const attestations = await this.validatorClient.collectAttestations(proposal, numberOfRequiredAttestations, attestationDeadline, this.checkpointNumber);
1126
1463
  collectedAttestationsCount = attestations.length;
1127
1464
  // Trim attestations to minimum required to save L1 calldata gas
1128
1465
  const localAddresses = this.validatorClient.getValidatorAddresses();
@@ -1132,20 +1469,55 @@ _dec = trackSpan('CheckpointProposalJob.execute'), _dec1 = trackSpan('Checkpoint
1132
1469
  }
1133
1470
  // Rollup contract requires that the signatures are provided in the order of the committee
1134
1471
  const sorted = orderAttestations(trimmed, committee);
1472
+ this.logCheckpointAttestations('collected', committee, attestations, numberOfRequiredAttestations, {
1473
+ submittedCount: trimmed.length
1474
+ });
1135
1475
  // Manipulate the attestations if we've been configured to do so
1136
1476
  if (this.config.injectFakeAttestation || this.config.injectHighSValueAttestation || this.config.injectUnrecoverableSignatureAttestation || this.config.shuffleAttestationOrdering) {
1137
1477
  return this.manipulateAttestations(proposal.slotNumber, epoch, seed, committee, sorted);
1138
1478
  }
1139
- return new CommitteeAttestationsAndSigners(sorted);
1479
+ return new CommitteeAttestationsAndSigners(sorted, this.getSignatureContext());
1140
1480
  } catch (err) {
1141
1481
  if (err && err instanceof AttestationTimeoutError) {
1142
1482
  collectedAttestationsCount = err.collectedCount;
1483
+ this.logCheckpointAttestations('failed', committee, undefined, numberOfRequiredAttestations, {
1484
+ collectedCount: collectedAttestationsCount,
1485
+ reason: 'timeout'
1486
+ });
1487
+ this.log.error(`Timeout while waiting for attestations for checkpoint proposal at slot ${proposal.slotNumber} (collected ${collectedAttestationsCount}/${numberOfRequiredAttestations})`, err);
1488
+ } else {
1489
+ this.logCheckpointAttestations('failed', committee, undefined, numberOfRequiredAttestations, {
1490
+ collectedCount: collectedAttestationsCount,
1491
+ reason: err instanceof Error ? err.message : String(err)
1492
+ });
1493
+ this.log.error(`Error collecting attestations for checkpoint proposal at slot ${proposal.slotNumber}`, err);
1143
1494
  }
1144
- throw err;
1495
+ return undefined;
1145
1496
  } finally{
1146
1497
  this.metrics.recordCollectedAttestations(collectedAttestationsCount, collectAttestationsTimer.ms());
1147
1498
  }
1148
1499
  }
1500
+ logCheckpointAttestations(status, committee, attestations, requiredAttestations, opts = {}) {
1501
+ const signedValidators = attestations?.map((attestation)=>attestation.getSender()?.toString()).filter((address)=>address !== undefined) ?? [];
1502
+ const collectedCount = opts.collectedCount ?? new Set(signedValidators).size;
1503
+ const missingValidatorCount = status === 'failed' ? Math.max(0, requiredAttestations - collectedCount) : undefined;
1504
+ this.logCheckpointEvent(`attestations-${status}`, `Checkpoint attestations ${status} for slot ${this.targetSlot}`, {
1505
+ slot: this.targetSlot,
1506
+ checkpointNumber: this.checkpointNumber,
1507
+ committeeSize: committee.length,
1508
+ requiredAttestations,
1509
+ collectedAttestations: collectedCount,
1510
+ ...opts.submittedCount !== undefined && {
1511
+ submittedAttestations: opts.submittedCount
1512
+ },
1513
+ ...missingValidatorCount !== undefined && {
1514
+ missingValidatorCount
1515
+ },
1516
+ ...opts.reason !== undefined && {
1517
+ reason: opts.reason
1518
+ }
1519
+ });
1520
+ }
1149
1521
  /** Breaks the attestations before publishing based on attack configs */ manipulateAttestations(slotNumber, epoch, seed, committee, attestations) {
1150
1522
  // Compute the proposer index in the committee, since we dont want to tweak it.
1151
1523
  // Otherwise, the L1 rollup contract will reject the block outright.
@@ -1171,7 +1543,7 @@ _dec = trackSpan('CheckpointProposalJob.execute'), _dec1 = trackSpan('Checkpoint
1171
1543
  unfreeze(attestations[targetIndex]).signature = generateRecoverableSignature();
1172
1544
  }
1173
1545
  }
1174
- return new CommitteeAttestationsAndSigners(attestations);
1546
+ return new CommitteeAttestationsAndSigners(attestations, this.getSignatureContext());
1175
1547
  }
1176
1548
  if (this.config.shuffleAttestationOrdering) {
1177
1549
  this.log.warn(`Shuffling attestation ordering in checkpoint for slot ${slotNumber} (proposer #${proposerIndex})`);
@@ -1197,10 +1569,10 @@ _dec = trackSpan('CheckpointProposalJob.execute'), _dec1 = trackSpan('Checkpoint
1197
1569
  shuffled[i]
1198
1570
  ];
1199
1571
  }
1200
- const signers = new CommitteeAttestationsAndSigners(attestations).getSigners();
1201
- return new MaliciousCommitteeAttestationsAndSigners(shuffled, signers);
1572
+ const signers = new CommitteeAttestationsAndSigners(attestations, this.getSignatureContext()).getSigners();
1573
+ return new MaliciousCommitteeAttestationsAndSigners(shuffled, signers, this.getSignatureContext());
1202
1574
  }
1203
- return new CommitteeAttestationsAndSigners(attestations);
1575
+ return new CommitteeAttestationsAndSigners(attestations, this.getSignatureContext());
1204
1576
  }
1205
1577
  async dropFailedTxsFromP2P(failedTxs) {
1206
1578
  if (failedTxs.length === 0) {
@@ -1215,8 +1587,12 @@ _dec = trackSpan('CheckpointProposalJob.execute'), _dec1 = trackSpan('Checkpoint
1215
1587
  * Adds the proposed block to the archiver so it's available via P2P.
1216
1588
  * Gossip doesn't echo messages back to the sender, so the proposer's archiver/world-state
1217
1589
  * would never receive its own block without this explicit sync.
1590
+ *
1591
+ * In fisherman mode we skip this push: the fisherman builds blocks locally for validation
1592
+ * and fee analysis only, and pushing them to the archiver causes spurious reorg cascades
1593
+ * whenever the real proposer's block arrives from L1.
1218
1594
  */ async syncProposedBlockToArchiver(block) {
1219
- if (this.config.skipPushProposedBlocksToArchiver !== false) {
1595
+ if (this.config.skipPushProposedBlocksToArchiver || this.config.fishermanMode) {
1220
1596
  this.log.warn(`Skipping push of proposed block ${block.number} to archiver`, {
1221
1597
  blockNumber: block.number,
1222
1598
  slot: block.header.globalVariables.slotNumber
@@ -1277,7 +1653,7 @@ _dec = trackSpan('CheckpointProposalJob.execute'), _dec1 = trackSpan('Checkpoint
1277
1653
  await sleep(TXS_POLLING_MS);
1278
1654
  }
1279
1655
  getSlotStartBuildTimestamp() {
1280
- return getSlotStartBuildTimestamp(this.slot, this.l1Constants);
1656
+ return getSlotStartBuildTimestamp(this.slotNow, this.l1Constants);
1281
1657
  }
1282
1658
  getSecondsIntoSlot() {
1283
1659
  const slotStartTimestamp = this.getSlotStartBuildTimestamp();