@aztec/sequencer-client 0.0.1-commit.e588bc7e5 → 0.0.1-commit.e5a3663dd

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. package/dest/client/sequencer-client.d.ts +3 -1
  2. package/dest/client/sequencer-client.d.ts.map +1 -1
  3. package/dest/client/sequencer-client.js +3 -4
  4. package/dest/config.d.ts +2 -1
  5. package/dest/config.d.ts.map +1 -1
  6. package/dest/config.js +16 -14
  7. package/dest/global_variable_builder/fee_predictor.d.ts +37 -0
  8. package/dest/global_variable_builder/fee_predictor.d.ts.map +1 -0
  9. package/dest/global_variable_builder/fee_predictor.js +128 -0
  10. package/dest/global_variable_builder/fee_provider.d.ts +21 -0
  11. package/dest/global_variable_builder/fee_provider.d.ts.map +1 -0
  12. package/dest/global_variable_builder/fee_provider.js +58 -0
  13. package/dest/global_variable_builder/global_builder.d.ts +4 -9
  14. package/dest/global_variable_builder/global_builder.d.ts.map +1 -1
  15. package/dest/global_variable_builder/global_builder.js +3 -41
  16. package/dest/global_variable_builder/index.d.ts +3 -1
  17. package/dest/global_variable_builder/index.d.ts.map +1 -1
  18. package/dest/global_variable_builder/index.js +2 -0
  19. package/dest/publisher/config.d.ts +1 -1
  20. package/dest/publisher/config.d.ts.map +1 -1
  21. package/dest/publisher/config.js +2 -2
  22. package/dest/publisher/l1_tx_failed_store/failed_tx_store.d.ts +3 -4
  23. package/dest/publisher/l1_tx_failed_store/failed_tx_store.d.ts.map +1 -1
  24. package/dest/publisher/sequencer-publisher-factory.d.ts +1 -3
  25. package/dest/publisher/sequencer-publisher-factory.d.ts.map +1 -1
  26. package/dest/publisher/sequencer-publisher-factory.js +0 -1
  27. package/dest/publisher/sequencer-publisher.d.ts +16 -35
  28. package/dest/publisher/sequencer-publisher.d.ts.map +1 -1
  29. package/dest/publisher/sequencer-publisher.js +62 -94
  30. package/dest/sequencer/chain_state_overrides.d.ts +25 -0
  31. package/dest/sequencer/chain_state_overrides.d.ts.map +1 -0
  32. package/dest/sequencer/chain_state_overrides.js +39 -0
  33. package/dest/sequencer/checkpoint_proposal_job.d.ts +33 -22
  34. package/dest/sequencer/checkpoint_proposal_job.d.ts.map +1 -1
  35. package/dest/sequencer/checkpoint_proposal_job.js +479 -170
  36. package/dest/sequencer/checkpoint_proposal_job_metrics.d.ts +34 -0
  37. package/dest/sequencer/checkpoint_proposal_job_metrics.d.ts.map +1 -0
  38. package/dest/sequencer/checkpoint_proposal_job_metrics.js +72 -0
  39. package/dest/sequencer/events.d.ts +6 -1
  40. package/dest/sequencer/events.d.ts.map +1 -1
  41. package/dest/sequencer/metrics.d.ts +9 -10
  42. package/dest/sequencer/metrics.d.ts.map +1 -1
  43. package/dest/sequencer/metrics.js +34 -20
  44. package/dest/sequencer/sequencer.d.ts +16 -5
  45. package/dest/sequencer/sequencer.d.ts.map +1 -1
  46. package/dest/sequencer/sequencer.js +63 -30
  47. package/dest/sequencer/timetable.d.ts +14 -1
  48. package/dest/sequencer/timetable.d.ts.map +1 -1
  49. package/dest/sequencer/timetable.js +45 -36
  50. package/dest/test/utils.d.ts +1 -1
  51. package/dest/test/utils.d.ts.map +1 -1
  52. package/dest/test/utils.js +7 -6
  53. package/package.json +27 -27
  54. package/src/client/sequencer-client.ts +5 -7
  55. package/src/config.ts +15 -11
  56. package/src/global_variable_builder/README.md +44 -0
  57. package/src/global_variable_builder/fee_predictor.ts +172 -0
  58. package/src/global_variable_builder/fee_provider.ts +75 -0
  59. package/src/global_variable_builder/global_builder.ts +7 -54
  60. package/src/global_variable_builder/index.ts +2 -0
  61. package/src/publisher/config.ts +6 -4
  62. package/src/publisher/l1_tx_failed_store/failed_tx_store.ts +3 -1
  63. package/src/publisher/sequencer-publisher-factory.ts +0 -3
  64. package/src/publisher/sequencer-publisher.ts +108 -156
  65. package/src/sequencer/README.md +82 -13
  66. package/src/sequencer/chain_state_overrides.ts +87 -0
  67. package/src/sequencer/checkpoint_proposal_job.ts +572 -218
  68. package/src/sequencer/checkpoint_proposal_job_metrics.ts +128 -0
  69. package/src/sequencer/events.ts +5 -0
  70. package/src/sequencer/metrics.ts +43 -24
  71. package/src/sequencer/sequencer.ts +81 -35
  72. package/src/sequencer/timetable.ts +57 -45
  73. package/src/test/utils.ts +28 -10
@@ -436,13 +436,12 @@ function _apply_decs_2203_r(targetClass, memberDecs, classDecs, parentClass) {
436
436
  return (_apply_decs_2203_r = applyDecs2203RFactory())(targetClass, memberDecs, classDecs, parentClass);
437
437
  }
438
438
  var _dec, _dec1, _dec2, _dec3, _dec4, _dec5, _dec6, _dec7, _initProto;
439
- import { RollupContract } from '@aztec/ethereum/contracts';
440
439
  import { BlockNumber, CheckpointNumber, IndexWithinCheckpoint } from '@aztec/foundation/branded-types';
441
440
  import { randomInt } from '@aztec/foundation/crypto/random';
442
441
  import { flipSignature, generateRecoverableSignature, generateUnrecoverableSignature } from '@aztec/foundation/crypto/secp256k1-signer';
443
- import { Signature } from '@aztec/foundation/eth-signature';
444
442
  import { filter } from '@aztec/foundation/iterator';
445
443
  import { createLogger } from '@aztec/foundation/log';
444
+ import { retryUntil } from '@aztec/foundation/retry';
446
445
  import { sleep, sleepUntil } from '@aztec/foundation/sleep';
447
446
  import { Timer } from '@aztec/foundation/timer';
448
447
  import { isErrorClass, unfreeze } from '@aztec/foundation/types';
@@ -456,6 +455,7 @@ import { orderAttestations, trimAttestations } from '@aztec/stdlib/p2p';
456
455
  import { AttestationTimeoutError } from '@aztec/stdlib/validators';
457
456
  import { Attributes, trackSpan } from '@aztec/telemetry-client';
458
457
  import { DutyAlreadySignedError, SlashingProtectionError } from '@aztec/validator-ha-signer/errors';
458
+ import { buildPipelinedParentSimulationOverridesPlan, buildSubmissionSimulationOverridesPlan } from './chain_state_overrides.js';
459
459
  import { CheckpointVoter } from './checkpoint_voter.js';
460
460
  import { SequencerInterruptedError } from './errors.js';
461
461
  import { SequencerState } from './utils.js';
@@ -491,12 +491,14 @@ _dec = trackSpan('CheckpointProposalJob.execute'), _dec1 = trackSpan('Checkpoint
491
491
  checkpointsBuilder;
492
492
  blockSink;
493
493
  l1Constants;
494
+ signatureContext;
494
495
  config;
495
496
  timetable;
496
497
  slasherClient;
497
498
  epochCache;
498
499
  dateProvider;
499
500
  metrics;
501
+ checkpointMetrics;
500
502
  eventEmitter;
501
503
  setStateFn;
502
504
  tracer;
@@ -546,10 +548,14 @@ _dec = trackSpan('CheckpointProposalJob.execute'), _dec1 = trackSpan('Checkpoint
546
548
  ], []));
547
549
  }
548
550
  log;
551
+ checkpointEventLog;
549
552
  /** Tracks the fire-and-forget L1 submission promise so it can be awaited during shutdown. */ pendingL1Submission;
550
- /** Fee header override computed during proposeCheckpoint, reused in enqueueCheckpointForSubmission. */ computedForceProposedFeeHeader;
553
+ /** Pipelined parent chain state used while building and later submitting this checkpoint. */ pipelinedParentSimulationOverridesPlan;
554
+ getSignatureContext() {
555
+ return this.signatureContext;
556
+ }
551
557
  constructor(slotNow, targetSlot, targetEpoch, checkpointNumber, syncedToBlockNumber, // TODO(palla/mbps): Can we remove the proposer in favor of attestorAddress? Need to check fisherman-node flows.
552
- proposer, publisher, attestorAddress, invalidateCheckpoint, validatorClient, globalsBuilder, p2pClient, worldState, l1ToL2MessageSource, l2BlockSource, checkpointsBuilder, blockSink, l1Constants, config, timetable, slasherClient, epochCache, dateProvider, metrics, eventEmitter, setStateFn, tracer, bindings, proposedCheckpointData){
558
+ proposer, publisher, attestorAddress, invalidateCheckpoint, validatorClient, globalsBuilder, p2pClient, worldState, l1ToL2MessageSource, l2BlockSource, checkpointsBuilder, blockSink, l1Constants, signatureContext, config, timetable, slasherClient, epochCache, dateProvider, metrics, checkpointMetrics, eventEmitter, setStateFn, tracer, bindings, proposedCheckpointData){
553
559
  this.slotNow = slotNow;
554
560
  this.targetSlot = targetSlot;
555
561
  this.targetEpoch = targetEpoch;
@@ -568,12 +574,14 @@ _dec = trackSpan('CheckpointProposalJob.execute'), _dec1 = trackSpan('Checkpoint
568
574
  this.checkpointsBuilder = checkpointsBuilder;
569
575
  this.blockSink = blockSink;
570
576
  this.l1Constants = l1Constants;
577
+ this.signatureContext = signatureContext;
571
578
  this.config = config;
572
579
  this.timetable = timetable;
573
580
  this.slasherClient = slasherClient;
574
581
  this.epochCache = epochCache;
575
582
  this.dateProvider = dateProvider;
576
583
  this.metrics = metrics;
584
+ this.checkpointMetrics = checkpointMetrics;
577
585
  this.eventEmitter = eventEmitter;
578
586
  this.setStateFn = setStateFn;
579
587
  this.tracer = tracer;
@@ -583,81 +591,152 @@ _dec = trackSpan('CheckpointProposalJob.execute'), _dec1 = trackSpan('Checkpoint
583
591
  ...bindings,
584
592
  instanceId: `slot-${this.slotNow}`
585
593
  });
594
+ this.checkpointEventLog = createLogger('sequencer:checkpoint-events', {
595
+ ...bindings,
596
+ instanceId: `slot-${this.slotNow}`
597
+ });
586
598
  }
587
599
  /** Awaits the pending L1 submission if one is in progress. Call during shutdown. */ async awaitPendingSubmission() {
588
600
  this.log.info('Awaiting pending L1 payload submission');
589
601
  await this.pendingL1Submission;
590
602
  }
603
+ logCheckpointEvent(eventName, message, fields) {
604
+ this.checkpointEventLog.debug(message, {
605
+ eventName: `sequencer-checkpoint-${eventName}`,
606
+ ...fields
607
+ });
608
+ }
591
609
  /**
592
610
  * Executes the checkpoint proposal job.
593
- * Builds blocks, collects attestations, enqueues requests, and schedules L1 submission as a
594
- * background task so the work loop can return to IDLE immediately.
611
+ * Builds blocks, assembles checkpoint, and broadcasts the proposal (blocking).
612
+ * Attestation collection, signing, and L1 submission are backgrounded so the
613
+ * work loop can return to IDLE immediately for consecutive slot proposals.
595
614
  * Returns the built checkpoint if successful, undefined otherwise.
596
615
  */ async execute() {
597
616
  // Enqueue governance and slashing votes (returns promises that will be awaited later)
598
617
  // In fisherman mode, we simulate slashing but don't actually publish to L1
599
618
  // These are constant for the whole slot, so we only enqueue them once
600
619
  const votesPromises = new CheckpointVoter(this.targetSlot, this.publisher, this.attestorAddress, this.validatorClient, this.slasherClient, this.l1Constants, this.config, this.metrics, this.log).enqueueVotes();
601
- // Build and propose the checkpoint. Builds blocks, broadcasts, collects attestations, and signs.
602
- // Does NOT enqueue to L1 yet that happens after the pipeline sleep.
603
- const proposalResult = await this.proposeCheckpoint();
604
- const checkpoint = proposalResult?.checkpoint;
605
- // Wait until the voting promises have resolved, so all requests are enqueued (not sent)
606
- await Promise.all(votesPromises);
607
- if (checkpoint) {
608
- this.metrics.recordCheckpointProposalSuccess();
620
+ // Build blocks, assemble checkpoint, and broadcast proposal (BLOCKING).
621
+ // Returns after broadcastattestation collection is deferred.
622
+ const broadcast = await this.proposeCheckpoint();
623
+ if (!broadcast) {
624
+ await Promise.all(votesPromises);
625
+ // Still submit votes even without a checkpoint
626
+ if (!this.config.fishermanMode) {
627
+ this.pendingL1Submission = this.publisher.sendRequestsAt(this.dateProvider.nowAsDate()).then(()=>{});
628
+ }
629
+ return undefined;
609
630
  }
631
+ const { checkpoint } = broadcast;
632
+ this.metrics.recordCheckpointProposalSuccess();
610
633
  // Do not post anything to L1 if we are fishermen, but do perform L1 fee analysis
611
634
  if (this.config.fishermanMode) {
612
635
  await this.handleCheckpointEndAsFisherman(checkpoint);
613
- return;
636
+ return checkpoint;
614
637
  }
615
- // Enqueue the checkpoint for L1 submission
616
- if (proposalResult) {
617
- try {
618
- await this.enqueueCheckpointForSubmission(proposalResult);
619
- } catch (err) {
620
- this.log.error(`Failed to enqueue checkpoint for L1 submission at slot ${this.targetSlot}`, err);
621
- // Continue to sendRequestsAt so votes are still sent
638
+ // Background the attestation signing → L1 pipeline so the work loop is unblocked
639
+ this.pendingL1Submission = this.waitForAttestationsAndEnqueueSubmissionAsync(broadcast, votesPromises);
640
+ // Return the built checkpoint immediately — the work loop is now unblocked
641
+ return checkpoint;
642
+ }
643
+ /**
644
+ * Background pipeline: collects attestations, signs them, enqueues the checkpoint, and submits to L1.
645
+ * Runs as a fire-and-forget task stored in `pendingL1Submission` so the work loop is unblocked.
646
+ */ async waitForAttestationsAndEnqueueSubmissionAsync(broadcast, votesPromises) {
647
+ const { checkpoint } = broadcast;
648
+ const isPipelining = this.epochCache.isProposerPipeliningEnabled();
649
+ try {
650
+ // Wait for all votes actions, enqueued at the beginning, to resolve
651
+ await Promise.all(votesPromises);
652
+ // Try to collect attestations from the committee
653
+ const signedAttestations = await this.getSignedCommitteeAttestations(broadcast);
654
+ // If pipelining, wait for the previous checkpoint to land on L1 before submitting,
655
+ // so we can check it matches the proposed checkpoint we used as parent, and has valid attestations.
656
+ if (signedAttestations && (!isPipelining || await this.waitForValidParentCheckpointOnL1())) {
657
+ await this.enqueueCheckpointForSubmission({
658
+ checkpoint,
659
+ ...signedAttestations
660
+ });
622
661
  }
623
- }
624
- // Compute the earliest time to submit: pipeline slot start when pipelining, now otherwise.
625
- const submitAfter = this.epochCache.isProposerPipeliningEnabled() ? new Date(Number(getTimestampForSlot(this.targetSlot, this.l1Constants)) * 1000) : new Date(this.dateProvider.now());
626
- // TODO(https://github.com/AztecProtocol/aztec-packages/pull/21250): should discard the pending submission if a reorg occurs underneath
627
- // Schedule L1 submission in the background so the work loop returns immediately.
628
- // The publisher will sleep until submitAfter, then send the bundled requests.
629
- // The promise is stored so it can be awaited during shutdown.
630
- this.pendingL1Submission = this.publisher.sendRequestsAt(submitAfter).then(async (l1Response)=>{
662
+ // If we failed to collect attestations, at least check if we need to issue an invalidation
663
+ // Note that if we are not pipelining, we enqueued the invalidation at the beginning
664
+ if (!signedAttestations && isPipelining && await this.waitForSyncedL2SlotNumber(this.slotNow)) {
665
+ const validationStatus = await this.l2BlockSource.getPendingChainValidationStatus();
666
+ if (!validationStatus.valid) {
667
+ this.log.warn(`Checkpoint ${validationStatus.checkpoint.checkpointNumber} has invalid attestations, enqueuing invalidation in spite of attestation collection failure`, {
668
+ checkpoint: validationStatus.checkpoint,
669
+ reason: validationStatus.reason
670
+ });
671
+ await this.enqueueInvalidation(validationStatus);
672
+ }
673
+ }
674
+ // Send whatever was enqueued: votes + (propose | invalidation | nothing).
675
+ // Compute the earliest time to submit: pipeline slot start when pipelining, now otherwise.
676
+ const submitAfter = isPipelining ? new Date(Number(getTimestampForSlot(this.targetSlot, this.l1Constants)) * 1000) : new Date(this.dateProvider.now());
677
+ const l1Response = await this.publisher.sendRequestsAt(submitAfter);
631
678
  const proposedAction = l1Response?.successfulActions.find((a)=>a === 'propose');
632
679
  if (proposedAction) {
680
+ this.logCheckpointEvent('published', `Checkpoint published for slot ${this.targetSlot}`, {
681
+ slot: this.targetSlot,
682
+ checkpointNumber: this.checkpointNumber,
683
+ successfulActions: l1Response?.successfulActions,
684
+ sentActions: l1Response?.sentActions
685
+ });
633
686
  this.eventEmitter.emit('checkpoint-published', {
634
687
  checkpoint: this.checkpointNumber,
635
688
  slot: this.targetSlot
636
689
  });
637
- const coinbase = checkpoint?.header.coinbase;
690
+ const coinbase = checkpoint.header.coinbase;
638
691
  await this.metrics.incFilledSlot(this.publisher.getSenderAddress().toString(), coinbase);
639
- } else if (checkpoint) {
692
+ } else {
693
+ this.logCheckpointEvent('publish-failed', `Checkpoint publish failed for slot ${this.targetSlot}`, {
694
+ slot: this.targetSlot,
695
+ checkpointNumber: this.checkpointNumber,
696
+ successfulActions: l1Response?.successfulActions,
697
+ failedActions: l1Response?.failedActions,
698
+ sentActions: l1Response?.sentActions,
699
+ expiredActions: l1Response?.expiredActions,
700
+ reason: 'propose_action_not_successful'
701
+ });
702
+ this.log.warn(`Checkpoint publish failed for slot ${this.targetSlot}`, {
703
+ slot: this.targetSlot,
704
+ checkpointNumber: this.checkpointNumber,
705
+ successfulActions: l1Response?.successfulActions,
706
+ failedActions: l1Response?.failedActions,
707
+ sentActions: l1Response?.sentActions,
708
+ expiredActions: l1Response?.expiredActions,
709
+ reason: 'propose_action_not_successful'
710
+ });
640
711
  this.eventEmitter.emit('checkpoint-publish-failed', {
641
712
  ...l1Response,
642
713
  slot: this.targetSlot
643
714
  });
644
- if (this.epochCache.isProposerPipeliningEnabled()) {
715
+ if (isPipelining) {
645
716
  this.metrics.recordPipelineDiscard();
646
717
  }
647
718
  }
648
- }).catch((err)=>{
649
- this.log.error(`Background L1 submission failed for slot ${this.targetSlot}`, err);
650
- if (checkpoint) {
651
- this.eventEmitter.emit('checkpoint-publish-failed', {
652
- slot: this.targetSlot
653
- });
654
- if (this.epochCache.isProposerPipeliningEnabled()) {
655
- this.metrics.recordPipelineDiscard();
656
- }
719
+ } catch (err) {
720
+ if (err instanceof SequencerInterruptedError) {
721
+ return;
657
722
  }
658
- });
659
- // Return the built checkpoint immediately — the work loop is now unblocked
660
- return checkpoint;
723
+ this.logCheckpointEvent('publish-failed', `Checkpoint publish failed for slot ${this.targetSlot}`, {
724
+ slot: this.targetSlot,
725
+ checkpointNumber: this.checkpointNumber,
726
+ reason: err instanceof Error ? err.message : String(err)
727
+ });
728
+ this.log.error(`Background attestation/L1 pipeline failed for slot ${this.targetSlot}`, err, {
729
+ slot: this.targetSlot,
730
+ checkpointNumber: this.checkpointNumber,
731
+ reason: err instanceof Error ? err.message : String(err)
732
+ });
733
+ this.eventEmitter.emit('checkpoint-publish-failed', {
734
+ slot: this.targetSlot
735
+ });
736
+ if (isPipelining) {
737
+ this.metrics.recordPipelineDiscard();
738
+ }
739
+ }
661
740
  }
662
741
  /** Enqueues the checkpoint for L1 submission. Called after pipeline sleep in execute(). */ async enqueueCheckpointForSubmission(result) {
663
742
  const { checkpoint, attestations, attestationsSignature } = result;
@@ -673,12 +752,135 @@ _dec = trackSpan('CheckpointProposalJob.execute'), _dec1 = trackSpan('Checkpoint
673
752
  return;
674
753
  }
675
754
  }
755
+ const isPipelining = this.epochCache.isProposerPipeliningEnabled();
756
+ const submissionSimulationOverridesPlan = buildSubmissionSimulationOverridesPlan({
757
+ pipelinedParentPlan: this.pipelinedParentSimulationOverridesPlan,
758
+ invalidateToPendingCheckpointNumber: this.invalidateCheckpoint?.forcePendingCheckpointNumber,
759
+ lastArchiveRoot: checkpoint.header.lastArchiveRoot,
760
+ pipeliningEnabled: isPipelining
761
+ });
676
762
  await this.publisher.enqueueProposeCheckpoint(checkpoint, attestations, attestationsSignature, {
677
763
  txTimeoutAt,
678
- forcePendingCheckpointNumber: this.invalidateCheckpoint?.forcePendingCheckpointNumber,
679
- forceProposedFeeHeader: this.computedForceProposedFeeHeader
764
+ ...submissionSimulationOverridesPlan ? {
765
+ simulationOverridesPlan: submissionSimulationOverridesPlan
766
+ } : {}
767
+ });
768
+ }
769
+ /**
770
+ * Wait until the archiver syncs past the given L2 slot number.
771
+ * The deadline is the end of `this.targetSlot`, beyond which any pipelined work would miss its
772
+ * L1 submission window and is no longer useful.
773
+ */ async waitForSyncedL2SlotNumber(waitForSlot) {
774
+ const targetSlotStart = Number(getTimestampForSlot(this.targetSlot, this.l1Constants));
775
+ const targetSlotEndMs = (targetSlotStart + this.l1Constants.slotDuration) * 1000;
776
+ const syncDelayTolerance = this.l1Constants.ethereumSlotDuration * 2 * 1000;
777
+ const timeoutSeconds = Math.max(0.1, (targetSlotEndMs + syncDelayTolerance - this.dateProvider.now()) / 1000);
778
+ try {
779
+ return await retryUntil(async ()=>{
780
+ const syncedSlot = await this.l2BlockSource.getSyncedL2SlotNumber();
781
+ return syncedSlot !== undefined && syncedSlot >= waitForSlot;
782
+ }, `archiver sync past slot ${waitForSlot}`, timeoutSeconds, 0.2);
783
+ } catch {
784
+ this.log.warn(`Archiver did not sync L1 past slot ${waitForSlot} before slot ${this.targetSlot} expired, discarding pipelined work`, {
785
+ checkpointNumber: this.checkpointNumber
786
+ });
787
+ this.emitPipelinedCheckpointDiscarded('archiver-sync-timeout');
788
+ return false;
789
+ }
790
+ }
791
+ /**
792
+ * Waits for the parent checkpoint to land on L1 before submitting a pipelined checkpoint.
793
+ * Polls until the archiver has synced L1 past the parent's slot, then verifies:
794
+ * - If we built on a proposed parent: it must have landed on L1 with matching hash and valid attestations.
795
+ * - If we built without a proposed parent: no new checkpoint must have appeared for that slot.
796
+ * If the parent has invalid attestations, enqueues an invalidation. Returns whether to proceed with the proposal.
797
+ */ async waitForValidParentCheckpointOnL1() {
798
+ const parentCheckpointNumber = CheckpointNumber(this.checkpointNumber - 1);
799
+ // Wait until archiver has synced L1 past the parent's slot (slotNow)
800
+ if (!await this.waitForSyncedL2SlotNumber(this.slotNow)) {
801
+ return false;
802
+ }
803
+ const tips = await this.l2BlockSource.getL2Tips();
804
+ const checkpointedNumber = tips.checkpointed.checkpoint.number;
805
+ // We built on top of a proposed checkpoint. Verify it landed on L1 as expected.
806
+ if (this.proposedCheckpointData) {
807
+ // After syncing from L1 we see the chain tip has invalid attestations. This means the parent checkpoint was posted
808
+ // with invalid attestations, or it built on top of something with invalid attestations and didnt invalidate them.
809
+ // Either way, we thought our parent would be valid, so we have to throw away our work. But at least we'll try and
810
+ // invalidate on L1 so we clean up the chain for the next proposer. And we'll slash them, but that's handled elsewhere.
811
+ const validationStatus = await this.l2BlockSource.getPendingChainValidationStatus();
812
+ if (!validationStatus.valid) {
813
+ this.log.warn(`Parent checkpoint ${parentCheckpointNumber} has invalid attestations, discarding pipelined work`, {
814
+ checkpointNumber: this.checkpointNumber,
815
+ reason: validationStatus.reason
816
+ });
817
+ this.emitPipelinedCheckpointDiscarded('parent-invalid-attestations');
818
+ await this.enqueueInvalidation(validationStatus);
819
+ return false;
820
+ }
821
+ // The pending chain is valid. But did the parent checkpoint land on L1 at all?
822
+ if (checkpointedNumber < parentCheckpointNumber) {
823
+ this.log.warn(`Parent checkpoint ${parentCheckpointNumber} did not land on L1, discarding pipelined work`, {
824
+ checkpointNumber: this.checkpointNumber,
825
+ checkpointedNumber
826
+ });
827
+ this.emitPipelinedCheckpointDiscarded('parent-not-on-l1');
828
+ return false;
829
+ }
830
+ // It landed. But is it the one we were expecting?
831
+ const expectedHash = this.proposedCheckpointData.header.hash().toString();
832
+ if (tips.checkpointed.checkpoint.hash !== expectedHash) {
833
+ this.log.warn(`Parent checkpoint ${parentCheckpointNumber} hash mismatch on L1, discarding pipelined work`, {
834
+ checkpointNumber: this.checkpointNumber,
835
+ expectedHash,
836
+ actualHash: tips.checkpointed.checkpoint.hash
837
+ });
838
+ this.emitPipelinedCheckpointDiscarded('parent-hash-mismatch');
839
+ return false;
840
+ }
841
+ return true;
842
+ } else {
843
+ // We didn't see a proposed checkpoint at build time, so we built on checkpointed parent from two slots ago.
844
+ // But if a new checkpoint for the previous slot appeared on L1 in the meantime, our checkpoint assumed the wrong parent,
845
+ // so we have to discard our work. This can happen if we're somehow cut off from p2p and fail to see the checkpoint
846
+ // proposal for the previous slot.
847
+ if (checkpointedNumber > parentCheckpointNumber) {
848
+ this.log.warn(`Unexpected checkpoint ${checkpointedNumber} landed on L1 after we built on top of parent ${parentCheckpointNumber}, discarding pipelined work`, {
849
+ checkpointNumber: this.checkpointNumber,
850
+ checkpointedNumber
851
+ });
852
+ this.emitPipelinedCheckpointDiscarded('unexpected-parent-appeared');
853
+ return false;
854
+ }
855
+ return true;
856
+ }
857
+ }
858
+ /** Emits the pipelined-checkpoint-discarded event and records the metric. */ emitPipelinedCheckpointDiscarded(reason) {
859
+ this.metrics.recordPipelineParentCheckpointMismatch(reason);
860
+ this.eventEmitter.emit('pipelined-checkpoint-discarded', {
861
+ slot: this.targetSlot,
862
+ checkpointNumber: this.checkpointNumber,
863
+ reason
680
864
  });
681
865
  }
866
+ /** Simulates and enqueues an invalidation request for the invalid parent checkpoint. */ async enqueueInvalidation(validationStatus) {
867
+ if (this.config.skipInvalidateBlockAsProposer) {
868
+ this.log.warn(`Skipping checkpoint invalidation as proposer due to test configuration`);
869
+ return;
870
+ }
871
+ const invalidateRequest = await this.publisher.simulateInvalidateCheckpoint(validationStatus);
872
+ if (invalidateRequest) {
873
+ const submissionSlotStart = Number(getTimestampForSlot(this.targetSlot, this.l1Constants));
874
+ const txTimeoutAt = new Date((submissionSlotStart + this.l1Constants.slotDuration) * 1000);
875
+ this.publisher.enqueueInvalidateCheckpoint(invalidateRequest, {
876
+ txTimeoutAt
877
+ });
878
+ } else {
879
+ this.log.info(`Invalidation simulation returned undefined, checkpoint may have been removed already`, {
880
+ checkpointNumber: this.checkpointNumber
881
+ });
882
+ }
883
+ }
682
884
  async proposeCheckpoint() {
683
885
  try {
684
886
  const env = {
@@ -687,16 +889,29 @@ _dec = trackSpan('CheckpointProposalJob.execute'), _dec1 = trackSpan('Checkpoint
687
889
  hasError: false
688
890
  };
689
891
  try {
892
+ const now = this.dateProvider.now();
893
+ if (this.epochCache.isProposerPipeliningEnabled() && this.proposedCheckpointData) {
894
+ // Measure against the wall-clock slot whose build window we are currently using.
895
+ // In pipelining mode `targetSlot` is intentionally one slot ahead, which makes the
896
+ // target-slot boundary a full slot away from the actual build start time.
897
+ const slotBoundaryMs = Number(getTimestampForSlot(this.slotNow, this.l1Constants)) * 1000;
898
+ this.checkpointMetrics.recordPipelinedCheckpointBuildStartOffsetFromSlotBoundary(now - slotBoundaryMs);
899
+ }
900
+ this.checkpointMetrics.startCheckpointTiming(now);
690
901
  // Get operator configured coinbase and fee recipient for this attestor
691
902
  const coinbase = this.validatorClient.getCoinbaseForAttestor(this.attestorAddress);
692
903
  const feeRecipient = this.validatorClient.getFeeRecipientForAttestor(this.attestorAddress);
693
904
  // Start the checkpoint
694
905
  this.setStateFn(SequencerState.INITIALIZING_CHECKPOINT, this.targetSlot);
695
- this.log.info(`Starting checkpoint proposal`, {
906
+ this.logCheckpointEvent('slot-started', `Starting checkpoint proposal for slot ${this.targetSlot}`, {
696
907
  buildSlot: this.slotNow,
697
908
  submissionSlot: this.targetSlot,
909
+ slot: this.targetSlot,
910
+ checkpointNumber: this.checkpointNumber,
698
911
  pipelining: this.epochCache.isProposerPipeliningEnabled(),
699
912
  proposer: this.proposer?.toString(),
913
+ attestorAddress: this.attestorAddress.toString(),
914
+ publisherAddress: this.publisher.getSenderAddress().toString(),
700
915
  coinbase: coinbase.toString()
701
916
  });
702
917
  this.metrics.incOpenSlot(this.targetSlot, this.proposer?.toString() ?? 'unknown');
@@ -708,15 +923,13 @@ _dec = trackSpan('CheckpointProposalJob.execute'), _dec1 = trackSpan('Checkpoint
708
923
  // When pipelining, force the proposed checkpoint number and fee header to our parent so the
709
924
  // fee computation sees the same chain tip that L1 will see once the previous pipelined checkpoint lands.
710
925
  const isPipelining = this.epochCache.isProposerPipeliningEnabled();
711
- const parentCheckpointNumber = isPipelining ? CheckpointNumber(this.checkpointNumber - 1) : undefined;
712
- // Compute the parent's fee header override when pipelining
713
- if (isPipelining && this.proposedCheckpointData) {
714
- this.computedForceProposedFeeHeader = await this.computeForceProposedFeeHeader(parentCheckpointNumber);
715
- }
716
- const checkpointGlobalVariables = await this.globalsBuilder.buildCheckpointGlobalVariables(coinbase, feeRecipient, this.targetSlot, {
717
- forcePendingCheckpointNumber: parentCheckpointNumber,
718
- forceProposedFeeHeader: this.computedForceProposedFeeHeader
719
- });
926
+ this.pipelinedParentSimulationOverridesPlan = isPipelining ? await buildPipelinedParentSimulationOverridesPlan({
927
+ checkpointNumber: this.checkpointNumber,
928
+ proposedCheckpointData: this.proposedCheckpointData,
929
+ rollup: this.publisher.rollupContract,
930
+ log: this.log
931
+ }) : undefined;
932
+ const checkpointGlobalVariables = await this.globalsBuilder.buildCheckpointGlobalVariables(coinbase, feeRecipient, this.targetSlot, this.pipelinedParentSimulationOverridesPlan);
720
933
  // Collect L1 to L2 messages for the checkpoint and compute their hash
721
934
  const l1ToL2Messages = await this.l1ToL2MessageSource.getL1ToL2Messages(this.checkpointNumber);
722
935
  const inHash = computeInHashFromL1ToL2Messages(l1ToL2Messages);
@@ -756,8 +969,15 @@ _dec = trackSpan('CheckpointProposalJob.execute'), _dec1 = trackSpan('Checkpoint
756
969
  throw err;
757
970
  }
758
971
  if (blocksInCheckpoint.length === 0) {
972
+ this.logCheckpointEvent('build-failed', `Checkpoint build failed for slot ${this.targetSlot}`, {
973
+ slot: this.targetSlot,
974
+ checkpointNumber: this.checkpointNumber,
975
+ reason: 'no_blocks_built'
976
+ });
759
977
  this.log.warn(`No blocks were built for slot ${this.targetSlot}`, {
760
- slot: this.targetSlot
978
+ slot: this.targetSlot,
979
+ checkpointNumber: this.checkpointNumber,
980
+ reason: 'no_blocks_built'
761
981
  });
762
982
  this.eventEmitter.emit('checkpoint-empty', {
763
983
  slot: this.targetSlot
@@ -766,10 +986,19 @@ _dec = trackSpan('CheckpointProposalJob.execute'), _dec1 = trackSpan('Checkpoint
766
986
  }
767
987
  const minBlocksForCheckpoint = this.config.minBlocksForCheckpoint;
768
988
  if (minBlocksForCheckpoint !== undefined && blocksInCheckpoint.length < minBlocksForCheckpoint) {
989
+ this.logCheckpointEvent('build-failed', `Checkpoint build failed for slot ${this.targetSlot}`, {
990
+ slot: this.targetSlot,
991
+ checkpointNumber: this.checkpointNumber,
992
+ blocksBuilt: blocksInCheckpoint.length,
993
+ minBlocksForCheckpoint,
994
+ reason: 'min_blocks_not_met'
995
+ });
769
996
  this.log.warn(`Checkpoint has fewer blocks than minimum (${blocksInCheckpoint.length} < ${minBlocksForCheckpoint}), skipping proposal`, {
770
997
  slot: this.targetSlot,
998
+ checkpointNumber: this.checkpointNumber,
771
999
  blocksBuilt: blocksInCheckpoint.length,
772
- minBlocksForCheckpoint
1000
+ minBlocksForCheckpoint,
1001
+ reason: 'min_blocks_not_met'
773
1002
  });
774
1003
  return undefined;
775
1004
  }
@@ -788,14 +1017,36 @@ _dec = trackSpan('CheckpointProposalJob.execute'), _dec1 = trackSpan('Checkpoint
788
1017
  maxTxsPerCheckpoint: this.config.maxTxsPerCheckpoint
789
1018
  });
790
1019
  } catch (err) {
1020
+ this.logCheckpointEvent('build-failed', `Checkpoint build failed for slot ${this.targetSlot}`, {
1021
+ slot: this.targetSlot,
1022
+ checkpointNumber: this.checkpointNumber,
1023
+ blocksBuilt: blocksInCheckpoint.length,
1024
+ reason: 'invalid_checkpoint',
1025
+ checkpoint: checkpoint.header.toInspect()
1026
+ });
791
1027
  this.log.error(`Built an invalid checkpoint at slot ${this.slotNow} (skipping proposal)`, err, {
1028
+ slot: this.targetSlot,
1029
+ checkpointNumber: this.checkpointNumber,
1030
+ blocksBuilt: blocksInCheckpoint.length,
1031
+ reason: 'invalid_checkpoint',
792
1032
  checkpoint: checkpoint.header.toInspect()
793
1033
  });
794
1034
  return undefined;
795
1035
  }
796
1036
  // Record checkpoint-level build metrics
797
- this.metrics.recordCheckpointBuild(checkpointBuildTimer.ms(), blocksInCheckpoint.length, checkpoint.getStats().txCount, Number(checkpoint.header.totalManaUsed.toBigInt()));
798
- // Do not collect attestations nor publish to L1 in fisherman mode
1037
+ this.checkpointMetrics.recordCheckpointBuild(checkpointBuildTimer.ms(), blocksInCheckpoint.length, checkpoint.getStats().txCount, Number(checkpoint.header.totalManaUsed.toBigInt()));
1038
+ this.logCheckpointEvent('built', `Checkpoint built for slot ${this.targetSlot}`, {
1039
+ slot: this.targetSlot,
1040
+ buildSlot: this.slotNow,
1041
+ checkpointNumber: this.checkpointNumber,
1042
+ proposer: this.proposer?.toString(),
1043
+ attestorAddress: this.attestorAddress.toString(),
1044
+ publisherAddress: this.publisher.getSenderAddress().toString(),
1045
+ blocksBuilt: blocksInCheckpoint.length,
1046
+ txCount: checkpoint.getStats().txCount,
1047
+ totalMana: Number(checkpoint.header.totalManaUsed.toBigInt())
1048
+ });
1049
+ // In fisherman mode, return the checkpoint without broadcasting or collecting attestations
799
1050
  if (this.config.fishermanMode) {
800
1051
  this.log.info(`Built checkpoint for slot ${this.targetSlot} with ${blocksInCheckpoint.length} blocks. ` + `Skipping proposal in fisherman mode.`, {
801
1052
  slot: this.targetSlot,
@@ -803,44 +1054,25 @@ _dec = trackSpan('CheckpointProposalJob.execute'), _dec1 = trackSpan('Checkpoint
803
1054
  blocksBuilt: blocksInCheckpoint.length
804
1055
  });
805
1056
  this.metrics.recordCheckpointSuccess();
1057
+ // Return a broadcast result with a dummy proposal — fisherman mode skips attestation collection
806
1058
  return {
807
1059
  checkpoint,
808
- attestations: CommitteeAttestationsAndSigners.empty(),
809
- attestationsSignature: Signature.empty()
1060
+ proposal: undefined,
1061
+ blockProposedAt: this.dateProvider.now()
810
1062
  };
811
1063
  }
812
- // Include the block pending broadcast in the checkpoint proposal if any
813
- const lastBlock = blockPendingBroadcast && {
814
- blockHeader: blockPendingBroadcast.block.header,
815
- indexWithinCheckpoint: blockPendingBroadcast.block.indexWithinCheckpoint,
816
- txs: blockPendingBroadcast.txs
817
- };
818
1064
  // Create the checkpoint proposal and broadcast it
819
- const proposal = await this.validatorClient.createCheckpointProposal(checkpoint.header, checkpoint.archive.root, feeAssetPriceModifier, lastBlock, this.proposer, checkpointProposalOptions);
1065
+ const proposal = await this.validatorClient.createCheckpointProposal(checkpoint.header, checkpoint.archive.root, this.checkpointNumber, feeAssetPriceModifier, blockPendingBroadcast, this.proposer, checkpointProposalOptions);
820
1066
  const blockProposedAt = this.dateProvider.now();
821
- await this.p2pClient.broadcastCheckpointProposal(proposal);
822
- this.setStateFn(SequencerState.COLLECTING_ATTESTATIONS, this.targetSlot);
823
- const attestations = await this.waitForAttestations(proposal);
824
- const blockAttestedAt = this.dateProvider.now();
825
- this.metrics.recordCheckpointAttestationDelay(blockAttestedAt - blockProposedAt);
826
- // Proposer must sign over the attestations before pushing them to L1
827
- const signer = this.proposer ?? this.publisher.getSenderAddress();
828
- let attestationsSignature;
829
- try {
830
- attestationsSignature = await this.validatorClient.signAttestationsAndSigners(attestations, signer, this.targetSlot, this.checkpointNumber);
831
- } catch (err) {
832
- // We shouldn't really get here since we yield to another HA node
833
- // as soon as we see these errors when creating block or checkpoint proposals.
834
- if (this.handleHASigningError(err, 'Attestations signature')) {
835
- return undefined;
836
- }
837
- throw err;
1067
+ if (!this.config.skipBroadcastProposals) {
1068
+ await this.p2pClient.broadcastCheckpointProposal(proposal);
1069
+ this.checkpointMetrics.noteCheckpointBroadcast(this.dateProvider.now());
838
1070
  }
839
- // Return the result for the caller to enqueue after the pipeline sleep
1071
+ // Return immediately after broadcast attestation collection happens in the background
840
1072
  return {
841
1073
  checkpoint,
842
- attestations,
843
- attestationsSignature
1074
+ proposal,
1075
+ blockProposedAt
844
1076
  };
845
1077
  } catch (e) {
846
1078
  env.error = e;
@@ -870,6 +1102,14 @@ _dec = trackSpan('CheckpointProposalJob.execute'), _dec1 = trackSpan('Checkpoint
870
1102
  const blocksBuilt = blocksInCheckpoint.length;
871
1103
  const indexWithinCheckpoint = IndexWithinCheckpoint(blocksBuilt);
872
1104
  const blockNumber = BlockNumber(initialBlockNumber + blocksBuilt);
1105
+ if (blocksBuilt >= this.config.maxBlocksPerCheckpoint) {
1106
+ this.log.debug(`Reached max blocks per checkpoint`, {
1107
+ slot: this.targetSlot,
1108
+ blocksBuilt,
1109
+ maxBlocksPerCheckpoint: this.config.maxBlocksPerCheckpoint
1110
+ });
1111
+ break;
1112
+ }
873
1113
  const secondsIntoSlot = this.getSecondsIntoSlot();
874
1114
  const timingInfo = this.timetable.canStartNextBlock(secondsIntoSlot);
875
1115
  if (!timingInfo.canStart) {
@@ -891,17 +1131,19 @@ _dec = trackSpan('CheckpointProposalJob.execute'), _dec1 = trackSpan('Checkpoint
891
1131
  indexWithinCheckpoint,
892
1132
  txHashesAlreadyIncluded
893
1133
  });
894
- // TODO(palla/mbps): Review these conditions. We may want to keep trying in some scenarios.
895
- if (!buildResult && timingInfo.isLastBlock) {
896
- break;
897
- } else if (!buildResult && timingInfo.deadline !== undefined) {
898
- // But if there is still time for more blocks, wait until the next subslot and try again
1134
+ // If we failed to build the block due to insufficient txs, we try again if there is still time left in the slot
1135
+ if ('failure' in buildResult) {
1136
+ // If this was the last subslot, or we're running with a single block per slot, we're done
1137
+ if (timingInfo.isLastBlock || timingInfo.deadline === undefined) {
1138
+ break;
1139
+ }
1140
+ // Otherwise, if there is still time for more blocks, we wait until the next subslot and try again
899
1141
  await this.waitUntilNextSubslot(timingInfo.deadline);
900
1142
  continue;
901
- } else if (!buildResult) {
902
- break;
903
- } else if ('error' in buildResult) {
904
- // If there was an error building the block, just exit the loop and give up the rest of the slot
1143
+ }
1144
+ // If there was an error building the block, we just exit the loop and give up the rest of the slot.
1145
+ // We don't want to risk building more blocks if something went wrong.
1146
+ if ('error' in buildResult) {
905
1147
  if (!(buildResult.error instanceof SequencerInterruptedError)) {
906
1148
  this.log.warn(`Halting block building for slot ${this.targetSlot}`, {
907
1149
  slot: this.targetSlot,
@@ -912,34 +1154,33 @@ _dec = trackSpan('CheckpointProposalJob.execute'), _dec1 = trackSpan('Checkpoint
912
1154
  break;
913
1155
  }
914
1156
  const { block, usedTxs } = buildResult;
1157
+ this.checkpointMetrics.noteCheckpointBlockBuilt(this.dateProvider.now(), {
1158
+ isFirstBlock: blocksBuilt === 0,
1159
+ isLastBlock: timingInfo.isLastBlock
1160
+ });
915
1161
  blocksInCheckpoint.push(block);
916
1162
  usedTxs.forEach((tx)=>txHashesAlreadyIncluded.add(tx.txHash.toString()));
917
- // If this is the last block, sync it to the archiver and exit the loop
918
- // so we can build the checkpoint and start collecting attestations.
1163
+ // Sign the block proposal. This will throw if HA signing fails.
1164
+ const proposal = await this.createBlockProposal(block, inHash, usedTxs, blockProposalOptions);
1165
+ // Sync the proposed block to the archiver to make it available, only after we've managed to sign the proposal,
1166
+ // so we avoid polluting our archive with a block that would fail.
1167
+ // We wait for the sync to succeed, as this helps catch consistency errors, even if it means we lose some time for block-building.
1168
+ // If this throws, we abort the entire checkpoint.
1169
+ await this.syncProposedBlockToArchiver(block);
1170
+ // If this is the last block, do not broadcast it, since it will be included in the checkpoint proposal.
919
1171
  if (timingInfo.isLastBlock) {
920
- await this.syncProposedBlockToArchiver(block);
921
1172
  this.log.verbose(`Completed final block ${blockNumber} for slot ${this.targetSlot}`, {
922
1173
  slot: this.targetSlot,
923
1174
  blockNumber,
924
1175
  blocksBuilt
925
1176
  });
926
- blockPendingBroadcast = {
927
- block,
928
- txs: usedTxs
929
- };
1177
+ blockPendingBroadcast = proposal;
930
1178
  break;
931
1179
  }
932
- // Broadcast the block proposal (unless we're in fisherman mode) unless the block is the last one,
933
- // in which case we'll broadcast it along with the checkpoint at the end of the loop.
934
- // Note that we only send the block to the archiver if we manage to create the proposal, so if there's
935
- // a HA error we don't pollute our archiver with a block that won't make it to the chain.
936
- const proposal = await this.createBlockProposal(block, inHash, usedTxs, blockProposalOptions);
937
- // Sync the proposed block to the archiver to make it available, only after we've managed to sign the proposal.
938
- // We wait for the sync to succeed, as this helps catch consistency errors, even if it means we lose some time for block-building.
939
- // If this throws, we abort the entire checkpoint.
940
- await this.syncProposedBlockToArchiver(block);
941
1180
  // Once we have a signed proposal and the archiver agreed with our proposed block, then we broadcast it.
942
- proposal && await this.p2pClient.broadcastProposal(proposal);
1181
+ if (proposal && !this.config.skipBroadcastProposals) {
1182
+ await this.p2pClient.broadcastProposal(proposal);
1183
+ }
943
1184
  // Wait until the next block's start time
944
1185
  await this.waitUntilNextSubslot(timingInfo.deadline);
945
1186
  }
@@ -957,7 +1198,7 @@ _dec = trackSpan('CheckpointProposalJob.execute'), _dec1 = trackSpan('Checkpoint
957
1198
  this.log.info(`Skipping block proposal for block ${block.number} in fisherman mode`);
958
1199
  return Promise.resolve(undefined);
959
1200
  }
960
- return this.validatorClient.createBlockProposal(block.header, block.indexWithinCheckpoint, inHash, block.archive.root, usedTxs, this.proposer, blockProposalOptions);
1201
+ return this.validatorClient.createBlockProposal(block.header, this.checkpointNumber, block.indexWithinCheckpoint, inHash, block.archive.root, usedTxs, this.proposer, blockProposalOptions);
961
1202
  }
962
1203
  /** Sleeps until it is time to produce the next block in the slot */ async waitUntilNextSubslot(nextSubslotStart) {
963
1204
  this.setStateFn(SequencerState.WAITING_UNTIL_NEXT_BLOCK, this.targetSlot);
@@ -976,10 +1217,23 @@ _dec = trackSpan('CheckpointProposalJob.execute'), _dec1 = trackSpan('Checkpoint
976
1217
  // Wait until we have enough txs to build the block
977
1218
  const { availableTxs, canStartBuilding, minTxs } = await this.waitForMinTxs(opts);
978
1219
  if (!canStartBuilding) {
1220
+ this.logCheckpointEvent('block-build-failed', `Block build failed for slot ${this.targetSlot}`, {
1221
+ reason: 'insufficient_txs',
1222
+ blockNumber,
1223
+ slot: this.targetSlot,
1224
+ checkpointNumber: this.checkpointNumber,
1225
+ indexWithinCheckpoint,
1226
+ availableTxs,
1227
+ minTxs
1228
+ });
979
1229
  this.log.warn(`Not enough txs to build block ${blockNumber} at index ${indexWithinCheckpoint} in slot ${this.targetSlot} (got ${availableTxs} txs but needs ${minTxs})`, {
1230
+ reason: 'insufficient_txs',
980
1231
  blockNumber,
981
1232
  slot: this.targetSlot,
982
- indexWithinCheckpoint
1233
+ checkpointNumber: this.checkpointNumber,
1234
+ indexWithinCheckpoint,
1235
+ availableTxs,
1236
+ minTxs
983
1237
  });
984
1238
  this.eventEmitter.emit('block-tx-count-check-failed', {
985
1239
  minTxs,
@@ -987,7 +1241,9 @@ _dec = trackSpan('CheckpointProposalJob.execute'), _dec1 = trackSpan('Checkpoint
987
1241
  slot: this.targetSlot
988
1242
  });
989
1243
  this.metrics.recordBlockProposalFailed('insufficient_txs');
990
- return undefined;
1244
+ return {
1245
+ failure: 'insufficient-txs'
1246
+ };
991
1247
  }
992
1248
  // Create iterator to pending txs. We filter out txs already included in previous blocks in the checkpoint
993
1249
  // just in case p2p failed to sync the provisional block and didn't get to remove those txs from the mempool yet.
@@ -1018,8 +1274,19 @@ _dec = trackSpan('CheckpointProposalJob.execute'), _dec1 = trackSpan('Checkpoint
1018
1274
  // If any txs failed during execution, drop them from the mempool so we don't pick them up again
1019
1275
  await this.dropFailedTxsFromP2P(buildResult.failedTxs);
1020
1276
  if (buildResult.status === 'insufficient-valid-txs') {
1277
+ this.logCheckpointEvent('block-build-failed', `Block build failed for slot ${this.targetSlot}`, {
1278
+ reason: 'insufficient_valid_txs',
1279
+ slot: this.targetSlot,
1280
+ checkpointNumber: this.checkpointNumber,
1281
+ blockNumber,
1282
+ numTxs: buildResult.processedCount,
1283
+ indexWithinCheckpoint,
1284
+ minValidTxs
1285
+ });
1021
1286
  this.log.warn(`Block ${blockNumber} at index ${indexWithinCheckpoint} on slot ${this.targetSlot} has too few valid txs to be proposed`, {
1287
+ reason: 'insufficient_valid_txs',
1022
1288
  slot: this.targetSlot,
1289
+ checkpointNumber: this.checkpointNumber,
1023
1290
  blockNumber,
1024
1291
  numTxs: buildResult.processedCount,
1025
1292
  indexWithinCheckpoint,
@@ -1030,7 +1297,9 @@ _dec = trackSpan('CheckpointProposalJob.execute'), _dec1 = trackSpan('Checkpoint
1030
1297
  slot: this.targetSlot
1031
1298
  });
1032
1299
  this.metrics.recordBlockProposalFailed('insufficient_valid_txs');
1033
- return undefined;
1300
+ return {
1301
+ failure: 'insufficient-valid-txs'
1302
+ };
1034
1303
  }
1035
1304
  // Block creation succeeded, emit stats and metrics
1036
1305
  const { block, publicProcessorDuration, usedTxs, blockBuildDuration, numTxs } = buildResult;
@@ -1056,7 +1325,7 @@ _dec = trackSpan('CheckpointProposalJob.execute'), _dec1 = trackSpan('Checkpoint
1056
1325
  slot: this.targetSlot,
1057
1326
  buildSlot: this.slotNow
1058
1327
  });
1059
- this.metrics.recordBuiltBlock(blockBuildDuration, block.header.totalManaUsed.toNumberUnsafe());
1328
+ this.metrics.recordBuiltBlock(blockBuildDuration, block.header.totalManaUsed.toNumberUnsafe(), this.targetSlot);
1060
1329
  return {
1061
1330
  block,
1062
1331
  usedTxs
@@ -1066,9 +1335,17 @@ _dec = trackSpan('CheckpointProposalJob.execute'), _dec1 = trackSpan('Checkpoint
1066
1335
  reason: err.message,
1067
1336
  slot: this.targetSlot
1068
1337
  });
1338
+ this.logCheckpointEvent('block-build-failed', `Block build failed for slot ${this.targetSlot}`, {
1339
+ reason: err instanceof Error ? err.message : String(err),
1340
+ slot: this.targetSlot,
1341
+ checkpointNumber: this.checkpointNumber,
1342
+ blockNumber
1343
+ });
1069
1344
  this.log.error(`Error building block`, err, {
1070
- blockNumber,
1071
- slot: this.targetSlot
1345
+ reason: err instanceof Error ? err.message : String(err),
1346
+ slot: this.targetSlot,
1347
+ checkpointNumber: this.checkpointNumber,
1348
+ blockNumber
1072
1349
  });
1073
1350
  this.metrics.recordBlockProposalFailed(err.name || 'unknown_error');
1074
1351
  this.metrics.recordFailedBlock();
@@ -1131,13 +1408,37 @@ _dec = trackSpan('CheckpointProposalJob.execute'), _dec1 = trackSpan('Checkpoint
1131
1408
  minTxs
1132
1409
  };
1133
1410
  }
1411
+ async getSignedCommitteeAttestations(broadcast) {
1412
+ const { proposal, blockProposedAt } = broadcast;
1413
+ this.setStateFn(SequencerState.COLLECTING_ATTESTATIONS, this.targetSlot);
1414
+ const attestations = await this.waitForAttestations(proposal);
1415
+ if (!attestations) {
1416
+ return undefined;
1417
+ }
1418
+ this.checkpointMetrics.recordCheckpointAttestationDelay(this.dateProvider.now() - blockProposedAt);
1419
+ // Proposer must sign over the attestations before pushing them to L1
1420
+ const signer = this.proposer ?? this.publisher.getSenderAddress();
1421
+ try {
1422
+ const attestationsSignature = await this.validatorClient.signAttestationsAndSigners(attestations, signer, this.targetSlot, this.checkpointNumber);
1423
+ return {
1424
+ attestations,
1425
+ attestationsSignature
1426
+ };
1427
+ } catch (err) {
1428
+ if (this.handleHASigningError(err, 'Attestations signature')) {
1429
+ return;
1430
+ }
1431
+ this.log.error(`Error signing attestations for checkpoint proposal at slot ${proposal.slotNumber}`, err);
1432
+ return undefined;
1433
+ }
1434
+ }
1134
1435
  /**
1135
1436
  * Waits for enough attestations to be collected via p2p.
1136
1437
  * This is run after all blocks for the checkpoint have been built.
1137
1438
  */ async waitForAttestations(proposal) {
1138
1439
  if (this.config.fishermanMode) {
1139
1440
  this.log.debug('Skipping attestation collection in fisherman mode');
1140
- return CommitteeAttestationsAndSigners.empty();
1441
+ return CommitteeAttestationsAndSigners.empty(this.getSignatureContext());
1141
1442
  }
1142
1443
  const slotNumber = proposal.slotNumber;
1143
1444
  const { committee, seed, epoch } = await this.epochCache.getCommittee(slotNumber);
@@ -1145,7 +1446,7 @@ _dec = trackSpan('CheckpointProposalJob.execute'), _dec1 = trackSpan('Checkpoint
1145
1446
  throw new Error('No committee when collecting attestations');
1146
1447
  } else if (committee.length === 0) {
1147
1448
  this.log.verbose(`Attesting committee is empty`);
1148
- return CommitteeAttestationsAndSigners.empty();
1449
+ return CommitteeAttestationsAndSigners.empty(this.getSignatureContext());
1149
1450
  } else {
1150
1451
  this.log.debug(`Attesting committee length is ${committee.length}`, {
1151
1452
  committee
@@ -1154,16 +1455,19 @@ _dec = trackSpan('CheckpointProposalJob.execute'), _dec1 = trackSpan('Checkpoint
1154
1455
  const numberOfRequiredAttestations = computeQuorum(committee.length);
1155
1456
  if (this.config.skipCollectingAttestations) {
1156
1457
  this.log.warn('Skipping attestation collection as per config (attesting with own keys only)');
1157
- const attestations = await this.validatorClient?.collectOwnAttestations(proposal);
1158
- return new CommitteeAttestationsAndSigners(orderAttestations(attestations ?? [], committee));
1458
+ const attestations = await this.validatorClient?.collectOwnAttestations(proposal, this.checkpointNumber);
1459
+ this.logCheckpointAttestations('collected', committee, attestations ?? [], numberOfRequiredAttestations, {
1460
+ reason: 'collect_own_only'
1461
+ });
1462
+ return new CommitteeAttestationsAndSigners(orderAttestations(attestations ?? [], committee), this.getSignatureContext());
1159
1463
  }
1160
- const attestationTimeAllowed = this.config.enforceTimeTable ? this.timetable.getMaxAllowedTime(SequencerState.PUBLISHING_CHECKPOINT) : this.l1Constants.slotDuration;
1464
+ const attestationTimeAllowed = this.config.enforceTimeTable ? this.timetable.getCheckpointAttestationDeadline() : this.l1Constants.slotDuration;
1161
1465
  const attestationDeadline = new Date((this.getSlotStartBuildTimestamp() + attestationTimeAllowed) * 1000);
1162
1466
  this.metrics.recordRequiredAttestations(numberOfRequiredAttestations, attestationTimeAllowed);
1163
1467
  const collectAttestationsTimer = new Timer();
1164
1468
  let collectedAttestationsCount = 0;
1165
1469
  try {
1166
- const attestations = await this.validatorClient.collectAttestations(proposal, numberOfRequiredAttestations, attestationDeadline);
1470
+ const attestations = await this.validatorClient.collectAttestations(proposal, numberOfRequiredAttestations, attestationDeadline, this.checkpointNumber);
1167
1471
  collectedAttestationsCount = attestations.length;
1168
1472
  // Trim attestations to minimum required to save L1 calldata gas
1169
1473
  const localAddresses = this.validatorClient.getValidatorAddresses();
@@ -1173,20 +1477,55 @@ _dec = trackSpan('CheckpointProposalJob.execute'), _dec1 = trackSpan('Checkpoint
1173
1477
  }
1174
1478
  // Rollup contract requires that the signatures are provided in the order of the committee
1175
1479
  const sorted = orderAttestations(trimmed, committee);
1480
+ this.logCheckpointAttestations('collected', committee, attestations, numberOfRequiredAttestations, {
1481
+ submittedCount: trimmed.length
1482
+ });
1176
1483
  // Manipulate the attestations if we've been configured to do so
1177
1484
  if (this.config.injectFakeAttestation || this.config.injectHighSValueAttestation || this.config.injectUnrecoverableSignatureAttestation || this.config.shuffleAttestationOrdering) {
1178
1485
  return this.manipulateAttestations(proposal.slotNumber, epoch, seed, committee, sorted);
1179
1486
  }
1180
- return new CommitteeAttestationsAndSigners(sorted);
1487
+ return new CommitteeAttestationsAndSigners(sorted, this.getSignatureContext());
1181
1488
  } catch (err) {
1182
1489
  if (err && err instanceof AttestationTimeoutError) {
1183
1490
  collectedAttestationsCount = err.collectedCount;
1491
+ this.logCheckpointAttestations('failed', committee, undefined, numberOfRequiredAttestations, {
1492
+ collectedCount: collectedAttestationsCount,
1493
+ reason: 'timeout'
1494
+ });
1495
+ this.log.error(`Timeout while waiting for attestations for checkpoint proposal at slot ${proposal.slotNumber} (collected ${collectedAttestationsCount}/${numberOfRequiredAttestations})`, err);
1496
+ } else {
1497
+ this.logCheckpointAttestations('failed', committee, undefined, numberOfRequiredAttestations, {
1498
+ collectedCount: collectedAttestationsCount,
1499
+ reason: err instanceof Error ? err.message : String(err)
1500
+ });
1501
+ this.log.error(`Error collecting attestations for checkpoint proposal at slot ${proposal.slotNumber}`, err);
1184
1502
  }
1185
- throw err;
1503
+ return undefined;
1186
1504
  } finally{
1187
1505
  this.metrics.recordCollectedAttestations(collectedAttestationsCount, collectAttestationsTimer.ms());
1188
1506
  }
1189
1507
  }
1508
+ logCheckpointAttestations(status, committee, attestations, requiredAttestations, opts = {}) {
1509
+ const signedValidators = attestations?.map((attestation)=>attestation.getSender()?.toString()).filter((address)=>address !== undefined) ?? [];
1510
+ const collectedCount = opts.collectedCount ?? new Set(signedValidators).size;
1511
+ const missingValidatorCount = status === 'failed' ? Math.max(0, requiredAttestations - collectedCount) : undefined;
1512
+ this.logCheckpointEvent(`attestations-${status}`, `Checkpoint attestations ${status} for slot ${this.targetSlot}`, {
1513
+ slot: this.targetSlot,
1514
+ checkpointNumber: this.checkpointNumber,
1515
+ committeeSize: committee.length,
1516
+ requiredAttestations,
1517
+ collectedAttestations: collectedCount,
1518
+ ...opts.submittedCount !== undefined && {
1519
+ submittedAttestations: opts.submittedCount
1520
+ },
1521
+ ...missingValidatorCount !== undefined && {
1522
+ missingValidatorCount
1523
+ },
1524
+ ...opts.reason !== undefined && {
1525
+ reason: opts.reason
1526
+ }
1527
+ });
1528
+ }
1190
1529
  /** Breaks the attestations before publishing based on attack configs */ manipulateAttestations(slotNumber, epoch, seed, committee, attestations) {
1191
1530
  // Compute the proposer index in the committee, since we dont want to tweak it.
1192
1531
  // Otherwise, the L1 rollup contract will reject the block outright.
@@ -1212,7 +1551,7 @@ _dec = trackSpan('CheckpointProposalJob.execute'), _dec1 = trackSpan('Checkpoint
1212
1551
  unfreeze(attestations[targetIndex]).signature = generateRecoverableSignature();
1213
1552
  }
1214
1553
  }
1215
- return new CommitteeAttestationsAndSigners(attestations);
1554
+ return new CommitteeAttestationsAndSigners(attestations, this.getSignatureContext());
1216
1555
  }
1217
1556
  if (this.config.shuffleAttestationOrdering) {
1218
1557
  this.log.warn(`Shuffling attestation ordering in checkpoint for slot ${slotNumber} (proposer #${proposerIndex})`);
@@ -1238,10 +1577,10 @@ _dec = trackSpan('CheckpointProposalJob.execute'), _dec1 = trackSpan('Checkpoint
1238
1577
  shuffled[i]
1239
1578
  ];
1240
1579
  }
1241
- const signers = new CommitteeAttestationsAndSigners(attestations).getSigners();
1242
- return new MaliciousCommitteeAttestationsAndSigners(shuffled, signers);
1580
+ const signers = new CommitteeAttestationsAndSigners(attestations, this.getSignatureContext()).getSigners();
1581
+ return new MaliciousCommitteeAttestationsAndSigners(shuffled, signers, this.getSignatureContext());
1243
1582
  }
1244
- return new CommitteeAttestationsAndSigners(attestations);
1583
+ return new CommitteeAttestationsAndSigners(attestations, this.getSignatureContext());
1245
1584
  }
1246
1585
  async dropFailedTxsFromP2P(failedTxs) {
1247
1586
  if (failedTxs.length === 0) {
@@ -1256,8 +1595,12 @@ _dec = trackSpan('CheckpointProposalJob.execute'), _dec1 = trackSpan('Checkpoint
1256
1595
  * Adds the proposed block to the archiver so it's available via P2P.
1257
1596
  * Gossip doesn't echo messages back to the sender, so the proposer's archiver/world-state
1258
1597
  * would never receive its own block without this explicit sync.
1598
+ *
1599
+ * In fisherman mode we skip this push: the fisherman builds blocks locally for validation
1600
+ * and fee analysis only, and pushing them to the archiver causes spurious reorg cascades
1601
+ * whenever the real proposer's block arrives from L1.
1259
1602
  */ async syncProposedBlockToArchiver(block) {
1260
- if (this.config.skipPushProposedBlocksToArchiver) {
1603
+ if (this.config.skipPushProposedBlocksToArchiver || this.config.fishermanMode) {
1261
1604
  this.log.warn(`Skipping push of proposed block ${block.number} to archiver`, {
1262
1605
  blockNumber: block.number,
1263
1606
  slot: block.header.globalVariables.slotNumber
@@ -1309,40 +1652,6 @@ _dec = trackSpan('CheckpointProposalJob.execute'), _dec1 = trackSpan('Checkpoint
1309
1652
  }
1310
1653
  return false;
1311
1654
  }
1312
- /**
1313
- * In times of congestion we need to simulate using the correct fee header override for the previous block
1314
- * We calculate the correct fee header values.
1315
- *
1316
- * If we are in block 1, or the checkpoint we are querying does not exist, we return undefined. However
1317
- * If we are pipelining - where this function is called, the grandparentCheckpointNumber should always exist
1318
- * @param parentCheckpointNumber
1319
- * @returns
1320
- */ async computeForceProposedFeeHeader(parentCheckpointNumber) {
1321
- if (!this.proposedCheckpointData) {
1322
- return undefined;
1323
- }
1324
- const rollup = this.publisher.rollupContract;
1325
- const grandparentCheckpointNumber = CheckpointNumber(this.checkpointNumber - 2);
1326
- try {
1327
- const [grandparentCheckpoint, manaTarget] = await Promise.all([
1328
- rollup.getCheckpoint(grandparentCheckpointNumber),
1329
- rollup.getManaTarget()
1330
- ]);
1331
- if (!grandparentCheckpoint || !grandparentCheckpoint.feeHeader) {
1332
- this.log.error(`Grandparent checkpoint or its feeHeader is undefined for checkpointNumber=${grandparentCheckpointNumber.toString()}`);
1333
- return undefined;
1334
- } else {
1335
- const parentFeeHeader = RollupContract.computeChildFeeHeader(grandparentCheckpoint.feeHeader, this.proposedCheckpointData.totalManaUsed, this.proposedCheckpointData.feeAssetPriceModifier, manaTarget);
1336
- return {
1337
- checkpointNumber: parentCheckpointNumber,
1338
- feeHeader: parentFeeHeader
1339
- };
1340
- }
1341
- } catch (err) {
1342
- this.log.error(`Failed to fetch grandparent checkpoint or mana target for checkpointNumber=${grandparentCheckpointNumber.toString()}: ${err}`);
1343
- return undefined;
1344
- }
1345
- }
1346
1655
  /** Waits until a specific time within the current slot */ async waitUntilTimeInSlot(targetSecondsIntoSlot) {
1347
1656
  const slotStartTimestamp = this.getSlotStartBuildTimestamp();
1348
1657
  const targetTimestamp = slotStartTimestamp + targetSecondsIntoSlot;