@aztec/sequencer-client 0.0.1-commit.e558bd1c → 0.0.1-commit.e57c76e

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (125) hide show
  1. package/README.md +281 -21
  2. package/dest/client/sequencer-client.d.ts +19 -9
  3. package/dest/client/sequencer-client.d.ts.map +1 -1
  4. package/dest/client/sequencer-client.js +65 -32
  5. package/dest/config.d.ts +28 -6
  6. package/dest/config.d.ts.map +1 -1
  7. package/dest/config.js +76 -37
  8. package/dest/global_variable_builder/fee_predictor.d.ts +37 -0
  9. package/dest/global_variable_builder/fee_predictor.d.ts.map +1 -0
  10. package/dest/global_variable_builder/fee_predictor.js +128 -0
  11. package/dest/global_variable_builder/fee_provider.d.ts +21 -0
  12. package/dest/global_variable_builder/fee_provider.d.ts.map +1 -0
  13. package/dest/global_variable_builder/fee_provider.js +58 -0
  14. package/dest/global_variable_builder/global_builder.d.ts +14 -16
  15. package/dest/global_variable_builder/global_builder.d.ts.map +1 -1
  16. package/dest/global_variable_builder/global_builder.js +16 -50
  17. package/dest/global_variable_builder/index.d.ts +4 -2
  18. package/dest/global_variable_builder/index.d.ts.map +1 -1
  19. package/dest/global_variable_builder/index.js +2 -0
  20. package/dest/index.d.ts +2 -2
  21. package/dest/index.d.ts.map +1 -1
  22. package/dest/index.js +1 -1
  23. package/dest/publisher/config.d.ts +47 -17
  24. package/dest/publisher/config.d.ts.map +1 -1
  25. package/dest/publisher/config.js +121 -42
  26. package/dest/publisher/index.d.ts +2 -1
  27. package/dest/publisher/index.d.ts.map +1 -1
  28. package/dest/publisher/l1_tx_failed_store/factory.d.ts +11 -0
  29. package/dest/publisher/l1_tx_failed_store/factory.d.ts.map +1 -0
  30. package/dest/publisher/l1_tx_failed_store/factory.js +22 -0
  31. package/dest/publisher/l1_tx_failed_store/failed_tx_store.d.ts +58 -0
  32. package/dest/publisher/l1_tx_failed_store/failed_tx_store.d.ts.map +1 -0
  33. package/dest/publisher/l1_tx_failed_store/failed_tx_store.js +1 -0
  34. package/dest/publisher/l1_tx_failed_store/file_store_failed_tx_store.d.ts +15 -0
  35. package/dest/publisher/l1_tx_failed_store/file_store_failed_tx_store.d.ts.map +1 -0
  36. package/dest/publisher/l1_tx_failed_store/file_store_failed_tx_store.js +34 -0
  37. package/dest/publisher/l1_tx_failed_store/index.d.ts +4 -0
  38. package/dest/publisher/l1_tx_failed_store/index.d.ts.map +1 -0
  39. package/dest/publisher/l1_tx_failed_store/index.js +2 -0
  40. package/dest/publisher/sequencer-bundle-simulator.d.ts +96 -0
  41. package/dest/publisher/sequencer-bundle-simulator.d.ts.map +1 -0
  42. package/dest/publisher/sequencer-bundle-simulator.js +198 -0
  43. package/dest/publisher/sequencer-publisher-factory.d.ts +11 -5
  44. package/dest/publisher/sequencer-publisher-factory.d.ts.map +1 -1
  45. package/dest/publisher/sequencer-publisher-factory.js +27 -3
  46. package/dest/publisher/sequencer-publisher.d.ts +94 -67
  47. package/dest/publisher/sequencer-publisher.d.ts.map +1 -1
  48. package/dest/publisher/sequencer-publisher.js +439 -378
  49. package/dest/sequencer/automine/automine_factory.d.ts +54 -0
  50. package/dest/sequencer/automine/automine_factory.d.ts.map +1 -0
  51. package/dest/sequencer/automine/automine_factory.js +84 -0
  52. package/dest/sequencer/automine/automine_sequencer.d.ts +151 -0
  53. package/dest/sequencer/automine/automine_sequencer.d.ts.map +1 -0
  54. package/dest/sequencer/automine/automine_sequencer.js +509 -0
  55. package/dest/sequencer/chain_state_overrides.d.ts +61 -0
  56. package/dest/sequencer/chain_state_overrides.d.ts.map +1 -0
  57. package/dest/sequencer/chain_state_overrides.js +98 -0
  58. package/dest/sequencer/checkpoint_proposal_job.d.ts +60 -12
  59. package/dest/sequencer/checkpoint_proposal_job.d.ts.map +1 -1
  60. package/dest/sequencer/checkpoint_proposal_job.js +830 -229
  61. package/dest/sequencer/checkpoint_proposal_job_metrics.d.ts +34 -0
  62. package/dest/sequencer/checkpoint_proposal_job_metrics.d.ts.map +1 -0
  63. package/dest/sequencer/checkpoint_proposal_job_metrics.js +72 -0
  64. package/dest/sequencer/checkpoint_voter.d.ts +1 -2
  65. package/dest/sequencer/checkpoint_voter.d.ts.map +1 -1
  66. package/dest/sequencer/checkpoint_voter.js +2 -5
  67. package/dest/sequencer/events.d.ts +48 -2
  68. package/dest/sequencer/events.d.ts.map +1 -1
  69. package/dest/sequencer/index.d.ts +3 -1
  70. package/dest/sequencer/index.d.ts.map +1 -1
  71. package/dest/sequencer/index.js +2 -0
  72. package/dest/sequencer/metrics.d.ts +23 -8
  73. package/dest/sequencer/metrics.d.ts.map +1 -1
  74. package/dest/sequencer/metrics.js +117 -21
  75. package/dest/sequencer/sequencer.d.ts +83 -22
  76. package/dest/sequencer/sequencer.d.ts.map +1 -1
  77. package/dest/sequencer/sequencer.js +378 -139
  78. package/dest/sequencer/timetable.d.ts +14 -6
  79. package/dest/sequencer/timetable.d.ts.map +1 -1
  80. package/dest/sequencer/timetable.js +47 -46
  81. package/dest/sequencer/types.d.ts +2 -2
  82. package/dest/sequencer/types.d.ts.map +1 -1
  83. package/dest/test/index.d.ts +3 -5
  84. package/dest/test/index.d.ts.map +1 -1
  85. package/dest/test/mock_checkpoint_builder.d.ts +11 -11
  86. package/dest/test/mock_checkpoint_builder.d.ts.map +1 -1
  87. package/dest/test/mock_checkpoint_builder.js +45 -34
  88. package/dest/test/utils.d.ts +3 -3
  89. package/dest/test/utils.d.ts.map +1 -1
  90. package/dest/test/utils.js +10 -8
  91. package/package.json +27 -28
  92. package/src/client/sequencer-client.ts +91 -35
  93. package/src/config.ts +98 -46
  94. package/src/global_variable_builder/README.md +44 -0
  95. package/src/global_variable_builder/fee_predictor.ts +172 -0
  96. package/src/global_variable_builder/fee_provider.ts +75 -0
  97. package/src/global_variable_builder/global_builder.ts +26 -63
  98. package/src/global_variable_builder/index.ts +3 -1
  99. package/src/index.ts +10 -1
  100. package/src/publisher/config.ts +157 -45
  101. package/src/publisher/index.ts +3 -0
  102. package/src/publisher/l1_tx_failed_store/factory.ts +32 -0
  103. package/src/publisher/l1_tx_failed_store/failed_tx_store.ts +57 -0
  104. package/src/publisher/l1_tx_failed_store/file_store_failed_tx_store.ts +46 -0
  105. package/src/publisher/l1_tx_failed_store/index.ts +3 -0
  106. package/src/publisher/sequencer-bundle-simulator.ts +254 -0
  107. package/src/publisher/sequencer-publisher-factory.ts +38 -9
  108. package/src/publisher/sequencer-publisher.ts +532 -469
  109. package/src/sequencer/README.md +162 -450
  110. package/src/sequencer/automine/README.md +46 -0
  111. package/src/sequencer/automine/automine_factory.ts +145 -0
  112. package/src/sequencer/automine/automine_sequencer.ts +595 -0
  113. package/src/sequencer/chain_state_overrides.ts +162 -0
  114. package/src/sequencer/checkpoint_proposal_job.ts +1003 -248
  115. package/src/sequencer/checkpoint_proposal_job_metrics.ts +128 -0
  116. package/src/sequencer/checkpoint_voter.ts +1 -12
  117. package/src/sequencer/events.ts +51 -2
  118. package/src/sequencer/index.ts +2 -0
  119. package/src/sequencer/metrics.ts +132 -25
  120. package/src/sequencer/sequencer.ts +466 -158
  121. package/src/sequencer/timetable.ts +62 -56
  122. package/src/sequencer/types.ts +1 -1
  123. package/src/test/index.ts +2 -4
  124. package/src/test/mock_checkpoint_builder.ts +63 -49
  125. package/src/test/utils.ts +31 -10
@@ -372,22 +372,23 @@ function _apply_decs_2203_r(targetClass, memberDecs, classDecs, parentClass) {
372
372
  }
373
373
  var _dec, _dec1, _dec2, _dec3, _initProto;
374
374
  import { getKzg } from '@aztec/blob-lib';
375
- import { INITIAL_L2_BLOCK_NUM } from '@aztec/constants';
375
+ import { PROPOSER_PIPELINING_SLOT_OFFSET } from '@aztec/epoch-cache';
376
376
  import { NoCommitteeError } from '@aztec/ethereum/contracts';
377
- import { BlockNumber, CheckpointNumber } from '@aztec/foundation/branded-types';
377
+ import { CheckpointNumber, SlotNumber } from '@aztec/foundation/branded-types';
378
378
  import { merge, omit, pick } from '@aztec/foundation/collection';
379
- import { Fr } from '@aztec/foundation/curves/bn254';
380
379
  import { EthAddress } from '@aztec/foundation/eth-address';
381
380
  import { createLogger } from '@aztec/foundation/log';
382
381
  import { RunningPromise } from '@aztec/foundation/running-promise';
383
- import { getSlotAtTimestamp, getSlotStartBuildTimestamp } from '@aztec/stdlib/epoch-helpers';
382
+ import { getEpochAtSlot, getSlotStartBuildTimestamp, getTimestampForSlot } from '@aztec/stdlib/epoch-helpers';
384
383
  import { SequencerConfigSchema } from '@aztec/stdlib/interfaces/server';
385
384
  import { pickFromSchema } from '@aztec/stdlib/schemas';
386
- import { MerkleTreeId } from '@aztec/stdlib/trees';
385
+ import { MIN_EXECUTION_TIME } from '@aztec/stdlib/timetable';
387
386
  import { Attributes, getTelemetryClient, trackSpan } from '@aztec/telemetry-client';
388
387
  import EventEmitter from 'node:events';
389
388
  import { DefaultSequencerConfig } from '../config.js';
389
+ import { buildCheckpointSimulationOverridesPlan } from './chain_state_overrides.js';
390
390
  import { CheckpointProposalJob } from './checkpoint_proposal_job.js';
391
+ import { CheckpointProposalJobMetrics } from './checkpoint_proposal_job_metrics.js';
391
392
  import { CheckpointVoter } from './checkpoint_voter.js';
392
393
  import { SequencerInterruptedError, SequencerTooSlowError } from './errors.js';
393
394
  import { SequencerMetrics } from './metrics.js';
@@ -448,46 +449,51 @@ _dec = trackSpan('Sequencer.work'), _dec1 = trackSpan('Sequencer.prepareCheckpoi
448
449
  }
449
450
  runningPromise;
450
451
  state;
452
+ stateSlotNumber;
453
+ stateEnteredAtMs;
451
454
  metrics;
455
+ checkpointProposalJobMetrics;
456
+ stateLog;
452
457
  /** The last slot for which we attempted to perform our voting duties with degraded block production */ lastSlotForFallbackVote;
453
458
  /** The last slot for which we logged "no committee" warning, to avoid spam */ lastSlotForNoCommitteeWarning;
454
459
  /** The last slot for which we triggered a checkpoint proposal job, to prevent duplicate attempts. */ lastSlotForCheckpointProposalJob;
455
460
  /** Last successful checkpoint proposed */ lastCheckpointProposed;
456
461
  /** The last epoch for which we logged strategy comparison in fisherman mode. */ lastEpochForStrategyComparison;
462
+ /** The last checkpoint proposal job, tracked so we can await its pending L1 submission during shutdown. */ lastCheckpointProposalJob;
457
463
  /** The maximum number of seconds that the sequencer can be into a slot to transition to a particular state. */ timetable;
458
- // This shouldn't be here as this gets re-created each time we build/propose a block.
459
- // But we have a number of tests that abuse/rely on this class having a permanent publisher.
460
- // As long as those tests only configure a single publisher they will continue to work.
461
- // This will get re-assigned every time the sequencer goes to build a new block to a publisher that is valid
462
- // for the block proposer.
463
- // TODO(palla/mbps): Remove this field and fix tests
464
- publisher;
465
464
  /** Config for the sequencer */ config;
465
+ signatureContext;
466
466
  constructor(publisherFactory, validatorClient, globalsBuilder, p2pClient, worldState, slasherClient, l2BlockSource, l1ToL2MessageSource, checkpointsBuilder, l1Constants, dateProvider, epochCache, rollupContract, config, telemetry = getTelemetryClient(), log = createLogger('sequencer')){
467
- super(), this.publisherFactory = publisherFactory, this.validatorClient = validatorClient, this.globalsBuilder = globalsBuilder, this.p2pClient = p2pClient, this.worldState = worldState, this.slasherClient = slasherClient, this.l2BlockSource = l2BlockSource, this.l1ToL2MessageSource = l1ToL2MessageSource, this.checkpointsBuilder = checkpointsBuilder, this.l1Constants = l1Constants, this.dateProvider = dateProvider, this.epochCache = epochCache, this.rollupContract = rollupContract, this.telemetry = telemetry, this.log = log, this.state = (_initProto(this), SequencerState.STOPPED), this.config = DefaultSequencerConfig;
467
+ super(), this.publisherFactory = publisherFactory, this.validatorClient = validatorClient, this.globalsBuilder = globalsBuilder, this.p2pClient = p2pClient, this.worldState = worldState, this.slasherClient = slasherClient, this.l2BlockSource = l2BlockSource, this.l1ToL2MessageSource = l1ToL2MessageSource, this.checkpointsBuilder = checkpointsBuilder, this.l1Constants = l1Constants, this.dateProvider = dateProvider, this.epochCache = epochCache, this.rollupContract = rollupContract, this.telemetry = telemetry, this.log = log, this.state = (_initProto(this), SequencerState.STOPPED), this.stateEnteredAtMs = performance.now(), this.config = DefaultSequencerConfig;
468
+ this.stateLog = log.createChild('state');
468
469
  // Add [FISHERMAN] prefix to logger if in fisherman mode
469
470
  if (config.fishermanMode) {
470
471
  this.log = log.createChild('[FISHERMAN]');
471
472
  }
473
+ this.signatureContext = {
474
+ chainId: config.l1ChainId,
475
+ rollupAddress: config.rollupAddress
476
+ };
472
477
  this.metrics = new SequencerMetrics(telemetry, this.rollupContract, 'Sequencer');
478
+ this.checkpointProposalJobMetrics = new CheckpointProposalJobMetrics(telemetry);
473
479
  this.updateConfig(config);
474
480
  }
475
481
  /** Updates sequencer config by the defined values and updates the timetable */ updateConfig(config) {
476
482
  const filteredConfig = pickFromSchema(config, SequencerConfigSchema);
477
- this.log.info(`Updated sequencer config`, omit(filteredConfig, 'txPublicSetupAllowList'));
483
+ this.log.info(`Updated sequencer config`, omit(filteredConfig, 'txPublicSetupAllowListExtend'));
478
484
  this.config = merge(this.config, filteredConfig);
485
+ const p2pPropagationTime = this.config.attestationPropagationTime;
479
486
  this.timetable = new SequencerTimetable({
480
487
  ethereumSlotDuration: this.l1Constants.ethereumSlotDuration,
481
488
  aztecSlotDuration: this.aztecSlotDuration,
482
489
  l1PublishingTime: this.l1PublishingTime,
483
- p2pPropagationTime: this.config.attestationPropagationTime,
490
+ p2pPropagationTime,
484
491
  blockDurationMs: this.config.blockDurationMs,
485
492
  enforce: this.config.enforceTimeTable
486
493
  }, this.metrics, this.log);
487
494
  }
488
- /** Initializes the sequencer (precomputes tables and creates a publisher). Takes about 3s. */ async init() {
495
+ /** Initializes the sequencer (precomputes tables). Takes about 3s. */ init() {
489
496
  getKzg();
490
- this.publisher = (await this.publisherFactory.create(undefined)).publisher;
491
497
  }
492
498
  /** Starts the sequencer and moves to IDLE state. */ start() {
493
499
  this.runningPromise = new RunningPromise(this.safeWork.bind(this), this.log, this.config.sequencerPollingIntervalMS);
@@ -497,13 +503,18 @@ _dec = trackSpan('Sequencer.work'), _dec1 = trackSpan('Sequencer.prepareCheckpoi
497
503
  this.runningPromise.start();
498
504
  this.log.info('Started sequencer');
499
505
  }
506
+ /** Triggers an immediate run of the sequencer, bypassing the polling interval. */ trigger() {
507
+ return this.runningPromise?.trigger();
508
+ }
500
509
  /** Stops the sequencer from building blocks and moves to STOPPED state. */ async stop() {
501
510
  this.log.info(`Stopping sequencer`);
502
511
  this.setState(SequencerState.STOPPING, undefined, {
503
512
  force: true
504
513
  });
505
- this.publisher?.interrupt();
514
+ this.lastCheckpointProposalJob?.interrupt();
515
+ await this.publisherFactory.stopAll();
506
516
  await this.runningPromise?.stop();
517
+ await this.lastCheckpointProposalJob?.awaitPendingSubmission();
507
518
  this.setState(SequencerState.STOPPED, undefined, {
508
519
  force: true
509
520
  });
@@ -513,21 +524,19 @@ _dec = trackSpan('Sequencer.work'), _dec1 = trackSpan('Sequencer.prepareCheckpoi
513
524
  try {
514
525
  await this.work();
515
526
  } catch (err) {
516
- this.emit('checkpoint-error', {
517
- error: err
518
- });
519
- if (err instanceof SequencerTooSlowError) {
520
- // TODO(palla/mbps): Add missing states
521
- // Log as warn only if we had to abort halfway through the block proposal
522
- const logLvl = [
523
- SequencerState.INITIALIZING_CHECKPOINT,
524
- SequencerState.PROPOSER_CHECK
525
- ].includes(err.proposedState) ? 'debug' : 'warn';
526
- this.log[logLvl](err.message, {
527
+ if (err instanceof SequencerTooSlowError && [
528
+ SequencerState.INITIALIZING_CHECKPOINT,
529
+ SequencerState.PROPOSER_CHECK
530
+ ].includes(err.proposedState)) {
531
+ // No need to alert if we just didn't get to start in time
532
+ this.log.debug(err.message, {
527
533
  now: this.dateProvider.nowInSeconds()
528
534
  });
529
535
  } else {
530
536
  // Re-throw other errors
537
+ this.emit('checkpoint-error', {
538
+ error: err
539
+ });
531
540
  throw err;
532
541
  }
533
542
  } finally{
@@ -548,40 +557,62 @@ _dec = trackSpan('Sequencer.work'), _dec1 = trackSpan('Sequencer.prepareCheckpoi
548
557
  * - Submit checkpoint
549
558
  */ async work() {
550
559
  this.setState(SequencerState.SYNCHRONIZING, undefined);
551
- const { slot, ts, now, epoch } = this.epochCache.getEpochAndSlotInNextL1Slot();
560
+ const { slot, targetSlot, epoch, targetEpoch, ts, nowSeconds } = this.getSlotContextInNextL1Slot();
552
561
  // Check if we are synced and it's our slot, grab a publisher, check previous block invalidation, etc
553
- const checkpointProposalJob = await this.prepareCheckpointProposal(epoch, slot, ts, now);
562
+ const checkpointProposalJob = await this.prepareCheckpointProposal(slot, targetSlot, epoch, targetEpoch, ts, nowSeconds);
554
563
  if (!checkpointProposalJob) {
555
564
  return;
556
565
  }
566
+ // Track the job so we can await its pending L1 submission during shutdown
567
+ this.lastCheckpointProposalJob = checkpointProposalJob;
557
568
  // Execute the checkpoint proposal job
558
569
  const checkpoint = await checkpointProposalJob.execute();
559
570
  // Update last checkpoint proposed (currently unused)
560
571
  if (checkpoint) {
561
572
  this.lastCheckpointProposed = checkpoint;
562
573
  }
563
- // Log fee strategy comparison if on fisherman
564
- if (this.config.fishermanMode && (this.lastEpochForStrategyComparison === undefined || epoch > this.lastEpochForStrategyComparison)) {
565
- this.logStrategyComparison(epoch, checkpointProposalJob.getPublisher());
566
- this.lastEpochForStrategyComparison = epoch;
574
+ // Log fee strategy comparison if on fisherman (uses target epoch since we mirror the proposer's perspective)
575
+ if (this.config.fishermanMode && (this.lastEpochForStrategyComparison === undefined || targetEpoch > this.lastEpochForStrategyComparison)) {
576
+ this.logStrategyComparison(targetEpoch, checkpointProposalJob.getPublisher());
577
+ this.lastEpochForStrategyComparison = targetEpoch;
567
578
  }
568
579
  return checkpoint;
569
580
  }
581
+ /** Returns slot and target slot from a single clock snapshot. */ getSlotContextInNextL1Slot() {
582
+ const { slot, ts, nowSeconds, epoch } = this.epochCache.getEpochAndSlotInNextL1Slot();
583
+ const targetSlot = SlotNumber(slot + PROPOSER_PIPELINING_SLOT_OFFSET);
584
+ return {
585
+ slot,
586
+ targetSlot,
587
+ epoch,
588
+ targetEpoch: getEpochAtSlot(targetSlot, this.l1Constants),
589
+ ts,
590
+ nowSeconds
591
+ };
592
+ }
570
593
  /**
571
594
  * Prepares the checkpoint proposal by performing all necessary checks and setup.
572
595
  * This is the initial step in the main loop.
573
596
  * @returns CheckpointProposalJob if successful, undefined if we are not yet synced or are not the proposer.
574
- */ async prepareCheckpointProposal(epoch, slot, ts, now) {
575
- // Check we have not already processed this slot (cheapest check)
597
+ */ async prepareCheckpointProposal(slot, targetSlot, epoch, targetEpoch, ts, nowSeconds) {
598
+ // Check we have not already processed this target slot (cheapest check)
576
599
  // We only check this if enforce timetable is set, since we want to keep processing the same slot if we are not
577
600
  // running against actual time (eg when we use sandbox-style automining)
578
- if (this.lastSlotForCheckpointProposalJob && this.lastSlotForCheckpointProposalJob >= slot && this.config.enforceTimeTable) {
579
- this.log.trace(`Slot ${slot} has already been processed`);
601
+ if (this.lastSlotForCheckpointProposalJob && this.lastSlotForCheckpointProposalJob >= targetSlot && this.config.enforceTimeTable) {
602
+ this.log.trace(`Target slot ${targetSlot} has already been processed`);
603
+ return undefined;
604
+ }
605
+ // But if we have already proposed for this slot, then we definitely have to skip it, automining or not
606
+ if (this.lastCheckpointProposed && this.lastCheckpointProposed.header.slotNumber >= targetSlot) {
607
+ this.log.trace(`Slot ${targetSlot} has already been published as checkpoint ${this.lastCheckpointProposed.number}`);
580
608
  return undefined;
581
609
  }
582
- // But if we have already proposed for this slot, the we definitely have to skip it, automining or not
583
- if (this.lastCheckpointProposed && this.lastCheckpointProposed.header.slotNumber >= slot) {
584
- this.log.trace(`Slot ${slot} has already been published as checkpoint ${this.lastCheckpointProposed.number}`);
610
+ // Test-only: skip proposing for explicitly paused slots. Attestation paths run in the validator
611
+ // client and are not gated by this hook, so paused proposers still attest to others' proposals.
612
+ if (this.config.pauseProposingForSlots?.some((s)=>s === targetSlot)) {
613
+ this.log.warn(`Skipping proposal for paused slot ${targetSlot} (test-only pauseProposingForSlots hook)`, {
614
+ targetSlot
615
+ });
585
616
  return undefined;
586
617
  }
587
618
  // Check all components are synced to latest as seen by the archiver (queries all subsystems)
@@ -592,19 +623,22 @@ _dec = trackSpan('Sequencer.work'), _dec1 = trackSpan('Sequencer.prepareCheckpoi
592
623
  if (!syncedTo) {
593
624
  await this.tryVoteWhenSyncFails({
594
625
  slot,
626
+ targetSlot,
595
627
  ts
596
628
  });
597
629
  return undefined;
598
630
  }
599
- // If escape hatch is open for this epoch, do not start checkpoint proposal work and do not attempt invalidations.
631
+ // If escape hatch is open for the target epoch, do not start checkpoint proposal work and do not attempt invalidations.
600
632
  // Still perform governance/slashing voting (as proposer) once per slot.
601
- const isEscapeHatchOpen = await this.epochCache.isEscapeHatchOpen(epoch);
633
+ // When pipelining, we check the target epoch (slot+1's epoch) since that's the epoch we're building for.
634
+ const isEscapeHatchOpen = await this.epochCache.isEscapeHatchOpen(targetEpoch);
602
635
  if (isEscapeHatchOpen) {
603
636
  this.setState(SequencerState.PROPOSER_CHECK, slot);
604
- const [canPropose, proposer] = await this.checkCanPropose(slot);
637
+ const [canPropose, proposer] = await this.checkCanPropose(targetSlot);
605
638
  if (canPropose) {
606
639
  await this.tryVoteWhenEscapeHatchOpen({
607
640
  slot,
641
+ targetSlot,
608
642
  proposer
609
643
  });
610
644
  } else {
@@ -619,29 +653,36 @@ _dec = trackSpan('Sequencer.work'), _dec1 = trackSpan('Sequencer.prepareCheckpoi
619
653
  // Next checkpoint follows from the last synced one
620
654
  const checkpointNumber = CheckpointNumber(syncedTo.checkpointNumber + 1);
621
655
  const logCtx = {
622
- now,
623
- syncedToL1Ts: syncedTo.l1Timestamp,
624
- syncedToL2Slot: getSlotAtTimestamp(syncedTo.l1Timestamp, this.l1Constants),
656
+ nowSeconds,
657
+ syncedToL2Slot: syncedTo.syncedL2Slot,
625
658
  slot,
659
+ targetSlot,
626
660
  slotTs: ts,
627
661
  checkpointNumber,
628
662
  isPendingChainValid: pick(syncedTo.pendingChainValidationStatus, 'valid', 'reason', 'invalidIndex')
629
663
  };
630
- // Check that we are a proposer for the next slot
664
+ // Check that we are a proposer for the target slot.
631
665
  this.setState(SequencerState.PROPOSER_CHECK, slot);
632
- const [canPropose, proposer] = await this.checkCanPropose(slot);
666
+ const [canPropose, proposer] = await this.checkCanPropose(targetSlot);
633
667
  // If we are not a proposer check if we should invalidate an invalid checkpoint, and bail
634
668
  if (!canPropose) {
635
669
  await this.considerInvalidatingCheckpoint(syncedTo, slot);
636
670
  return undefined;
637
671
  }
638
- // Check that the slot is not taken by a block already (should never happen, since only us can propose for this slot)
639
- if (syncedTo.block && syncedTo.block.header.getSlot() >= slot) {
640
- this.log.warn(`Cannot propose block at next L2 slot ${slot} since that slot was taken by block ${syncedTo.blockNumber}`, {
672
+ // Guard: don't exceed 1-deep pipeline. Without a proposed checkpoint, we can only build
673
+ // confirmed + 1. With a proposed checkpoint, we can build confirmed + 2.
674
+ const confirmedCkpt = syncedTo.checkpointedCheckpointNumber;
675
+ if (checkpointNumber > confirmedCkpt + 2) {
676
+ this.log.verbose(`Skipping slot ${targetSlot}: checkpoint ${checkpointNumber} exceeds max pipeline depth (confirmed=${confirmedCkpt})`);
677
+ return undefined;
678
+ }
679
+ // Check that the target slot is not taken by a block already (should never happen, since only us can propose for this slot)
680
+ if (syncedTo.blockData && syncedTo.blockData.header.getSlot() >= targetSlot) {
681
+ this.log.warn(`Cannot propose block at target slot ${targetSlot} since that slot was taken by block ${syncedTo.blockNumber}`, {
641
682
  ...logCtx,
642
- block: syncedTo.block.header.toInspect()
683
+ block: syncedTo.blockData.header.toInspect()
643
684
  });
644
- this.metrics.recordBlockProposalPrecheckFailed('slot_already_taken');
685
+ this.metrics.recordCheckpointPrecheckFailed('slot_already_taken');
645
686
  return undefined;
646
687
  }
647
688
  // We now need to get ourselves a publisher.
@@ -651,42 +692,109 @@ _dec = trackSpan('Sequencer.work'), _dec1 = trackSpan('Sequencer.prepareCheckpoi
651
692
  const proposerForPublisher = this.config.fishermanMode ? undefined : proposer;
652
693
  const { attestorAddress, publisher } = await this.publisherFactory.create(proposerForPublisher);
653
694
  this.log.verbose(`Created publisher at address ${publisher.getSenderAddress()} for attestor ${attestorAddress}`);
654
- this.publisher = publisher;
655
- // In fisherman mode, set the actual proposer's address for simulations
656
- if (this.config.fishermanMode && proposer) {
657
- publisher.setProposerAddressForSimulation(proposer);
658
- this.log.debug(`Set proposer address ${proposer} for simulation in fisherman mode`);
695
+ // Prepare invalidation request if the pending chain is invalid (returns undefined if no need).
696
+ // Only simulate invalidation when there's no proposed parent, since we assume the proposed parent
697
+ // will invalidate the currently invalid checkpoint on L1.
698
+ const invalidateCheckpoint = syncedTo.hasProposedCheckpoint || syncedTo.pendingChainValidationStatus.valid ? undefined : await publisher.simulateInvalidateCheckpoint(syncedTo.pendingChainValidationStatus);
699
+ // Determine the correct archive and L1 state overrides for the canProposeAt check.
700
+ // The L1 contract reads archives[proposedCheckpointNumber] and compares it with the provided archive.
701
+ // When invalidating or pipelining, the local archive may differ from L1's, so we adjust accordingly.
702
+ let archiveForCheck = syncedTo.archive;
703
+ if (syncedTo.hasProposedCheckpoint) {
704
+ this.metrics.recordPipelineDepth(syncedTo.checkpointNumber - syncedTo.checkpointedCheckpointNumber);
705
+ this.log.verbose(`Building on top of proposed checkpoint (pending=${syncedTo.proposedCheckpointData?.checkpointNumber}) for target slot ${targetSlot}`, {
706
+ targetSlot,
707
+ parentCheckpointNumber: CheckpointNumber(checkpointNumber - 1)
708
+ });
709
+ // Match what L1 will see at archives[pending] once the proposed parent lands: the parent's
710
+ // own archive root from the gossiped proposal. `syncedTo.archive` is the world-state-local
711
+ // view and can transiently diverge from the proposed parent (e.g. before the proposed
712
+ // parent's blocks have been applied locally); diverging here would cause the canProposeAt
713
+ // override to set archives[pending] to one value while we present another for comparison.
714
+ archiveForCheck = syncedTo.proposedCheckpointData.archive.root;
715
+ } else if (invalidateCheckpoint) {
716
+ // After invalidation, L1 will roll back to checkpoint N-1. The archive at N-1 already
717
+ // exists on L1, so we just pass the matching archive (the lastArchive of the invalid checkpoint).
718
+ archiveForCheck = invalidateCheckpoint.lastArchive;
719
+ this.metrics.recordPipelineDepth(0);
720
+ } else {
721
+ this.metrics.recordPipelineDepth(0);
659
722
  }
660
- // Prepare invalidation request if the pending chain is invalid (returns undefined if no need)
661
- const invalidateCheckpoint = await publisher.simulateInvalidateCheckpoint(syncedTo.pendingChainValidationStatus);
662
- // Check with the rollup contract if we can indeed propose at the next L2 slot. This check should not fail
663
- // if all the previous checks are good, but we do it just in case.
664
- const canProposeCheck = await publisher.canProposeAtNextEthBlock(syncedTo.archive, proposer ?? EthAddress.ZERO, invalidateCheckpoint);
723
+ // Build the simulation plan: pending/proven override from pipelining or invalidation (or the
724
+ // current snapshot when neither applies, to short-circuit any pending prune in simulation),
725
+ // plus the parent checkpoint cell and fee header when pipelining.
726
+ const simulationOverridesPlan = await buildCheckpointSimulationOverridesPlan({
727
+ checkpointNumber,
728
+ proposedCheckpointData: syncedTo.hasProposedCheckpoint ? syncedTo.proposedCheckpointData : undefined,
729
+ invalidateToPendingCheckpointNumber: invalidateCheckpoint?.forcePendingCheckpointNumber,
730
+ checkpointedCheckpointNumber: syncedTo.checkpointedCheckpointNumber,
731
+ rollup: this.rollupContract,
732
+ signatureContext: this.signatureContext,
733
+ log: this.log
734
+ });
735
+ // The plan always pins both pending/proven (to short-circuit `canPruneAtTime` in simulation),
736
+ // so `provenOverride` always reflects the assumed proven checkpoint we are pinning the
737
+ // simulation to. We additionally warn when the pin is load-bearing — i.e. when a prune would
738
+ // actually fire at the target slot without it — so observers can spot "we are building
739
+ // optimistically across a pruning boundary" in the logs.
740
+ const provenOverride = simulationOverridesPlan?.chainTipsOverride?.proven;
741
+ if (provenOverride !== undefined && await this.l2BlockSource.isPruneDueAtSlot(targetSlot)) {
742
+ this.log.warn(`Assuming proof for epoch ending at checkpoint ${provenOverride} lands by target slot ${targetSlot}`, {
743
+ checkpointNumber,
744
+ slot,
745
+ targetSlot,
746
+ provenOverride
747
+ });
748
+ }
749
+ this.emit('preparing-checkpoint', {
750
+ targetSlot,
751
+ checkpointNumber,
752
+ hadProposedParent: syncedTo.hasProposedCheckpoint,
753
+ provenOverride,
754
+ simulatedPending: simulationOverridesPlan?.chainTipsOverride?.pending
755
+ });
756
+ const canProposeCheck = await publisher.canProposeAt(archiveForCheck, proposer ?? EthAddress.ZERO, simulationOverridesPlan);
757
+ const proposeContext = {
758
+ hasProposedCheckpoint: syncedTo.hasProposedCheckpoint,
759
+ proposedCheckpointNumber: syncedTo.proposedCheckpointData?.checkpointNumber,
760
+ checkpointedCheckpointNumber: syncedTo.checkpointedCheckpointNumber,
761
+ isInvalidating: !!invalidateCheckpoint,
762
+ invalidatingCheckpointNumber: invalidateCheckpoint?.checkpointNumber,
763
+ archiveForCheck: archiveForCheck.toString(),
764
+ overridePendingCheckpointNumber: simulationOverridesPlan?.chainTipsOverride?.pending,
765
+ overrideArchive: simulationOverridesPlan?.pendingCheckpointState?.archive,
766
+ overrideFeeHeader: simulationOverridesPlan?.pendingCheckpointState?.feeHeader
767
+ };
665
768
  if (canProposeCheck === undefined) {
666
- this.log.warn(`Cannot propose checkpoint ${checkpointNumber} at slot ${slot} due to failed rollup contract check`, logCtx);
769
+ this.log.warn(`Cannot propose checkpoint ${checkpointNumber} at slot ${slot} due to failed rollup contract check`, {
770
+ ...logCtx,
771
+ ...proposeContext
772
+ });
667
773
  this.emit('proposer-rollup-check-failed', {
668
774
  reason: 'Rollup contract check failed',
669
775
  slot
670
776
  });
671
- this.metrics.recordBlockProposalPrecheckFailed('rollup_contract_check_failed');
777
+ this.metrics.recordCheckpointPrecheckFailed('rollup_contract_check_failed');
672
778
  return undefined;
673
779
  }
674
- if (canProposeCheck.slot !== slot) {
675
- this.log.warn(`Cannot propose block due to slot mismatch with rollup contract (this can be caused by a clock out of sync). Expected slot ${slot} but got ${canProposeCheck.slot}.`, {
780
+ if (canProposeCheck.slot !== targetSlot) {
781
+ this.log.warn(`Cannot propose block due to slot mismatch with rollup contract (this can be caused by a clock out of sync). Expected slot ${targetSlot} but got ${canProposeCheck.slot}.`, {
676
782
  ...logCtx,
783
+ ...proposeContext,
677
784
  rollup: canProposeCheck,
678
- expectedSlot: slot
785
+ expectedSlot: targetSlot
679
786
  });
680
787
  this.emit('proposer-rollup-check-failed', {
681
788
  reason: 'Slot mismatch',
682
789
  slot
683
790
  });
684
- this.metrics.recordBlockProposalPrecheckFailed('slot_mismatch');
791
+ this.metrics.recordCheckpointPrecheckFailed('slot_mismatch');
685
792
  return undefined;
686
793
  }
687
794
  if (canProposeCheck.checkpointNumber !== checkpointNumber) {
688
795
  this.log.warn(`Cannot propose due to block mismatch with rollup contract (this can be caused by a pending archiver sync). Expected checkpoint ${checkpointNumber} but got ${canProposeCheck.checkpointNumber}.`, {
689
796
  ...logCtx,
797
+ ...proposeContext,
690
798
  rollup: canProposeCheck,
691
799
  expectedSlot: slot
692
800
  });
@@ -694,25 +802,35 @@ _dec = trackSpan('Sequencer.work'), _dec1 = trackSpan('Sequencer.prepareCheckpoi
694
802
  reason: 'Block mismatch',
695
803
  slot
696
804
  });
697
- this.metrics.recordBlockProposalPrecheckFailed('block_number_mismatch');
805
+ this.metrics.recordCheckpointPrecheckFailed('block_number_mismatch');
698
806
  return undefined;
699
807
  }
700
- this.lastSlotForCheckpointProposalJob = slot;
701
- this.log.info(`Preparing checkpoint proposal ${checkpointNumber} at slot ${slot}`, {
808
+ this.lastSlotForCheckpointProposalJob = targetSlot;
809
+ await this.p2pClient.prepareForSlot(targetSlot);
810
+ this.log.info(`Preparing checkpoint proposal ${checkpointNumber} for target slot ${targetSlot} during wall-clock slot ${slot}`, {
702
811
  ...logCtx,
812
+ ...proposeContext,
703
813
  proposer
704
814
  });
705
815
  // Create and return the checkpoint proposal job
706
- return this.createCheckpointProposalJob(epoch, slot, checkpointNumber, syncedTo.blockNumber, proposer, publisher, attestorAddress, invalidateCheckpoint);
816
+ return this.createCheckpointProposalJob(slot, targetSlot, targetEpoch, checkpointNumber, syncedTo.blockNumber, syncedTo.checkpointedCheckpointNumber, proposer, publisher, attestorAddress, invalidateCheckpoint, syncedTo.proposedCheckpointData);
707
817
  }
708
- createCheckpointProposalJob(epoch, slot, checkpointNumber, syncedToBlockNumber, proposer, publisher, attestorAddress, invalidateCheckpoint) {
709
- return new CheckpointProposalJob(epoch, slot, checkpointNumber, syncedToBlockNumber, proposer, publisher, attestorAddress, invalidateCheckpoint, this.validatorClient, this.globalsBuilder, this.p2pClient, this.worldState, this.l1ToL2MessageSource, this.l2BlockSource, this.checkpointsBuilder, this.l2BlockSource, this.l1Constants, this.config, this.timetable, this.slasherClient, this.epochCache, this.dateProvider, this.metrics, this, this.setState.bind(this), this.tracer, this.log.getBindings());
818
+ createCheckpointProposalJob(slot, targetSlot, targetEpoch, checkpointNumber, syncedToBlockNumber, checkpointedCheckpointNumber, proposer, publisher, attestorAddress, invalidateCheckpoint, proposedCheckpointData) {
819
+ return new CheckpointProposalJob(slot, targetSlot, targetEpoch, checkpointNumber, syncedToBlockNumber, checkpointedCheckpointNumber, proposer, publisher, attestorAddress, invalidateCheckpoint, this.validatorClient, this.globalsBuilder, this.p2pClient, this.worldState, this.l1ToL2MessageSource, this.l2BlockSource, this.checkpointsBuilder, this.l2BlockSource, this.l1Constants, this.signatureContext, this.config, this.timetable, this.slasherClient, this.epochCache, this.dateProvider, this.metrics, this.checkpointProposalJobMetrics.createRecorder(), this, (state, slot, timeReferenceSlot)=>this.setState(state, slot, {
820
+ timeReferenceSlot
821
+ }), this.tracer, this.log.getBindings(), proposedCheckpointData);
822
+ }
823
+ /**
824
+ * Returns the current sequencer state.
825
+ */ getState() {
826
+ return this.state;
710
827
  }
711
828
  /**
712
829
  * Internal helper for setting the sequencer state and checks if we have enough time left in the slot to transition to the new state.
713
830
  * @param proposedState - The new state to transition to.
714
- * @param slotNumber - The current slot number.
831
+ * @param slotNumber - The logical slot associated with the new state.
715
832
  * @param force - Whether to force the transition even if the sequencer is stopped.
833
+ * @param timeReferenceSlot - The slot to use for deadline checks when it differs from the logical slot.
716
834
  */ setState(proposedState, slotNumber, opts = {}) {
717
835
  if (this.state === SequencerState.STOPPING && proposedState !== SequencerState.STOPPED && !opts.force) {
718
836
  this.log.warn(`Cannot set sequencer to ${proposedState} as it is stopping.`);
@@ -723,41 +841,59 @@ _dec = trackSpan('Sequencer.work'), _dec1 = trackSpan('Sequencer.prepareCheckpoi
723
841
  return;
724
842
  }
725
843
  let secondsIntoSlot = undefined;
726
- if (slotNumber !== undefined) {
727
- secondsIntoSlot = this.getSecondsIntoSlot(slotNumber);
844
+ const timeReferenceSlot = opts.timeReferenceSlot ?? slotNumber;
845
+ if (timeReferenceSlot !== undefined) {
846
+ secondsIntoSlot = this.getSecondsIntoSlot(timeReferenceSlot);
728
847
  this.timetable.assertTimeLeft(proposedState, secondsIntoSlot);
729
848
  }
849
+ const oldState = this.state;
850
+ const oldStateSlotNumber = this.stateSlotNumber;
851
+ const stateChanged = proposedState !== oldState;
852
+ const transitionAtMs = performance.now();
853
+ const stateDurationMs = transitionAtMs - this.stateEnteredAtMs;
730
854
  const boringStates = [
731
855
  SequencerState.IDLE,
732
856
  SequencerState.SYNCHRONIZING
733
857
  ];
734
- const logLevel = boringStates.includes(proposedState) && boringStates.includes(this.state) ? 'trace' : 'debug';
735
- this.log[logLevel](`Transitioning from ${this.state} to ${proposedState}`, {
858
+ const logLevel = boringStates.includes(proposedState) && boringStates.includes(oldState) ? 'trace' : 'debug';
859
+ this.stateLog[logLevel](`Transitioning from ${oldState} to ${proposedState}`, {
860
+ oldState,
861
+ newState: proposedState,
736
862
  slotNumber,
737
- secondsIntoSlot
863
+ timeReferenceSlot,
864
+ stateSlotNumber: oldStateSlotNumber,
865
+ secondsIntoSlot,
866
+ ...stateChanged && {
867
+ stateDurationMs: Math.ceil(stateDurationMs)
868
+ }
738
869
  });
739
870
  this.emit('state-changed', {
740
- oldState: this.state,
871
+ oldState,
741
872
  newState: proposedState,
742
873
  secondsIntoSlot,
743
- slot: slotNumber
874
+ slot: slotNumber,
875
+ timeReferenceSlot
744
876
  });
877
+ if (stateChanged) {
878
+ this.metrics.recordStateDuration(stateDurationMs, oldState);
879
+ this.stateEnteredAtMs = transitionAtMs;
880
+ this.stateSlotNumber = slotNumber;
881
+ }
745
882
  this.state = proposedState;
746
883
  }
747
884
  /**
748
885
  * Returns whether all dependencies have caught up.
749
886
  * We don't check against the previous block submitted since it may have been reorg'd out.
750
887
  */ async checkSync(args) {
751
- // Check that the archiver and dependencies have synced to the previous L1 slot at least
752
- // TODO(#14766): Archiver reports L1 timestamp based on L1 blocks seen, which means that a missed L1 block will
753
- // cause the archiver L1 timestamp to fall behind, and cause this sequencer to start processing one L1 slot later.
754
- const l1Timestamp = await this.l2BlockSource.getL1Timestamp();
755
- const { slot, ts } = args;
756
- if (l1Timestamp === undefined || l1Timestamp + BigInt(this.l1Constants.ethereumSlotDuration) < ts) {
888
+ // Check that the archiver has fully synced the L2 slot before the one we want to propose in.
889
+ // The archiver reports sync progress via L1 block timestamps and synced checkpoint slots.
890
+ // See getSyncedL2SlotNumber for how missed L1 blocks are handled.
891
+ const syncedL2Slot = await this.l2BlockSource.getSyncedL2SlotNumber();
892
+ const { slot } = args;
893
+ if (syncedL2Slot === undefined || syncedL2Slot + 1 < slot) {
757
894
  this.log.debug(`Cannot propose block at next L2 slot ${slot} due to pending sync from L1`, {
758
895
  slot,
759
- ts,
760
- l1Timestamp
896
+ syncedL2Slot
761
897
  });
762
898
  return undefined;
763
899
  }
@@ -766,71 +902,142 @@ _dec = trackSpan('Sequencer.work'), _dec1 = trackSpan('Sequencer.prepareCheckpoi
766
902
  number: syncSummary.latestBlockNumber,
767
903
  hash: syncSummary.latestBlockHash
768
904
  })),
769
- this.l2BlockSource.getL2Tips().then((t)=>t.proposed),
905
+ this.l2BlockSource.getL2Tips().then((t)=>({
906
+ proposed: t.proposed,
907
+ checkpointed: t.checkpointed,
908
+ proposedCheckpoint: t.proposedCheckpoint
909
+ })),
770
910
  this.p2pClient.getStatus().then((p2p)=>p2p.syncedToL2Block),
771
- this.l1ToL2MessageSource.getL2Tips().then((t)=>t.proposed),
772
- this.l2BlockSource.getPendingChainValidationStatus()
911
+ this.l1ToL2MessageSource.getL2Tips().then((t)=>({
912
+ proposed: t.proposed,
913
+ checkpointed: t.checkpointed
914
+ })),
915
+ this.l2BlockSource.getPendingChainValidationStatus(),
916
+ this.l2BlockSource.getProposedCheckpointData()
773
917
  ]);
774
- const [worldState, l2BlockSource, p2p, l1ToL2MessageSource, pendingChainValidationStatus] = syncedBlocks;
775
- // Handle zero as a special case, since the block hash won't match across services if we're changing the prefilled data for the genesis block,
776
- // as the world state can compute the new genesis block hash, but other components use the hardcoded constant.
777
- // TODO(palla/mbps): Fix the above. All components should be able to handle dynamic genesis block hashes.
778
- const result = l2BlockSource.number === 0 && worldState.number === 0 && p2p.number === 0 && l1ToL2MessageSource.number === 0 || worldState.hash === l2BlockSource.hash && p2p.hash === l2BlockSource.hash && l1ToL2MessageSource.hash === l2BlockSource.hash;
918
+ const [worldState, l2Tips, p2p, l1ToL2MessageSourceTips, pendingChainValidationStatus, proposedCheckpointData] = syncedBlocks;
919
+ const result = worldState.hash === l2Tips.proposed.hash && p2p.hash === l2Tips.proposed.hash && l1ToL2MessageSourceTips.proposed.hash === l2Tips.proposed.hash && l1ToL2MessageSourceTips.checkpointed.block.hash === l2Tips.checkpointed.block.hash && l1ToL2MessageSourceTips.checkpointed.checkpoint.hash === l2Tips.checkpointed.checkpoint.hash;
779
920
  if (!result) {
780
921
  this.log.debug(`Sequencer sync check failed`, {
781
922
  worldState,
782
- l2BlockSource,
923
+ l2BlockSource: l2Tips.proposed,
783
924
  p2p,
784
- l1ToL2MessageSource
925
+ l1ToL2MessageSourceTips
785
926
  });
786
927
  return undefined;
787
928
  }
788
- // Special case for genesis state
789
929
  const blockNumber = worldState.number;
790
- if (blockNumber < INITIAL_L2_BLOCK_NUM) {
791
- const archive = new Fr((await this.worldState.getCommitted().getTreeInfo(MerkleTreeId.ARCHIVE)).root);
792
- return {
793
- checkpointNumber: CheckpointNumber.ZERO,
794
- blockNumber: BlockNumber.ZERO,
795
- archive,
796
- l1Timestamp,
797
- pendingChainValidationStatus
930
+ const blockData = await this.l2BlockSource.getBlockData({
931
+ number: blockNumber
932
+ });
933
+ if (!blockData) {
934
+ this.log.warn(`Sequencer sync check failed: failed to get L2 block data ${blockNumber} from the archiver`, {
935
+ blockNumber,
936
+ l2Tips,
937
+ syncedL2Slot,
938
+ ...args
939
+ });
940
+ return undefined;
941
+ }
942
+ // Refuse to build a checkpoint on top of a proposed block whose enclosing checkpoint was never
943
+ // proposed. Under pipelining we may have received and reexecuted such a block locally — advancing
944
+ // our world-state tip past the checkpointed tip — while the proposing node never published the
945
+ // matching proposed checkpoint (e.g. it crashed before assembling it). Building on this orphan block
946
+ // would fork the chain off a tip no other node can follow. The archiver prunes these orphan blocks
947
+ // once their build slot ends; this guard is the correctness barrier during the grace window before.
948
+ if (blockData.checkpointNumber > l2Tips.checkpointed.checkpoint.number && (l2Tips.proposedCheckpoint.checkpoint.number !== blockData.checkpointNumber || proposedCheckpointData?.checkpointNumber !== blockData.checkpointNumber)) {
949
+ const logCtx = {
950
+ blockCheckpointNumber: blockData.checkpointNumber,
951
+ checkpointedCheckpointNumber: l2Tips.checkpointed.checkpoint.number,
952
+ proposedCheckpointTipNumber: l2Tips.proposedCheckpoint.checkpoint.number,
953
+ proposedCheckpointDataNumber: proposedCheckpointData?.checkpointNumber,
954
+ blockNumber: blockData.header.getBlockNumber(),
955
+ blockSlot: blockData.header.getSlot(),
956
+ syncedL2Slot,
957
+ ...args
798
958
  };
959
+ // Under pipelining the block proposal for a checkpoint leads its checkpoint proposal by up to one
960
+ // slot, so a world-state tip sitting in an as-yet-unproposed checkpoint is the expected steady state
961
+ // until that checkpoint is due. Only treat it as abnormal — and warn — once the checkpoint is overdue
962
+ // by the same deadline the archiver uses to prune the orphan block (see pruneOrphanProposedBlocks).
963
+ // Before then this is normal pipelining and we wait it out quietly.
964
+ if (this.isProposedCheckpointOverdue(blockData.header.getSlot())) {
965
+ this.log.warn(`Sequencer sync check failed: proposed block has no matching proposed checkpoint`, logCtx);
966
+ } else {
967
+ this.log.debug(`Waiting for proposed checkpoint to catch up with reexecuted block`, logCtx);
968
+ }
969
+ return undefined;
970
+ }
971
+ const hasProposedCheckpoint = l2Tips.proposedCheckpoint.checkpoint.number > l2Tips.checkpointed.checkpoint.number;
972
+ // The l2Tips and proposedCheckpointData reads above come from independent archiver snapshots
973
+ // (a JS-side tips cache vs. a direct store read on `#proposedCheckpoints`). A concurrent archiver
974
+ // write that mutates both can be observed split, leaving us with `hasProposedCheckpoint=true` but
975
+ // no proposedCheckpointData (or one whose number doesn't match the tip). Refuse to proceed in that
976
+ // window — the next checkSync tick will see a coherent snapshot.
977
+ if (hasProposedCheckpoint && (!proposedCheckpointData || proposedCheckpointData.checkpointNumber !== l2Tips.proposedCheckpoint.checkpoint.number)) {
978
+ this.log.warn(`Sequencer sync check failed: inconsistent proposed-checkpoint state`, {
979
+ proposedCheckpointTipNumber: l2Tips.proposedCheckpoint.checkpoint.number,
980
+ checkpointedTipNumber: l2Tips.checkpointed.checkpoint.number,
981
+ proposedCheckpointDataNumber: proposedCheckpointData?.checkpointNumber,
982
+ syncedL2Slot,
983
+ ...args
984
+ });
985
+ return undefined;
799
986
  }
800
- const block = await this.l2BlockSource.getL2Block(blockNumber);
801
- if (!block) {
802
- // this shouldn't really happen because a moment ago we checked that all components were in sync
803
- this.log.error(`Failed to get L2 block ${blockNumber} from the archiver with all components in sync`);
987
+ // Check that the proposed checkpoint is indeed the parent of the checkpoint we'll be building
988
+ // The checkpoint number to build is derived as blockData.checkpointNumber + 1
989
+ if (proposedCheckpointData && proposedCheckpointData.checkpointNumber !== blockData.checkpointNumber) {
990
+ this.log.warn(`Sequencer sync check failed: proposed checkpoint number mismatch`, {
991
+ proposedCheckpointNumber: proposedCheckpointData.checkpointNumber,
992
+ blockCheckpointNumber: blockData.checkpointNumber,
993
+ syncedL2Slot,
994
+ ...args
995
+ });
804
996
  return undefined;
805
997
  }
806
998
  return {
807
- block,
808
- blockNumber: block.number,
809
- checkpointNumber: block.checkpointNumber,
810
- archive: block.archive.root,
811
- l1Timestamp,
999
+ blockData,
1000
+ blockNumber: blockData.header.getBlockNumber(),
1001
+ checkpointNumber: blockData.checkpointNumber,
1002
+ checkpointedCheckpointNumber: l2Tips.checkpointed.checkpoint.number,
1003
+ archive: blockData.archive.root,
1004
+ hasProposedCheckpoint,
1005
+ proposedCheckpointData,
1006
+ syncedL2Slot,
812
1007
  pendingChainValidationStatus
813
1008
  };
814
1009
  }
815
1010
  /**
1011
+ * Whether the enclosing checkpoint of a reexecuted block is overdue: past the deadline by which a
1012
+ * well-behaved proposer should have published it. Mirrors the archiver's orphan-prune deadline (the
1013
+ * start of the slot after the block's build slot, plus a grace period) so the sequencer only warns
1014
+ * about a missing proposed checkpoint once the archiver itself would prune the orphan block. The grace
1015
+ * is derived from the block build duration the same way the archiver defaults it at node wiring.
1016
+ */ isProposedCheckpointOverdue(blockSlot) {
1017
+ const expectedBySlot = SlotNumber(Number(blockSlot) - PROPOSER_PIPELINING_SLOT_OFFSET + 1);
1018
+ const graceSeconds = this.config.blockDurationMs !== undefined ? Math.ceil(this.config.blockDurationMs / 1000) : MIN_EXECUTION_TIME;
1019
+ const expectedByTime = getTimestampForSlot(expectedBySlot, this.l1Constants) + BigInt(graceSeconds);
1020
+ return BigInt(this.dateProvider.nowInSeconds()) >= expectedByTime;
1021
+ }
1022
+ /**
816
1023
  * Checks if we are the proposer for the next slot.
817
1024
  * @returns True if we can propose, and the proposer address (undefined if anyone can propose)
818
- */ async checkCanPropose(slot) {
1025
+ */ async checkCanPropose(targetSlot) {
819
1026
  let proposer;
820
1027
  try {
821
- proposer = await this.epochCache.getProposerAttesterAddressInSlot(slot);
1028
+ proposer = await this.epochCache.getProposerAttesterAddressInSlot(targetSlot);
822
1029
  } catch (e) {
823
1030
  if (e instanceof NoCommitteeError) {
824
- if (this.lastSlotForNoCommitteeWarning !== slot) {
825
- this.lastSlotForNoCommitteeWarning = slot;
826
- this.log.warn(`Cannot propose at next L2 slot ${slot} since the committee does not exist on L1`);
1031
+ if (this.lastSlotForNoCommitteeWarning !== targetSlot) {
1032
+ this.lastSlotForNoCommitteeWarning = targetSlot;
1033
+ this.log.warn(`Cannot propose at target slot ${targetSlot} since the committee does not exist on L1`);
827
1034
  }
828
1035
  return [
829
1036
  false,
830
1037
  undefined
831
1038
  ];
832
1039
  }
833
- this.log.error(`Error getting proposer for slot ${slot}`, e);
1040
+ this.log.error(`Error getting proposer for target slot ${targetSlot}`, e);
834
1041
  return [
835
1042
  false,
836
1043
  undefined
@@ -853,7 +1060,8 @@ _dec = trackSpan('Sequencer.work'), _dec1 = trackSpan('Sequencer.prepareCheckpoi
853
1060
  const validatorAddresses = this.validatorClient.getValidatorAddresses();
854
1061
  const weAreProposer = validatorAddresses.some((addr)=>addr.equals(proposer));
855
1062
  if (!weAreProposer) {
856
- this.log.debug(`Cannot propose at slot ${slot} since we are not a proposer`, {
1063
+ this.log.debug(`Cannot propose at target slot ${targetSlot} since we are not a proposer`, {
1064
+ targetSlot,
857
1065
  validatorAddresses,
858
1066
  proposer
859
1067
  });
@@ -862,6 +1070,10 @@ _dec = trackSpan('Sequencer.work'), _dec1 = trackSpan('Sequencer.prepareCheckpoi
862
1070
  proposer
863
1071
  ];
864
1072
  }
1073
+ this.log.info(`We are the proposer for pipeline slot ${targetSlot}`, {
1074
+ targetSlot,
1075
+ proposer
1076
+ });
865
1077
  return [
866
1078
  true,
867
1079
  proposer
@@ -871,7 +1083,7 @@ _dec = trackSpan('Sequencer.work'), _dec1 = trackSpan('Sequencer.prepareCheckpoi
871
1083
  * Tries to vote on slashing actions and governance when the sync check fails but we're past the max time for initializing a proposal.
872
1084
  * This allows the sequencer to participate in governance/slashing votes even when it cannot build blocks.
873
1085
  */ async tryVoteWhenSyncFails(args) {
874
- const { slot } = args;
1086
+ const { slot, targetSlot } = args;
875
1087
  // Prevent duplicate attempts in the same slot
876
1088
  if (this.lastSlotForFallbackVote === slot) {
877
1089
  this.log.trace(`Already attempted to vote in slot ${slot} (skipping)`);
@@ -894,7 +1106,7 @@ _dec = trackSpan('Sequencer.work'), _dec1 = trackSpan('Sequencer.prepareCheckpoi
894
1106
  maxAllowedTime
895
1107
  });
896
1108
  // Check if we're a proposer or proposal is open
897
- const [canPropose, proposer] = await this.checkCanPropose(slot);
1109
+ const [canPropose, proposer] = await this.checkCanPropose(targetSlot);
898
1110
  if (!canPropose) {
899
1111
  this.log.trace(`Cannot vote in slot ${slot} since we are not a proposer`, {
900
1112
  slot,
@@ -910,8 +1122,8 @@ _dec = trackSpan('Sequencer.work'), _dec1 = trackSpan('Sequencer.prepareCheckpoi
910
1122
  attestorAddress,
911
1123
  slot
912
1124
  });
913
- // Enqueue governance and slashing votes
914
- const voter = new CheckpointVoter(slot, publisher, attestorAddress, this.validatorClient, this.slasherClient, this.l1Constants, this.config, this.metrics, this.log);
1125
+ // Enqueue governance and slashing votes (voter uses the target slot for L1 submission)
1126
+ const voter = new CheckpointVoter(targetSlot, publisher, attestorAddress, this.validatorClient, this.slasherClient, this.l1Constants, this.config, this.metrics, this.log);
915
1127
  const votesPromises = voter.enqueueVotes();
916
1128
  const votes = await Promise.all(votesPromises);
917
1129
  if (votes.every((p)=>!p)) {
@@ -921,13 +1133,21 @@ _dec = trackSpan('Sequencer.work'), _dec1 = trackSpan('Sequencer.prepareCheckpoi
921
1133
  this.log.info(`Voting in slot ${slot} despite sync failure`, {
922
1134
  slot
923
1135
  });
924
- await publisher.sendRequests();
1136
+ // Votes are EIP-712-signed for `targetSlot` (the pipelined slot in which the multicall is
1137
+ // expected to mine). Delay submission to the start of `targetSlot` so the tx mines in the
1138
+ // slot the votes were signed for. We fire-and-forget so we don't block the sequencer's
1139
+ // work loop while waiting for the target slot to start.
1140
+ void publisher.sendRequestsAt(targetSlot).catch((err)=>{
1141
+ this.log.error(`Failed to publish votes despite sync failure for slot ${slot}`, err, {
1142
+ slot
1143
+ });
1144
+ });
925
1145
  }
926
1146
  /**
927
1147
  * Tries to vote on slashing actions and governance proposals when escape hatch is open.
928
1148
  * This allows the sequencer to participate in voting without performing checkpoint proposal work.
929
1149
  */ async tryVoteWhenEscapeHatchOpen(args) {
930
- const { slot, proposer } = args;
1150
+ const { slot, targetSlot, proposer } = args;
931
1151
  // Prevent duplicate attempts in the same slot
932
1152
  if (this.lastSlotForFallbackVote === slot) {
933
1153
  this.log.trace(`Already attempted to vote in slot ${slot} (escape hatch open, skipping)`);
@@ -938,9 +1158,14 @@ _dec = trackSpan('Sequencer.work'), _dec1 = trackSpan('Sequencer.prepareCheckpoi
938
1158
  const { attestorAddress, publisher } = await this.publisherFactory.create(proposer);
939
1159
  this.log.debug(`Escape hatch open for slot ${slot}, attempting vote-only actions`, {
940
1160
  slot,
1161
+ targetSlot,
941
1162
  attestorAddress
942
1163
  });
943
- const voter = new CheckpointVoter(slot, publisher, attestorAddress, this.validatorClient, this.slasherClient, this.l1Constants, this.config, this.metrics, this.log);
1164
+ // Under proposer pipelining, the multicall is expected to mine in `targetSlot` (slot + 1).
1165
+ // Governance and slashing votes are EIP-712-signed against the slot they will mine in, and the
1166
+ // L1 contract checks `msg.sender == getCurrentProposer()` using the mining slot. So we must
1167
+ // sign for `targetSlot` and delay submission to the start of `targetSlot`.
1168
+ const voter = new CheckpointVoter(targetSlot, publisher, attestorAddress, this.validatorClient, this.slasherClient, this.l1Constants, this.config, this.metrics, this.log);
944
1169
  const votesPromises = voter.enqueueVotes();
945
1170
  const votes = await Promise.all(votesPromises);
946
1171
  if (votes.every((p)=>!p)) {
@@ -948,9 +1173,20 @@ _dec = trackSpan('Sequencer.work'), _dec1 = trackSpan('Sequencer.prepareCheckpoi
948
1173
  return;
949
1174
  }
950
1175
  this.log.info(`Voting in slot ${slot} (escape hatch open)`, {
951
- slot
1176
+ slot,
1177
+ targetSlot
1178
+ });
1179
+ // Votes are EIP-712-signed for `targetSlot`. Delay submission to the start of `targetSlot` so
1180
+ // the multicall mines in the slot the votes were signed for; otherwise the L1 contract reads
1181
+ // `signaler = getCurrentProposer()` against the wrong slot and signature verification fails
1182
+ // silently inside Multicall3. Fire-and-forget so we don't block the sequencer's work loop while
1183
+ // waiting for the target slot to start, mirroring tryVoteWhenSyncFails.
1184
+ void publisher.sendRequestsAt(targetSlot).catch((err)=>{
1185
+ this.log.error(`Failed to publish escape-hatch votes for slot ${slot}`, err, {
1186
+ slot,
1187
+ targetSlot
1188
+ });
952
1189
  });
953
- await publisher.sendRequests();
954
1190
  }
955
1191
  /**
956
1192
  * Considers invalidating a block if the pending chain is invalid. Depends on how long the invalid block
@@ -958,7 +1194,7 @@ _dec = trackSpan('Sequencer.work'), _dec1 = trackSpan('Sequencer.prepareCheckpoi
958
1194
  * have the proposer try to invalidate, but if they fail, the sequencers in the committee are expected to try,
959
1195
  * and if they fail, any sequencer will try as well.
960
1196
  */ async considerInvalidatingCheckpoint(syncedTo, currentSlot) {
961
- const { pendingChainValidationStatus, l1Timestamp } = syncedTo;
1197
+ const { pendingChainValidationStatus, syncedL2Slot } = syncedTo;
962
1198
  if (pendingChainValidationStatus.valid) {
963
1199
  return;
964
1200
  }
@@ -969,7 +1205,7 @@ _dec = trackSpan('Sequencer.work'), _dec1 = trackSpan('Sequencer.prepareCheckpoi
969
1205
  const { secondsBeforeInvalidatingBlockAsCommitteeMember, secondsBeforeInvalidatingBlockAsNonCommitteeMember } = this.config;
970
1206
  const logData = {
971
1207
  invalidL1Timestamp: invalidCheckpointTimestamp,
972
- l1Timestamp,
1208
+ syncedL2Slot,
973
1209
  invalidCheckpoint: pendingChainValidationStatus.checkpoint,
974
1210
  secondsBeforeInvalidatingBlockAsCommitteeMember,
975
1211
  secondsBeforeInvalidatingBlockAsNonCommitteeMember,
@@ -1060,6 +1296,9 @@ _dec = trackSpan('Sequencer.work'), _dec1 = trackSpan('Sequencer.prepareCheckpoi
1060
1296
  getValidatorAddresses() {
1061
1297
  return this.validatorClient?.getValidatorAddresses();
1062
1298
  }
1299
+ /** Updates the publisher factory's node keystore adapter after a keystore reload. */ updatePublisherNodeKeyStore(adapter) {
1300
+ this.publisherFactory.updateNodeKeyStore(adapter);
1301
+ }
1063
1302
  getConfig() {
1064
1303
  return this.config;
1065
1304
  }