@aztec/sequencer-client 0.0.1-commit.023c3e5 → 0.0.1-commit.031e54b

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (125) hide show
  1. package/README.md +281 -21
  2. package/dest/client/sequencer-client.d.ts +19 -9
  3. package/dest/client/sequencer-client.d.ts.map +1 -1
  4. package/dest/client/sequencer-client.js +65 -32
  5. package/dest/config.d.ts +28 -6
  6. package/dest/config.d.ts.map +1 -1
  7. package/dest/config.js +76 -37
  8. package/dest/global_variable_builder/fee_predictor.d.ts +37 -0
  9. package/dest/global_variable_builder/fee_predictor.d.ts.map +1 -0
  10. package/dest/global_variable_builder/fee_predictor.js +128 -0
  11. package/dest/global_variable_builder/fee_provider.d.ts +21 -0
  12. package/dest/global_variable_builder/fee_provider.d.ts.map +1 -0
  13. package/dest/global_variable_builder/fee_provider.js +58 -0
  14. package/dest/global_variable_builder/global_builder.d.ts +14 -16
  15. package/dest/global_variable_builder/global_builder.d.ts.map +1 -1
  16. package/dest/global_variable_builder/global_builder.js +16 -50
  17. package/dest/global_variable_builder/index.d.ts +4 -2
  18. package/dest/global_variable_builder/index.d.ts.map +1 -1
  19. package/dest/global_variable_builder/index.js +2 -0
  20. package/dest/index.d.ts +2 -2
  21. package/dest/index.d.ts.map +1 -1
  22. package/dest/index.js +1 -1
  23. package/dest/publisher/config.d.ts +47 -17
  24. package/dest/publisher/config.d.ts.map +1 -1
  25. package/dest/publisher/config.js +121 -42
  26. package/dest/publisher/index.d.ts +2 -1
  27. package/dest/publisher/index.d.ts.map +1 -1
  28. package/dest/publisher/l1_tx_failed_store/factory.d.ts +11 -0
  29. package/dest/publisher/l1_tx_failed_store/factory.d.ts.map +1 -0
  30. package/dest/publisher/l1_tx_failed_store/factory.js +22 -0
  31. package/dest/publisher/l1_tx_failed_store/failed_tx_store.d.ts +58 -0
  32. package/dest/publisher/l1_tx_failed_store/failed_tx_store.d.ts.map +1 -0
  33. package/dest/publisher/l1_tx_failed_store/failed_tx_store.js +1 -0
  34. package/dest/publisher/l1_tx_failed_store/file_store_failed_tx_store.d.ts +15 -0
  35. package/dest/publisher/l1_tx_failed_store/file_store_failed_tx_store.d.ts.map +1 -0
  36. package/dest/publisher/l1_tx_failed_store/file_store_failed_tx_store.js +34 -0
  37. package/dest/publisher/l1_tx_failed_store/index.d.ts +4 -0
  38. package/dest/publisher/l1_tx_failed_store/index.d.ts.map +1 -0
  39. package/dest/publisher/l1_tx_failed_store/index.js +2 -0
  40. package/dest/publisher/sequencer-bundle-simulator.d.ts +96 -0
  41. package/dest/publisher/sequencer-bundle-simulator.d.ts.map +1 -0
  42. package/dest/publisher/sequencer-bundle-simulator.js +198 -0
  43. package/dest/publisher/sequencer-publisher-factory.d.ts +11 -5
  44. package/dest/publisher/sequencer-publisher-factory.d.ts.map +1 -1
  45. package/dest/publisher/sequencer-publisher-factory.js +27 -3
  46. package/dest/publisher/sequencer-publisher.d.ts +94 -67
  47. package/dest/publisher/sequencer-publisher.d.ts.map +1 -1
  48. package/dest/publisher/sequencer-publisher.js +439 -378
  49. package/dest/sequencer/automine/automine_factory.d.ts +54 -0
  50. package/dest/sequencer/automine/automine_factory.d.ts.map +1 -0
  51. package/dest/sequencer/automine/automine_factory.js +84 -0
  52. package/dest/sequencer/automine/automine_sequencer.d.ts +151 -0
  53. package/dest/sequencer/automine/automine_sequencer.d.ts.map +1 -0
  54. package/dest/sequencer/automine/automine_sequencer.js +509 -0
  55. package/dest/sequencer/chain_state_overrides.d.ts +61 -0
  56. package/dest/sequencer/chain_state_overrides.d.ts.map +1 -0
  57. package/dest/sequencer/chain_state_overrides.js +98 -0
  58. package/dest/sequencer/checkpoint_proposal_job.d.ts +65 -11
  59. package/dest/sequencer/checkpoint_proposal_job.d.ts.map +1 -1
  60. package/dest/sequencer/checkpoint_proposal_job.js +849 -249
  61. package/dest/sequencer/checkpoint_proposal_job_metrics.d.ts +34 -0
  62. package/dest/sequencer/checkpoint_proposal_job_metrics.d.ts.map +1 -0
  63. package/dest/sequencer/checkpoint_proposal_job_metrics.js +72 -0
  64. package/dest/sequencer/checkpoint_voter.d.ts +1 -2
  65. package/dest/sequencer/checkpoint_voter.d.ts.map +1 -1
  66. package/dest/sequencer/checkpoint_voter.js +2 -5
  67. package/dest/sequencer/events.d.ts +48 -2
  68. package/dest/sequencer/events.d.ts.map +1 -1
  69. package/dest/sequencer/index.d.ts +3 -1
  70. package/dest/sequencer/index.d.ts.map +1 -1
  71. package/dest/sequencer/index.js +2 -0
  72. package/dest/sequencer/metrics.d.ts +23 -8
  73. package/dest/sequencer/metrics.d.ts.map +1 -1
  74. package/dest/sequencer/metrics.js +117 -21
  75. package/dest/sequencer/sequencer.d.ts +85 -22
  76. package/dest/sequencer/sequencer.d.ts.map +1 -1
  77. package/dest/sequencer/sequencer.js +380 -137
  78. package/dest/sequencer/timetable.d.ts +14 -6
  79. package/dest/sequencer/timetable.d.ts.map +1 -1
  80. package/dest/sequencer/timetable.js +47 -46
  81. package/dest/sequencer/types.d.ts +2 -2
  82. package/dest/sequencer/types.d.ts.map +1 -1
  83. package/dest/test/index.d.ts +3 -5
  84. package/dest/test/index.d.ts.map +1 -1
  85. package/dest/test/mock_checkpoint_builder.d.ts +11 -11
  86. package/dest/test/mock_checkpoint_builder.d.ts.map +1 -1
  87. package/dest/test/mock_checkpoint_builder.js +45 -34
  88. package/dest/test/utils.d.ts +3 -3
  89. package/dest/test/utils.d.ts.map +1 -1
  90. package/dest/test/utils.js +10 -8
  91. package/package.json +27 -28
  92. package/src/client/sequencer-client.ts +91 -35
  93. package/src/config.ts +98 -46
  94. package/src/global_variable_builder/README.md +44 -0
  95. package/src/global_variable_builder/fee_predictor.ts +172 -0
  96. package/src/global_variable_builder/fee_provider.ts +75 -0
  97. package/src/global_variable_builder/global_builder.ts +26 -63
  98. package/src/global_variable_builder/index.ts +3 -1
  99. package/src/index.ts +10 -1
  100. package/src/publisher/config.ts +157 -45
  101. package/src/publisher/index.ts +3 -0
  102. package/src/publisher/l1_tx_failed_store/factory.ts +32 -0
  103. package/src/publisher/l1_tx_failed_store/failed_tx_store.ts +57 -0
  104. package/src/publisher/l1_tx_failed_store/file_store_failed_tx_store.ts +46 -0
  105. package/src/publisher/l1_tx_failed_store/index.ts +3 -0
  106. package/src/publisher/sequencer-bundle-simulator.ts +254 -0
  107. package/src/publisher/sequencer-publisher-factory.ts +38 -9
  108. package/src/publisher/sequencer-publisher.ts +532 -469
  109. package/src/sequencer/README.md +162 -450
  110. package/src/sequencer/automine/README.md +46 -0
  111. package/src/sequencer/automine/automine_factory.ts +145 -0
  112. package/src/sequencer/automine/automine_sequencer.ts +595 -0
  113. package/src/sequencer/chain_state_overrides.ts +162 -0
  114. package/src/sequencer/checkpoint_proposal_job.ts +1026 -268
  115. package/src/sequencer/checkpoint_proposal_job_metrics.ts +128 -0
  116. package/src/sequencer/checkpoint_voter.ts +1 -12
  117. package/src/sequencer/events.ts +51 -2
  118. package/src/sequencer/index.ts +2 -0
  119. package/src/sequencer/metrics.ts +132 -25
  120. package/src/sequencer/sequencer.ts +470 -156
  121. package/src/sequencer/timetable.ts +62 -56
  122. package/src/sequencer/types.ts +1 -1
  123. package/src/test/index.ts +2 -4
  124. package/src/test/mock_checkpoint_builder.ts +63 -49
  125. package/src/test/utils.ts +31 -10
@@ -372,22 +372,23 @@ function _apply_decs_2203_r(targetClass, memberDecs, classDecs, parentClass) {
372
372
  }
373
373
  var _dec, _dec1, _dec2, _dec3, _initProto;
374
374
  import { getKzg } from '@aztec/blob-lib';
375
- import { INITIAL_L2_BLOCK_NUM } from '@aztec/constants';
375
+ import { PROPOSER_PIPELINING_SLOT_OFFSET } from '@aztec/epoch-cache';
376
376
  import { NoCommitteeError } from '@aztec/ethereum/contracts';
377
- import { BlockNumber, CheckpointNumber } from '@aztec/foundation/branded-types';
377
+ import { CheckpointNumber, SlotNumber } from '@aztec/foundation/branded-types';
378
378
  import { merge, omit, pick } from '@aztec/foundation/collection';
379
- import { Fr } from '@aztec/foundation/curves/bn254';
380
379
  import { EthAddress } from '@aztec/foundation/eth-address';
381
380
  import { createLogger } from '@aztec/foundation/log';
382
381
  import { RunningPromise } from '@aztec/foundation/running-promise';
383
- import { getSlotAtTimestamp, getSlotStartBuildTimestamp } from '@aztec/stdlib/epoch-helpers';
382
+ import { getEpochAtSlot, getSlotStartBuildTimestamp, getTimestampForSlot } from '@aztec/stdlib/epoch-helpers';
384
383
  import { SequencerConfigSchema } from '@aztec/stdlib/interfaces/server';
385
384
  import { pickFromSchema } from '@aztec/stdlib/schemas';
386
- import { MerkleTreeId } from '@aztec/stdlib/trees';
385
+ import { MIN_EXECUTION_TIME } from '@aztec/stdlib/timetable';
387
386
  import { Attributes, getTelemetryClient, trackSpan } from '@aztec/telemetry-client';
388
387
  import EventEmitter from 'node:events';
389
388
  import { DefaultSequencerConfig } from '../config.js';
389
+ import { buildCheckpointSimulationOverridesPlan } from './chain_state_overrides.js';
390
390
  import { CheckpointProposalJob } from './checkpoint_proposal_job.js';
391
+ import { CheckpointProposalJobMetrics } from './checkpoint_proposal_job_metrics.js';
391
392
  import { CheckpointVoter } from './checkpoint_voter.js';
392
393
  import { SequencerInterruptedError, SequencerTooSlowError } from './errors.js';
393
394
  import { SequencerMetrics } from './metrics.js';
@@ -448,45 +449,51 @@ _dec = trackSpan('Sequencer.work'), _dec1 = trackSpan('Sequencer.prepareCheckpoi
448
449
  }
449
450
  runningPromise;
450
451
  state;
452
+ stateSlotNumber;
453
+ stateEnteredAtMs;
451
454
  metrics;
455
+ checkpointProposalJobMetrics;
456
+ stateLog;
452
457
  /** The last slot for which we attempted to perform our voting duties with degraded block production */ lastSlotForFallbackVote;
458
+ /** The last slot for which we logged "no committee" warning, to avoid spam */ lastSlotForNoCommitteeWarning;
453
459
  /** The last slot for which we triggered a checkpoint proposal job, to prevent duplicate attempts. */ lastSlotForCheckpointProposalJob;
454
460
  /** Last successful checkpoint proposed */ lastCheckpointProposed;
455
461
  /** The last epoch for which we logged strategy comparison in fisherman mode. */ lastEpochForStrategyComparison;
462
+ /** The last checkpoint proposal job, tracked so we can await its pending L1 submission during shutdown. */ lastCheckpointProposalJob;
456
463
  /** The maximum number of seconds that the sequencer can be into a slot to transition to a particular state. */ timetable;
457
- // This shouldn't be here as this gets re-created each time we build/propose a block.
458
- // But we have a number of tests that abuse/rely on this class having a permanent publisher.
459
- // As long as those tests only configure a single publisher they will continue to work.
460
- // This will get re-assigned every time the sequencer goes to build a new block to a publisher that is valid
461
- // for the block proposer.
462
- // TODO(palla/mbps): Remove this field and fix tests
463
- publisher;
464
464
  /** Config for the sequencer */ config;
465
+ signatureContext;
465
466
  constructor(publisherFactory, validatorClient, globalsBuilder, p2pClient, worldState, slasherClient, l2BlockSource, l1ToL2MessageSource, checkpointsBuilder, l1Constants, dateProvider, epochCache, rollupContract, config, telemetry = getTelemetryClient(), log = createLogger('sequencer')){
466
- super(), this.publisherFactory = publisherFactory, this.validatorClient = validatorClient, this.globalsBuilder = globalsBuilder, this.p2pClient = p2pClient, this.worldState = worldState, this.slasherClient = slasherClient, this.l2BlockSource = l2BlockSource, this.l1ToL2MessageSource = l1ToL2MessageSource, this.checkpointsBuilder = checkpointsBuilder, this.l1Constants = l1Constants, this.dateProvider = dateProvider, this.epochCache = epochCache, this.rollupContract = rollupContract, this.telemetry = telemetry, this.log = log, this.state = (_initProto(this), SequencerState.STOPPED), this.config = DefaultSequencerConfig;
467
+ super(), this.publisherFactory = publisherFactory, this.validatorClient = validatorClient, this.globalsBuilder = globalsBuilder, this.p2pClient = p2pClient, this.worldState = worldState, this.slasherClient = slasherClient, this.l2BlockSource = l2BlockSource, this.l1ToL2MessageSource = l1ToL2MessageSource, this.checkpointsBuilder = checkpointsBuilder, this.l1Constants = l1Constants, this.dateProvider = dateProvider, this.epochCache = epochCache, this.rollupContract = rollupContract, this.telemetry = telemetry, this.log = log, this.state = (_initProto(this), SequencerState.STOPPED), this.stateEnteredAtMs = performance.now(), this.config = DefaultSequencerConfig;
468
+ this.stateLog = log.createChild('state');
467
469
  // Add [FISHERMAN] prefix to logger if in fisherman mode
468
470
  if (config.fishermanMode) {
469
471
  this.log = log.createChild('[FISHERMAN]');
470
472
  }
473
+ this.signatureContext = {
474
+ chainId: config.l1ChainId,
475
+ rollupAddress: config.rollupAddress
476
+ };
471
477
  this.metrics = new SequencerMetrics(telemetry, this.rollupContract, 'Sequencer');
478
+ this.checkpointProposalJobMetrics = new CheckpointProposalJobMetrics(telemetry);
472
479
  this.updateConfig(config);
473
480
  }
474
481
  /** Updates sequencer config by the defined values and updates the timetable */ updateConfig(config) {
475
482
  const filteredConfig = pickFromSchema(config, SequencerConfigSchema);
476
- this.log.info(`Updated sequencer config`, omit(filteredConfig, 'txPublicSetupAllowList'));
483
+ this.log.info(`Updated sequencer config`, omit(filteredConfig, 'txPublicSetupAllowListExtend'));
477
484
  this.config = merge(this.config, filteredConfig);
485
+ const p2pPropagationTime = this.config.attestationPropagationTime;
478
486
  this.timetable = new SequencerTimetable({
479
487
  ethereumSlotDuration: this.l1Constants.ethereumSlotDuration,
480
488
  aztecSlotDuration: this.aztecSlotDuration,
481
489
  l1PublishingTime: this.l1PublishingTime,
482
- p2pPropagationTime: this.config.attestationPropagationTime,
490
+ p2pPropagationTime,
483
491
  blockDurationMs: this.config.blockDurationMs,
484
492
  enforce: this.config.enforceTimeTable
485
493
  }, this.metrics, this.log);
486
494
  }
487
- /** Initializes the sequencer (precomputes tables and creates a publisher). Takes about 3s. */ async init() {
495
+ /** Initializes the sequencer (precomputes tables). Takes about 3s. */ init() {
488
496
  getKzg();
489
- this.publisher = (await this.publisherFactory.create(undefined)).publisher;
490
497
  }
491
498
  /** Starts the sequencer and moves to IDLE state. */ start() {
492
499
  this.runningPromise = new RunningPromise(this.safeWork.bind(this), this.log, this.config.sequencerPollingIntervalMS);
@@ -496,13 +503,18 @@ _dec = trackSpan('Sequencer.work'), _dec1 = trackSpan('Sequencer.prepareCheckpoi
496
503
  this.runningPromise.start();
497
504
  this.log.info('Started sequencer');
498
505
  }
506
+ /** Triggers an immediate run of the sequencer, bypassing the polling interval. */ trigger() {
507
+ return this.runningPromise?.trigger();
508
+ }
499
509
  /** Stops the sequencer from building blocks and moves to STOPPED state. */ async stop() {
500
510
  this.log.info(`Stopping sequencer`);
501
511
  this.setState(SequencerState.STOPPING, undefined, {
502
512
  force: true
503
513
  });
504
- this.publisher?.interrupt();
514
+ this.lastCheckpointProposalJob?.interrupt();
515
+ await this.publisherFactory.stopAll();
505
516
  await this.runningPromise?.stop();
517
+ await this.lastCheckpointProposalJob?.awaitPendingSubmission();
506
518
  this.setState(SequencerState.STOPPED, undefined, {
507
519
  force: true
508
520
  });
@@ -512,21 +524,19 @@ _dec = trackSpan('Sequencer.work'), _dec1 = trackSpan('Sequencer.prepareCheckpoi
512
524
  try {
513
525
  await this.work();
514
526
  } catch (err) {
515
- this.emit('checkpoint-error', {
516
- error: err
517
- });
518
- if (err instanceof SequencerTooSlowError) {
519
- // TODO(palla/mbps): Add missing states
520
- // Log as warn only if we had to abort halfway through the block proposal
521
- const logLvl = [
522
- SequencerState.INITIALIZING_CHECKPOINT,
523
- SequencerState.PROPOSER_CHECK
524
- ].includes(err.proposedState) ? 'debug' : 'warn';
525
- this.log[logLvl](err.message, {
527
+ if (err instanceof SequencerTooSlowError && [
528
+ SequencerState.INITIALIZING_CHECKPOINT,
529
+ SequencerState.PROPOSER_CHECK
530
+ ].includes(err.proposedState)) {
531
+ // No need to alert if we just didn't get to start in time
532
+ this.log.debug(err.message, {
526
533
  now: this.dateProvider.nowInSeconds()
527
534
  });
528
535
  } else {
529
536
  // Re-throw other errors
537
+ this.emit('checkpoint-error', {
538
+ error: err
539
+ });
530
540
  throw err;
531
541
  }
532
542
  } finally{
@@ -547,40 +557,62 @@ _dec = trackSpan('Sequencer.work'), _dec1 = trackSpan('Sequencer.prepareCheckpoi
547
557
  * - Submit checkpoint
548
558
  */ async work() {
549
559
  this.setState(SequencerState.SYNCHRONIZING, undefined);
550
- const { slot, ts, now, epoch } = this.epochCache.getEpochAndSlotInNextL1Slot();
560
+ const { slot, targetSlot, epoch, targetEpoch, ts, nowSeconds } = this.getSlotContextInNextL1Slot();
551
561
  // Check if we are synced and it's our slot, grab a publisher, check previous block invalidation, etc
552
- const checkpointProposalJob = await this.prepareCheckpointProposal(epoch, slot, ts, now);
562
+ const checkpointProposalJob = await this.prepareCheckpointProposal(slot, targetSlot, epoch, targetEpoch, ts, nowSeconds);
553
563
  if (!checkpointProposalJob) {
554
564
  return;
555
565
  }
566
+ // Track the job so we can await its pending L1 submission during shutdown
567
+ this.lastCheckpointProposalJob = checkpointProposalJob;
556
568
  // Execute the checkpoint proposal job
557
569
  const checkpoint = await checkpointProposalJob.execute();
558
570
  // Update last checkpoint proposed (currently unused)
559
571
  if (checkpoint) {
560
572
  this.lastCheckpointProposed = checkpoint;
561
573
  }
562
- // Log fee strategy comparison if on fisherman
563
- if (this.config.fishermanMode && (this.lastEpochForStrategyComparison === undefined || epoch > this.lastEpochForStrategyComparison)) {
564
- this.logStrategyComparison(epoch, checkpointProposalJob.getPublisher());
565
- this.lastEpochForStrategyComparison = epoch;
574
+ // Log fee strategy comparison if on fisherman (uses target epoch since we mirror the proposer's perspective)
575
+ if (this.config.fishermanMode && (this.lastEpochForStrategyComparison === undefined || targetEpoch > this.lastEpochForStrategyComparison)) {
576
+ this.logStrategyComparison(targetEpoch, checkpointProposalJob.getPublisher());
577
+ this.lastEpochForStrategyComparison = targetEpoch;
566
578
  }
567
579
  return checkpoint;
568
580
  }
581
+ /** Returns slot and target slot from a single clock snapshot. */ getSlotContextInNextL1Slot() {
582
+ const { slot, ts, nowSeconds, epoch } = this.epochCache.getEpochAndSlotInNextL1Slot();
583
+ const targetSlot = SlotNumber(slot + PROPOSER_PIPELINING_SLOT_OFFSET);
584
+ return {
585
+ slot,
586
+ targetSlot,
587
+ epoch,
588
+ targetEpoch: getEpochAtSlot(targetSlot, this.l1Constants),
589
+ ts,
590
+ nowSeconds
591
+ };
592
+ }
569
593
  /**
570
594
  * Prepares the checkpoint proposal by performing all necessary checks and setup.
571
595
  * This is the initial step in the main loop.
572
596
  * @returns CheckpointProposalJob if successful, undefined if we are not yet synced or are not the proposer.
573
- */ async prepareCheckpointProposal(epoch, slot, ts, now) {
574
- // Check we have not already processed this slot (cheapest check)
597
+ */ async prepareCheckpointProposal(slot, targetSlot, epoch, targetEpoch, ts, nowSeconds) {
598
+ // Check we have not already processed this target slot (cheapest check)
575
599
  // We only check this if enforce timetable is set, since we want to keep processing the same slot if we are not
576
600
  // running against actual time (eg when we use sandbox-style automining)
577
- if (this.lastSlotForCheckpointProposalJob && this.lastSlotForCheckpointProposalJob >= slot && this.config.enforceTimeTable) {
578
- this.log.trace(`Slot ${slot} has already been processed`);
601
+ if (this.lastSlotForCheckpointProposalJob && this.lastSlotForCheckpointProposalJob >= targetSlot && this.config.enforceTimeTable) {
602
+ this.log.trace(`Target slot ${targetSlot} has already been processed`);
603
+ return undefined;
604
+ }
605
+ // But if we have already proposed for this slot, then we definitely have to skip it, automining or not
606
+ if (this.lastCheckpointProposed && this.lastCheckpointProposed.header.slotNumber >= targetSlot) {
607
+ this.log.trace(`Slot ${targetSlot} has already been published as checkpoint ${this.lastCheckpointProposed.number}`);
579
608
  return undefined;
580
609
  }
581
- // But if we have already proposed for this slot, the we definitely have to skip it, automining or not
582
- if (this.lastCheckpointProposed && this.lastCheckpointProposed.header.slotNumber >= slot) {
583
- this.log.trace(`Slot ${slot} has already been published as checkpoint ${this.lastCheckpointProposed.number}`);
610
+ // Test-only: skip proposing for explicitly paused slots. Attestation paths run in the validator
611
+ // client and are not gated by this hook, so paused proposers still attest to others' proposals.
612
+ if (this.config.pauseProposingForSlots?.some((s)=>s === targetSlot)) {
613
+ this.log.warn(`Skipping proposal for paused slot ${targetSlot} (test-only pauseProposingForSlots hook)`, {
614
+ targetSlot
615
+ });
584
616
  return undefined;
585
617
  }
586
618
  // Check all components are synced to latest as seen by the archiver (queries all subsystems)
@@ -591,19 +623,22 @@ _dec = trackSpan('Sequencer.work'), _dec1 = trackSpan('Sequencer.prepareCheckpoi
591
623
  if (!syncedTo) {
592
624
  await this.tryVoteWhenSyncFails({
593
625
  slot,
626
+ targetSlot,
594
627
  ts
595
628
  });
596
629
  return undefined;
597
630
  }
598
- // If escape hatch is open for this epoch, do not start checkpoint proposal work and do not attempt invalidations.
631
+ // If escape hatch is open for the target epoch, do not start checkpoint proposal work and do not attempt invalidations.
599
632
  // Still perform governance/slashing voting (as proposer) once per slot.
600
- const isEscapeHatchOpen = await this.epochCache.isEscapeHatchOpen(epoch);
633
+ // When pipelining, we check the target epoch (slot+1's epoch) since that's the epoch we're building for.
634
+ const isEscapeHatchOpen = await this.epochCache.isEscapeHatchOpen(targetEpoch);
601
635
  if (isEscapeHatchOpen) {
602
636
  this.setState(SequencerState.PROPOSER_CHECK, slot);
603
- const [canPropose, proposer] = await this.checkCanPropose(slot);
637
+ const [canPropose, proposer] = await this.checkCanPropose(targetSlot);
604
638
  if (canPropose) {
605
639
  await this.tryVoteWhenEscapeHatchOpen({
606
640
  slot,
641
+ targetSlot,
607
642
  proposer
608
643
  });
609
644
  } else {
@@ -618,29 +653,36 @@ _dec = trackSpan('Sequencer.work'), _dec1 = trackSpan('Sequencer.prepareCheckpoi
618
653
  // Next checkpoint follows from the last synced one
619
654
  const checkpointNumber = CheckpointNumber(syncedTo.checkpointNumber + 1);
620
655
  const logCtx = {
621
- now,
622
- syncedToL1Ts: syncedTo.l1Timestamp,
623
- syncedToL2Slot: getSlotAtTimestamp(syncedTo.l1Timestamp, this.l1Constants),
656
+ nowSeconds,
657
+ syncedToL2Slot: syncedTo.syncedL2Slot,
624
658
  slot,
659
+ targetSlot,
625
660
  slotTs: ts,
626
661
  checkpointNumber,
627
662
  isPendingChainValid: pick(syncedTo.pendingChainValidationStatus, 'valid', 'reason', 'invalidIndex')
628
663
  };
629
- // Check that we are a proposer for the next slot
664
+ // Check that we are a proposer for the target slot.
630
665
  this.setState(SequencerState.PROPOSER_CHECK, slot);
631
- const [canPropose, proposer] = await this.checkCanPropose(slot);
666
+ const [canPropose, proposer] = await this.checkCanPropose(targetSlot);
632
667
  // If we are not a proposer check if we should invalidate an invalid checkpoint, and bail
633
668
  if (!canPropose) {
634
669
  await this.considerInvalidatingCheckpoint(syncedTo, slot);
635
670
  return undefined;
636
671
  }
637
- // Check that the slot is not taken by a block already (should never happen, since only us can propose for this slot)
638
- if (syncedTo.block && syncedTo.block.header.getSlot() >= slot) {
639
- this.log.warn(`Cannot propose block at next L2 slot ${slot} since that slot was taken by block ${syncedTo.blockNumber}`, {
672
+ // Guard: don't exceed 1-deep pipeline. Without a proposed checkpoint, we can only build
673
+ // confirmed + 1. With a proposed checkpoint, we can build confirmed + 2.
674
+ const confirmedCkpt = syncedTo.checkpointedCheckpointNumber;
675
+ if (checkpointNumber > confirmedCkpt + 2) {
676
+ this.log.verbose(`Skipping slot ${targetSlot}: checkpoint ${checkpointNumber} exceeds max pipeline depth (confirmed=${confirmedCkpt})`);
677
+ return undefined;
678
+ }
679
+ // Check that the target slot is not taken by a block already (should never happen, since only us can propose for this slot)
680
+ if (syncedTo.blockData && syncedTo.blockData.header.getSlot() >= targetSlot) {
681
+ this.log.warn(`Cannot propose block at target slot ${targetSlot} since that slot was taken by block ${syncedTo.blockNumber}`, {
640
682
  ...logCtx,
641
- block: syncedTo.block.header.toInspect()
683
+ block: syncedTo.blockData.header.toInspect()
642
684
  });
643
- this.metrics.recordBlockProposalPrecheckFailed('slot_already_taken');
685
+ this.metrics.recordCheckpointPrecheckFailed('slot_already_taken');
644
686
  return undefined;
645
687
  }
646
688
  // We now need to get ourselves a publisher.
@@ -650,42 +692,109 @@ _dec = trackSpan('Sequencer.work'), _dec1 = trackSpan('Sequencer.prepareCheckpoi
650
692
  const proposerForPublisher = this.config.fishermanMode ? undefined : proposer;
651
693
  const { attestorAddress, publisher } = await this.publisherFactory.create(proposerForPublisher);
652
694
  this.log.verbose(`Created publisher at address ${publisher.getSenderAddress()} for attestor ${attestorAddress}`);
653
- this.publisher = publisher;
654
- // In fisherman mode, set the actual proposer's address for simulations
655
- if (this.config.fishermanMode && proposer) {
656
- publisher.setProposerAddressForSimulation(proposer);
657
- this.log.debug(`Set proposer address ${proposer} for simulation in fisherman mode`);
695
+ // Prepare invalidation request if the pending chain is invalid (returns undefined if no need).
696
+ // Only simulate invalidation when there's no proposed parent, since we assume the proposed parent
697
+ // will invalidate the currently invalid checkpoint on L1.
698
+ const invalidateCheckpoint = syncedTo.hasProposedCheckpoint || syncedTo.pendingChainValidationStatus.valid ? undefined : await publisher.simulateInvalidateCheckpoint(syncedTo.pendingChainValidationStatus);
699
+ // Determine the correct archive and L1 state overrides for the canProposeAt check.
700
+ // The L1 contract reads archives[proposedCheckpointNumber] and compares it with the provided archive.
701
+ // When invalidating or pipelining, the local archive may differ from L1's, so we adjust accordingly.
702
+ let archiveForCheck = syncedTo.archive;
703
+ if (syncedTo.hasProposedCheckpoint) {
704
+ this.metrics.recordPipelineDepth(syncedTo.checkpointNumber - syncedTo.checkpointedCheckpointNumber);
705
+ this.log.verbose(`Building on top of proposed checkpoint (pending=${syncedTo.proposedCheckpointData?.checkpointNumber}) for target slot ${targetSlot}`, {
706
+ targetSlot,
707
+ parentCheckpointNumber: CheckpointNumber(checkpointNumber - 1)
708
+ });
709
+ // Match what L1 will see at archives[pending] once the proposed parent lands: the parent's
710
+ // own archive root from the gossiped proposal. `syncedTo.archive` is the world-state-local
711
+ // view and can transiently diverge from the proposed parent (e.g. before the proposed
712
+ // parent's blocks have been applied locally); diverging here would cause the canProposeAt
713
+ // override to set archives[pending] to one value while we present another for comparison.
714
+ archiveForCheck = syncedTo.proposedCheckpointData.archive.root;
715
+ } else if (invalidateCheckpoint) {
716
+ // After invalidation, L1 will roll back to checkpoint N-1. The archive at N-1 already
717
+ // exists on L1, so we just pass the matching archive (the lastArchive of the invalid checkpoint).
718
+ archiveForCheck = invalidateCheckpoint.lastArchive;
719
+ this.metrics.recordPipelineDepth(0);
720
+ } else {
721
+ this.metrics.recordPipelineDepth(0);
658
722
  }
659
- // Prepare invalidation request if the pending chain is invalid (returns undefined if no need)
660
- const invalidateCheckpoint = await publisher.simulateInvalidateCheckpoint(syncedTo.pendingChainValidationStatus);
661
- // Check with the rollup contract if we can indeed propose at the next L2 slot. This check should not fail
662
- // if all the previous checks are good, but we do it just in case.
663
- const canProposeCheck = await publisher.canProposeAtNextEthBlock(syncedTo.archive, proposer ?? EthAddress.ZERO, invalidateCheckpoint);
723
+ // Build the simulation plan: pending/proven override from pipelining or invalidation (or the
724
+ // current snapshot when neither applies, to short-circuit any pending prune in simulation),
725
+ // plus the parent checkpoint cell and fee header when pipelining.
726
+ const simulationOverridesPlan = await buildCheckpointSimulationOverridesPlan({
727
+ checkpointNumber,
728
+ proposedCheckpointData: syncedTo.hasProposedCheckpoint ? syncedTo.proposedCheckpointData : undefined,
729
+ invalidateToPendingCheckpointNumber: invalidateCheckpoint?.forcePendingCheckpointNumber,
730
+ checkpointedCheckpointNumber: syncedTo.checkpointedCheckpointNumber,
731
+ rollup: this.rollupContract,
732
+ signatureContext: this.signatureContext,
733
+ log: this.log
734
+ });
735
+ // The plan always pins both pending/proven (to short-circuit `canPruneAtTime` in simulation),
736
+ // so `provenOverride` always reflects the assumed proven checkpoint we are pinning the
737
+ // simulation to. We additionally warn when the pin is load-bearing — i.e. when a prune would
738
+ // actually fire at the target slot without it — so observers can spot "we are building
739
+ // optimistically across a pruning boundary" in the logs.
740
+ const provenOverride = simulationOverridesPlan?.chainTipsOverride?.proven;
741
+ if (provenOverride !== undefined && await this.l2BlockSource.isPruneDueAtSlot(targetSlot)) {
742
+ this.log.warn(`Assuming proof for epoch ending at checkpoint ${provenOverride} lands by target slot ${targetSlot}`, {
743
+ checkpointNumber,
744
+ slot,
745
+ targetSlot,
746
+ provenOverride
747
+ });
748
+ }
749
+ this.emit('preparing-checkpoint', {
750
+ targetSlot,
751
+ checkpointNumber,
752
+ hadProposedParent: syncedTo.hasProposedCheckpoint,
753
+ provenOverride,
754
+ simulatedPending: simulationOverridesPlan?.chainTipsOverride?.pending
755
+ });
756
+ const canProposeCheck = await publisher.canProposeAt(archiveForCheck, proposer ?? EthAddress.ZERO, simulationOverridesPlan);
757
+ const proposeContext = {
758
+ hasProposedCheckpoint: syncedTo.hasProposedCheckpoint,
759
+ proposedCheckpointNumber: syncedTo.proposedCheckpointData?.checkpointNumber,
760
+ checkpointedCheckpointNumber: syncedTo.checkpointedCheckpointNumber,
761
+ isInvalidating: !!invalidateCheckpoint,
762
+ invalidatingCheckpointNumber: invalidateCheckpoint?.checkpointNumber,
763
+ archiveForCheck: archiveForCheck.toString(),
764
+ overridePendingCheckpointNumber: simulationOverridesPlan?.chainTipsOverride?.pending,
765
+ overrideArchive: simulationOverridesPlan?.pendingCheckpointState?.archive,
766
+ overrideFeeHeader: simulationOverridesPlan?.pendingCheckpointState?.feeHeader
767
+ };
664
768
  if (canProposeCheck === undefined) {
665
- this.log.warn(`Cannot propose checkpoint ${checkpointNumber} at slot ${slot} due to failed rollup contract check`, logCtx);
769
+ this.log.warn(`Cannot propose checkpoint ${checkpointNumber} at slot ${slot} due to failed rollup contract check`, {
770
+ ...logCtx,
771
+ ...proposeContext
772
+ });
666
773
  this.emit('proposer-rollup-check-failed', {
667
774
  reason: 'Rollup contract check failed',
668
775
  slot
669
776
  });
670
- this.metrics.recordBlockProposalPrecheckFailed('rollup_contract_check_failed');
777
+ this.metrics.recordCheckpointPrecheckFailed('rollup_contract_check_failed');
671
778
  return undefined;
672
779
  }
673
- if (canProposeCheck.slot !== slot) {
674
- this.log.warn(`Cannot propose block due to slot mismatch with rollup contract (this can be caused by a clock out of sync). Expected slot ${slot} but got ${canProposeCheck.slot}.`, {
780
+ if (canProposeCheck.slot !== targetSlot) {
781
+ this.log.warn(`Cannot propose block due to slot mismatch with rollup contract (this can be caused by a clock out of sync). Expected slot ${targetSlot} but got ${canProposeCheck.slot}.`, {
675
782
  ...logCtx,
783
+ ...proposeContext,
676
784
  rollup: canProposeCheck,
677
- expectedSlot: slot
785
+ expectedSlot: targetSlot
678
786
  });
679
787
  this.emit('proposer-rollup-check-failed', {
680
788
  reason: 'Slot mismatch',
681
789
  slot
682
790
  });
683
- this.metrics.recordBlockProposalPrecheckFailed('slot_mismatch');
791
+ this.metrics.recordCheckpointPrecheckFailed('slot_mismatch');
684
792
  return undefined;
685
793
  }
686
794
  if (canProposeCheck.checkpointNumber !== checkpointNumber) {
687
795
  this.log.warn(`Cannot propose due to block mismatch with rollup contract (this can be caused by a pending archiver sync). Expected checkpoint ${checkpointNumber} but got ${canProposeCheck.checkpointNumber}.`, {
688
796
  ...logCtx,
797
+ ...proposeContext,
689
798
  rollup: canProposeCheck,
690
799
  expectedSlot: slot
691
800
  });
@@ -693,25 +802,35 @@ _dec = trackSpan('Sequencer.work'), _dec1 = trackSpan('Sequencer.prepareCheckpoi
693
802
  reason: 'Block mismatch',
694
803
  slot
695
804
  });
696
- this.metrics.recordBlockProposalPrecheckFailed('block_number_mismatch');
805
+ this.metrics.recordCheckpointPrecheckFailed('block_number_mismatch');
697
806
  return undefined;
698
807
  }
699
- this.lastSlotForCheckpointProposalJob = slot;
700
- this.log.info(`Preparing checkpoint proposal ${checkpointNumber} at slot ${slot}`, {
808
+ this.lastSlotForCheckpointProposalJob = targetSlot;
809
+ await this.p2pClient.prepareForSlot(targetSlot);
810
+ this.log.info(`Preparing checkpoint proposal ${checkpointNumber} for target slot ${targetSlot} during wall-clock slot ${slot}`, {
701
811
  ...logCtx,
812
+ ...proposeContext,
702
813
  proposer
703
814
  });
704
815
  // Create and return the checkpoint proposal job
705
- return this.createCheckpointProposalJob(epoch, slot, checkpointNumber, syncedTo.blockNumber, proposer, publisher, attestorAddress, invalidateCheckpoint);
816
+ return this.createCheckpointProposalJob(slot, targetSlot, targetEpoch, checkpointNumber, syncedTo.blockNumber, syncedTo.checkpointedCheckpointNumber, proposer, publisher, attestorAddress, invalidateCheckpoint, syncedTo.proposedCheckpointData);
706
817
  }
707
- createCheckpointProposalJob(epoch, slot, checkpointNumber, syncedToBlockNumber, proposer, publisher, attestorAddress, invalidateCheckpoint) {
708
- return new CheckpointProposalJob(epoch, slot, checkpointNumber, syncedToBlockNumber, proposer, publisher, attestorAddress, invalidateCheckpoint, this.validatorClient, this.globalsBuilder, this.p2pClient, this.worldState, this.l1ToL2MessageSource, this.l2BlockSource, this.checkpointsBuilder, this.l2BlockSource, this.l1Constants, this.config, this.timetable, this.slasherClient, this.epochCache, this.dateProvider, this.metrics, this, this.setState.bind(this), this.tracer, this.log.getBindings());
818
+ createCheckpointProposalJob(slot, targetSlot, targetEpoch, checkpointNumber, syncedToBlockNumber, checkpointedCheckpointNumber, proposer, publisher, attestorAddress, invalidateCheckpoint, proposedCheckpointData) {
819
+ return new CheckpointProposalJob(slot, targetSlot, targetEpoch, checkpointNumber, syncedToBlockNumber, checkpointedCheckpointNumber, proposer, publisher, attestorAddress, invalidateCheckpoint, this.validatorClient, this.globalsBuilder, this.p2pClient, this.worldState, this.l1ToL2MessageSource, this.l2BlockSource, this.checkpointsBuilder, this.l2BlockSource, this.l1Constants, this.signatureContext, this.config, this.timetable, this.slasherClient, this.epochCache, this.dateProvider, this.metrics, this.checkpointProposalJobMetrics.createRecorder(), this, (state, slot, timeReferenceSlot)=>this.setState(state, slot, {
820
+ timeReferenceSlot
821
+ }), this.tracer, this.log.getBindings(), proposedCheckpointData);
822
+ }
823
+ /**
824
+ * Returns the current sequencer state.
825
+ */ getState() {
826
+ return this.state;
709
827
  }
710
828
  /**
711
829
  * Internal helper for setting the sequencer state and checks if we have enough time left in the slot to transition to the new state.
712
830
  * @param proposedState - The new state to transition to.
713
- * @param slotNumber - The current slot number.
831
+ * @param slotNumber - The logical slot associated with the new state.
714
832
  * @param force - Whether to force the transition even if the sequencer is stopped.
833
+ * @param timeReferenceSlot - The slot to use for deadline checks when it differs from the logical slot.
715
834
  */ setState(proposedState, slotNumber, opts = {}) {
716
835
  if (this.state === SequencerState.STOPPING && proposedState !== SequencerState.STOPPED && !opts.force) {
717
836
  this.log.warn(`Cannot set sequencer to ${proposedState} as it is stopping.`);
@@ -722,41 +841,59 @@ _dec = trackSpan('Sequencer.work'), _dec1 = trackSpan('Sequencer.prepareCheckpoi
722
841
  return;
723
842
  }
724
843
  let secondsIntoSlot = undefined;
725
- if (slotNumber !== undefined) {
726
- secondsIntoSlot = this.getSecondsIntoSlot(slotNumber);
844
+ const timeReferenceSlot = opts.timeReferenceSlot ?? slotNumber;
845
+ if (timeReferenceSlot !== undefined) {
846
+ secondsIntoSlot = this.getSecondsIntoSlot(timeReferenceSlot);
727
847
  this.timetable.assertTimeLeft(proposedState, secondsIntoSlot);
728
848
  }
849
+ const oldState = this.state;
850
+ const oldStateSlotNumber = this.stateSlotNumber;
851
+ const stateChanged = proposedState !== oldState;
852
+ const transitionAtMs = performance.now();
853
+ const stateDurationMs = transitionAtMs - this.stateEnteredAtMs;
729
854
  const boringStates = [
730
855
  SequencerState.IDLE,
731
856
  SequencerState.SYNCHRONIZING
732
857
  ];
733
- const logLevel = boringStates.includes(proposedState) && boringStates.includes(this.state) ? 'trace' : 'debug';
734
- this.log[logLevel](`Transitioning from ${this.state} to ${proposedState}`, {
858
+ const logLevel = boringStates.includes(proposedState) && boringStates.includes(oldState) ? 'trace' : 'debug';
859
+ this.stateLog[logLevel](`Transitioning from ${oldState} to ${proposedState}`, {
860
+ oldState,
861
+ newState: proposedState,
735
862
  slotNumber,
736
- secondsIntoSlot
863
+ timeReferenceSlot,
864
+ stateSlotNumber: oldStateSlotNumber,
865
+ secondsIntoSlot,
866
+ ...stateChanged && {
867
+ stateDurationMs: Math.ceil(stateDurationMs)
868
+ }
737
869
  });
738
870
  this.emit('state-changed', {
739
- oldState: this.state,
871
+ oldState,
740
872
  newState: proposedState,
741
873
  secondsIntoSlot,
742
- slot: slotNumber
874
+ slot: slotNumber,
875
+ timeReferenceSlot
743
876
  });
877
+ if (stateChanged) {
878
+ this.metrics.recordStateDuration(stateDurationMs, oldState);
879
+ this.stateEnteredAtMs = transitionAtMs;
880
+ this.stateSlotNumber = slotNumber;
881
+ }
744
882
  this.state = proposedState;
745
883
  }
746
884
  /**
747
885
  * Returns whether all dependencies have caught up.
748
886
  * We don't check against the previous block submitted since it may have been reorg'd out.
749
887
  */ async checkSync(args) {
750
- // Check that the archiver and dependencies have synced to the previous L1 slot at least
751
- // TODO(#14766): Archiver reports L1 timestamp based on L1 blocks seen, which means that a missed L1 block will
752
- // cause the archiver L1 timestamp to fall behind, and cause this sequencer to start processing one L1 slot later.
753
- const l1Timestamp = await this.l2BlockSource.getL1Timestamp();
754
- const { slot, ts } = args;
755
- if (l1Timestamp === undefined || l1Timestamp + BigInt(this.l1Constants.ethereumSlotDuration) < ts) {
888
+ // Check that the archiver has fully synced the L2 slot before the one we want to propose in.
889
+ // The archiver reports sync progress via L1 block timestamps and synced checkpoint slots.
890
+ // See getSyncedL2SlotNumber for how missed L1 blocks are handled.
891
+ const syncedL2Slot = await this.l2BlockSource.getSyncedL2SlotNumber();
892
+ const { slot } = args;
893
+ if (syncedL2Slot === undefined || syncedL2Slot + 1 < slot) {
756
894
  this.log.debug(`Cannot propose block at next L2 slot ${slot} due to pending sync from L1`, {
757
895
  slot,
758
- ts,
759
- l1Timestamp
896
+ syncedL2Slot
760
897
  });
761
898
  return undefined;
762
899
  }
@@ -765,68 +902,142 @@ _dec = trackSpan('Sequencer.work'), _dec1 = trackSpan('Sequencer.prepareCheckpoi
765
902
  number: syncSummary.latestBlockNumber,
766
903
  hash: syncSummary.latestBlockHash
767
904
  })),
768
- this.l2BlockSource.getL2Tips().then((t)=>t.proposed),
905
+ this.l2BlockSource.getL2Tips().then((t)=>({
906
+ proposed: t.proposed,
907
+ checkpointed: t.checkpointed,
908
+ proposedCheckpoint: t.proposedCheckpoint
909
+ })),
769
910
  this.p2pClient.getStatus().then((p2p)=>p2p.syncedToL2Block),
770
- this.l1ToL2MessageSource.getL2Tips().then((t)=>t.proposed),
771
- this.l2BlockSource.getPendingChainValidationStatus()
911
+ this.l1ToL2MessageSource.getL2Tips().then((t)=>({
912
+ proposed: t.proposed,
913
+ checkpointed: t.checkpointed
914
+ })),
915
+ this.l2BlockSource.getPendingChainValidationStatus(),
916
+ this.l2BlockSource.getProposedCheckpointData()
772
917
  ]);
773
- const [worldState, l2BlockSource, p2p, l1ToL2MessageSource, pendingChainValidationStatus] = syncedBlocks;
774
- // Handle zero as a special case, since the block hash won't match across services if we're changing the prefilled data for the genesis block,
775
- // as the world state can compute the new genesis block hash, but other components use the hardcoded constant.
776
- // TODO(palla/mbps): Fix the above. All components should be able to handle dynamic genesis block hashes.
777
- const result = l2BlockSource.number === 0 && worldState.number === 0 && p2p.number === 0 && l1ToL2MessageSource.number === 0 || worldState.hash === l2BlockSource.hash && p2p.hash === l2BlockSource.hash && l1ToL2MessageSource.hash === l2BlockSource.hash;
918
+ const [worldState, l2Tips, p2p, l1ToL2MessageSourceTips, pendingChainValidationStatus, proposedCheckpointData] = syncedBlocks;
919
+ const result = worldState.hash === l2Tips.proposed.hash && p2p.hash === l2Tips.proposed.hash && l1ToL2MessageSourceTips.proposed.hash === l2Tips.proposed.hash && l1ToL2MessageSourceTips.checkpointed.block.hash === l2Tips.checkpointed.block.hash && l1ToL2MessageSourceTips.checkpointed.checkpoint.hash === l2Tips.checkpointed.checkpoint.hash;
778
920
  if (!result) {
779
921
  this.log.debug(`Sequencer sync check failed`, {
780
922
  worldState,
781
- l2BlockSource,
923
+ l2BlockSource: l2Tips.proposed,
782
924
  p2p,
783
- l1ToL2MessageSource
925
+ l1ToL2MessageSourceTips
784
926
  });
785
927
  return undefined;
786
928
  }
787
- // Special case for genesis state
788
929
  const blockNumber = worldState.number;
789
- if (blockNumber < INITIAL_L2_BLOCK_NUM) {
790
- const archive = new Fr((await this.worldState.getCommitted().getTreeInfo(MerkleTreeId.ARCHIVE)).root);
791
- return {
792
- checkpointNumber: CheckpointNumber.ZERO,
793
- blockNumber: BlockNumber.ZERO,
794
- archive,
795
- l1Timestamp,
796
- pendingChainValidationStatus
930
+ const blockData = await this.l2BlockSource.getBlockData({
931
+ number: blockNumber
932
+ });
933
+ if (!blockData) {
934
+ this.log.warn(`Sequencer sync check failed: failed to get L2 block data ${blockNumber} from the archiver`, {
935
+ blockNumber,
936
+ l2Tips,
937
+ syncedL2Slot,
938
+ ...args
939
+ });
940
+ return undefined;
941
+ }
942
+ // Refuse to build a checkpoint on top of a proposed block whose enclosing checkpoint was never
943
+ // proposed. Under pipelining we may have received and reexecuted such a block locally — advancing
944
+ // our world-state tip past the checkpointed tip — while the proposing node never published the
945
+ // matching proposed checkpoint (e.g. it crashed before assembling it). Building on this orphan block
946
+ // would fork the chain off a tip no other node can follow. The archiver prunes these orphan blocks
947
+ // once their build slot ends; this guard is the correctness barrier during the grace window before.
948
+ if (blockData.checkpointNumber > l2Tips.checkpointed.checkpoint.number && (l2Tips.proposedCheckpoint.checkpoint.number !== blockData.checkpointNumber || proposedCheckpointData?.checkpointNumber !== blockData.checkpointNumber)) {
949
+ const logCtx = {
950
+ blockCheckpointNumber: blockData.checkpointNumber,
951
+ checkpointedCheckpointNumber: l2Tips.checkpointed.checkpoint.number,
952
+ proposedCheckpointTipNumber: l2Tips.proposedCheckpoint.checkpoint.number,
953
+ proposedCheckpointDataNumber: proposedCheckpointData?.checkpointNumber,
954
+ blockNumber: blockData.header.getBlockNumber(),
955
+ blockSlot: blockData.header.getSlot(),
956
+ syncedL2Slot,
957
+ ...args
797
958
  };
959
+ // Under pipelining the block proposal for a checkpoint leads its checkpoint proposal by up to one
960
+ // slot, so a world-state tip sitting in an as-yet-unproposed checkpoint is the expected steady state
961
+ // until that checkpoint is due. Only treat it as abnormal — and warn — once the checkpoint is overdue
962
+ // by the same deadline the archiver uses to prune the orphan block (see pruneOrphanProposedBlocks).
963
+ // Before then this is normal pipelining and we wait it out quietly.
964
+ if (this.isProposedCheckpointOverdue(blockData.header.getSlot())) {
965
+ this.log.warn(`Sequencer sync check failed: proposed block has no matching proposed checkpoint`, logCtx);
966
+ } else {
967
+ this.log.debug(`Waiting for proposed checkpoint to catch up with reexecuted block`, logCtx);
968
+ }
969
+ return undefined;
798
970
  }
799
- const block = await this.l2BlockSource.getL2Block(blockNumber);
800
- if (!block) {
801
- // this shouldn't really happen because a moment ago we checked that all components were in sync
802
- this.log.error(`Failed to get L2 block ${blockNumber} from the archiver with all components in sync`);
971
+ const hasProposedCheckpoint = l2Tips.proposedCheckpoint.checkpoint.number > l2Tips.checkpointed.checkpoint.number;
972
+ // The l2Tips and proposedCheckpointData reads above come from independent archiver snapshots
973
+ // (a JS-side tips cache vs. a direct store read on `#proposedCheckpoints`). A concurrent archiver
974
+ // write that mutates both can be observed split, leaving us with `hasProposedCheckpoint=true` but
975
+ // no proposedCheckpointData (or one whose number doesn't match the tip). Refuse to proceed in that
976
+ // window — the next checkSync tick will see a coherent snapshot.
977
+ if (hasProposedCheckpoint && (!proposedCheckpointData || proposedCheckpointData.checkpointNumber !== l2Tips.proposedCheckpoint.checkpoint.number)) {
978
+ this.log.warn(`Sequencer sync check failed: inconsistent proposed-checkpoint state`, {
979
+ proposedCheckpointTipNumber: l2Tips.proposedCheckpoint.checkpoint.number,
980
+ checkpointedTipNumber: l2Tips.checkpointed.checkpoint.number,
981
+ proposedCheckpointDataNumber: proposedCheckpointData?.checkpointNumber,
982
+ syncedL2Slot,
983
+ ...args
984
+ });
985
+ return undefined;
986
+ }
987
+ // Check that the proposed checkpoint is indeed the parent of the checkpoint we'll be building
988
+ // The checkpoint number to build is derived as blockData.checkpointNumber + 1
989
+ if (proposedCheckpointData && proposedCheckpointData.checkpointNumber !== blockData.checkpointNumber) {
990
+ this.log.warn(`Sequencer sync check failed: proposed checkpoint number mismatch`, {
991
+ proposedCheckpointNumber: proposedCheckpointData.checkpointNumber,
992
+ blockCheckpointNumber: blockData.checkpointNumber,
993
+ syncedL2Slot,
994
+ ...args
995
+ });
803
996
  return undefined;
804
997
  }
805
998
  return {
806
- block,
807
- blockNumber: block.number,
808
- checkpointNumber: block.checkpointNumber,
809
- archive: block.archive.root,
810
- l1Timestamp,
999
+ blockData,
1000
+ blockNumber: blockData.header.getBlockNumber(),
1001
+ checkpointNumber: blockData.checkpointNumber,
1002
+ checkpointedCheckpointNumber: l2Tips.checkpointed.checkpoint.number,
1003
+ archive: blockData.archive.root,
1004
+ hasProposedCheckpoint,
1005
+ proposedCheckpointData,
1006
+ syncedL2Slot,
811
1007
  pendingChainValidationStatus
812
1008
  };
813
1009
  }
814
1010
  /**
1011
+ * Whether the enclosing checkpoint of a reexecuted block is overdue: past the deadline by which a
1012
+ * well-behaved proposer should have published it. Mirrors the archiver's orphan-prune deadline (the
1013
+ * start of the slot after the block's build slot, plus a grace period) so the sequencer only warns
1014
+ * about a missing proposed checkpoint once the archiver itself would prune the orphan block. The grace
1015
+ * is derived from the block build duration the same way the archiver defaults it at node wiring.
1016
+ */ isProposedCheckpointOverdue(blockSlot) {
1017
+ const expectedBySlot = SlotNumber(Number(blockSlot) - PROPOSER_PIPELINING_SLOT_OFFSET + 1);
1018
+ const graceSeconds = this.config.blockDurationMs !== undefined ? Math.ceil(this.config.blockDurationMs / 1000) : MIN_EXECUTION_TIME;
1019
+ const expectedByTime = getTimestampForSlot(expectedBySlot, this.l1Constants) + BigInt(graceSeconds);
1020
+ return BigInt(this.dateProvider.nowInSeconds()) >= expectedByTime;
1021
+ }
1022
+ /**
815
1023
  * Checks if we are the proposer for the next slot.
816
1024
  * @returns True if we can propose, and the proposer address (undefined if anyone can propose)
817
- */ async checkCanPropose(slot) {
1025
+ */ async checkCanPropose(targetSlot) {
818
1026
  let proposer;
819
1027
  try {
820
- proposer = await this.epochCache.getProposerAttesterAddressInSlot(slot);
1028
+ proposer = await this.epochCache.getProposerAttesterAddressInSlot(targetSlot);
821
1029
  } catch (e) {
822
1030
  if (e instanceof NoCommitteeError) {
823
- this.log.warn(`Cannot propose at next L2 slot ${slot} since the committee does not exist on L1`);
1031
+ if (this.lastSlotForNoCommitteeWarning !== targetSlot) {
1032
+ this.lastSlotForNoCommitteeWarning = targetSlot;
1033
+ this.log.warn(`Cannot propose at target slot ${targetSlot} since the committee does not exist on L1`);
1034
+ }
824
1035
  return [
825
1036
  false,
826
1037
  undefined
827
1038
  ];
828
1039
  }
829
- this.log.error(`Error getting proposer for slot ${slot}`, e);
1040
+ this.log.error(`Error getting proposer for target slot ${targetSlot}`, e);
830
1041
  return [
831
1042
  false,
832
1043
  undefined
@@ -849,7 +1060,8 @@ _dec = trackSpan('Sequencer.work'), _dec1 = trackSpan('Sequencer.prepareCheckpoi
849
1060
  const validatorAddresses = this.validatorClient.getValidatorAddresses();
850
1061
  const weAreProposer = validatorAddresses.some((addr)=>addr.equals(proposer));
851
1062
  if (!weAreProposer) {
852
- this.log.debug(`Cannot propose at slot ${slot} since we are not a proposer`, {
1063
+ this.log.debug(`Cannot propose at target slot ${targetSlot} since we are not a proposer`, {
1064
+ targetSlot,
853
1065
  validatorAddresses,
854
1066
  proposer
855
1067
  });
@@ -858,6 +1070,10 @@ _dec = trackSpan('Sequencer.work'), _dec1 = trackSpan('Sequencer.prepareCheckpoi
858
1070
  proposer
859
1071
  ];
860
1072
  }
1073
+ this.log.info(`We are the proposer for pipeline slot ${targetSlot}`, {
1074
+ targetSlot,
1075
+ proposer
1076
+ });
861
1077
  return [
862
1078
  true,
863
1079
  proposer
@@ -867,7 +1083,7 @@ _dec = trackSpan('Sequencer.work'), _dec1 = trackSpan('Sequencer.prepareCheckpoi
867
1083
  * Tries to vote on slashing actions and governance when the sync check fails but we're past the max time for initializing a proposal.
868
1084
  * This allows the sequencer to participate in governance/slashing votes even when it cannot build blocks.
869
1085
  */ async tryVoteWhenSyncFails(args) {
870
- const { slot } = args;
1086
+ const { slot, targetSlot } = args;
871
1087
  // Prevent duplicate attempts in the same slot
872
1088
  if (this.lastSlotForFallbackVote === slot) {
873
1089
  this.log.trace(`Already attempted to vote in slot ${slot} (skipping)`);
@@ -890,7 +1106,7 @@ _dec = trackSpan('Sequencer.work'), _dec1 = trackSpan('Sequencer.prepareCheckpoi
890
1106
  maxAllowedTime
891
1107
  });
892
1108
  // Check if we're a proposer or proposal is open
893
- const [canPropose, proposer] = await this.checkCanPropose(slot);
1109
+ const [canPropose, proposer] = await this.checkCanPropose(targetSlot);
894
1110
  if (!canPropose) {
895
1111
  this.log.trace(`Cannot vote in slot ${slot} since we are not a proposer`, {
896
1112
  slot,
@@ -906,8 +1122,8 @@ _dec = trackSpan('Sequencer.work'), _dec1 = trackSpan('Sequencer.prepareCheckpoi
906
1122
  attestorAddress,
907
1123
  slot
908
1124
  });
909
- // Enqueue governance and slashing votes
910
- const voter = new CheckpointVoter(slot, publisher, attestorAddress, this.validatorClient, this.slasherClient, this.l1Constants, this.config, this.metrics, this.log);
1125
+ // Enqueue governance and slashing votes (voter uses the target slot for L1 submission)
1126
+ const voter = new CheckpointVoter(targetSlot, publisher, attestorAddress, this.validatorClient, this.slasherClient, this.l1Constants, this.config, this.metrics, this.log);
911
1127
  const votesPromises = voter.enqueueVotes();
912
1128
  const votes = await Promise.all(votesPromises);
913
1129
  if (votes.every((p)=>!p)) {
@@ -917,13 +1133,21 @@ _dec = trackSpan('Sequencer.work'), _dec1 = trackSpan('Sequencer.prepareCheckpoi
917
1133
  this.log.info(`Voting in slot ${slot} despite sync failure`, {
918
1134
  slot
919
1135
  });
920
- await publisher.sendRequests();
1136
+ // Votes are EIP-712-signed for `targetSlot` (the pipelined slot in which the multicall is
1137
+ // expected to mine). Delay submission to the start of `targetSlot` so the tx mines in the
1138
+ // slot the votes were signed for. We fire-and-forget so we don't block the sequencer's
1139
+ // work loop while waiting for the target slot to start.
1140
+ void publisher.sendRequestsAt(targetSlot).catch((err)=>{
1141
+ this.log.error(`Failed to publish votes despite sync failure for slot ${slot}`, err, {
1142
+ slot
1143
+ });
1144
+ });
921
1145
  }
922
1146
  /**
923
1147
  * Tries to vote on slashing actions and governance proposals when escape hatch is open.
924
1148
  * This allows the sequencer to participate in voting without performing checkpoint proposal work.
925
1149
  */ async tryVoteWhenEscapeHatchOpen(args) {
926
- const { slot, proposer } = args;
1150
+ const { slot, targetSlot, proposer } = args;
927
1151
  // Prevent duplicate attempts in the same slot
928
1152
  if (this.lastSlotForFallbackVote === slot) {
929
1153
  this.log.trace(`Already attempted to vote in slot ${slot} (escape hatch open, skipping)`);
@@ -934,9 +1158,14 @@ _dec = trackSpan('Sequencer.work'), _dec1 = trackSpan('Sequencer.prepareCheckpoi
934
1158
  const { attestorAddress, publisher } = await this.publisherFactory.create(proposer);
935
1159
  this.log.debug(`Escape hatch open for slot ${slot}, attempting vote-only actions`, {
936
1160
  slot,
1161
+ targetSlot,
937
1162
  attestorAddress
938
1163
  });
939
- const voter = new CheckpointVoter(slot, publisher, attestorAddress, this.validatorClient, this.slasherClient, this.l1Constants, this.config, this.metrics, this.log);
1164
+ // Under proposer pipelining, the multicall is expected to mine in `targetSlot` (slot + 1).
1165
+ // Governance and slashing votes are EIP-712-signed against the slot they will mine in, and the
1166
+ // L1 contract checks `msg.sender == getCurrentProposer()` using the mining slot. So we must
1167
+ // sign for `targetSlot` and delay submission to the start of `targetSlot`.
1168
+ const voter = new CheckpointVoter(targetSlot, publisher, attestorAddress, this.validatorClient, this.slasherClient, this.l1Constants, this.config, this.metrics, this.log);
940
1169
  const votesPromises = voter.enqueueVotes();
941
1170
  const votes = await Promise.all(votesPromises);
942
1171
  if (votes.every((p)=>!p)) {
@@ -944,9 +1173,20 @@ _dec = trackSpan('Sequencer.work'), _dec1 = trackSpan('Sequencer.prepareCheckpoi
944
1173
  return;
945
1174
  }
946
1175
  this.log.info(`Voting in slot ${slot} (escape hatch open)`, {
947
- slot
1176
+ slot,
1177
+ targetSlot
1178
+ });
1179
+ // Votes are EIP-712-signed for `targetSlot`. Delay submission to the start of `targetSlot` so
1180
+ // the multicall mines in the slot the votes were signed for; otherwise the L1 contract reads
1181
+ // `signaler = getCurrentProposer()` against the wrong slot and signature verification fails
1182
+ // silently inside Multicall3. Fire-and-forget so we don't block the sequencer's work loop while
1183
+ // waiting for the target slot to start, mirroring tryVoteWhenSyncFails.
1184
+ void publisher.sendRequestsAt(targetSlot).catch((err)=>{
1185
+ this.log.error(`Failed to publish escape-hatch votes for slot ${slot}`, err, {
1186
+ slot,
1187
+ targetSlot
1188
+ });
948
1189
  });
949
- await publisher.sendRequests();
950
1190
  }
951
1191
  /**
952
1192
  * Considers invalidating a block if the pending chain is invalid. Depends on how long the invalid block
@@ -954,7 +1194,7 @@ _dec = trackSpan('Sequencer.work'), _dec1 = trackSpan('Sequencer.prepareCheckpoi
954
1194
  * have the proposer try to invalidate, but if they fail, the sequencers in the committee are expected to try,
955
1195
  * and if they fail, any sequencer will try as well.
956
1196
  */ async considerInvalidatingCheckpoint(syncedTo, currentSlot) {
957
- const { pendingChainValidationStatus, l1Timestamp } = syncedTo;
1197
+ const { pendingChainValidationStatus, syncedL2Slot } = syncedTo;
958
1198
  if (pendingChainValidationStatus.valid) {
959
1199
  return;
960
1200
  }
@@ -965,7 +1205,7 @@ _dec = trackSpan('Sequencer.work'), _dec1 = trackSpan('Sequencer.prepareCheckpoi
965
1205
  const { secondsBeforeInvalidatingBlockAsCommitteeMember, secondsBeforeInvalidatingBlockAsNonCommitteeMember } = this.config;
966
1206
  const logData = {
967
1207
  invalidL1Timestamp: invalidCheckpointTimestamp,
968
- l1Timestamp,
1208
+ syncedL2Slot,
969
1209
  invalidCheckpoint: pendingChainValidationStatus.checkpoint,
970
1210
  secondsBeforeInvalidatingBlockAsCommitteeMember,
971
1211
  secondsBeforeInvalidatingBlockAsNonCommitteeMember,
@@ -1056,6 +1296,9 @@ _dec = trackSpan('Sequencer.work'), _dec1 = trackSpan('Sequencer.prepareCheckpoi
1056
1296
  getValidatorAddresses() {
1057
1297
  return this.validatorClient?.getValidatorAddresses();
1058
1298
  }
1299
+ /** Updates the publisher factory's node keystore adapter after a keystore reload. */ updatePublisherNodeKeyStore(adapter) {
1300
+ this.publisherFactory.updateNodeKeyStore(adapter);
1301
+ }
1059
1302
  getConfig() {
1060
1303
  return this.config;
1061
1304
  }