@aztec/sequencer-client 0.0.1-commit.96bb3f7 → 0.0.1-commit.993d240

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (133) hide show
  1. package/README.md +282 -21
  2. package/dest/client/sequencer-client.d.ts +19 -9
  3. package/dest/client/sequencer-client.d.ts.map +1 -1
  4. package/dest/client/sequencer-client.js +67 -33
  5. package/dest/config.d.ts +29 -7
  6. package/dest/config.d.ts.map +1 -1
  7. package/dest/config.js +77 -39
  8. package/dest/global_variable_builder/fee_predictor.d.ts +37 -0
  9. package/dest/global_variable_builder/fee_predictor.d.ts.map +1 -0
  10. package/dest/global_variable_builder/fee_predictor.js +128 -0
  11. package/dest/global_variable_builder/fee_provider.d.ts +21 -0
  12. package/dest/global_variable_builder/fee_provider.d.ts.map +1 -0
  13. package/dest/global_variable_builder/fee_provider.js +58 -0
  14. package/dest/global_variable_builder/global_builder.d.ts +14 -16
  15. package/dest/global_variable_builder/global_builder.d.ts.map +1 -1
  16. package/dest/global_variable_builder/global_builder.js +16 -50
  17. package/dest/global_variable_builder/index.d.ts +4 -2
  18. package/dest/global_variable_builder/index.d.ts.map +1 -1
  19. package/dest/global_variable_builder/index.js +2 -0
  20. package/dest/index.d.ts +2 -2
  21. package/dest/index.d.ts.map +1 -1
  22. package/dest/index.js +1 -1
  23. package/dest/publisher/config.d.ts +47 -17
  24. package/dest/publisher/config.d.ts.map +1 -1
  25. package/dest/publisher/config.js +121 -42
  26. package/dest/publisher/index.d.ts +2 -1
  27. package/dest/publisher/index.d.ts.map +1 -1
  28. package/dest/publisher/l1_tx_failed_store/factory.d.ts +11 -0
  29. package/dest/publisher/l1_tx_failed_store/factory.d.ts.map +1 -0
  30. package/dest/publisher/l1_tx_failed_store/factory.js +22 -0
  31. package/dest/publisher/l1_tx_failed_store/failed_tx_store.d.ts +58 -0
  32. package/dest/publisher/l1_tx_failed_store/failed_tx_store.d.ts.map +1 -0
  33. package/dest/publisher/l1_tx_failed_store/failed_tx_store.js +1 -0
  34. package/dest/publisher/l1_tx_failed_store/file_store_failed_tx_store.d.ts +15 -0
  35. package/dest/publisher/l1_tx_failed_store/file_store_failed_tx_store.d.ts.map +1 -0
  36. package/dest/publisher/l1_tx_failed_store/file_store_failed_tx_store.js +34 -0
  37. package/dest/publisher/l1_tx_failed_store/index.d.ts +4 -0
  38. package/dest/publisher/l1_tx_failed_store/index.d.ts.map +1 -0
  39. package/dest/publisher/l1_tx_failed_store/index.js +2 -0
  40. package/dest/publisher/sequencer-bundle-simulator.d.ts +96 -0
  41. package/dest/publisher/sequencer-bundle-simulator.d.ts.map +1 -0
  42. package/dest/publisher/sequencer-bundle-simulator.js +198 -0
  43. package/dest/publisher/sequencer-publisher-factory.d.ts +11 -5
  44. package/dest/publisher/sequencer-publisher-factory.d.ts.map +1 -1
  45. package/dest/publisher/sequencer-publisher-factory.js +27 -3
  46. package/dest/publisher/sequencer-publisher-metrics.d.ts +1 -1
  47. package/dest/publisher/sequencer-publisher-metrics.d.ts.map +1 -1
  48. package/dest/publisher/sequencer-publisher-metrics.js +12 -4
  49. package/dest/publisher/sequencer-publisher.d.ts +94 -68
  50. package/dest/publisher/sequencer-publisher.d.ts.map +1 -1
  51. package/dest/publisher/sequencer-publisher.js +446 -363
  52. package/dest/sequencer/automine/automine_factory.d.ts +54 -0
  53. package/dest/sequencer/automine/automine_factory.d.ts.map +1 -0
  54. package/dest/sequencer/automine/automine_factory.js +84 -0
  55. package/dest/sequencer/automine/automine_sequencer.d.ts +151 -0
  56. package/dest/sequencer/automine/automine_sequencer.d.ts.map +1 -0
  57. package/dest/sequencer/automine/automine_sequencer.js +506 -0
  58. package/dest/sequencer/chain_state_overrides.d.ts +61 -0
  59. package/dest/sequencer/chain_state_overrides.d.ts.map +1 -0
  60. package/dest/sequencer/chain_state_overrides.js +98 -0
  61. package/dest/sequencer/checkpoint_proposal_job.d.ts +81 -13
  62. package/dest/sequencer/checkpoint_proposal_job.d.ts.map +1 -1
  63. package/dest/sequencer/checkpoint_proposal_job.js +874 -195
  64. package/dest/sequencer/checkpoint_proposal_job_metrics.d.ts +34 -0
  65. package/dest/sequencer/checkpoint_proposal_job_metrics.d.ts.map +1 -0
  66. package/dest/sequencer/checkpoint_proposal_job_metrics.js +72 -0
  67. package/dest/sequencer/checkpoint_voter.d.ts +3 -3
  68. package/dest/sequencer/checkpoint_voter.d.ts.map +1 -1
  69. package/dest/sequencer/checkpoint_voter.js +34 -13
  70. package/dest/sequencer/events.d.ts +47 -2
  71. package/dest/sequencer/events.d.ts.map +1 -1
  72. package/dest/sequencer/index.d.ts +3 -2
  73. package/dest/sequencer/index.d.ts.map +1 -1
  74. package/dest/sequencer/index.js +2 -1
  75. package/dest/sequencer/metrics.d.ts +23 -8
  76. package/dest/sequencer/metrics.d.ts.map +1 -1
  77. package/dest/sequencer/metrics.js +142 -36
  78. package/dest/sequencer/sequencer.d.ts +87 -26
  79. package/dest/sequencer/sequencer.d.ts.map +1 -1
  80. package/dest/sequencer/sequencer.js +408 -135
  81. package/dest/sequencer/timetable.d.ts +17 -6
  82. package/dest/sequencer/timetable.d.ts.map +1 -1
  83. package/dest/sequencer/timetable.js +51 -46
  84. package/dest/sequencer/types.d.ts +2 -2
  85. package/dest/sequencer/types.d.ts.map +1 -1
  86. package/dest/test/index.d.ts +3 -5
  87. package/dest/test/index.d.ts.map +1 -1
  88. package/dest/test/mock_checkpoint_builder.d.ts +23 -19
  89. package/dest/test/mock_checkpoint_builder.d.ts.map +1 -1
  90. package/dest/test/mock_checkpoint_builder.js +67 -38
  91. package/dest/test/utils.d.ts +8 -8
  92. package/dest/test/utils.d.ts.map +1 -1
  93. package/dest/test/utils.js +17 -15
  94. package/package.json +29 -28
  95. package/src/client/sequencer-client.ts +92 -35
  96. package/src/config.ts +103 -49
  97. package/src/global_variable_builder/README.md +44 -0
  98. package/src/global_variable_builder/fee_predictor.ts +172 -0
  99. package/src/global_variable_builder/fee_provider.ts +75 -0
  100. package/src/global_variable_builder/global_builder.ts +26 -63
  101. package/src/global_variable_builder/index.ts +3 -1
  102. package/src/index.ts +5 -1
  103. package/src/publisher/config.ts +157 -45
  104. package/src/publisher/index.ts +3 -0
  105. package/src/publisher/l1_tx_failed_store/factory.ts +32 -0
  106. package/src/publisher/l1_tx_failed_store/failed_tx_store.ts +57 -0
  107. package/src/publisher/l1_tx_failed_store/file_store_failed_tx_store.ts +46 -0
  108. package/src/publisher/l1_tx_failed_store/index.ts +3 -0
  109. package/src/publisher/sequencer-bundle-simulator.ts +254 -0
  110. package/src/publisher/sequencer-publisher-factory.ts +38 -9
  111. package/src/publisher/sequencer-publisher-metrics.ts +7 -3
  112. package/src/publisher/sequencer-publisher.ts +540 -460
  113. package/src/sequencer/README.md +152 -450
  114. package/src/sequencer/automine/README.md +46 -0
  115. package/src/sequencer/automine/automine_factory.ts +145 -0
  116. package/src/sequencer/automine/automine_sequencer.ts +592 -0
  117. package/src/sequencer/chain_state_overrides.ts +162 -0
  118. package/src/sequencer/checkpoint_proposal_job.ts +1088 -219
  119. package/src/sequencer/checkpoint_proposal_job_metrics.ts +128 -0
  120. package/src/sequencer/checkpoint_voter.ts +31 -17
  121. package/src/sequencer/events.ts +50 -2
  122. package/src/sequencer/index.ts +2 -1
  123. package/src/sequencer/metrics.ts +164 -39
  124. package/src/sequencer/sequencer.ts +509 -147
  125. package/src/sequencer/timetable.ts +70 -57
  126. package/src/sequencer/types.ts +1 -1
  127. package/src/test/index.ts +2 -4
  128. package/src/test/mock_checkpoint_builder.ts +122 -78
  129. package/src/test/utils.ts +50 -22
  130. package/dest/sequencer/block_builder.d.ts +0 -26
  131. package/dest/sequencer/block_builder.d.ts.map +0 -1
  132. package/dest/sequencer/block_builder.js +0 -129
  133. package/src/sequencer/block_builder.ts +0 -216
@@ -370,24 +370,24 @@ function applyDecs2203RFactory() {
370
370
  function _apply_decs_2203_r(targetClass, memberDecs, classDecs, parentClass) {
371
371
  return (_apply_decs_2203_r = applyDecs2203RFactory())(targetClass, memberDecs, classDecs, parentClass);
372
372
  }
373
- var _dec, _dec1, _dec2, _initProto;
373
+ var _dec, _dec1, _dec2, _dec3, _initProto;
374
374
  import { getKzg } from '@aztec/blob-lib';
375
- import { INITIAL_L2_BLOCK_NUM } from '@aztec/constants';
375
+ import { PROPOSER_PIPELINING_SLOT_OFFSET } from '@aztec/epoch-cache';
376
376
  import { NoCommitteeError } from '@aztec/ethereum/contracts';
377
- import { BlockNumber, CheckpointNumber } from '@aztec/foundation/branded-types';
377
+ import { CheckpointNumber, SlotNumber } from '@aztec/foundation/branded-types';
378
378
  import { merge, omit, pick } from '@aztec/foundation/collection';
379
- import { Fr } from '@aztec/foundation/curves/bn254';
380
379
  import { EthAddress } from '@aztec/foundation/eth-address';
381
380
  import { createLogger } from '@aztec/foundation/log';
382
381
  import { RunningPromise } from '@aztec/foundation/running-promise';
383
- import { getSlotAtTimestamp, getSlotStartBuildTimestamp } from '@aztec/stdlib/epoch-helpers';
382
+ import { getEpochAtSlot, getSlotStartBuildTimestamp } from '@aztec/stdlib/epoch-helpers';
384
383
  import { SequencerConfigSchema } from '@aztec/stdlib/interfaces/server';
385
384
  import { pickFromSchema } from '@aztec/stdlib/schemas';
386
- import { MerkleTreeId } from '@aztec/stdlib/trees';
387
385
  import { Attributes, getTelemetryClient, trackSpan } from '@aztec/telemetry-client';
388
386
  import EventEmitter from 'node:events';
389
387
  import { DefaultSequencerConfig } from '../config.js';
388
+ import { buildCheckpointSimulationOverridesPlan } from './chain_state_overrides.js';
390
389
  import { CheckpointProposalJob } from './checkpoint_proposal_job.js';
390
+ import { CheckpointProposalJobMetrics } from './checkpoint_proposal_job_metrics.js';
391
391
  import { CheckpointVoter } from './checkpoint_voter.js';
392
392
  import { SequencerInterruptedError, SequencerTooSlowError } from './errors.js';
393
393
  import { SequencerMetrics } from './metrics.js';
@@ -396,6 +396,8 @@ import { SequencerState } from './utils.js';
396
396
  export { SequencerState };
397
397
  _dec = trackSpan('Sequencer.work'), _dec1 = trackSpan('Sequencer.prepareCheckpointProposal'), _dec2 = trackSpan('Seqeuencer.tryVoteWhenSyncFails', ({ slot })=>({
398
398
  [Attributes.SLOT_NUMBER]: slot
399
+ })), _dec3 = trackSpan('Sequencer.tryVoteWhenEscapeHatchOpen', ({ slot })=>({
400
+ [Attributes.SLOT_NUMBER]: slot
399
401
  }));
400
402
  /**
401
403
  * Sequencer client
@@ -436,50 +438,62 @@ _dec = trackSpan('Sequencer.work'), _dec1 = trackSpan('Sequencer.prepareCheckpoi
436
438
  _dec2,
437
439
  2,
438
440
  "tryVoteWhenSyncFails"
441
+ ],
442
+ [
443
+ _dec3,
444
+ 2,
445
+ "tryVoteWhenEscapeHatchOpen"
439
446
  ]
440
447
  ], []));
441
448
  }
442
449
  runningPromise;
443
450
  state;
451
+ stateSlotNumber;
452
+ stateEnteredAtMs;
444
453
  metrics;
445
- /** The last slot for which we attempted to vote when sync failed, to prevent duplicate attempts. */ lastSlotForVoteWhenSyncFailed;
454
+ checkpointProposalJobMetrics;
455
+ stateLog;
456
+ /** The last slot for which we attempted to perform our voting duties with degraded block production */ lastSlotForFallbackVote;
457
+ /** The last slot for which we logged "no committee" warning, to avoid spam */ lastSlotForNoCommitteeWarning;
446
458
  /** The last slot for which we triggered a checkpoint proposal job, to prevent duplicate attempts. */ lastSlotForCheckpointProposalJob;
447
459
  /** Last successful checkpoint proposed */ lastCheckpointProposed;
448
460
  /** The last epoch for which we logged strategy comparison in fisherman mode. */ lastEpochForStrategyComparison;
461
+ /** The last checkpoint proposal job, tracked so we can await its pending L1 submission during shutdown. */ lastCheckpointProposalJob;
449
462
  /** The maximum number of seconds that the sequencer can be into a slot to transition to a particular state. */ timetable;
450
- // This shouldn't be here as this gets re-created each time we build/propose a block.
451
- // But we have a number of tests that abuse/rely on this class having a permanent publisher.
452
- // As long as those tests only configure a single publisher they will continue to work.
453
- // This will get re-assigned every time the sequencer goes to build a new block to a publisher that is valid
454
- // for the block proposer.
455
- // TODO(palla/mbps): Remove this field and fix tests
456
- publisher;
457
463
  /** Config for the sequencer */ config;
464
+ signatureContext;
458
465
  constructor(publisherFactory, validatorClient, globalsBuilder, p2pClient, worldState, slasherClient, l2BlockSource, l1ToL2MessageSource, checkpointsBuilder, l1Constants, dateProvider, epochCache, rollupContract, config, telemetry = getTelemetryClient(), log = createLogger('sequencer')){
459
- super(), this.publisherFactory = publisherFactory, this.validatorClient = validatorClient, this.globalsBuilder = globalsBuilder, this.p2pClient = p2pClient, this.worldState = worldState, this.slasherClient = slasherClient, this.l2BlockSource = l2BlockSource, this.l1ToL2MessageSource = l1ToL2MessageSource, this.checkpointsBuilder = checkpointsBuilder, this.l1Constants = l1Constants, this.dateProvider = dateProvider, this.epochCache = epochCache, this.rollupContract = rollupContract, this.telemetry = telemetry, this.log = log, this.state = (_initProto(this), SequencerState.STOPPED), this.config = DefaultSequencerConfig;
466
+ super(), this.publisherFactory = publisherFactory, this.validatorClient = validatorClient, this.globalsBuilder = globalsBuilder, this.p2pClient = p2pClient, this.worldState = worldState, this.slasherClient = slasherClient, this.l2BlockSource = l2BlockSource, this.l1ToL2MessageSource = l1ToL2MessageSource, this.checkpointsBuilder = checkpointsBuilder, this.l1Constants = l1Constants, this.dateProvider = dateProvider, this.epochCache = epochCache, this.rollupContract = rollupContract, this.telemetry = telemetry, this.log = log, this.state = (_initProto(this), SequencerState.STOPPED), this.stateEnteredAtMs = performance.now(), this.config = DefaultSequencerConfig;
467
+ this.stateLog = log.createChild('state');
460
468
  // Add [FISHERMAN] prefix to logger if in fisherman mode
461
469
  if (config.fishermanMode) {
462
470
  this.log = log.createChild('[FISHERMAN]');
463
471
  }
472
+ this.signatureContext = {
473
+ chainId: config.l1ChainId,
474
+ rollupAddress: config.rollupAddress
475
+ };
464
476
  this.metrics = new SequencerMetrics(telemetry, this.rollupContract, 'Sequencer');
477
+ this.checkpointProposalJobMetrics = new CheckpointProposalJobMetrics(telemetry);
465
478
  this.updateConfig(config);
466
479
  }
467
480
  /** Updates sequencer config by the defined values and updates the timetable */ updateConfig(config) {
468
481
  const filteredConfig = pickFromSchema(config, SequencerConfigSchema);
469
- this.log.info(`Updated sequencer config`, omit(filteredConfig, 'txPublicSetupAllowList'));
482
+ this.log.info(`Updated sequencer config`, omit(filteredConfig, 'txPublicSetupAllowListExtend'));
470
483
  this.config = merge(this.config, filteredConfig);
484
+ const p2pPropagationTime = this.config.attestationPropagationTime;
471
485
  this.timetable = new SequencerTimetable({
472
486
  ethereumSlotDuration: this.l1Constants.ethereumSlotDuration,
473
487
  aztecSlotDuration: this.aztecSlotDuration,
474
488
  l1PublishingTime: this.l1PublishingTime,
475
- p2pPropagationTime: this.config.attestationPropagationTime,
489
+ p2pPropagationTime,
476
490
  blockDurationMs: this.config.blockDurationMs,
477
- enforce: this.config.enforceTimeTable
491
+ enforce: this.config.enforceTimeTable,
492
+ pipelining: this.epochCache.isProposerPipeliningEnabled()
478
493
  }, this.metrics, this.log);
479
494
  }
480
- /** Initializes the sequencer (precomputes tables and creates a publisher). Takes about 3s. */ async init() {
495
+ /** Initializes the sequencer (precomputes tables). Takes about 3s. */ init() {
481
496
  getKzg();
482
- this.publisher = (await this.publisherFactory.create(undefined)).publisher;
483
497
  }
484
498
  /** Starts the sequencer and moves to IDLE state. */ start() {
485
499
  this.runningPromise = new RunningPromise(this.safeWork.bind(this), this.log, this.config.sequencerPollingIntervalMS);
@@ -489,13 +503,17 @@ _dec = trackSpan('Sequencer.work'), _dec1 = trackSpan('Sequencer.prepareCheckpoi
489
503
  this.runningPromise.start();
490
504
  this.log.info('Started sequencer');
491
505
  }
506
+ /** Triggers an immediate run of the sequencer, bypassing the polling interval. */ trigger() {
507
+ return this.runningPromise?.trigger();
508
+ }
492
509
  /** Stops the sequencer from building blocks and moves to STOPPED state. */ async stop() {
493
510
  this.log.info(`Stopping sequencer`);
494
511
  this.setState(SequencerState.STOPPING, undefined, {
495
512
  force: true
496
513
  });
497
- this.publisher?.interrupt();
514
+ await this.publisherFactory.stopAll();
498
515
  await this.runningPromise?.stop();
516
+ await this.lastCheckpointProposalJob?.awaitPendingSubmission();
499
517
  this.setState(SequencerState.STOPPED, undefined, {
500
518
  force: true
501
519
  });
@@ -505,21 +523,19 @@ _dec = trackSpan('Sequencer.work'), _dec1 = trackSpan('Sequencer.prepareCheckpoi
505
523
  try {
506
524
  await this.work();
507
525
  } catch (err) {
508
- this.emit('checkpoint-error', {
509
- error: err
510
- });
511
- if (err instanceof SequencerTooSlowError) {
512
- // TODO(palla/mbps): Add missing states
513
- // Log as warn only if we had to abort halfway through the block proposal
514
- const logLvl = [
515
- SequencerState.INITIALIZING_CHECKPOINT,
516
- SequencerState.PROPOSER_CHECK
517
- ].includes(err.proposedState) ? 'debug' : 'warn';
518
- this.log[logLvl](err.message, {
526
+ if (err instanceof SequencerTooSlowError && [
527
+ SequencerState.INITIALIZING_CHECKPOINT,
528
+ SequencerState.PROPOSER_CHECK
529
+ ].includes(err.proposedState)) {
530
+ // No need to alert if we just didn't get to start in time
531
+ this.log.debug(err.message, {
519
532
  now: this.dateProvider.nowInSeconds()
520
533
  });
521
534
  } else {
522
535
  // Re-throw other errors
536
+ this.emit('checkpoint-error', {
537
+ error: err
538
+ });
523
539
  throw err;
524
540
  }
525
541
  } finally{
@@ -540,40 +556,62 @@ _dec = trackSpan('Sequencer.work'), _dec1 = trackSpan('Sequencer.prepareCheckpoi
540
556
  * - Submit checkpoint
541
557
  */ async work() {
542
558
  this.setState(SequencerState.SYNCHRONIZING, undefined);
543
- const { slot, ts, now, epoch } = this.epochCache.getEpochAndSlotInNextL1Slot();
559
+ const { slot, targetSlot, epoch, targetEpoch, ts, nowSeconds } = this.getSlotContextInNextL1Slot();
544
560
  // Check if we are synced and it's our slot, grab a publisher, check previous block invalidation, etc
545
- const checkpointProposalJob = await this.prepareCheckpointProposal(slot, ts, now);
561
+ const checkpointProposalJob = await this.prepareCheckpointProposal(slot, targetSlot, epoch, targetEpoch, ts, nowSeconds);
546
562
  if (!checkpointProposalJob) {
547
563
  return;
548
564
  }
565
+ // Track the job so we can await its pending L1 submission during shutdown
566
+ this.lastCheckpointProposalJob = checkpointProposalJob;
549
567
  // Execute the checkpoint proposal job
550
568
  const checkpoint = await checkpointProposalJob.execute();
551
569
  // Update last checkpoint proposed (currently unused)
552
570
  if (checkpoint) {
553
571
  this.lastCheckpointProposed = checkpoint;
554
572
  }
555
- // Log fee strategy comparison if on fisherman
556
- if (this.config.fishermanMode && (this.lastEpochForStrategyComparison === undefined || epoch > this.lastEpochForStrategyComparison)) {
557
- this.logStrategyComparison(epoch, checkpointProposalJob.getPublisher());
558
- this.lastEpochForStrategyComparison = epoch;
573
+ // Log fee strategy comparison if on fisherman (uses target epoch since we mirror the proposer's perspective)
574
+ if (this.config.fishermanMode && (this.lastEpochForStrategyComparison === undefined || targetEpoch > this.lastEpochForStrategyComparison)) {
575
+ this.logStrategyComparison(targetEpoch, checkpointProposalJob.getPublisher());
576
+ this.lastEpochForStrategyComparison = targetEpoch;
559
577
  }
560
578
  return checkpoint;
561
579
  }
580
+ /** Returns slot and target slot from a single clock snapshot. */ getSlotContextInNextL1Slot() {
581
+ const { slot, ts, nowSeconds, epoch } = this.epochCache.getEpochAndSlotInNextL1Slot();
582
+ const targetSlot = SlotNumber(slot + (this.epochCache.isProposerPipeliningEnabled() ? PROPOSER_PIPELINING_SLOT_OFFSET : 0));
583
+ return {
584
+ slot,
585
+ targetSlot,
586
+ epoch,
587
+ targetEpoch: getEpochAtSlot(targetSlot, this.l1Constants),
588
+ ts,
589
+ nowSeconds
590
+ };
591
+ }
562
592
  /**
563
593
  * Prepares the checkpoint proposal by performing all necessary checks and setup.
564
594
  * This is the initial step in the main loop.
565
595
  * @returns CheckpointProposalJob if successful, undefined if we are not yet synced or are not the proposer.
566
- */ async prepareCheckpointProposal(slot, ts, now) {
567
- // Check we have not already processed this slot (cheapest check)
596
+ */ async prepareCheckpointProposal(slot, targetSlot, epoch, targetEpoch, ts, nowSeconds) {
597
+ // Check we have not already processed this target slot (cheapest check)
568
598
  // We only check this if enforce timetable is set, since we want to keep processing the same slot if we are not
569
599
  // running against actual time (eg when we use sandbox-style automining)
570
- if (this.lastSlotForCheckpointProposalJob && this.lastSlotForCheckpointProposalJob >= slot && this.config.enforceTimeTable) {
571
- this.log.trace(`Slot ${slot} has already been processed`);
600
+ if (this.lastSlotForCheckpointProposalJob && this.lastSlotForCheckpointProposalJob >= targetSlot && this.config.enforceTimeTable) {
601
+ this.log.trace(`Target slot ${targetSlot} has already been processed`);
602
+ return undefined;
603
+ }
604
+ // But if we have already proposed for this slot, then we definitely have to skip it, automining or not
605
+ if (this.lastCheckpointProposed && this.lastCheckpointProposed.header.slotNumber >= targetSlot) {
606
+ this.log.trace(`Slot ${targetSlot} has already been published as checkpoint ${this.lastCheckpointProposed.number}`);
572
607
  return undefined;
573
608
  }
574
- // But if we have already proposed for this slot, the we definitely have to skip it, automining or not
575
- if (this.lastCheckpointProposed && this.lastCheckpointProposed.header.slotNumber >= slot) {
576
- this.log.trace(`Slot ${slot} has already been published as checkpoint ${this.lastCheckpointProposed.number}`);
609
+ // Test-only: skip proposing for explicitly paused slots. Attestation paths run in the validator
610
+ // client and are not gated by this hook, so paused proposers still attest to others' proposals.
611
+ if (this.config.pauseProposingForSlots?.some((s)=>s === targetSlot)) {
612
+ this.log.warn(`Skipping proposal for paused slot ${targetSlot} (test-only pauseProposingForSlots hook)`, {
613
+ targetSlot
614
+ });
577
615
  return undefined;
578
616
  }
579
617
  // Check all components are synced to latest as seen by the archiver (queries all subsystems)
@@ -584,36 +622,66 @@ _dec = trackSpan('Sequencer.work'), _dec1 = trackSpan('Sequencer.prepareCheckpoi
584
622
  if (!syncedTo) {
585
623
  await this.tryVoteWhenSyncFails({
586
624
  slot,
625
+ targetSlot,
587
626
  ts
588
627
  });
589
628
  return undefined;
590
629
  }
630
+ // If escape hatch is open for the target epoch, do not start checkpoint proposal work and do not attempt invalidations.
631
+ // Still perform governance/slashing voting (as proposer) once per slot.
632
+ // When pipelining, we check the target epoch (slot+1's epoch) since that's the epoch we're building for.
633
+ const isEscapeHatchOpen = await this.epochCache.isEscapeHatchOpen(targetEpoch);
634
+ if (isEscapeHatchOpen) {
635
+ this.setState(SequencerState.PROPOSER_CHECK, slot);
636
+ const [canPropose, proposer] = await this.checkCanPropose(targetSlot);
637
+ if (canPropose) {
638
+ await this.tryVoteWhenEscapeHatchOpen({
639
+ slot,
640
+ targetSlot,
641
+ proposer
642
+ });
643
+ } else {
644
+ this.log.trace(`Escape hatch open but we are not proposer, skipping vote-only actions`, {
645
+ slot,
646
+ epoch,
647
+ proposer
648
+ });
649
+ }
650
+ return undefined;
651
+ }
591
652
  // Next checkpoint follows from the last synced one
592
653
  const checkpointNumber = CheckpointNumber(syncedTo.checkpointNumber + 1);
593
654
  const logCtx = {
594
- now,
595
- syncedToL1Ts: syncedTo.l1Timestamp,
596
- syncedToL2Slot: getSlotAtTimestamp(syncedTo.l1Timestamp, this.l1Constants),
655
+ nowSeconds,
656
+ syncedToL2Slot: syncedTo.syncedL2Slot,
597
657
  slot,
658
+ targetSlot,
598
659
  slotTs: ts,
599
660
  checkpointNumber,
600
661
  isPendingChainValid: pick(syncedTo.pendingChainValidationStatus, 'valid', 'reason', 'invalidIndex')
601
662
  };
602
- // Check that we are a proposer for the next slot
663
+ // Check that we are a proposer for the target slot.
603
664
  this.setState(SequencerState.PROPOSER_CHECK, slot);
604
- const [canPropose, proposer] = await this.checkCanPropose(slot);
665
+ const [canPropose, proposer] = await this.checkCanPropose(targetSlot);
605
666
  // If we are not a proposer check if we should invalidate an invalid checkpoint, and bail
606
667
  if (!canPropose) {
607
668
  await this.considerInvalidatingCheckpoint(syncedTo, slot);
608
669
  return undefined;
609
670
  }
610
- // Check that the slot is not taken by a block already (should never happen, since only us can propose for this slot)
611
- if (syncedTo.block && syncedTo.block.header.getSlot() >= slot) {
612
- this.log.warn(`Cannot propose block at next L2 slot ${slot} since that slot was taken by block ${syncedTo.blockNumber}`, {
671
+ // Guard: don't exceed 1-deep pipeline. Without a proposed checkpoint, we can only build
672
+ // confirmed + 1. With a proposed checkpoint, we can build confirmed + 2.
673
+ const confirmedCkpt = syncedTo.checkpointedCheckpointNumber;
674
+ if (checkpointNumber > confirmedCkpt + 2) {
675
+ this.log.verbose(`Skipping slot ${targetSlot}: checkpoint ${checkpointNumber} exceeds max pipeline depth (confirmed=${confirmedCkpt})`);
676
+ return undefined;
677
+ }
678
+ // Check that the target slot is not taken by a block already (should never happen, since only us can propose for this slot)
679
+ if (syncedTo.blockData && syncedTo.blockData.header.getSlot() >= targetSlot) {
680
+ this.log.warn(`Cannot propose block at target slot ${targetSlot} since that slot was taken by block ${syncedTo.blockNumber}`, {
613
681
  ...logCtx,
614
- block: syncedTo.block.header.toInspect()
682
+ block: syncedTo.blockData.header.toInspect()
615
683
  });
616
- this.metrics.recordBlockProposalPrecheckFailed('slot_already_taken');
684
+ this.metrics.recordCheckpointPrecheckFailed('slot_already_taken');
617
685
  return undefined;
618
686
  }
619
687
  // We now need to get ourselves a publisher.
@@ -623,42 +691,110 @@ _dec = trackSpan('Sequencer.work'), _dec1 = trackSpan('Sequencer.prepareCheckpoi
623
691
  const proposerForPublisher = this.config.fishermanMode ? undefined : proposer;
624
692
  const { attestorAddress, publisher } = await this.publisherFactory.create(proposerForPublisher);
625
693
  this.log.verbose(`Created publisher at address ${publisher.getSenderAddress()} for attestor ${attestorAddress}`);
626
- this.publisher = publisher;
627
- // In fisherman mode, set the actual proposer's address for simulations
628
- if (this.config.fishermanMode && proposer) {
629
- publisher.setProposerAddressForSimulation(proposer);
630
- this.log.debug(`Set proposer address ${proposer} for simulation in fisherman mode`);
631
- }
632
- // Prepare invalidation request if the pending chain is invalid (returns undefined if no need)
633
- const invalidateCheckpoint = await publisher.simulateInvalidateCheckpoint(syncedTo.pendingChainValidationStatus);
634
- // Check with the rollup contract if we can indeed propose at the next L2 slot. This check should not fail
635
- // if all the previous checks are good, but we do it just in case.
636
- const canProposeCheck = await publisher.canProposeAtNextEthBlock(syncedTo.archive, proposer ?? EthAddress.ZERO, invalidateCheckpoint);
694
+ const isPipelining = this.epochCache.isProposerPipeliningEnabled();
695
+ // Prepare invalidation request if the pending chain is invalid (returns undefined if no need).
696
+ // Only simulate invalidation when there's no proposed parent, since we assume the proposed parent
697
+ // will invalidate the currently invalid checkpoint on L1.
698
+ const invalidateCheckpoint = isPipelining && syncedTo.hasProposedCheckpoint || syncedTo.pendingChainValidationStatus.valid ? undefined : await publisher.simulateInvalidateCheckpoint(syncedTo.pendingChainValidationStatus);
699
+ // Determine the correct archive and L1 state overrides for the canProposeAt check.
700
+ // The L1 contract reads archives[proposedCheckpointNumber] and compares it with the provided archive.
701
+ // When invalidating or pipelining, the local archive may differ from L1's, so we adjust accordingly.
702
+ let archiveForCheck = syncedTo.archive;
703
+ if (isPipelining && syncedTo.hasProposedCheckpoint) {
704
+ this.metrics.recordPipelineDepth(syncedTo.checkpointNumber - syncedTo.checkpointedCheckpointNumber);
705
+ this.log.verbose(`Building on top of proposed checkpoint (pending=${syncedTo.proposedCheckpointData?.checkpointNumber}) for target slot ${targetSlot}`, {
706
+ targetSlot,
707
+ parentCheckpointNumber: CheckpointNumber(checkpointNumber - 1)
708
+ });
709
+ // Match what L1 will see at archives[pending] once the proposed parent lands: the parent's
710
+ // own archive root from the gossiped proposal. `syncedTo.archive` is the world-state-local
711
+ // view and can transiently diverge from the proposed parent (e.g. before the proposed
712
+ // parent's blocks have been applied locally); diverging here would cause the canProposeAt
713
+ // override to set archives[pending] to one value while we present another for comparison.
714
+ archiveForCheck = syncedTo.proposedCheckpointData.archive.root;
715
+ } else if (invalidateCheckpoint) {
716
+ // After invalidation, L1 will roll back to checkpoint N-1. The archive at N-1 already
717
+ // exists on L1, so we just pass the matching archive (the lastArchive of the invalid checkpoint).
718
+ archiveForCheck = invalidateCheckpoint.lastArchive;
719
+ this.metrics.recordPipelineDepth(0);
720
+ } else {
721
+ this.metrics.recordPipelineDepth(0);
722
+ }
723
+ // Build the simulation plan: pending/proven override from pipelining or invalidation (or the
724
+ // current snapshot when neither applies, to short-circuit any pending prune in simulation),
725
+ // plus the parent checkpoint cell and fee header when pipelining.
726
+ const simulationOverridesPlan = await buildCheckpointSimulationOverridesPlan({
727
+ checkpointNumber,
728
+ proposedCheckpointData: isPipelining && syncedTo.hasProposedCheckpoint ? syncedTo.proposedCheckpointData : undefined,
729
+ invalidateToPendingCheckpointNumber: invalidateCheckpoint?.forcePendingCheckpointNumber,
730
+ checkpointedCheckpointNumber: syncedTo.checkpointedCheckpointNumber,
731
+ rollup: this.rollupContract,
732
+ signatureContext: this.signatureContext,
733
+ log: this.log
734
+ });
735
+ // The plan always pins both pending/proven (to short-circuit `canPruneAtTime` in simulation),
736
+ // so `provenOverride` always reflects the assumed proven checkpoint we are pinning the
737
+ // simulation to. We additionally warn when the pin is load-bearing — i.e. when a prune would
738
+ // actually fire at the target slot without it — so observers can spot "we are building
739
+ // optimistically across a pruning boundary" in the logs.
740
+ const provenOverride = simulationOverridesPlan?.chainTipsOverride?.proven;
741
+ if (provenOverride !== undefined && await this.l2BlockSource.isPruneDueAtSlot(targetSlot)) {
742
+ this.log.warn(`Assuming proof for epoch ending at checkpoint ${provenOverride} lands by target slot ${targetSlot}`, {
743
+ checkpointNumber,
744
+ slot,
745
+ targetSlot,
746
+ provenOverride
747
+ });
748
+ }
749
+ this.emit('preparing-checkpoint', {
750
+ targetSlot,
751
+ checkpointNumber,
752
+ hadProposedParent: this.epochCache.isProposerPipeliningEnabled() && syncedTo.hasProposedCheckpoint,
753
+ provenOverride,
754
+ simulatedPending: simulationOverridesPlan?.chainTipsOverride?.pending
755
+ });
756
+ const canProposeCheck = await publisher.canProposeAt(archiveForCheck, proposer ?? EthAddress.ZERO, simulationOverridesPlan);
757
+ const proposeContext = {
758
+ hasProposedCheckpoint: syncedTo.hasProposedCheckpoint,
759
+ proposedCheckpointNumber: syncedTo.proposedCheckpointData?.checkpointNumber,
760
+ checkpointedCheckpointNumber: syncedTo.checkpointedCheckpointNumber,
761
+ isInvalidating: !!invalidateCheckpoint,
762
+ invalidatingCheckpointNumber: invalidateCheckpoint?.checkpointNumber,
763
+ archiveForCheck: archiveForCheck.toString(),
764
+ overridePendingCheckpointNumber: simulationOverridesPlan?.chainTipsOverride?.pending,
765
+ overrideArchive: simulationOverridesPlan?.pendingCheckpointState?.archive,
766
+ overrideFeeHeader: simulationOverridesPlan?.pendingCheckpointState?.feeHeader
767
+ };
637
768
  if (canProposeCheck === undefined) {
638
- this.log.warn(`Cannot propose checkpoint ${checkpointNumber} at slot ${slot} due to failed rollup contract check`, logCtx);
769
+ this.log.warn(`Cannot propose checkpoint ${checkpointNumber} at slot ${slot} due to failed rollup contract check`, {
770
+ ...logCtx,
771
+ ...proposeContext
772
+ });
639
773
  this.emit('proposer-rollup-check-failed', {
640
774
  reason: 'Rollup contract check failed',
641
775
  slot
642
776
  });
643
- this.metrics.recordBlockProposalPrecheckFailed('rollup_contract_check_failed');
777
+ this.metrics.recordCheckpointPrecheckFailed('rollup_contract_check_failed');
644
778
  return undefined;
645
779
  }
646
- if (canProposeCheck.slot !== slot) {
647
- this.log.warn(`Cannot propose block due to slot mismatch with rollup contract (this can be caused by a clock out of sync). Expected slot ${slot} but got ${canProposeCheck.slot}.`, {
780
+ if (canProposeCheck.slot !== targetSlot) {
781
+ this.log.warn(`Cannot propose block due to slot mismatch with rollup contract (this can be caused by a clock out of sync). Expected slot ${targetSlot} but got ${canProposeCheck.slot}.`, {
648
782
  ...logCtx,
783
+ ...proposeContext,
649
784
  rollup: canProposeCheck,
650
- expectedSlot: slot
785
+ expectedSlot: targetSlot
651
786
  });
652
787
  this.emit('proposer-rollup-check-failed', {
653
788
  reason: 'Slot mismatch',
654
789
  slot
655
790
  });
656
- this.metrics.recordBlockProposalPrecheckFailed('slot_mismatch');
791
+ this.metrics.recordCheckpointPrecheckFailed('slot_mismatch');
657
792
  return undefined;
658
793
  }
659
794
  if (canProposeCheck.checkpointNumber !== checkpointNumber) {
660
795
  this.log.warn(`Cannot propose due to block mismatch with rollup contract (this can be caused by a pending archiver sync). Expected checkpoint ${checkpointNumber} but got ${canProposeCheck.checkpointNumber}.`, {
661
796
  ...logCtx,
797
+ ...proposeContext,
662
798
  rollup: canProposeCheck,
663
799
  expectedSlot: slot
664
800
  });
@@ -666,19 +802,27 @@ _dec = trackSpan('Sequencer.work'), _dec1 = trackSpan('Sequencer.prepareCheckpoi
666
802
  reason: 'Block mismatch',
667
803
  slot
668
804
  });
669
- this.metrics.recordBlockProposalPrecheckFailed('block_number_mismatch');
805
+ this.metrics.recordCheckpointPrecheckFailed('block_number_mismatch');
670
806
  return undefined;
671
807
  }
672
- this.lastSlotForCheckpointProposalJob = slot;
673
- this.log.info(`Preparing checkpoint proposal ${checkpointNumber} at slot ${slot}`, {
808
+ this.lastSlotForCheckpointProposalJob = targetSlot;
809
+ await this.p2pClient.prepareForSlot(targetSlot);
810
+ this.log.info(`Preparing checkpoint proposal ${checkpointNumber} for target slot ${targetSlot} during wall-clock slot ${slot}`, {
674
811
  ...logCtx,
675
- proposer
812
+ ...proposeContext,
813
+ proposer,
814
+ pipeliningEnabled: this.epochCache.isProposerPipeliningEnabled()
676
815
  });
677
816
  // Create and return the checkpoint proposal job
678
- return this.createCheckpointProposalJob(slot, checkpointNumber, syncedTo.blockNumber, proposer, publisher, attestorAddress, invalidateCheckpoint);
817
+ return this.createCheckpointProposalJob(slot, targetSlot, targetEpoch, checkpointNumber, syncedTo.blockNumber, syncedTo.checkpointedCheckpointNumber, proposer, publisher, attestorAddress, invalidateCheckpoint, syncedTo.proposedCheckpointData);
818
+ }
819
+ createCheckpointProposalJob(slot, targetSlot, targetEpoch, checkpointNumber, syncedToBlockNumber, checkpointedCheckpointNumber, proposer, publisher, attestorAddress, invalidateCheckpoint, proposedCheckpointData) {
820
+ return new CheckpointProposalJob(slot, targetSlot, targetEpoch, checkpointNumber, syncedToBlockNumber, checkpointedCheckpointNumber, proposer, publisher, attestorAddress, invalidateCheckpoint, this.validatorClient, this.globalsBuilder, this.p2pClient, this.worldState, this.l1ToL2MessageSource, this.l2BlockSource, this.checkpointsBuilder, this.l2BlockSource, this.l1Constants, this.signatureContext, this.config, this.timetable, this.slasherClient, this.epochCache, this.dateProvider, this.metrics, this.checkpointProposalJobMetrics.createRecorder(), this, this.setState.bind(this), this.tracer, this.log.getBindings(), proposedCheckpointData);
679
821
  }
680
- createCheckpointProposalJob(slot, checkpointNumber, syncedToBlockNumber, proposer, publisher, attestorAddress, invalidateCheckpoint) {
681
- return new CheckpointProposalJob(slot, checkpointNumber, syncedToBlockNumber, proposer, publisher, attestorAddress, invalidateCheckpoint, this.validatorClient, this.globalsBuilder, this.p2pClient, this.worldState, this.l1ToL2MessageSource, this.checkpointsBuilder, this.l2BlockSource, this.l1Constants, this.config, this.timetable, this.slasherClient, this.epochCache, this.dateProvider, this.metrics, this, this.setState.bind(this), this.log, this.tracer);
822
+ /**
823
+ * Returns the current sequencer state.
824
+ */ getState() {
825
+ return this.state;
682
826
  }
683
827
  /**
684
828
  * Internal helper for setting the sequencer state and checks if we have enough time left in the slot to transition to the new state.
@@ -699,37 +843,52 @@ _dec = trackSpan('Sequencer.work'), _dec1 = trackSpan('Sequencer.prepareCheckpoi
699
843
  secondsIntoSlot = this.getSecondsIntoSlot(slotNumber);
700
844
  this.timetable.assertTimeLeft(proposedState, secondsIntoSlot);
701
845
  }
846
+ const oldState = this.state;
847
+ const oldStateSlotNumber = this.stateSlotNumber;
848
+ const stateChanged = proposedState !== oldState;
849
+ const transitionAtMs = performance.now();
850
+ const stateDurationMs = transitionAtMs - this.stateEnteredAtMs;
702
851
  const boringStates = [
703
852
  SequencerState.IDLE,
704
853
  SequencerState.SYNCHRONIZING
705
854
  ];
706
- const logLevel = boringStates.includes(proposedState) && boringStates.includes(this.state) ? 'trace' : 'debug';
707
- this.log[logLevel](`Transitioning from ${this.state} to ${proposedState}`, {
855
+ const logLevel = boringStates.includes(proposedState) && boringStates.includes(oldState) ? 'trace' : 'debug';
856
+ this.stateLog[logLevel](`Transitioning from ${oldState} to ${proposedState}`, {
857
+ oldState,
858
+ newState: proposedState,
708
859
  slotNumber,
709
- secondsIntoSlot
860
+ stateSlotNumber: oldStateSlotNumber,
861
+ secondsIntoSlot,
862
+ ...stateChanged && {
863
+ stateDurationMs: Math.ceil(stateDurationMs)
864
+ }
710
865
  });
711
866
  this.emit('state-changed', {
712
- oldState: this.state,
867
+ oldState,
713
868
  newState: proposedState,
714
869
  secondsIntoSlot,
715
870
  slot: slotNumber
716
871
  });
872
+ if (stateChanged) {
873
+ this.metrics.recordStateDuration(stateDurationMs, oldState);
874
+ this.stateEnteredAtMs = transitionAtMs;
875
+ this.stateSlotNumber = slotNumber;
876
+ }
717
877
  this.state = proposedState;
718
878
  }
719
879
  /**
720
880
  * Returns whether all dependencies have caught up.
721
881
  * We don't check against the previous block submitted since it may have been reorg'd out.
722
882
  */ async checkSync(args) {
723
- // Check that the archiver and dependencies have synced to the previous L1 slot at least
724
- // TODO(#14766): Archiver reports L1 timestamp based on L1 blocks seen, which means that a missed L1 block will
725
- // cause the archiver L1 timestamp to fall behind, and cause this sequencer to start processing one L1 slot later.
726
- const l1Timestamp = await this.l2BlockSource.getL1Timestamp();
727
- const { slot, ts } = args;
728
- if (l1Timestamp === undefined || l1Timestamp + BigInt(this.l1Constants.ethereumSlotDuration) < ts) {
883
+ // Check that the archiver has fully synced the L2 slot before the one we want to propose in.
884
+ // The archiver reports sync progress via L1 block timestamps and synced checkpoint slots.
885
+ // See getSyncedL2SlotNumber for how missed L1 blocks are handled.
886
+ const syncedL2Slot = await this.l2BlockSource.getSyncedL2SlotNumber();
887
+ const { slot } = args;
888
+ if (syncedL2Slot === undefined || syncedL2Slot + 1 < slot) {
729
889
  this.log.debug(`Cannot propose block at next L2 slot ${slot} due to pending sync from L1`, {
730
890
  slot,
731
- ts,
732
- l1Timestamp
891
+ syncedL2Slot
733
892
  });
734
893
  return undefined;
735
894
  }
@@ -738,68 +897,120 @@ _dec = trackSpan('Sequencer.work'), _dec1 = trackSpan('Sequencer.prepareCheckpoi
738
897
  number: syncSummary.latestBlockNumber,
739
898
  hash: syncSummary.latestBlockHash
740
899
  })),
741
- this.l2BlockSource.getL2Tips().then((t)=>t.proposed),
900
+ this.l2BlockSource.getL2Tips().then((t)=>({
901
+ proposed: t.proposed,
902
+ checkpointed: t.checkpointed,
903
+ proposedCheckpoint: t.proposedCheckpoint
904
+ })),
742
905
  this.p2pClient.getStatus().then((p2p)=>p2p.syncedToL2Block),
743
- this.l1ToL2MessageSource.getL2Tips().then((t)=>t.proposed),
744
- this.l2BlockSource.getPendingChainValidationStatus()
906
+ this.l1ToL2MessageSource.getL2Tips().then((t)=>({
907
+ proposed: t.proposed,
908
+ checkpointed: t.checkpointed
909
+ })),
910
+ this.l2BlockSource.getPendingChainValidationStatus(),
911
+ this.l2BlockSource.getProposedCheckpointData()
745
912
  ]);
746
- const [worldState, l2BlockSource, p2p, l1ToL2MessageSource, pendingChainValidationStatus] = syncedBlocks;
747
- // Handle zero as a special case, since the block hash won't match across services if we're changing the prefilled data for the genesis block,
748
- // as the world state can compute the new genesis block hash, but other components use the hardcoded constant.
749
- // TODO(palla/mbps): Fix the above. All components should be able to handle dynamic genesis block hashes.
750
- const result = l2BlockSource.number === 0 && worldState.number === 0 && p2p.number === 0 && l1ToL2MessageSource.number === 0 || worldState.hash === l2BlockSource.hash && p2p.hash === l2BlockSource.hash && l1ToL2MessageSource.hash === l2BlockSource.hash;
913
+ const [worldState, l2Tips, p2p, l1ToL2MessageSourceTips, pendingChainValidationStatus, proposedCheckpointData] = syncedBlocks;
914
+ const result = worldState.hash === l2Tips.proposed.hash && p2p.hash === l2Tips.proposed.hash && l1ToL2MessageSourceTips.proposed.hash === l2Tips.proposed.hash && l1ToL2MessageSourceTips.checkpointed.block.hash === l2Tips.checkpointed.block.hash && l1ToL2MessageSourceTips.checkpointed.checkpoint.hash === l2Tips.checkpointed.checkpoint.hash;
751
915
  if (!result) {
752
916
  this.log.debug(`Sequencer sync check failed`, {
753
917
  worldState,
754
- l2BlockSource,
918
+ l2BlockSource: l2Tips.proposed,
755
919
  p2p,
756
- l1ToL2MessageSource
920
+ l1ToL2MessageSourceTips
757
921
  });
758
922
  return undefined;
759
923
  }
760
- // Special case for genesis state
761
924
  const blockNumber = worldState.number;
762
- if (blockNumber < INITIAL_L2_BLOCK_NUM) {
763
- const archive = new Fr((await this.worldState.getCommitted().getTreeInfo(MerkleTreeId.ARCHIVE)).root);
764
- return {
765
- checkpointNumber: CheckpointNumber.ZERO,
766
- blockNumber: BlockNumber.ZERO,
767
- archive,
768
- l1Timestamp,
769
- pendingChainValidationStatus
770
- };
925
+ const blockData = await this.l2BlockSource.getBlockData({
926
+ number: blockNumber
927
+ });
928
+ if (!blockData) {
929
+ this.log.warn(`Sequencer sync check failed: failed to get L2 block data ${blockNumber} from the archiver`, {
930
+ blockNumber,
931
+ l2Tips,
932
+ syncedL2Slot,
933
+ ...args
934
+ });
935
+ return undefined;
936
+ }
937
+ // Refuse to build a checkpoint on top of a proposed block whose enclosing checkpoint was never
938
+ // proposed. Under pipelining we may have received and reexecuted such a block locally — advancing
939
+ // our world-state tip past the checkpointed tip — while the proposing node never published the
940
+ // matching proposed checkpoint (e.g. it crashed before assembling it). Building on this orphan block
941
+ // would fork the chain off a tip no other node can follow. The archiver prunes these orphan blocks
942
+ // once their build slot ends; this guard is the correctness barrier during the grace window before.
943
+ if (blockData.checkpointNumber > l2Tips.checkpointed.checkpoint.number && (l2Tips.proposedCheckpoint.checkpoint.number !== blockData.checkpointNumber || proposedCheckpointData?.checkpointNumber !== blockData.checkpointNumber)) {
944
+ this.log.warn(`Sequencer sync check failed: proposed block has no matching proposed checkpoint`, {
945
+ blockCheckpointNumber: blockData.checkpointNumber,
946
+ checkpointedCheckpointNumber: l2Tips.checkpointed.checkpoint.number,
947
+ proposedCheckpointTipNumber: l2Tips.proposedCheckpoint.checkpoint.number,
948
+ proposedCheckpointDataNumber: proposedCheckpointData?.checkpointNumber,
949
+ blockNumber: blockData.header.getBlockNumber(),
950
+ blockSlot: blockData.header.getSlot(),
951
+ syncedL2Slot,
952
+ ...args
953
+ });
954
+ return undefined;
955
+ }
956
+ const hasProposedCheckpoint = l2Tips.proposedCheckpoint.checkpoint.number > l2Tips.checkpointed.checkpoint.number;
957
+ // The l2Tips and proposedCheckpointData reads above come from independent archiver snapshots
958
+ // (a JS-side tips cache vs. a direct store read on `#proposedCheckpoints`). A concurrent archiver
959
+ // write that mutates both can be observed split, leaving us with `hasProposedCheckpoint=true` but
960
+ // no proposedCheckpointData (or one whose number doesn't match the tip). Refuse to proceed in that
961
+ // window — the next checkSync tick will see a coherent snapshot.
962
+ if (hasProposedCheckpoint && (!proposedCheckpointData || proposedCheckpointData.checkpointNumber !== l2Tips.proposedCheckpoint.checkpoint.number)) {
963
+ this.log.warn(`Sequencer sync check failed: inconsistent proposed-checkpoint state`, {
964
+ proposedCheckpointTipNumber: l2Tips.proposedCheckpoint.checkpoint.number,
965
+ checkpointedTipNumber: l2Tips.checkpointed.checkpoint.number,
966
+ proposedCheckpointDataNumber: proposedCheckpointData?.checkpointNumber,
967
+ syncedL2Slot,
968
+ ...args
969
+ });
970
+ return undefined;
771
971
  }
772
- const block = await this.l2BlockSource.getL2BlockNew(blockNumber);
773
- if (!block) {
774
- // this shouldn't really happen because a moment ago we checked that all components were in sync
775
- this.log.error(`Failed to get L2 block ${blockNumber} from the archiver with all components in sync`);
972
+ // Check that the proposed checkpoint is indeed the parent of the checkpoint we'll be building
973
+ // The checkpoint number to build is derived as blockData.checkpointNumber + 1
974
+ if (proposedCheckpointData && proposedCheckpointData.checkpointNumber !== blockData.checkpointNumber) {
975
+ this.log.warn(`Sequencer sync check failed: proposed checkpoint number mismatch`, {
976
+ proposedCheckpointNumber: proposedCheckpointData.checkpointNumber,
977
+ blockCheckpointNumber: blockData.checkpointNumber,
978
+ syncedL2Slot,
979
+ ...args
980
+ });
776
981
  return undefined;
777
982
  }
778
983
  return {
779
- block,
780
- blockNumber: block.number,
781
- checkpointNumber: block.checkpointNumber,
782
- archive: block.archive.root,
783
- l1Timestamp,
984
+ blockData,
985
+ blockNumber: blockData.header.getBlockNumber(),
986
+ checkpointNumber: blockData.checkpointNumber,
987
+ checkpointedCheckpointNumber: l2Tips.checkpointed.checkpoint.number,
988
+ archive: blockData.archive.root,
989
+ hasProposedCheckpoint,
990
+ proposedCheckpointData,
991
+ syncedL2Slot,
784
992
  pendingChainValidationStatus
785
993
  };
786
994
  }
787
995
  /**
788
996
  * Checks if we are the proposer for the next slot.
789
997
  * @returns True if we can propose, and the proposer address (undefined if anyone can propose)
790
- */ async checkCanPropose(slot) {
998
+ */ async checkCanPropose(targetSlot) {
791
999
  let proposer;
792
1000
  try {
793
- proposer = await this.epochCache.getProposerAttesterAddressInSlot(slot);
1001
+ proposer = await this.epochCache.getProposerAttesterAddressInSlot(targetSlot);
794
1002
  } catch (e) {
795
1003
  if (e instanceof NoCommitteeError) {
796
- this.log.warn(`Cannot propose at next L2 slot ${slot} since the committee does not exist on L1`);
1004
+ if (this.lastSlotForNoCommitteeWarning !== targetSlot) {
1005
+ this.lastSlotForNoCommitteeWarning = targetSlot;
1006
+ this.log.warn(`Cannot propose at target slot ${targetSlot} since the committee does not exist on L1`);
1007
+ }
797
1008
  return [
798
1009
  false,
799
1010
  undefined
800
1011
  ];
801
1012
  }
802
- this.log.error(`Error getting proposer for slot ${slot}`, e);
1013
+ this.log.error(`Error getting proposer for target slot ${targetSlot}`, e);
803
1014
  return [
804
1015
  false,
805
1016
  undefined
@@ -822,7 +1033,8 @@ _dec = trackSpan('Sequencer.work'), _dec1 = trackSpan('Sequencer.prepareCheckpoi
822
1033
  const validatorAddresses = this.validatorClient.getValidatorAddresses();
823
1034
  const weAreProposer = validatorAddresses.some((addr)=>addr.equals(proposer));
824
1035
  if (!weAreProposer) {
825
- this.log.debug(`Cannot propose at slot ${slot} since we are not a proposer`, {
1036
+ this.log.debug(`Cannot propose at target slot ${targetSlot} since we are not a proposer`, {
1037
+ targetSlot,
826
1038
  validatorAddresses,
827
1039
  proposer
828
1040
  });
@@ -831,6 +1043,10 @@ _dec = trackSpan('Sequencer.work'), _dec1 = trackSpan('Sequencer.prepareCheckpoi
831
1043
  proposer
832
1044
  ];
833
1045
  }
1046
+ this.log.info(`We are the proposer for pipeline slot ${targetSlot}`, {
1047
+ targetSlot,
1048
+ proposer
1049
+ });
834
1050
  return [
835
1051
  true,
836
1052
  proposer
@@ -840,9 +1056,9 @@ _dec = trackSpan('Sequencer.work'), _dec1 = trackSpan('Sequencer.prepareCheckpoi
840
1056
  * Tries to vote on slashing actions and governance when the sync check fails but we're past the max time for initializing a proposal.
841
1057
  * This allows the sequencer to participate in governance/slashing votes even when it cannot build blocks.
842
1058
  */ async tryVoteWhenSyncFails(args) {
843
- const { slot } = args;
1059
+ const { slot, targetSlot } = args;
844
1060
  // Prevent duplicate attempts in the same slot
845
- if (this.lastSlotForVoteWhenSyncFailed === slot) {
1061
+ if (this.lastSlotForFallbackVote === slot) {
846
1062
  this.log.trace(`Already attempted to vote in slot ${slot} (skipping)`);
847
1063
  return;
848
1064
  }
@@ -863,7 +1079,7 @@ _dec = trackSpan('Sequencer.work'), _dec1 = trackSpan('Sequencer.prepareCheckpoi
863
1079
  maxAllowedTime
864
1080
  });
865
1081
  // Check if we're a proposer or proposal is open
866
- const [canPropose, proposer] = await this.checkCanPropose(slot);
1082
+ const [canPropose, proposer] = await this.checkCanPropose(targetSlot);
867
1083
  if (!canPropose) {
868
1084
  this.log.trace(`Cannot vote in slot ${slot} since we are not a proposer`, {
869
1085
  slot,
@@ -872,15 +1088,15 @@ _dec = trackSpan('Sequencer.work'), _dec1 = trackSpan('Sequencer.prepareCheckpoi
872
1088
  return;
873
1089
  }
874
1090
  // Mark this slot as attempted
875
- this.lastSlotForVoteWhenSyncFailed = slot;
1091
+ this.lastSlotForFallbackVote = slot;
876
1092
  // Get a publisher for voting
877
1093
  const { attestorAddress, publisher } = await this.publisherFactory.create(proposer);
878
1094
  this.log.debug(`Attempting to vote despite sync failure at slot ${slot}`, {
879
1095
  attestorAddress,
880
1096
  slot
881
1097
  });
882
- // Enqueue governance and slashing votes
883
- const voter = new CheckpointVoter(slot, publisher, attestorAddress, this.validatorClient, this.slasherClient, this.l1Constants, this.config, this.metrics, this.log);
1098
+ // Enqueue governance and slashing votes (voter uses the target slot for L1 submission)
1099
+ const voter = new CheckpointVoter(targetSlot, publisher, attestorAddress, this.validatorClient, this.slasherClient, this.l1Constants, this.config, this.metrics, this.log);
884
1100
  const votesPromises = voter.enqueueVotes();
885
1101
  const votes = await Promise.all(votesPromises);
886
1102
  if (votes.every((p)=>!p)) {
@@ -890,15 +1106,69 @@ _dec = trackSpan('Sequencer.work'), _dec1 = trackSpan('Sequencer.prepareCheckpoi
890
1106
  this.log.info(`Voting in slot ${slot} despite sync failure`, {
891
1107
  slot
892
1108
  });
893
- await publisher.sendRequests();
1109
+ // Votes are EIP-712-signed for `targetSlot` (the pipelined slot in which the multicall is
1110
+ // expected to mine). Delay submission to the start of `targetSlot` so the tx mines in the
1111
+ // slot the votes were signed for. We fire-and-forget so we don't block the sequencer's
1112
+ // work loop while waiting for the target slot to start.
1113
+ void publisher.sendRequestsAt(targetSlot).catch((err)=>{
1114
+ this.log.error(`Failed to publish votes despite sync failure for slot ${slot}`, err, {
1115
+ slot
1116
+ });
1117
+ });
894
1118
  }
895
1119
  /**
896
- * Considers invalidating a checkpoint if the pending chain is invalid. Depends on how long the invalid checkpoint
1120
+ * Tries to vote on slashing actions and governance proposals when escape hatch is open.
1121
+ * This allows the sequencer to participate in voting without performing checkpoint proposal work.
1122
+ */ async tryVoteWhenEscapeHatchOpen(args) {
1123
+ const { slot, targetSlot, proposer } = args;
1124
+ // Prevent duplicate attempts in the same slot
1125
+ if (this.lastSlotForFallbackVote === slot) {
1126
+ this.log.trace(`Already attempted to vote in slot ${slot} (escape hatch open, skipping)`);
1127
+ return;
1128
+ }
1129
+ // Mark this slot as attempted
1130
+ this.lastSlotForFallbackVote = slot;
1131
+ const { attestorAddress, publisher } = await this.publisherFactory.create(proposer);
1132
+ this.log.debug(`Escape hatch open for slot ${slot}, attempting vote-only actions`, {
1133
+ slot,
1134
+ targetSlot,
1135
+ attestorAddress
1136
+ });
1137
+ // Under proposer pipelining, the multicall is expected to mine in `targetSlot` (slot + 1).
1138
+ // Governance and slashing votes are EIP-712-signed against the slot they will mine in, and the
1139
+ // L1 contract checks `msg.sender == getCurrentProposer()` using the mining slot. So we must
1140
+ // sign for `targetSlot` and delay submission to the start of `targetSlot`. When pipelining is
1141
+ // disabled `targetSlot == slot` and `sendRequestsAt` resolves with no extra sleep.
1142
+ const voter = new CheckpointVoter(targetSlot, publisher, attestorAddress, this.validatorClient, this.slasherClient, this.l1Constants, this.config, this.metrics, this.log);
1143
+ const votesPromises = voter.enqueueVotes();
1144
+ const votes = await Promise.all(votesPromises);
1145
+ if (votes.every((p)=>!p)) {
1146
+ this.log.debug(`No votes to enqueue for slot ${slot} (escape hatch open)`);
1147
+ return;
1148
+ }
1149
+ this.log.info(`Voting in slot ${slot} (escape hatch open)`, {
1150
+ slot,
1151
+ targetSlot
1152
+ });
1153
+ // Votes are EIP-712-signed for `targetSlot`. Delay submission to the start of `targetSlot` so
1154
+ // the multicall mines in the slot the votes were signed for; otherwise the L1 contract reads
1155
+ // `signaler = getCurrentProposer()` against the wrong slot and signature verification fails
1156
+ // silently inside Multicall3. Fire-and-forget so we don't block the sequencer's work loop while
1157
+ // waiting for the target slot to start, mirroring tryVoteWhenSyncFails.
1158
+ void publisher.sendRequestsAt(targetSlot).catch((err)=>{
1159
+ this.log.error(`Failed to publish escape-hatch votes for slot ${slot}`, err, {
1160
+ slot,
1161
+ targetSlot
1162
+ });
1163
+ });
1164
+ }
1165
+ /**
1166
+ * Considers invalidating a block if the pending chain is invalid. Depends on how long the invalid block
897
1167
  * has been there without being invalidated and whether the sequencer is in the committee or not. We always
898
1168
  * have the proposer try to invalidate, but if they fail, the sequencers in the committee are expected to try,
899
1169
  * and if they fail, any sequencer will try as well.
900
1170
  */ async considerInvalidatingCheckpoint(syncedTo, currentSlot) {
901
- const { pendingChainValidationStatus, l1Timestamp } = syncedTo;
1171
+ const { pendingChainValidationStatus, syncedL2Slot } = syncedTo;
902
1172
  if (pendingChainValidationStatus.valid) {
903
1173
  return;
904
1174
  }
@@ -909,7 +1179,7 @@ _dec = trackSpan('Sequencer.work'), _dec1 = trackSpan('Sequencer.prepareCheckpoi
909
1179
  const { secondsBeforeInvalidatingBlockAsCommitteeMember, secondsBeforeInvalidatingBlockAsNonCommitteeMember } = this.config;
910
1180
  const logData = {
911
1181
  invalidL1Timestamp: invalidCheckpointTimestamp,
912
- l1Timestamp,
1182
+ syncedL2Slot,
913
1183
  invalidCheckpoint: pendingChainValidationStatus.checkpoint,
914
1184
  secondsBeforeInvalidatingBlockAsCommitteeMember,
915
1185
  secondsBeforeInvalidatingBlockAsNonCommitteeMember,
@@ -1000,6 +1270,9 @@ _dec = trackSpan('Sequencer.work'), _dec1 = trackSpan('Sequencer.prepareCheckpoi
1000
1270
  getValidatorAddresses() {
1001
1271
  return this.validatorClient?.getValidatorAddresses();
1002
1272
  }
1273
+ /** Updates the publisher factory's node keystore adapter after a keystore reload. */ updatePublisherNodeKeyStore(adapter) {
1274
+ this.publisherFactory.updateNodeKeyStore(adapter);
1275
+ }
1003
1276
  getConfig() {
1004
1277
  return this.config;
1005
1278
  }