@aztec/sequencer-client 0.0.1-commit.03f7ef2 → 0.0.1-commit.04d373f

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (145) hide show
  1. package/README.md +282 -21
  2. package/dest/client/sequencer-client.d.ts +20 -11
  3. package/dest/client/sequencer-client.d.ts.map +1 -1
  4. package/dest/client/sequencer-client.js +67 -33
  5. package/dest/config.d.ts +29 -7
  6. package/dest/config.d.ts.map +1 -1
  7. package/dest/config.js +82 -37
  8. package/dest/global_variable_builder/fee_predictor.d.ts +37 -0
  9. package/dest/global_variable_builder/fee_predictor.d.ts.map +1 -0
  10. package/dest/global_variable_builder/fee_predictor.js +128 -0
  11. package/dest/global_variable_builder/fee_provider.d.ts +21 -0
  12. package/dest/global_variable_builder/fee_provider.d.ts.map +1 -0
  13. package/dest/global_variable_builder/fee_provider.js +58 -0
  14. package/dest/global_variable_builder/global_builder.d.ts +14 -16
  15. package/dest/global_variable_builder/global_builder.d.ts.map +1 -1
  16. package/dest/global_variable_builder/global_builder.js +16 -50
  17. package/dest/global_variable_builder/index.d.ts +4 -2
  18. package/dest/global_variable_builder/index.d.ts.map +1 -1
  19. package/dest/global_variable_builder/index.js +2 -0
  20. package/dest/index.d.ts +2 -3
  21. package/dest/index.d.ts.map +1 -1
  22. package/dest/index.js +1 -2
  23. package/dest/publisher/config.d.ts +47 -17
  24. package/dest/publisher/config.d.ts.map +1 -1
  25. package/dest/publisher/config.js +121 -42
  26. package/dest/publisher/index.d.ts +2 -1
  27. package/dest/publisher/index.d.ts.map +1 -1
  28. package/dest/publisher/l1_tx_failed_store/factory.d.ts +11 -0
  29. package/dest/publisher/l1_tx_failed_store/factory.d.ts.map +1 -0
  30. package/dest/publisher/l1_tx_failed_store/factory.js +22 -0
  31. package/dest/publisher/l1_tx_failed_store/failed_tx_store.d.ts +58 -0
  32. package/dest/publisher/l1_tx_failed_store/failed_tx_store.d.ts.map +1 -0
  33. package/dest/publisher/l1_tx_failed_store/failed_tx_store.js +1 -0
  34. package/dest/publisher/l1_tx_failed_store/file_store_failed_tx_store.d.ts +15 -0
  35. package/dest/publisher/l1_tx_failed_store/file_store_failed_tx_store.d.ts.map +1 -0
  36. package/dest/publisher/l1_tx_failed_store/file_store_failed_tx_store.js +34 -0
  37. package/dest/publisher/l1_tx_failed_store/index.d.ts +4 -0
  38. package/dest/publisher/l1_tx_failed_store/index.d.ts.map +1 -0
  39. package/dest/publisher/l1_tx_failed_store/index.js +2 -0
  40. package/dest/publisher/sequencer-bundle-simulator.d.ts +96 -0
  41. package/dest/publisher/sequencer-bundle-simulator.d.ts.map +1 -0
  42. package/dest/publisher/sequencer-bundle-simulator.js +198 -0
  43. package/dest/publisher/sequencer-publisher-factory.d.ts +12 -6
  44. package/dest/publisher/sequencer-publisher-factory.d.ts.map +1 -1
  45. package/dest/publisher/sequencer-publisher-factory.js +27 -3
  46. package/dest/publisher/sequencer-publisher-metrics.d.ts +1 -1
  47. package/dest/publisher/sequencer-publisher-metrics.d.ts.map +1 -1
  48. package/dest/publisher/sequencer-publisher-metrics.js +23 -86
  49. package/dest/publisher/sequencer-publisher.d.ts +108 -81
  50. package/dest/publisher/sequencer-publisher.d.ts.map +1 -1
  51. package/dest/publisher/sequencer-publisher.js +876 -403
  52. package/dest/sequencer/automine/automine_factory.d.ts +54 -0
  53. package/dest/sequencer/automine/automine_factory.d.ts.map +1 -0
  54. package/dest/sequencer/automine/automine_factory.js +84 -0
  55. package/dest/sequencer/automine/automine_sequencer.d.ts +151 -0
  56. package/dest/sequencer/automine/automine_sequencer.d.ts.map +1 -0
  57. package/dest/sequencer/automine/automine_sequencer.js +506 -0
  58. package/dest/sequencer/chain_state_overrides.d.ts +61 -0
  59. package/dest/sequencer/chain_state_overrides.d.ts.map +1 -0
  60. package/dest/sequencer/chain_state_overrides.js +98 -0
  61. package/dest/sequencer/checkpoint_proposal_job.d.ts +87 -16
  62. package/dest/sequencer/checkpoint_proposal_job.d.ts.map +1 -1
  63. package/dest/sequencer/checkpoint_proposal_job.js +1331 -203
  64. package/dest/sequencer/checkpoint_proposal_job_metrics.d.ts +34 -0
  65. package/dest/sequencer/checkpoint_proposal_job_metrics.d.ts.map +1 -0
  66. package/dest/sequencer/checkpoint_proposal_job_metrics.js +72 -0
  67. package/dest/sequencer/checkpoint_voter.d.ts +3 -3
  68. package/dest/sequencer/checkpoint_voter.d.ts.map +1 -1
  69. package/dest/sequencer/checkpoint_voter.js +34 -13
  70. package/dest/sequencer/events.d.ts +47 -2
  71. package/dest/sequencer/events.d.ts.map +1 -1
  72. package/dest/sequencer/index.d.ts +3 -3
  73. package/dest/sequencer/index.d.ts.map +1 -1
  74. package/dest/sequencer/index.js +2 -2
  75. package/dest/sequencer/metrics.d.ts +23 -8
  76. package/dest/sequencer/metrics.d.ts.map +1 -1
  77. package/dest/sequencer/metrics.js +158 -143
  78. package/dest/sequencer/sequencer.d.ts +92 -29
  79. package/dest/sequencer/sequencer.d.ts.map +1 -1
  80. package/dest/sequencer/sequencer.js +818 -157
  81. package/dest/sequencer/timetable.d.ts +17 -6
  82. package/dest/sequencer/timetable.d.ts.map +1 -1
  83. package/dest/sequencer/timetable.js +51 -46
  84. package/dest/sequencer/types.d.ts +2 -2
  85. package/dest/sequencer/types.d.ts.map +1 -1
  86. package/dest/test/index.d.ts +4 -7
  87. package/dest/test/index.d.ts.map +1 -1
  88. package/dest/test/mock_checkpoint_builder.d.ts +29 -17
  89. package/dest/test/mock_checkpoint_builder.d.ts.map +1 -1
  90. package/dest/test/mock_checkpoint_builder.js +86 -34
  91. package/dest/test/utils.d.ts +13 -9
  92. package/dest/test/utils.d.ts.map +1 -1
  93. package/dest/test/utils.js +31 -20
  94. package/package.json +29 -28
  95. package/src/client/sequencer-client.ts +94 -38
  96. package/src/config.ts +107 -46
  97. package/src/global_variable_builder/README.md +44 -0
  98. package/src/global_variable_builder/fee_predictor.ts +172 -0
  99. package/src/global_variable_builder/fee_provider.ts +75 -0
  100. package/src/global_variable_builder/global_builder.ts +26 -63
  101. package/src/global_variable_builder/index.ts +3 -1
  102. package/src/index.ts +5 -4
  103. package/src/publisher/config.ts +157 -45
  104. package/src/publisher/index.ts +3 -0
  105. package/src/publisher/l1_tx_failed_store/factory.ts +32 -0
  106. package/src/publisher/l1_tx_failed_store/failed_tx_store.ts +57 -0
  107. package/src/publisher/l1_tx_failed_store/file_store_failed_tx_store.ts +46 -0
  108. package/src/publisher/l1_tx_failed_store/index.ts +3 -0
  109. package/src/publisher/sequencer-bundle-simulator.ts +254 -0
  110. package/src/publisher/sequencer-publisher-factory.ts +39 -10
  111. package/src/publisher/sequencer-publisher-metrics.ts +17 -69
  112. package/src/publisher/sequencer-publisher.ts +611 -521
  113. package/src/sequencer/README.md +152 -450
  114. package/src/sequencer/automine/README.md +46 -0
  115. package/src/sequencer/automine/automine_factory.ts +145 -0
  116. package/src/sequencer/automine/automine_sequencer.ts +592 -0
  117. package/src/sequencer/chain_state_overrides.ts +162 -0
  118. package/src/sequencer/checkpoint_proposal_job.ts +1138 -230
  119. package/src/sequencer/checkpoint_proposal_job_metrics.ts +128 -0
  120. package/src/sequencer/checkpoint_voter.ts +31 -17
  121. package/src/sequencer/events.ts +50 -2
  122. package/src/sequencer/index.ts +2 -2
  123. package/src/sequencer/metrics.ts +168 -151
  124. package/src/sequencer/sequencer.ts +535 -162
  125. package/src/sequencer/timetable.ts +70 -57
  126. package/src/sequencer/types.ts +1 -1
  127. package/src/test/index.ts +3 -6
  128. package/src/test/mock_checkpoint_builder.ts +149 -73
  129. package/src/test/utils.ts +78 -30
  130. package/dest/sequencer/block_builder.d.ts +0 -26
  131. package/dest/sequencer/block_builder.d.ts.map +0 -1
  132. package/dest/sequencer/block_builder.js +0 -129
  133. package/dest/sequencer/checkpoint_builder.d.ts +0 -63
  134. package/dest/sequencer/checkpoint_builder.d.ts.map +0 -1
  135. package/dest/sequencer/checkpoint_builder.js +0 -131
  136. package/dest/tx_validator/nullifier_cache.d.ts +0 -14
  137. package/dest/tx_validator/nullifier_cache.d.ts.map +0 -1
  138. package/dest/tx_validator/nullifier_cache.js +0 -24
  139. package/dest/tx_validator/tx_validator_factory.d.ts +0 -18
  140. package/dest/tx_validator/tx_validator_factory.d.ts.map +0 -1
  141. package/dest/tx_validator/tx_validator_factory.js +0 -53
  142. package/src/sequencer/block_builder.ts +0 -217
  143. package/src/sequencer/checkpoint_builder.ts +0 -217
  144. package/src/tx_validator/nullifier_cache.ts +0 -30
  145. package/src/tx_validator/tx_validator_factory.ts +0 -133
@@ -1,20 +1,26 @@
1
1
  import { getKzg } from '@aztec/blob-lib';
2
- import { INITIAL_L2_BLOCK_NUM } from '@aztec/constants';
3
- import type { EpochCache } from '@aztec/epoch-cache';
2
+ import { type EpochCache, PROPOSER_PIPELINING_SLOT_OFFSET } from '@aztec/epoch-cache';
4
3
  import { NoCommitteeError, type RollupContract } from '@aztec/ethereum/contracts';
5
4
  import { BlockNumber, CheckpointNumber, EpochNumber, SlotNumber } from '@aztec/foundation/branded-types';
6
5
  import { merge, omit, pick } from '@aztec/foundation/collection';
7
6
  import { Fr } from '@aztec/foundation/curves/bn254';
8
7
  import { EthAddress } from '@aztec/foundation/eth-address';
9
- import { createLogger } from '@aztec/foundation/log';
8
+ import { type Logger, createLogger } from '@aztec/foundation/log';
10
9
  import { RunningPromise } from '@aztec/foundation/running-promise';
11
10
  import type { DateProvider } from '@aztec/foundation/timer';
12
11
  import type { TypedEventEmitter } from '@aztec/foundation/types';
13
12
  import type { P2P } from '@aztec/p2p';
14
13
  import type { SlasherClientInterface } from '@aztec/slasher';
15
- import type { L2BlockNew, L2BlockSource, ValidateBlockResult } from '@aztec/stdlib/block';
16
- import type { Checkpoint } from '@aztec/stdlib/checkpoint';
17
- import { getSlotAtTimestamp, getSlotStartBuildTimestamp } from '@aztec/stdlib/epoch-helpers';
14
+ import type {
15
+ BlockData,
16
+ L2BlockSink,
17
+ L2BlockSource,
18
+ ProposedCheckpointSink,
19
+ ValidateCheckpointResult,
20
+ } from '@aztec/stdlib/block';
21
+ import type { Checkpoint, ProposedCheckpointData } from '@aztec/stdlib/checkpoint';
22
+ import type { ChainConfig } from '@aztec/stdlib/config';
23
+ import { getEpochAtSlot, getSlotStartBuildTimestamp } from '@aztec/stdlib/epoch-helpers';
18
24
  import {
19
25
  type ResolvedSequencerConfig,
20
26
  type SequencerConfig,
@@ -22,19 +28,20 @@ import {
22
28
  type WorldStateSynchronizer,
23
29
  } from '@aztec/stdlib/interfaces/server';
24
30
  import type { L1ToL2MessageSource } from '@aztec/stdlib/messaging';
31
+ import type { CoordinationSignatureContext } from '@aztec/stdlib/p2p';
25
32
  import { pickFromSchema } from '@aztec/stdlib/schemas';
26
- import { MerkleTreeId } from '@aztec/stdlib/trees';
27
- import { type TelemetryClient, type Tracer, getTelemetryClient, trackSpan } from '@aztec/telemetry-client';
28
- import type { ValidatorClient } from '@aztec/validator-client';
33
+ import { Attributes, type TelemetryClient, type Tracer, getTelemetryClient, trackSpan } from '@aztec/telemetry-client';
34
+ import { FullNodeCheckpointsBuilder, NodeKeystoreAdapter, type ValidatorClient } from '@aztec/validator-client';
29
35
 
30
36
  import EventEmitter from 'node:events';
31
37
 
32
38
  import { DefaultSequencerConfig } from '../config.js';
33
39
  import type { GlobalVariableBuilder } from '../global_variable_builder/global_builder.js';
34
40
  import type { SequencerPublisherFactory } from '../publisher/sequencer-publisher-factory.js';
35
- import type { InvalidateBlockRequest, SequencerPublisher } from '../publisher/sequencer-publisher.js';
36
- import { FullNodeCheckpointsBuilder } from './checkpoint_builder.js';
41
+ import type { InvalidateCheckpointRequest, SequencerPublisher } from '../publisher/sequencer-publisher.js';
42
+ import { buildCheckpointSimulationOverridesPlan } from './chain_state_overrides.js';
37
43
  import { CheckpointProposalJob } from './checkpoint_proposal_job.js';
44
+ import { CheckpointProposalJobMetrics } from './checkpoint_proposal_job_metrics.js';
38
45
  import { CheckpointVoter } from './checkpoint_voter.js';
39
46
  import { SequencerInterruptedError, SequencerTooSlowError } from './errors.js';
40
47
  import type { SequencerEvents } from './events.js';
@@ -45,6 +52,16 @@ import { SequencerState } from './utils.js';
45
52
 
46
53
  export { SequencerState };
47
54
 
55
+ /** Slot snapshot used to prepare a checkpoint proposal. */
56
+ type SequencerSlotContext = {
57
+ slot: SlotNumber;
58
+ targetSlot: SlotNumber;
59
+ epoch: EpochNumber;
60
+ targetEpoch: EpochNumber;
61
+ ts: bigint;
62
+ nowSeconds: bigint;
63
+ };
64
+
48
65
  /**
49
66
  * Sequencer client
50
67
  * - Checks whether it is elected as proposer for the next slot
@@ -56,10 +73,17 @@ export { SequencerState };
56
73
  export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<SequencerEvents>) {
57
74
  private runningPromise?: RunningPromise;
58
75
  private state = SequencerState.STOPPED;
76
+ private stateSlotNumber: SlotNumber | undefined;
77
+ private stateEnteredAtMs = performance.now();
59
78
  private metrics: SequencerMetrics;
79
+ private checkpointProposalJobMetrics: CheckpointProposalJobMetrics;
80
+ private readonly stateLog: Logger;
60
81
 
61
- /** The last slot for which we attempted to vote when sync failed, to prevent duplicate attempts. */
62
- private lastSlotForVoteWhenSyncFailed: SlotNumber | undefined;
82
+ /** The last slot for which we attempted to perform our voting duties with degraded block production */
83
+ private lastSlotForFallbackVote: SlotNumber | undefined;
84
+
85
+ /** The last slot for which we logged "no committee" warning, to avoid spam */
86
+ private lastSlotForNoCommitteeWarning: SlotNumber | undefined;
63
87
 
64
88
  /** The last slot for which we triggered a checkpoint proposal job, to prevent duplicate attempts. */
65
89
  private lastSlotForCheckpointProposalJob: SlotNumber | undefined;
@@ -70,19 +94,15 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
70
94
  /** The last epoch for which we logged strategy comparison in fisherman mode. */
71
95
  private lastEpochForStrategyComparison: EpochNumber | undefined;
72
96
 
97
+ /** The last checkpoint proposal job, tracked so we can await its pending L1 submission during shutdown. */
98
+ protected lastCheckpointProposalJob: CheckpointProposalJob | undefined;
99
+
73
100
  /** The maximum number of seconds that the sequencer can be into a slot to transition to a particular state. */
74
101
  protected timetable!: SequencerTimetable;
75
102
 
76
- // This shouldn't be here as this gets re-created each time we build/propose a block.
77
- // But we have a number of tests that abuse/rely on this class having a permanent publisher.
78
- // As long as those tests only configure a single publisher they will continue to work.
79
- // This will get re-assigned every time the sequencer goes to build a new block to a publisher that is valid
80
- // for the block proposer.
81
- // TODO(palla/mbps): Remove this field and fix tests
82
- protected publisher: SequencerPublisher | undefined;
83
-
84
103
  /** Config for the sequencer */
85
104
  protected config: ResolvedSequencerConfig = DefaultSequencerConfig;
105
+ private readonly signatureContext: CoordinationSignatureContext;
86
106
 
87
107
  constructor(
88
108
  protected publisherFactory: SequencerPublisherFactory,
@@ -91,51 +111,58 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
91
111
  protected p2pClient: P2P,
92
112
  protected worldState: WorldStateSynchronizer,
93
113
  protected slasherClient: SlasherClientInterface | undefined,
94
- protected l2BlockSource: L2BlockSource,
114
+ protected l2BlockSource: L2BlockSource & L2BlockSink & ProposedCheckpointSink,
95
115
  protected l1ToL2MessageSource: L1ToL2MessageSource,
96
116
  protected checkpointsBuilder: FullNodeCheckpointsBuilder,
97
117
  protected l1Constants: SequencerRollupConstants,
98
118
  protected dateProvider: DateProvider,
99
119
  protected epochCache: EpochCache,
100
120
  protected rollupContract: RollupContract,
101
- config: SequencerConfig,
121
+ config: SequencerConfig & Pick<ChainConfig, 'l1ChainId' | 'rollupAddress'>,
102
122
  protected telemetry: TelemetryClient = getTelemetryClient(),
103
123
  protected log = createLogger('sequencer'),
104
124
  ) {
105
125
  super();
126
+ this.stateLog = log.createChild('state');
106
127
 
107
128
  // Add [FISHERMAN] prefix to logger if in fisherman mode
108
129
  if (config.fishermanMode) {
109
130
  this.log = log.createChild('[FISHERMAN]');
110
131
  }
111
132
 
133
+ this.signatureContext = {
134
+ chainId: config.l1ChainId,
135
+ rollupAddress: config.rollupAddress,
136
+ };
112
137
  this.metrics = new SequencerMetrics(telemetry, this.rollupContract, 'Sequencer');
138
+ this.checkpointProposalJobMetrics = new CheckpointProposalJobMetrics(telemetry);
113
139
  this.updateConfig(config);
114
140
  }
115
141
 
116
142
  /** Updates sequencer config by the defined values and updates the timetable */
117
143
  public updateConfig(config: Partial<SequencerConfig>) {
118
144
  const filteredConfig = pickFromSchema(config, SequencerConfigSchema);
119
- this.log.info(`Updated sequencer config`, omit(filteredConfig, 'txPublicSetupAllowList'));
145
+ this.log.info(`Updated sequencer config`, omit(filteredConfig, 'txPublicSetupAllowListExtend'));
120
146
  this.config = merge(this.config, filteredConfig);
147
+ const p2pPropagationTime = this.config.attestationPropagationTime;
121
148
  this.timetable = new SequencerTimetable(
122
149
  {
123
150
  ethereumSlotDuration: this.l1Constants.ethereumSlotDuration,
124
151
  aztecSlotDuration: this.aztecSlotDuration,
125
152
  l1PublishingTime: this.l1PublishingTime,
126
- p2pPropagationTime: this.config.attestationPropagationTime,
153
+ p2pPropagationTime,
127
154
  blockDurationMs: this.config.blockDurationMs,
128
155
  enforce: this.config.enforceTimeTable,
156
+ pipelining: this.epochCache.isProposerPipeliningEnabled(),
129
157
  },
130
158
  this.metrics,
131
159
  this.log,
132
160
  );
133
161
  }
134
162
 
135
- /** Initializes the sequencer (precomputes tables and creates a publisher). Takes about 3s. */
136
- public async init() {
163
+ /** Initializes the sequencer (precomputes tables). Takes about 3s. */
164
+ public init() {
137
165
  getKzg();
138
- this.publisher = (await this.publisherFactory.create(undefined)).publisher;
139
166
  }
140
167
 
141
168
  /** Starts the sequencer and moves to IDLE state. */
@@ -150,34 +177,36 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
150
177
  this.log.info('Started sequencer');
151
178
  }
152
179
 
180
+ /** Triggers an immediate run of the sequencer, bypassing the polling interval. */
181
+ public trigger() {
182
+ return this.runningPromise?.trigger();
183
+ }
184
+
153
185
  /** Stops the sequencer from building blocks and moves to STOPPED state. */
154
186
  public async stop(): Promise<void> {
155
187
  this.log.info(`Stopping sequencer`);
156
188
  this.setState(SequencerState.STOPPING, undefined, { force: true });
157
- this.publisher?.interrupt();
189
+ await this.publisherFactory.stopAll();
158
190
  await this.runningPromise?.stop();
191
+ await this.lastCheckpointProposalJob?.awaitPendingSubmission();
159
192
  this.setState(SequencerState.STOPPED, undefined, { force: true });
160
193
  this.log.info('Stopped sequencer');
161
194
  }
162
195
 
163
- @trackSpan('Sequencer.work')
164
196
  /** Main sequencer loop with a try/catch */
165
197
  protected async safeWork() {
166
198
  try {
167
199
  await this.work();
168
200
  } catch (err) {
169
- this.emit('checkpoint-error', { error: err as Error });
170
- if (err instanceof SequencerTooSlowError) {
171
- // TODO(palla/mbps): Add missing states
172
- // Log as warn only if we had to abort halfway through the block proposal
173
- const logLvl = [SequencerState.INITIALIZING_CHECKPOINT, SequencerState.PROPOSER_CHECK].includes(
174
- err.proposedState,
175
- )
176
- ? ('debug' as const)
177
- : ('warn' as const);
178
- this.log[logLvl](err.message, { now: this.dateProvider.nowInSeconds() });
201
+ if (
202
+ err instanceof SequencerTooSlowError &&
203
+ [SequencerState.INITIALIZING_CHECKPOINT, SequencerState.PROPOSER_CHECK].includes(err.proposedState)
204
+ ) {
205
+ // No need to alert if we just didn't get to start in time
206
+ this.log.debug(err.message, { now: this.dateProvider.nowInSeconds() });
179
207
  } else {
180
208
  // Re-throw other errors
209
+ this.emit('checkpoint-error', { error: err as Error });
181
210
  throw err;
182
211
  }
183
212
  } finally {
@@ -198,16 +227,27 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
198
227
  * - Collect attestations for the final block
199
228
  * - Submit checkpoint
200
229
  */
230
+ @trackSpan('Sequencer.work')
201
231
  protected async work() {
202
232
  this.setState(SequencerState.SYNCHRONIZING, undefined);
203
- const { slot, ts, now, epoch } = this.epochCache.getEpochAndSlotInNextL1Slot();
233
+ const { slot, targetSlot, epoch, targetEpoch, ts, nowSeconds } = this.getSlotContextInNextL1Slot();
204
234
 
205
235
  // Check if we are synced and it's our slot, grab a publisher, check previous block invalidation, etc
206
- const checkpointProposalJob = await this.prepareCheckpointProposal(slot, ts, now);
236
+ const checkpointProposalJob = await this.prepareCheckpointProposal(
237
+ slot,
238
+ targetSlot,
239
+ epoch,
240
+ targetEpoch,
241
+ ts,
242
+ nowSeconds,
243
+ );
207
244
  if (!checkpointProposalJob) {
208
245
  return;
209
246
  }
210
247
 
248
+ // Track the job so we can await its pending L1 submission during shutdown
249
+ this.lastCheckpointProposalJob = checkpointProposalJob;
250
+
211
251
  // Execute the checkpoint proposal job
212
252
  const checkpoint = await checkpointProposalJob.execute();
213
253
 
@@ -216,83 +256,137 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
216
256
  this.lastCheckpointProposed = checkpoint;
217
257
  }
218
258
 
219
- // Log fee strategy comparison if on fisherman
259
+ // Log fee strategy comparison if on fisherman (uses target epoch since we mirror the proposer's perspective)
220
260
  if (
221
261
  this.config.fishermanMode &&
222
- (this.lastEpochForStrategyComparison === undefined || epoch > this.lastEpochForStrategyComparison)
262
+ (this.lastEpochForStrategyComparison === undefined || targetEpoch > this.lastEpochForStrategyComparison)
223
263
  ) {
224
- this.logStrategyComparison(epoch, checkpointProposalJob.getPublisher());
225
- this.lastEpochForStrategyComparison = epoch;
264
+ this.logStrategyComparison(targetEpoch, checkpointProposalJob.getPublisher());
265
+ this.lastEpochForStrategyComparison = targetEpoch;
226
266
  }
227
267
 
228
268
  return checkpoint;
229
269
  }
230
270
 
271
+ /** Returns slot and target slot from a single clock snapshot. */
272
+ protected getSlotContextInNextL1Slot(): SequencerSlotContext {
273
+ const { slot, ts, nowSeconds, epoch } = this.epochCache.getEpochAndSlotInNextL1Slot();
274
+ const targetSlot = SlotNumber(
275
+ slot + (this.epochCache.isProposerPipeliningEnabled() ? PROPOSER_PIPELINING_SLOT_OFFSET : 0),
276
+ );
277
+ return { slot, targetSlot, epoch, targetEpoch: getEpochAtSlot(targetSlot, this.l1Constants), ts, nowSeconds };
278
+ }
279
+
231
280
  /**
232
281
  * Prepares the checkpoint proposal by performing all necessary checks and setup.
233
282
  * This is the initial step in the main loop.
234
283
  * @returns CheckpointProposalJob if successful, undefined if we are not yet synced or are not the proposer.
235
284
  */
236
- private async prepareCheckpointProposal(
285
+ @trackSpan('Sequencer.prepareCheckpointProposal')
286
+ protected async prepareCheckpointProposal(
237
287
  slot: SlotNumber,
288
+ targetSlot: SlotNumber,
289
+ epoch: EpochNumber,
290
+ targetEpoch: EpochNumber,
238
291
  ts: bigint,
239
- now: bigint,
292
+ nowSeconds: bigint,
240
293
  ): Promise<CheckpointProposalJob | undefined> {
241
- // Check we have not already processed this slot (cheapest check)
294
+ // Check we have not already processed this target slot (cheapest check)
242
295
  // We only check this if enforce timetable is set, since we want to keep processing the same slot if we are not
243
296
  // running against actual time (eg when we use sandbox-style automining)
244
297
  if (
245
298
  this.lastSlotForCheckpointProposalJob &&
246
- this.lastSlotForCheckpointProposalJob >= slot &&
299
+ this.lastSlotForCheckpointProposalJob >= targetSlot &&
247
300
  this.config.enforceTimeTable
248
301
  ) {
249
- this.log.trace(`Slot ${slot} has already been processed`);
302
+ this.log.trace(`Target slot ${targetSlot} has already been processed`);
250
303
  return undefined;
251
304
  }
252
305
 
253
- // But if we have already proposed for this slot, the we definitely have to skip it, automining or not
254
- if (this.lastCheckpointProposed && this.lastCheckpointProposed.header.slotNumber >= slot) {
255
- this.log.trace(`Slot ${slot} has already been published as checkpoint ${this.lastCheckpointProposed.number}`);
306
+ // But if we have already proposed for this slot, then we definitely have to skip it, automining or not
307
+ if (this.lastCheckpointProposed && this.lastCheckpointProposed.header.slotNumber >= targetSlot) {
308
+ this.log.trace(
309
+ `Slot ${targetSlot} has already been published as checkpoint ${this.lastCheckpointProposed.number}`,
310
+ );
311
+ return undefined;
312
+ }
313
+
314
+ // Test-only: skip proposing for explicitly paused slots. Attestation paths run in the validator
315
+ // client and are not gated by this hook, so paused proposers still attest to others' proposals.
316
+ if (this.config.pauseProposingForSlots?.some(s => s === targetSlot)) {
317
+ this.log.warn(`Skipping proposal for paused slot ${targetSlot} (test-only pauseProposingForSlots hook)`, {
318
+ targetSlot,
319
+ });
256
320
  return undefined;
257
321
  }
258
322
 
259
323
  // Check all components are synced to latest as seen by the archiver (queries all subsystems)
260
324
  const syncedTo = await this.checkSync({ ts, slot });
261
325
  if (!syncedTo) {
262
- await this.tryVoteWhenSyncFails({ slot, ts });
326
+ await this.tryVoteWhenSyncFails({ slot, targetSlot, ts });
263
327
  return undefined;
264
328
  }
265
329
 
266
- // TODO(palla/mbps): Compute proper checkpoint number
267
- const checkpointNumber = CheckpointNumber.fromBlockNumber(BlockNumber(syncedTo.blockNumber + 1));
330
+ // If escape hatch is open for the target epoch, do not start checkpoint proposal work and do not attempt invalidations.
331
+ // Still perform governance/slashing voting (as proposer) once per slot.
332
+ // When pipelining, we check the target epoch (slot+1's epoch) since that's the epoch we're building for.
333
+ const isEscapeHatchOpen = await this.epochCache.isEscapeHatchOpen(targetEpoch);
334
+
335
+ if (isEscapeHatchOpen) {
336
+ this.setState(SequencerState.PROPOSER_CHECK, slot);
337
+ const [canPropose, proposer] = await this.checkCanPropose(targetSlot);
338
+ if (canPropose) {
339
+ await this.tryVoteWhenEscapeHatchOpen({ slot, targetSlot, proposer });
340
+ } else {
341
+ this.log.trace(`Escape hatch open but we are not proposer, skipping vote-only actions`, {
342
+ slot,
343
+ epoch,
344
+ proposer,
345
+ });
346
+ }
347
+ return undefined;
348
+ }
349
+
350
+ // Next checkpoint follows from the last synced one
351
+ const checkpointNumber = CheckpointNumber(syncedTo.checkpointNumber + 1);
268
352
 
269
353
  const logCtx = {
270
- now,
271
- syncedToL1Ts: syncedTo.l1Timestamp,
272
- syncedToL2Slot: getSlotAtTimestamp(syncedTo.l1Timestamp, this.l1Constants),
354
+ nowSeconds,
355
+ syncedToL2Slot: syncedTo.syncedL2Slot,
273
356
  slot,
357
+ targetSlot,
274
358
  slotTs: ts,
275
359
  checkpointNumber,
276
360
  isPendingChainValid: pick(syncedTo.pendingChainValidationStatus, 'valid', 'reason', 'invalidIndex'),
277
361
  };
278
362
 
279
- // Check that we are a proposer for the next slot
363
+ // Check that we are a proposer for the target slot.
280
364
  this.setState(SequencerState.PROPOSER_CHECK, slot);
281
- const [canPropose, proposer] = await this.checkCanPropose(slot);
365
+ const [canPropose, proposer] = await this.checkCanPropose(targetSlot);
282
366
 
283
- // If we are not a proposer check if we should invalidate a invalid block, and bail
367
+ // If we are not a proposer check if we should invalidate an invalid checkpoint, and bail
284
368
  if (!canPropose) {
285
- await this.considerInvalidatingBlock(syncedTo, slot);
369
+ await this.considerInvalidatingCheckpoint(syncedTo, slot);
370
+ return undefined;
371
+ }
372
+
373
+ // Guard: don't exceed 1-deep pipeline. Without a proposed checkpoint, we can only build
374
+ // confirmed + 1. With a proposed checkpoint, we can build confirmed + 2.
375
+ const confirmedCkpt = syncedTo.checkpointedCheckpointNumber;
376
+ if (checkpointNumber > confirmedCkpt + 2) {
377
+ this.log.verbose(
378
+ `Skipping slot ${targetSlot}: checkpoint ${checkpointNumber} exceeds max pipeline depth (confirmed=${confirmedCkpt})`,
379
+ );
286
380
  return undefined;
287
381
  }
288
382
 
289
- // Check that the slot is not taken by a block already (should never happen, since only us can propose for this slot)
290
- if (syncedTo.block && syncedTo.block.header.getSlot() >= slot) {
383
+ // Check that the target slot is not taken by a block already (should never happen, since only us can propose for this slot)
384
+ if (syncedTo.blockData && syncedTo.blockData.header.getSlot() >= targetSlot) {
291
385
  this.log.warn(
292
- `Cannot propose block at next L2 slot ${slot} since that slot was taken by block ${syncedTo.blockNumber}`,
293
- { ...logCtx, block: syncedTo.block.header.toInspect() },
386
+ `Cannot propose block at target slot ${targetSlot} since that slot was taken by block ${syncedTo.blockNumber}`,
387
+ { ...logCtx, block: syncedTo.blockData.header.toInspect() },
294
388
  );
295
- this.metrics.recordBlockProposalPrecheckFailed('slot_already_taken');
389
+ this.metrics.recordCheckpointPrecheckFailed('slot_already_taken');
296
390
  return undefined;
297
391
  }
298
392
 
@@ -303,107 +397,211 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
303
397
  const proposerForPublisher = this.config.fishermanMode ? undefined : proposer;
304
398
  const { attestorAddress, publisher } = await this.publisherFactory.create(proposerForPublisher);
305
399
  this.log.verbose(`Created publisher at address ${publisher.getSenderAddress()} for attestor ${attestorAddress}`);
306
- this.publisher = publisher;
307
400
 
308
- // In fisherman mode, set the actual proposer's address for simulations
309
- if (this.config.fishermanMode && proposer) {
310
- publisher.setProposerAddressForSimulation(proposer);
311
- this.log.debug(`Set proposer address ${proposer} for simulation in fisherman mode`);
401
+ const isPipelining = this.epochCache.isProposerPipeliningEnabled();
402
+
403
+ // Prepare invalidation request if the pending chain is invalid (returns undefined if no need).
404
+ // Only simulate invalidation when there's no proposed parent, since we assume the proposed parent
405
+ // will invalidate the currently invalid checkpoint on L1.
406
+ const invalidateCheckpoint =
407
+ (isPipelining && syncedTo.hasProposedCheckpoint) || syncedTo.pendingChainValidationStatus.valid
408
+ ? undefined
409
+ : await publisher.simulateInvalidateCheckpoint(syncedTo.pendingChainValidationStatus);
410
+
411
+ // Determine the correct archive and L1 state overrides for the canProposeAt check.
412
+ // The L1 contract reads archives[proposedCheckpointNumber] and compares it with the provided archive.
413
+ // When invalidating or pipelining, the local archive may differ from L1's, so we adjust accordingly.
414
+ let archiveForCheck = syncedTo.archive;
415
+
416
+ if (isPipelining && syncedTo.hasProposedCheckpoint) {
417
+ this.metrics.recordPipelineDepth(syncedTo.checkpointNumber - syncedTo.checkpointedCheckpointNumber);
418
+ this.log.verbose(
419
+ `Building on top of proposed checkpoint (pending=${syncedTo.proposedCheckpointData?.checkpointNumber}) for target slot ${targetSlot}`,
420
+ { targetSlot, parentCheckpointNumber: CheckpointNumber(checkpointNumber - 1) },
421
+ );
422
+ // Match what L1 will see at archives[pending] once the proposed parent lands: the parent's
423
+ // own archive root from the gossiped proposal. `syncedTo.archive` is the world-state-local
424
+ // view and can transiently diverge from the proposed parent (e.g. before the proposed
425
+ // parent's blocks have been applied locally); diverging here would cause the canProposeAt
426
+ // override to set archives[pending] to one value while we present another for comparison.
427
+ archiveForCheck = syncedTo.proposedCheckpointData!.archive.root;
428
+ } else if (invalidateCheckpoint) {
429
+ // After invalidation, L1 will roll back to checkpoint N-1. The archive at N-1 already
430
+ // exists on L1, so we just pass the matching archive (the lastArchive of the invalid checkpoint).
431
+ archiveForCheck = invalidateCheckpoint.lastArchive;
432
+ this.metrics.recordPipelineDepth(0);
433
+ } else {
434
+ this.metrics.recordPipelineDepth(0);
312
435
  }
313
436
 
314
- // Prepare invalidation request if the pending chain is invalid (returns undefined if no need)
315
- // TODO(palla/mbps): We need to invalidate checkpoints, not blocks
316
- const invalidateBlock = await publisher.simulateInvalidateBlock(syncedTo.pendingChainValidationStatus);
437
+ // Build the simulation plan: pending/proven override from pipelining or invalidation (or the
438
+ // current snapshot when neither applies, to short-circuit any pending prune in simulation),
439
+ // plus the parent checkpoint cell and fee header when pipelining.
440
+ const simulationOverridesPlan = await buildCheckpointSimulationOverridesPlan({
441
+ checkpointNumber,
442
+ proposedCheckpointData:
443
+ isPipelining && syncedTo.hasProposedCheckpoint ? syncedTo.proposedCheckpointData : undefined,
444
+ invalidateToPendingCheckpointNumber: invalidateCheckpoint?.forcePendingCheckpointNumber,
445
+ checkpointedCheckpointNumber: syncedTo.checkpointedCheckpointNumber,
446
+ rollup: this.rollupContract,
447
+ signatureContext: this.signatureContext,
448
+ log: this.log,
449
+ });
450
+
451
+ // The plan always pins both pending/proven (to short-circuit `canPruneAtTime` in simulation),
452
+ // so `provenOverride` always reflects the assumed proven checkpoint we are pinning the
453
+ // simulation to. We additionally warn when the pin is load-bearing — i.e. when a prune would
454
+ // actually fire at the target slot without it — so observers can spot "we are building
455
+ // optimistically across a pruning boundary" in the logs.
456
+ const provenOverride = simulationOverridesPlan?.chainTipsOverride?.proven;
457
+ if (provenOverride !== undefined && (await this.l2BlockSource.isPruneDueAtSlot(targetSlot))) {
458
+ this.log.warn(
459
+ `Assuming proof for epoch ending at checkpoint ${provenOverride} lands by target slot ${targetSlot}`,
460
+ { checkpointNumber, slot, targetSlot, provenOverride },
461
+ );
462
+ }
317
463
 
318
- // Check with the rollup contract if we can indeed propose at the next L2 slot. This check should not fail
319
- // if all the previous checks are good, but we do it just in case.
320
- const canProposeCheck = await publisher.canProposeAtNextEthBlock(
321
- syncedTo.archive,
464
+ this.emit('preparing-checkpoint', {
465
+ targetSlot,
466
+ checkpointNumber,
467
+ hadProposedParent: this.epochCache.isProposerPipeliningEnabled() && syncedTo.hasProposedCheckpoint,
468
+ provenOverride,
469
+ simulatedPending: simulationOverridesPlan?.chainTipsOverride?.pending,
470
+ });
471
+
472
+ const canProposeCheck = await publisher.canProposeAt(
473
+ archiveForCheck,
322
474
  proposer ?? EthAddress.ZERO,
323
- invalidateBlock,
475
+ simulationOverridesPlan,
324
476
  );
325
477
 
478
+ const proposeContext = {
479
+ hasProposedCheckpoint: syncedTo.hasProposedCheckpoint,
480
+ proposedCheckpointNumber: syncedTo.proposedCheckpointData?.checkpointNumber,
481
+ checkpointedCheckpointNumber: syncedTo.checkpointedCheckpointNumber,
482
+ isInvalidating: !!invalidateCheckpoint,
483
+ invalidatingCheckpointNumber: invalidateCheckpoint?.checkpointNumber,
484
+ archiveForCheck: archiveForCheck.toString(),
485
+ overridePendingCheckpointNumber: simulationOverridesPlan?.chainTipsOverride?.pending,
486
+ overrideArchive: simulationOverridesPlan?.pendingCheckpointState?.archive,
487
+ overrideFeeHeader: simulationOverridesPlan?.pendingCheckpointState?.feeHeader,
488
+ };
489
+
326
490
  if (canProposeCheck === undefined) {
327
491
  this.log.warn(
328
492
  `Cannot propose checkpoint ${checkpointNumber} at slot ${slot} due to failed rollup contract check`,
329
- logCtx,
493
+ { ...logCtx, ...proposeContext },
330
494
  );
331
495
  this.emit('proposer-rollup-check-failed', { reason: 'Rollup contract check failed', slot });
332
- this.metrics.recordBlockProposalPrecheckFailed('rollup_contract_check_failed');
496
+ this.metrics.recordCheckpointPrecheckFailed('rollup_contract_check_failed');
333
497
  return undefined;
334
498
  }
335
499
 
336
- if (canProposeCheck.slot !== slot) {
500
+ if (canProposeCheck.slot !== targetSlot) {
337
501
  this.log.warn(
338
- `Cannot propose block due to slot mismatch with rollup contract (this can be caused by a clock out of sync). Expected slot ${slot} but got ${canProposeCheck.slot}.`,
339
- { ...logCtx, rollup: canProposeCheck, expectedSlot: slot },
502
+ `Cannot propose block due to slot mismatch with rollup contract (this can be caused by a clock out of sync). Expected slot ${targetSlot} but got ${canProposeCheck.slot}.`,
503
+ { ...logCtx, ...proposeContext, rollup: canProposeCheck, expectedSlot: targetSlot },
340
504
  );
341
505
  this.emit('proposer-rollup-check-failed', { reason: 'Slot mismatch', slot });
342
- this.metrics.recordBlockProposalPrecheckFailed('slot_mismatch');
506
+ this.metrics.recordCheckpointPrecheckFailed('slot_mismatch');
343
507
  return undefined;
344
508
  }
345
509
 
346
510
  if (canProposeCheck.checkpointNumber !== checkpointNumber) {
347
511
  this.log.warn(
348
512
  `Cannot propose due to block mismatch with rollup contract (this can be caused by a pending archiver sync). Expected checkpoint ${checkpointNumber} but got ${canProposeCheck.checkpointNumber}.`,
349
- { ...logCtx, rollup: canProposeCheck, expectedSlot: slot },
513
+ { ...logCtx, ...proposeContext, rollup: canProposeCheck, expectedSlot: slot },
350
514
  );
351
515
  this.emit('proposer-rollup-check-failed', { reason: 'Block mismatch', slot });
352
- this.metrics.recordBlockProposalPrecheckFailed('block_number_mismatch');
516
+ this.metrics.recordCheckpointPrecheckFailed('block_number_mismatch');
353
517
  return undefined;
354
518
  }
355
519
 
356
- this.lastSlotForCheckpointProposalJob = slot;
357
- this.log.info(`Preparing checkpoint proposal ${checkpointNumber} at slot ${slot}`, { ...logCtx, proposer });
520
+ this.lastSlotForCheckpointProposalJob = targetSlot;
521
+
522
+ await this.p2pClient.prepareForSlot(targetSlot);
523
+ this.log.info(
524
+ `Preparing checkpoint proposal ${checkpointNumber} for target slot ${targetSlot} during wall-clock slot ${slot}`,
525
+ {
526
+ ...logCtx,
527
+ ...proposeContext,
528
+ proposer,
529
+ pipeliningEnabled: this.epochCache.isProposerPipeliningEnabled(),
530
+ },
531
+ );
358
532
 
359
533
  // Create and return the checkpoint proposal job
360
534
  return this.createCheckpointProposalJob(
361
535
  slot,
536
+ targetSlot,
537
+ targetEpoch,
362
538
  checkpointNumber,
363
539
  syncedTo.blockNumber,
540
+ syncedTo.checkpointedCheckpointNumber,
364
541
  proposer,
365
542
  publisher,
366
543
  attestorAddress,
367
- invalidateBlock,
544
+ invalidateCheckpoint,
545
+ syncedTo.proposedCheckpointData,
368
546
  );
369
547
  }
370
548
 
371
549
  protected createCheckpointProposalJob(
372
550
  slot: SlotNumber,
551
+ targetSlot: SlotNumber,
552
+ targetEpoch: EpochNumber,
373
553
  checkpointNumber: CheckpointNumber,
374
554
  syncedToBlockNumber: BlockNumber,
555
+ checkpointedCheckpointNumber: CheckpointNumber,
375
556
  proposer: EthAddress | undefined,
376
557
  publisher: SequencerPublisher,
377
558
  attestorAddress: EthAddress,
378
- invalidateBlock: InvalidateBlockRequest | undefined,
559
+ invalidateCheckpoint: InvalidateCheckpointRequest | undefined,
560
+ proposedCheckpointData?: ProposedCheckpointData,
379
561
  ): CheckpointProposalJob {
380
562
  return new CheckpointProposalJob(
381
563
  slot,
564
+ targetSlot,
565
+ targetEpoch,
382
566
  checkpointNumber,
383
567
  syncedToBlockNumber,
568
+ checkpointedCheckpointNumber,
384
569
  proposer,
385
570
  publisher,
386
571
  attestorAddress,
387
- invalidateBlock,
572
+ invalidateCheckpoint,
388
573
  this.validatorClient,
389
574
  this.globalsBuilder,
390
575
  this.p2pClient,
391
576
  this.worldState,
392
577
  this.l1ToL2MessageSource,
578
+ this.l2BlockSource,
393
579
  this.checkpointsBuilder,
580
+ this.l2BlockSource,
394
581
  this.l1Constants,
582
+ this.signatureContext,
395
583
  this.config,
396
584
  this.timetable,
397
585
  this.slasherClient,
398
586
  this.epochCache,
399
587
  this.dateProvider,
400
588
  this.metrics,
589
+ this.checkpointProposalJobMetrics.createRecorder(),
401
590
  this,
402
591
  this.setState.bind(this),
403
- this.log,
592
+ this.tracer,
593
+ this.log.getBindings(),
594
+ proposedCheckpointData,
404
595
  );
405
596
  }
406
597
 
598
+ /**
599
+ * Returns the current sequencer state.
600
+ */
601
+ public getState(): SequencerState {
602
+ return this.state;
603
+ }
604
+
407
605
  /**
408
606
  * Internal helper for setting the sequencer state and checks if we have enough time left in the slot to transition to the new state.
409
607
  * @param proposedState - The new state to transition to.
@@ -429,19 +627,35 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
429
627
  this.timetable.assertTimeLeft(proposedState, secondsIntoSlot);
430
628
  }
431
629
 
630
+ const oldState = this.state;
631
+ const oldStateSlotNumber = this.stateSlotNumber;
632
+ const stateChanged = proposedState !== oldState;
633
+ const transitionAtMs = performance.now();
634
+ const stateDurationMs = transitionAtMs - this.stateEnteredAtMs;
635
+
432
636
  const boringStates = [SequencerState.IDLE, SequencerState.SYNCHRONIZING];
433
637
  const logLevel =
434
- boringStates.includes(proposedState) && boringStates.includes(this.state)
435
- ? ('trace' as const)
436
- : ('debug' as const);
437
- this.log[logLevel](`Transitioning from ${this.state} to ${proposedState}`, { slotNumber, secondsIntoSlot });
638
+ boringStates.includes(proposedState) && boringStates.includes(oldState) ? ('trace' as const) : ('debug' as const);
639
+ this.stateLog[logLevel](`Transitioning from ${oldState} to ${proposedState}`, {
640
+ oldState,
641
+ newState: proposedState,
642
+ slotNumber,
643
+ stateSlotNumber: oldStateSlotNumber,
644
+ secondsIntoSlot,
645
+ ...(stateChanged && { stateDurationMs: Math.ceil(stateDurationMs) }),
646
+ });
438
647
 
439
648
  this.emit('state-changed', {
440
- oldState: this.state,
649
+ oldState,
441
650
  newState: proposedState,
442
651
  secondsIntoSlot,
443
652
  slot: slotNumber,
444
653
  });
654
+ if (stateChanged) {
655
+ this.metrics.recordStateDuration(stateDurationMs, oldState);
656
+ this.stateEnteredAtMs = transitionAtMs;
657
+ this.stateSlotNumber = slotNumber;
658
+ }
445
659
  this.state = proposedState;
446
660
  }
447
661
 
@@ -450,16 +664,15 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
450
664
  * We don't check against the previous block submitted since it may have been reorg'd out.
451
665
  */
452
666
  protected async checkSync(args: { ts: bigint; slot: SlotNumber }): Promise<SequencerSyncCheckResult | undefined> {
453
- // Check that the archiver and dependencies have synced to the previous L1 slot at least
454
- // TODO(#14766): Archiver reports L1 timestamp based on L1 blocks seen, which means that a missed L1 block will
455
- // cause the archiver L1 timestamp to fall behind, and cause this sequencer to start processing one L1 slot later.
456
- const l1Timestamp = await this.l2BlockSource.getL1Timestamp();
457
- const { slot, ts } = args;
458
- if (l1Timestamp === undefined || l1Timestamp + BigInt(this.l1Constants.ethereumSlotDuration) < ts) {
667
+ // Check that the archiver has fully synced the L2 slot before the one we want to propose in.
668
+ // The archiver reports sync progress via L1 block timestamps and synced checkpoint slots.
669
+ // See getSyncedL2SlotNumber for how missed L1 blocks are handled.
670
+ const syncedL2Slot = await this.l2BlockSource.getSyncedL2SlotNumber();
671
+ const { slot } = args;
672
+ if (syncedL2Slot === undefined || syncedL2Slot + 1 < slot) {
459
673
  this.log.debug(`Cannot propose block at next L2 slot ${slot} due to pending sync from L1`, {
460
674
  slot,
461
- ts,
462
- l1Timestamp,
675
+ syncedL2Slot,
463
676
  });
464
677
  return undefined;
465
678
  }
@@ -469,46 +682,114 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
469
682
  number: syncSummary.latestBlockNumber,
470
683
  hash: syncSummary.latestBlockHash,
471
684
  })),
472
- this.l2BlockSource.getL2Tips().then(t => t.latest),
685
+ this.l2BlockSource
686
+ .getL2Tips()
687
+ .then(t => ({ proposed: t.proposed, checkpointed: t.checkpointed, proposedCheckpoint: t.proposedCheckpoint })),
473
688
  this.p2pClient.getStatus().then(p2p => p2p.syncedToL2Block),
474
- this.l1ToL2MessageSource.getL2Tips().then(t => t.latest),
689
+ this.l1ToL2MessageSource.getL2Tips().then(t => ({ proposed: t.proposed, checkpointed: t.checkpointed })),
475
690
  this.l2BlockSource.getPendingChainValidationStatus(),
691
+ this.l2BlockSource.getProposedCheckpointData(),
476
692
  ] as const);
477
693
 
478
- const [worldState, l2BlockSource, p2p, l1ToL2MessageSource, pendingChainValidationStatus] = syncedBlocks;
694
+ const [worldState, l2Tips, p2p, l1ToL2MessageSourceTips, pendingChainValidationStatus, proposedCheckpointData] =
695
+ syncedBlocks;
479
696
 
480
- // Handle zero as a special case, since the block hash won't match across services if we're changing the prefilled data for the genesis block,
481
- // as the world state can compute the new genesis block hash, but other components use the hardcoded constant.
482
697
  const result =
483
- (l2BlockSource.number === 0 && worldState.number === 0 && p2p.number === 0 && l1ToL2MessageSource.number === 0) ||
484
- (worldState.hash === l2BlockSource.hash &&
485
- p2p.hash === l2BlockSource.hash &&
486
- l1ToL2MessageSource.hash === l2BlockSource.hash);
698
+ worldState.hash === l2Tips.proposed.hash &&
699
+ p2p.hash === l2Tips.proposed.hash &&
700
+ l1ToL2MessageSourceTips.proposed.hash === l2Tips.proposed.hash &&
701
+ l1ToL2MessageSourceTips.checkpointed.block.hash === l2Tips.checkpointed.block.hash &&
702
+ l1ToL2MessageSourceTips.checkpointed.checkpoint.hash === l2Tips.checkpointed.checkpoint.hash;
487
703
 
488
704
  if (!result) {
489
- this.log.debug(`Sequencer sync check failed`, { worldState, l2BlockSource, p2p, l1ToL2MessageSource });
705
+ this.log.debug(`Sequencer sync check failed`, {
706
+ worldState,
707
+ l2BlockSource: l2Tips.proposed,
708
+ p2p,
709
+ l1ToL2MessageSourceTips,
710
+ });
490
711
  return undefined;
491
712
  }
492
713
 
493
- // Special case for genesis state
494
714
  const blockNumber = worldState.number;
495
- if (blockNumber < INITIAL_L2_BLOCK_NUM) {
496
- const archive = new Fr((await this.worldState.getCommitted().getTreeInfo(MerkleTreeId.ARCHIVE)).root);
497
- return { blockNumber: BlockNumber(INITIAL_L2_BLOCK_NUM - 1), archive, l1Timestamp, pendingChainValidationStatus };
715
+ const blockData = await this.l2BlockSource.getBlockData({ number: blockNumber });
716
+ if (!blockData) {
717
+ this.log.warn(`Sequencer sync check failed: failed to get L2 block data ${blockNumber} from the archiver`, {
718
+ blockNumber,
719
+ l2Tips,
720
+ syncedL2Slot,
721
+ ...args,
722
+ });
723
+ return undefined;
724
+ }
725
+
726
+ // Refuse to build a checkpoint on top of a proposed block whose enclosing checkpoint was never
727
+ // proposed. Under pipelining we may have received and reexecuted such a block locally — advancing
728
+ // our world-state tip past the checkpointed tip — while the proposing node never published the
729
+ // matching proposed checkpoint (e.g. it crashed before assembling it). Building on this orphan block
730
+ // would fork the chain off a tip no other node can follow. The archiver prunes these orphan blocks
731
+ // once their build slot ends; this guard is the correctness barrier during the grace window before.
732
+ if (
733
+ blockData.checkpointNumber > l2Tips.checkpointed.checkpoint.number &&
734
+ (l2Tips.proposedCheckpoint.checkpoint.number !== blockData.checkpointNumber ||
735
+ proposedCheckpointData?.checkpointNumber !== blockData.checkpointNumber)
736
+ ) {
737
+ this.log.warn(`Sequencer sync check failed: proposed block has no matching proposed checkpoint`, {
738
+ blockCheckpointNumber: blockData.checkpointNumber,
739
+ checkpointedCheckpointNumber: l2Tips.checkpointed.checkpoint.number,
740
+ proposedCheckpointTipNumber: l2Tips.proposedCheckpoint.checkpoint.number,
741
+ proposedCheckpointDataNumber: proposedCheckpointData?.checkpointNumber,
742
+ blockNumber: blockData.header.getBlockNumber(),
743
+ blockSlot: blockData.header.getSlot(),
744
+ syncedL2Slot,
745
+ ...args,
746
+ });
747
+ return undefined;
748
+ }
749
+
750
+ const hasProposedCheckpoint = l2Tips.proposedCheckpoint.checkpoint.number > l2Tips.checkpointed.checkpoint.number;
751
+
752
+ // The l2Tips and proposedCheckpointData reads above come from independent archiver snapshots
753
+ // (a JS-side tips cache vs. a direct store read on `#proposedCheckpoints`). A concurrent archiver
754
+ // write that mutates both can be observed split, leaving us with `hasProposedCheckpoint=true` but
755
+ // no proposedCheckpointData (or one whose number doesn't match the tip). Refuse to proceed in that
756
+ // window — the next checkSync tick will see a coherent snapshot.
757
+ if (
758
+ hasProposedCheckpoint &&
759
+ (!proposedCheckpointData ||
760
+ proposedCheckpointData.checkpointNumber !== l2Tips.proposedCheckpoint.checkpoint.number)
761
+ ) {
762
+ this.log.warn(`Sequencer sync check failed: inconsistent proposed-checkpoint state`, {
763
+ proposedCheckpointTipNumber: l2Tips.proposedCheckpoint.checkpoint.number,
764
+ checkpointedTipNumber: l2Tips.checkpointed.checkpoint.number,
765
+ proposedCheckpointDataNumber: proposedCheckpointData?.checkpointNumber,
766
+ syncedL2Slot,
767
+ ...args,
768
+ });
769
+ return undefined;
498
770
  }
499
771
 
500
- const block = await this.l2BlockSource.getL2BlockNew(blockNumber);
501
- if (!block) {
502
- // this shouldn't really happen because a moment ago we checked that all components were in sync
503
- this.log.error(`Failed to get L2 block ${blockNumber} from the archiver with all components in sync`);
772
+ // Check that the proposed checkpoint is indeed the parent of the checkpoint we'll be building
773
+ // The checkpoint number to build is derived as blockData.checkpointNumber + 1
774
+ if (proposedCheckpointData && proposedCheckpointData.checkpointNumber !== blockData.checkpointNumber) {
775
+ this.log.warn(`Sequencer sync check failed: proposed checkpoint number mismatch`, {
776
+ proposedCheckpointNumber: proposedCheckpointData.checkpointNumber,
777
+ blockCheckpointNumber: blockData.checkpointNumber,
778
+ syncedL2Slot,
779
+ ...args,
780
+ });
504
781
  return undefined;
505
782
  }
506
783
 
507
784
  return {
508
- block,
509
- blockNumber: block.number,
510
- archive: block.archive.root,
511
- l1Timestamp,
785
+ blockData,
786
+ blockNumber: blockData.header.getBlockNumber(),
787
+ checkpointNumber: blockData.checkpointNumber,
788
+ checkpointedCheckpointNumber: l2Tips.checkpointed.checkpoint.number,
789
+ archive: blockData.archive.root,
790
+ hasProposedCheckpoint,
791
+ proposedCheckpointData,
792
+ syncedL2Slot,
512
793
  pendingChainValidationStatus,
513
794
  };
514
795
  }
@@ -517,17 +798,20 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
517
798
  * Checks if we are the proposer for the next slot.
518
799
  * @returns True if we can propose, and the proposer address (undefined if anyone can propose)
519
800
  */
520
- protected async checkCanPropose(slot: SlotNumber): Promise<[boolean, EthAddress | undefined]> {
801
+ protected async checkCanPropose(targetSlot: SlotNumber): Promise<[boolean, EthAddress | undefined]> {
521
802
  let proposer: EthAddress | undefined;
522
803
 
523
804
  try {
524
- proposer = await this.epochCache.getProposerAttesterAddressInSlot(slot);
805
+ proposer = await this.epochCache.getProposerAttesterAddressInSlot(targetSlot);
525
806
  } catch (e) {
526
807
  if (e instanceof NoCommitteeError) {
527
- this.log.warn(`Cannot propose at next L2 slot ${slot} since the committee does not exist on L1`);
808
+ if (this.lastSlotForNoCommitteeWarning !== targetSlot) {
809
+ this.lastSlotForNoCommitteeWarning = targetSlot;
810
+ this.log.warn(`Cannot propose at target slot ${targetSlot} since the committee does not exist on L1`);
811
+ }
528
812
  return [false, undefined];
529
813
  }
530
- this.log.error(`Error getting proposer for slot ${slot}`, e);
814
+ this.log.error(`Error getting proposer for target slot ${targetSlot}`, e);
531
815
  return [false, undefined];
532
816
  }
533
817
 
@@ -544,10 +828,18 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
544
828
  const weAreProposer = validatorAddresses.some(addr => addr.equals(proposer));
545
829
 
546
830
  if (!weAreProposer) {
547
- this.log.debug(`Cannot propose at slot ${slot} since we are not a proposer`, { validatorAddresses, proposer });
831
+ this.log.debug(`Cannot propose at target slot ${targetSlot} since we are not a proposer`, {
832
+ targetSlot,
833
+ validatorAddresses,
834
+ proposer,
835
+ });
548
836
  return [false, proposer];
549
837
  }
550
838
 
839
+ this.log.info(`We are the proposer for pipeline slot ${targetSlot}`, {
840
+ targetSlot,
841
+ proposer,
842
+ });
551
843
  return [true, proposer];
552
844
  }
553
845
 
@@ -555,11 +847,12 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
555
847
  * Tries to vote on slashing actions and governance when the sync check fails but we're past the max time for initializing a proposal.
556
848
  * This allows the sequencer to participate in governance/slashing votes even when it cannot build blocks.
557
849
  */
558
- protected async tryVoteWhenSyncFails(args: { slot: SlotNumber; ts: bigint }): Promise<void> {
559
- const { slot } = args;
850
+ @trackSpan('Seqeuencer.tryVoteWhenSyncFails', ({ slot }) => ({ [Attributes.SLOT_NUMBER]: slot }))
851
+ protected async tryVoteWhenSyncFails(args: { slot: SlotNumber; targetSlot: SlotNumber; ts: bigint }): Promise<void> {
852
+ const { slot, targetSlot } = args;
560
853
 
561
854
  // Prevent duplicate attempts in the same slot
562
- if (this.lastSlotForVoteWhenSyncFailed === slot) {
855
+ if (this.lastSlotForFallbackVote === slot) {
563
856
  this.log.trace(`Already attempted to vote in slot ${slot} (skipping)`);
564
857
  return;
565
858
  }
@@ -584,14 +877,14 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
584
877
  });
585
878
 
586
879
  // Check if we're a proposer or proposal is open
587
- const [canPropose, proposer] = await this.checkCanPropose(slot);
880
+ const [canPropose, proposer] = await this.checkCanPropose(targetSlot);
588
881
  if (!canPropose) {
589
882
  this.log.trace(`Cannot vote in slot ${slot} since we are not a proposer`, { slot, proposer });
590
883
  return;
591
884
  }
592
885
 
593
886
  // Mark this slot as attempted
594
- this.lastSlotForVoteWhenSyncFailed = slot;
887
+ this.lastSlotForFallbackVote = slot;
595
888
 
596
889
  // Get a publisher for voting
597
890
  const { attestorAddress, publisher } = await this.publisherFactory.create(proposer);
@@ -601,9 +894,9 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
601
894
  slot,
602
895
  });
603
896
 
604
- // Enqueue governance and slashing votes
897
+ // Enqueue governance and slashing votes (voter uses the target slot for L1 submission)
605
898
  const voter = new CheckpointVoter(
606
- slot,
899
+ targetSlot,
607
900
  publisher,
608
901
  attestorAddress,
609
902
  this.validatorClient,
@@ -622,7 +915,78 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
622
915
  }
623
916
 
624
917
  this.log.info(`Voting in slot ${slot} despite sync failure`, { slot });
625
- await publisher.sendRequests();
918
+ // Votes are EIP-712-signed for `targetSlot` (the pipelined slot in which the multicall is
919
+ // expected to mine). Delay submission to the start of `targetSlot` so the tx mines in the
920
+ // slot the votes were signed for. We fire-and-forget so we don't block the sequencer's
921
+ // work loop while waiting for the target slot to start.
922
+ void publisher.sendRequestsAt(targetSlot).catch(err => {
923
+ this.log.error(`Failed to publish votes despite sync failure for slot ${slot}`, err, { slot });
924
+ });
925
+ }
926
+
927
+ /**
928
+ * Tries to vote on slashing actions and governance proposals when escape hatch is open.
929
+ * This allows the sequencer to participate in voting without performing checkpoint proposal work.
930
+ */
931
+ @trackSpan('Sequencer.tryVoteWhenEscapeHatchOpen', ({ slot }) => ({ [Attributes.SLOT_NUMBER]: slot }))
932
+ protected async tryVoteWhenEscapeHatchOpen(args: {
933
+ slot: SlotNumber;
934
+ targetSlot: SlotNumber;
935
+ proposer: EthAddress | undefined;
936
+ }): Promise<void> {
937
+ const { slot, targetSlot, proposer } = args;
938
+
939
+ // Prevent duplicate attempts in the same slot
940
+ if (this.lastSlotForFallbackVote === slot) {
941
+ this.log.trace(`Already attempted to vote in slot ${slot} (escape hatch open, skipping)`);
942
+ return;
943
+ }
944
+
945
+ // Mark this slot as attempted
946
+ this.lastSlotForFallbackVote = slot;
947
+
948
+ const { attestorAddress, publisher } = await this.publisherFactory.create(proposer);
949
+
950
+ this.log.debug(`Escape hatch open for slot ${slot}, attempting vote-only actions`, {
951
+ slot,
952
+ targetSlot,
953
+ attestorAddress,
954
+ });
955
+
956
+ // Under proposer pipelining, the multicall is expected to mine in `targetSlot` (slot + 1).
957
+ // Governance and slashing votes are EIP-712-signed against the slot they will mine in, and the
958
+ // L1 contract checks `msg.sender == getCurrentProposer()` using the mining slot. So we must
959
+ // sign for `targetSlot` and delay submission to the start of `targetSlot`. When pipelining is
960
+ // disabled `targetSlot == slot` and `sendRequestsAt` resolves with no extra sleep.
961
+ const voter = new CheckpointVoter(
962
+ targetSlot,
963
+ publisher,
964
+ attestorAddress,
965
+ this.validatorClient,
966
+ this.slasherClient,
967
+ this.l1Constants,
968
+ this.config,
969
+ this.metrics,
970
+ this.log,
971
+ );
972
+
973
+ const votesPromises = voter.enqueueVotes();
974
+ const votes = await Promise.all(votesPromises);
975
+
976
+ if (votes.every(p => !p)) {
977
+ this.log.debug(`No votes to enqueue for slot ${slot} (escape hatch open)`);
978
+ return;
979
+ }
980
+
981
+ this.log.info(`Voting in slot ${slot} (escape hatch open)`, { slot, targetSlot });
982
+ // Votes are EIP-712-signed for `targetSlot`. Delay submission to the start of `targetSlot` so
983
+ // the multicall mines in the slot the votes were signed for; otherwise the L1 contract reads
984
+ // `signaler = getCurrentProposer()` against the wrong slot and signature verification fails
985
+ // silently inside Multicall3. Fire-and-forget so we don't block the sequencer's work loop while
986
+ // waiting for the target slot to start, mirroring tryVoteWhenSyncFails.
987
+ void publisher.sendRequestsAt(targetSlot).catch(err => {
988
+ this.log.error(`Failed to publish escape-hatch votes for slot ${slot}`, err, { slot, targetSlot });
989
+ });
626
990
  }
627
991
 
628
992
  /**
@@ -631,27 +995,27 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
631
995
  * have the proposer try to invalidate, but if they fail, the sequencers in the committee are expected to try,
632
996
  * and if they fail, any sequencer will try as well.
633
997
  */
634
- protected async considerInvalidatingBlock(
998
+ protected async considerInvalidatingCheckpoint(
635
999
  syncedTo: SequencerSyncCheckResult,
636
1000
  currentSlot: SlotNumber,
637
1001
  ): Promise<void> {
638
- const { pendingChainValidationStatus, l1Timestamp } = syncedTo;
1002
+ const { pendingChainValidationStatus, syncedL2Slot } = syncedTo;
639
1003
  if (pendingChainValidationStatus.valid) {
640
1004
  return;
641
1005
  }
642
1006
 
643
- const invalidBlockNumber = pendingChainValidationStatus.block.blockNumber;
644
- const invalidBlockTimestamp = pendingChainValidationStatus.block.timestamp;
645
- const timeSinceChainInvalid = this.dateProvider.nowInSeconds() - Number(invalidBlockTimestamp);
1007
+ const invalidCheckpointNumber = pendingChainValidationStatus.checkpoint.checkpointNumber;
1008
+ const invalidCheckpointTimestamp = pendingChainValidationStatus.checkpoint.timestamp;
1009
+ const timeSinceChainInvalid = this.dateProvider.nowInSeconds() - Number(invalidCheckpointTimestamp);
646
1010
  const ourValidatorAddresses = this.validatorClient.getValidatorAddresses();
647
1011
 
648
1012
  const { secondsBeforeInvalidatingBlockAsCommitteeMember, secondsBeforeInvalidatingBlockAsNonCommitteeMember } =
649
1013
  this.config;
650
1014
 
651
1015
  const logData = {
652
- invalidL1Timestamp: invalidBlockTimestamp,
653
- l1Timestamp,
654
- invalidBlock: pendingChainValidationStatus.block,
1016
+ invalidL1Timestamp: invalidCheckpointTimestamp,
1017
+ syncedL2Slot,
1018
+ invalidCheckpoint: pendingChainValidationStatus.checkpoint,
655
1019
  secondsBeforeInvalidatingBlockAsCommitteeMember,
656
1020
  secondsBeforeInvalidatingBlockAsNonCommitteeMember,
657
1021
  ourValidatorAddresses,
@@ -697,25 +1061,25 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
697
1061
 
698
1062
  const { publisher } = await this.publisherFactory.create(validatorToUse);
699
1063
 
700
- const invalidateBlock = await publisher.simulateInvalidateBlock(pendingChainValidationStatus);
701
- if (!invalidateBlock) {
702
- this.log.warn(`Failed to simulate invalidate block`, logData);
1064
+ const invalidateCheckpoint = await publisher.simulateInvalidateCheckpoint(pendingChainValidationStatus);
1065
+ if (!invalidateCheckpoint) {
1066
+ this.log.warn(`Failed to simulate invalidate checkpoint`, logData);
703
1067
  return;
704
1068
  }
705
1069
 
706
1070
  this.log.info(
707
1071
  invalidateAsCommitteeMember
708
- ? `Invalidating block ${invalidBlockNumber} as committee member`
709
- : `Invalidating block ${invalidBlockNumber} as non-committee member`,
1072
+ ? `Invalidating checkpoint ${invalidCheckpointNumber} as committee member`
1073
+ : `Invalidating checkpoint ${invalidCheckpointNumber} as non-committee member`,
710
1074
  logData,
711
1075
  );
712
1076
 
713
- publisher.enqueueInvalidateBlock(invalidateBlock);
1077
+ publisher.enqueueInvalidateCheckpoint(invalidateCheckpoint);
714
1078
 
715
1079
  if (!this.config.fishermanMode) {
716
1080
  await publisher.sendRequests();
717
1081
  } else {
718
- this.log.info('Invalidating block in fisherman mode, clearing pending requests');
1082
+ this.log.info('Invalidating checkpoint in fisherman mode, clearing pending requests');
719
1083
  publisher.clearPendingRequests();
720
1084
  }
721
1085
  }
@@ -778,6 +1142,11 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
778
1142
  return this.validatorClient?.getValidatorAddresses();
779
1143
  }
780
1144
 
1145
+ /** Updates the publisher factory's node keystore adapter after a keystore reload. */
1146
+ public updatePublisherNodeKeyStore(adapter: NodeKeystoreAdapter): void {
1147
+ this.publisherFactory.updateNodeKeyStore(adapter);
1148
+ }
1149
+
781
1150
  public getConfig() {
782
1151
  return this.config;
783
1152
  }
@@ -788,9 +1157,13 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
788
1157
  }
789
1158
 
790
1159
  type SequencerSyncCheckResult = {
791
- block?: L2BlockNew;
1160
+ blockData?: BlockData;
1161
+ checkpointNumber: CheckpointNumber;
1162
+ checkpointedCheckpointNumber: CheckpointNumber;
792
1163
  blockNumber: BlockNumber;
793
1164
  archive: Fr;
794
- l1Timestamp: bigint;
795
- pendingChainValidationStatus: ValidateBlockResult;
1165
+ hasProposedCheckpoint: boolean;
1166
+ proposedCheckpointData?: ProposedCheckpointData;
1167
+ syncedL2Slot: SlotNumber;
1168
+ pendingChainValidationStatus: ValidateCheckpointResult;
796
1169
  };