@aztec/sequencer-client 0.0.1-commit.96bb3f7 → 0.0.1-commit.993d240

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (133) hide show
  1. package/README.md +282 -21
  2. package/dest/client/sequencer-client.d.ts +19 -9
  3. package/dest/client/sequencer-client.d.ts.map +1 -1
  4. package/dest/client/sequencer-client.js +67 -33
  5. package/dest/config.d.ts +29 -7
  6. package/dest/config.d.ts.map +1 -1
  7. package/dest/config.js +77 -39
  8. package/dest/global_variable_builder/fee_predictor.d.ts +37 -0
  9. package/dest/global_variable_builder/fee_predictor.d.ts.map +1 -0
  10. package/dest/global_variable_builder/fee_predictor.js +128 -0
  11. package/dest/global_variable_builder/fee_provider.d.ts +21 -0
  12. package/dest/global_variable_builder/fee_provider.d.ts.map +1 -0
  13. package/dest/global_variable_builder/fee_provider.js +58 -0
  14. package/dest/global_variable_builder/global_builder.d.ts +14 -16
  15. package/dest/global_variable_builder/global_builder.d.ts.map +1 -1
  16. package/dest/global_variable_builder/global_builder.js +16 -50
  17. package/dest/global_variable_builder/index.d.ts +4 -2
  18. package/dest/global_variable_builder/index.d.ts.map +1 -1
  19. package/dest/global_variable_builder/index.js +2 -0
  20. package/dest/index.d.ts +2 -2
  21. package/dest/index.d.ts.map +1 -1
  22. package/dest/index.js +1 -1
  23. package/dest/publisher/config.d.ts +47 -17
  24. package/dest/publisher/config.d.ts.map +1 -1
  25. package/dest/publisher/config.js +121 -42
  26. package/dest/publisher/index.d.ts +2 -1
  27. package/dest/publisher/index.d.ts.map +1 -1
  28. package/dest/publisher/l1_tx_failed_store/factory.d.ts +11 -0
  29. package/dest/publisher/l1_tx_failed_store/factory.d.ts.map +1 -0
  30. package/dest/publisher/l1_tx_failed_store/factory.js +22 -0
  31. package/dest/publisher/l1_tx_failed_store/failed_tx_store.d.ts +58 -0
  32. package/dest/publisher/l1_tx_failed_store/failed_tx_store.d.ts.map +1 -0
  33. package/dest/publisher/l1_tx_failed_store/failed_tx_store.js +1 -0
  34. package/dest/publisher/l1_tx_failed_store/file_store_failed_tx_store.d.ts +15 -0
  35. package/dest/publisher/l1_tx_failed_store/file_store_failed_tx_store.d.ts.map +1 -0
  36. package/dest/publisher/l1_tx_failed_store/file_store_failed_tx_store.js +34 -0
  37. package/dest/publisher/l1_tx_failed_store/index.d.ts +4 -0
  38. package/dest/publisher/l1_tx_failed_store/index.d.ts.map +1 -0
  39. package/dest/publisher/l1_tx_failed_store/index.js +2 -0
  40. package/dest/publisher/sequencer-bundle-simulator.d.ts +96 -0
  41. package/dest/publisher/sequencer-bundle-simulator.d.ts.map +1 -0
  42. package/dest/publisher/sequencer-bundle-simulator.js +198 -0
  43. package/dest/publisher/sequencer-publisher-factory.d.ts +11 -5
  44. package/dest/publisher/sequencer-publisher-factory.d.ts.map +1 -1
  45. package/dest/publisher/sequencer-publisher-factory.js +27 -3
  46. package/dest/publisher/sequencer-publisher-metrics.d.ts +1 -1
  47. package/dest/publisher/sequencer-publisher-metrics.d.ts.map +1 -1
  48. package/dest/publisher/sequencer-publisher-metrics.js +12 -4
  49. package/dest/publisher/sequencer-publisher.d.ts +94 -68
  50. package/dest/publisher/sequencer-publisher.d.ts.map +1 -1
  51. package/dest/publisher/sequencer-publisher.js +446 -363
  52. package/dest/sequencer/automine/automine_factory.d.ts +54 -0
  53. package/dest/sequencer/automine/automine_factory.d.ts.map +1 -0
  54. package/dest/sequencer/automine/automine_factory.js +84 -0
  55. package/dest/sequencer/automine/automine_sequencer.d.ts +151 -0
  56. package/dest/sequencer/automine/automine_sequencer.d.ts.map +1 -0
  57. package/dest/sequencer/automine/automine_sequencer.js +506 -0
  58. package/dest/sequencer/chain_state_overrides.d.ts +61 -0
  59. package/dest/sequencer/chain_state_overrides.d.ts.map +1 -0
  60. package/dest/sequencer/chain_state_overrides.js +98 -0
  61. package/dest/sequencer/checkpoint_proposal_job.d.ts +81 -13
  62. package/dest/sequencer/checkpoint_proposal_job.d.ts.map +1 -1
  63. package/dest/sequencer/checkpoint_proposal_job.js +874 -195
  64. package/dest/sequencer/checkpoint_proposal_job_metrics.d.ts +34 -0
  65. package/dest/sequencer/checkpoint_proposal_job_metrics.d.ts.map +1 -0
  66. package/dest/sequencer/checkpoint_proposal_job_metrics.js +72 -0
  67. package/dest/sequencer/checkpoint_voter.d.ts +3 -3
  68. package/dest/sequencer/checkpoint_voter.d.ts.map +1 -1
  69. package/dest/sequencer/checkpoint_voter.js +34 -13
  70. package/dest/sequencer/events.d.ts +47 -2
  71. package/dest/sequencer/events.d.ts.map +1 -1
  72. package/dest/sequencer/index.d.ts +3 -2
  73. package/dest/sequencer/index.d.ts.map +1 -1
  74. package/dest/sequencer/index.js +2 -1
  75. package/dest/sequencer/metrics.d.ts +23 -8
  76. package/dest/sequencer/metrics.d.ts.map +1 -1
  77. package/dest/sequencer/metrics.js +142 -36
  78. package/dest/sequencer/sequencer.d.ts +87 -26
  79. package/dest/sequencer/sequencer.d.ts.map +1 -1
  80. package/dest/sequencer/sequencer.js +408 -135
  81. package/dest/sequencer/timetable.d.ts +17 -6
  82. package/dest/sequencer/timetable.d.ts.map +1 -1
  83. package/dest/sequencer/timetable.js +51 -46
  84. package/dest/sequencer/types.d.ts +2 -2
  85. package/dest/sequencer/types.d.ts.map +1 -1
  86. package/dest/test/index.d.ts +3 -5
  87. package/dest/test/index.d.ts.map +1 -1
  88. package/dest/test/mock_checkpoint_builder.d.ts +23 -19
  89. package/dest/test/mock_checkpoint_builder.d.ts.map +1 -1
  90. package/dest/test/mock_checkpoint_builder.js +67 -38
  91. package/dest/test/utils.d.ts +8 -8
  92. package/dest/test/utils.d.ts.map +1 -1
  93. package/dest/test/utils.js +17 -15
  94. package/package.json +29 -28
  95. package/src/client/sequencer-client.ts +92 -35
  96. package/src/config.ts +103 -49
  97. package/src/global_variable_builder/README.md +44 -0
  98. package/src/global_variable_builder/fee_predictor.ts +172 -0
  99. package/src/global_variable_builder/fee_provider.ts +75 -0
  100. package/src/global_variable_builder/global_builder.ts +26 -63
  101. package/src/global_variable_builder/index.ts +3 -1
  102. package/src/index.ts +5 -1
  103. package/src/publisher/config.ts +157 -45
  104. package/src/publisher/index.ts +3 -0
  105. package/src/publisher/l1_tx_failed_store/factory.ts +32 -0
  106. package/src/publisher/l1_tx_failed_store/failed_tx_store.ts +57 -0
  107. package/src/publisher/l1_tx_failed_store/file_store_failed_tx_store.ts +46 -0
  108. package/src/publisher/l1_tx_failed_store/index.ts +3 -0
  109. package/src/publisher/sequencer-bundle-simulator.ts +254 -0
  110. package/src/publisher/sequencer-publisher-factory.ts +38 -9
  111. package/src/publisher/sequencer-publisher-metrics.ts +7 -3
  112. package/src/publisher/sequencer-publisher.ts +540 -460
  113. package/src/sequencer/README.md +152 -450
  114. package/src/sequencer/automine/README.md +46 -0
  115. package/src/sequencer/automine/automine_factory.ts +145 -0
  116. package/src/sequencer/automine/automine_sequencer.ts +592 -0
  117. package/src/sequencer/chain_state_overrides.ts +162 -0
  118. package/src/sequencer/checkpoint_proposal_job.ts +1088 -219
  119. package/src/sequencer/checkpoint_proposal_job_metrics.ts +128 -0
  120. package/src/sequencer/checkpoint_voter.ts +31 -17
  121. package/src/sequencer/events.ts +50 -2
  122. package/src/sequencer/index.ts +2 -1
  123. package/src/sequencer/metrics.ts +164 -39
  124. package/src/sequencer/sequencer.ts +509 -147
  125. package/src/sequencer/timetable.ts +70 -57
  126. package/src/sequencer/types.ts +1 -1
  127. package/src/test/index.ts +2 -4
  128. package/src/test/mock_checkpoint_builder.ts +122 -78
  129. package/src/test/utils.ts +50 -22
  130. package/dest/sequencer/block_builder.d.ts +0 -26
  131. package/dest/sequencer/block_builder.d.ts.map +0 -1
  132. package/dest/sequencer/block_builder.js +0 -129
  133. package/src/sequencer/block_builder.ts +0 -216
@@ -1,20 +1,26 @@
1
1
  import { getKzg } from '@aztec/blob-lib';
2
- import { INITIAL_L2_BLOCK_NUM } from '@aztec/constants';
3
- import type { EpochCache } from '@aztec/epoch-cache';
2
+ import { type EpochCache, PROPOSER_PIPELINING_SLOT_OFFSET } from '@aztec/epoch-cache';
4
3
  import { NoCommitteeError, type RollupContract } from '@aztec/ethereum/contracts';
5
4
  import { BlockNumber, CheckpointNumber, EpochNumber, SlotNumber } from '@aztec/foundation/branded-types';
6
5
  import { merge, omit, pick } from '@aztec/foundation/collection';
7
6
  import { Fr } from '@aztec/foundation/curves/bn254';
8
7
  import { EthAddress } from '@aztec/foundation/eth-address';
9
- import { createLogger } from '@aztec/foundation/log';
8
+ import { type Logger, createLogger } from '@aztec/foundation/log';
10
9
  import { RunningPromise } from '@aztec/foundation/running-promise';
11
10
  import type { DateProvider } from '@aztec/foundation/timer';
12
11
  import type { TypedEventEmitter } from '@aztec/foundation/types';
13
12
  import type { P2P } from '@aztec/p2p';
14
13
  import type { SlasherClientInterface } from '@aztec/slasher';
15
- import type { L2BlockNew, L2BlockSink, L2BlockSource, ValidateCheckpointResult } from '@aztec/stdlib/block';
16
- import type { Checkpoint } from '@aztec/stdlib/checkpoint';
17
- import { getSlotAtTimestamp, getSlotStartBuildTimestamp } from '@aztec/stdlib/epoch-helpers';
14
+ import type {
15
+ BlockData,
16
+ L2BlockSink,
17
+ L2BlockSource,
18
+ ProposedCheckpointSink,
19
+ ValidateCheckpointResult,
20
+ } from '@aztec/stdlib/block';
21
+ import type { Checkpoint, ProposedCheckpointData } from '@aztec/stdlib/checkpoint';
22
+ import type { ChainConfig } from '@aztec/stdlib/config';
23
+ import { getEpochAtSlot, getSlotStartBuildTimestamp } from '@aztec/stdlib/epoch-helpers';
18
24
  import {
19
25
  type ResolvedSequencerConfig,
20
26
  type SequencerConfig,
@@ -22,10 +28,10 @@ import {
22
28
  type WorldStateSynchronizer,
23
29
  } from '@aztec/stdlib/interfaces/server';
24
30
  import type { L1ToL2MessageSource } from '@aztec/stdlib/messaging';
31
+ import type { CoordinationSignatureContext } from '@aztec/stdlib/p2p';
25
32
  import { pickFromSchema } from '@aztec/stdlib/schemas';
26
- import { MerkleTreeId } from '@aztec/stdlib/trees';
27
33
  import { Attributes, type TelemetryClient, type Tracer, getTelemetryClient, trackSpan } from '@aztec/telemetry-client';
28
- import { FullNodeCheckpointsBuilder, type ValidatorClient } from '@aztec/validator-client';
34
+ import { FullNodeCheckpointsBuilder, NodeKeystoreAdapter, type ValidatorClient } from '@aztec/validator-client';
29
35
 
30
36
  import EventEmitter from 'node:events';
31
37
 
@@ -33,7 +39,9 @@ import { DefaultSequencerConfig } from '../config.js';
33
39
  import type { GlobalVariableBuilder } from '../global_variable_builder/global_builder.js';
34
40
  import type { SequencerPublisherFactory } from '../publisher/sequencer-publisher-factory.js';
35
41
  import type { InvalidateCheckpointRequest, SequencerPublisher } from '../publisher/sequencer-publisher.js';
42
+ import { buildCheckpointSimulationOverridesPlan } from './chain_state_overrides.js';
36
43
  import { CheckpointProposalJob } from './checkpoint_proposal_job.js';
44
+ import { CheckpointProposalJobMetrics } from './checkpoint_proposal_job_metrics.js';
37
45
  import { CheckpointVoter } from './checkpoint_voter.js';
38
46
  import { SequencerInterruptedError, SequencerTooSlowError } from './errors.js';
39
47
  import type { SequencerEvents } from './events.js';
@@ -44,6 +52,16 @@ import { SequencerState } from './utils.js';
44
52
 
45
53
  export { SequencerState };
46
54
 
55
+ /** Slot snapshot used to prepare a checkpoint proposal. */
56
+ type SequencerSlotContext = {
57
+ slot: SlotNumber;
58
+ targetSlot: SlotNumber;
59
+ epoch: EpochNumber;
60
+ targetEpoch: EpochNumber;
61
+ ts: bigint;
62
+ nowSeconds: bigint;
63
+ };
64
+
47
65
  /**
48
66
  * Sequencer client
49
67
  * - Checks whether it is elected as proposer for the next slot
@@ -55,10 +73,17 @@ export { SequencerState };
55
73
  export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<SequencerEvents>) {
56
74
  private runningPromise?: RunningPromise;
57
75
  private state = SequencerState.STOPPED;
76
+ private stateSlotNumber: SlotNumber | undefined;
77
+ private stateEnteredAtMs = performance.now();
58
78
  private metrics: SequencerMetrics;
79
+ private checkpointProposalJobMetrics: CheckpointProposalJobMetrics;
80
+ private readonly stateLog: Logger;
59
81
 
60
- /** The last slot for which we attempted to vote when sync failed, to prevent duplicate attempts. */
61
- private lastSlotForVoteWhenSyncFailed: SlotNumber | undefined;
82
+ /** The last slot for which we attempted to perform our voting duties with degraded block production */
83
+ private lastSlotForFallbackVote: SlotNumber | undefined;
84
+
85
+ /** The last slot for which we logged "no committee" warning, to avoid spam */
86
+ private lastSlotForNoCommitteeWarning: SlotNumber | undefined;
62
87
 
63
88
  /** The last slot for which we triggered a checkpoint proposal job, to prevent duplicate attempts. */
64
89
  private lastSlotForCheckpointProposalJob: SlotNumber | undefined;
@@ -69,19 +94,15 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
69
94
  /** The last epoch for which we logged strategy comparison in fisherman mode. */
70
95
  private lastEpochForStrategyComparison: EpochNumber | undefined;
71
96
 
97
+ /** The last checkpoint proposal job, tracked so we can await its pending L1 submission during shutdown. */
98
+ protected lastCheckpointProposalJob: CheckpointProposalJob | undefined;
99
+
72
100
  /** The maximum number of seconds that the sequencer can be into a slot to transition to a particular state. */
73
101
  protected timetable!: SequencerTimetable;
74
102
 
75
- // This shouldn't be here as this gets re-created each time we build/propose a block.
76
- // But we have a number of tests that abuse/rely on this class having a permanent publisher.
77
- // As long as those tests only configure a single publisher they will continue to work.
78
- // This will get re-assigned every time the sequencer goes to build a new block to a publisher that is valid
79
- // for the block proposer.
80
- // TODO(palla/mbps): Remove this field and fix tests
81
- protected publisher: SequencerPublisher | undefined;
82
-
83
103
  /** Config for the sequencer */
84
104
  protected config: ResolvedSequencerConfig = DefaultSequencerConfig;
105
+ private readonly signatureContext: CoordinationSignatureContext;
85
106
 
86
107
  constructor(
87
108
  protected publisherFactory: SequencerPublisherFactory,
@@ -90,51 +111,58 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
90
111
  protected p2pClient: P2P,
91
112
  protected worldState: WorldStateSynchronizer,
92
113
  protected slasherClient: SlasherClientInterface | undefined,
93
- protected l2BlockSource: L2BlockSource & L2BlockSink,
114
+ protected l2BlockSource: L2BlockSource & L2BlockSink & ProposedCheckpointSink,
94
115
  protected l1ToL2MessageSource: L1ToL2MessageSource,
95
116
  protected checkpointsBuilder: FullNodeCheckpointsBuilder,
96
117
  protected l1Constants: SequencerRollupConstants,
97
118
  protected dateProvider: DateProvider,
98
119
  protected epochCache: EpochCache,
99
120
  protected rollupContract: RollupContract,
100
- config: SequencerConfig,
121
+ config: SequencerConfig & Pick<ChainConfig, 'l1ChainId' | 'rollupAddress'>,
101
122
  protected telemetry: TelemetryClient = getTelemetryClient(),
102
123
  protected log = createLogger('sequencer'),
103
124
  ) {
104
125
  super();
126
+ this.stateLog = log.createChild('state');
105
127
 
106
128
  // Add [FISHERMAN] prefix to logger if in fisherman mode
107
129
  if (config.fishermanMode) {
108
130
  this.log = log.createChild('[FISHERMAN]');
109
131
  }
110
132
 
133
+ this.signatureContext = {
134
+ chainId: config.l1ChainId,
135
+ rollupAddress: config.rollupAddress,
136
+ };
111
137
  this.metrics = new SequencerMetrics(telemetry, this.rollupContract, 'Sequencer');
138
+ this.checkpointProposalJobMetrics = new CheckpointProposalJobMetrics(telemetry);
112
139
  this.updateConfig(config);
113
140
  }
114
141
 
115
142
  /** Updates sequencer config by the defined values and updates the timetable */
116
143
  public updateConfig(config: Partial<SequencerConfig>) {
117
144
  const filteredConfig = pickFromSchema(config, SequencerConfigSchema);
118
- this.log.info(`Updated sequencer config`, omit(filteredConfig, 'txPublicSetupAllowList'));
145
+ this.log.info(`Updated sequencer config`, omit(filteredConfig, 'txPublicSetupAllowListExtend'));
119
146
  this.config = merge(this.config, filteredConfig);
147
+ const p2pPropagationTime = this.config.attestationPropagationTime;
120
148
  this.timetable = new SequencerTimetable(
121
149
  {
122
150
  ethereumSlotDuration: this.l1Constants.ethereumSlotDuration,
123
151
  aztecSlotDuration: this.aztecSlotDuration,
124
152
  l1PublishingTime: this.l1PublishingTime,
125
- p2pPropagationTime: this.config.attestationPropagationTime,
153
+ p2pPropagationTime,
126
154
  blockDurationMs: this.config.blockDurationMs,
127
155
  enforce: this.config.enforceTimeTable,
156
+ pipelining: this.epochCache.isProposerPipeliningEnabled(),
128
157
  },
129
158
  this.metrics,
130
159
  this.log,
131
160
  );
132
161
  }
133
162
 
134
- /** Initializes the sequencer (precomputes tables and creates a publisher). Takes about 3s. */
135
- public async init() {
163
+ /** Initializes the sequencer (precomputes tables). Takes about 3s. */
164
+ public init() {
136
165
  getKzg();
137
- this.publisher = (await this.publisherFactory.create(undefined)).publisher;
138
166
  }
139
167
 
140
168
  /** Starts the sequencer and moves to IDLE state. */
@@ -149,12 +177,18 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
149
177
  this.log.info('Started sequencer');
150
178
  }
151
179
 
180
+ /** Triggers an immediate run of the sequencer, bypassing the polling interval. */
181
+ public trigger() {
182
+ return this.runningPromise?.trigger();
183
+ }
184
+
152
185
  /** Stops the sequencer from building blocks and moves to STOPPED state. */
153
186
  public async stop(): Promise<void> {
154
187
  this.log.info(`Stopping sequencer`);
155
188
  this.setState(SequencerState.STOPPING, undefined, { force: true });
156
- this.publisher?.interrupt();
189
+ await this.publisherFactory.stopAll();
157
190
  await this.runningPromise?.stop();
191
+ await this.lastCheckpointProposalJob?.awaitPendingSubmission();
158
192
  this.setState(SequencerState.STOPPED, undefined, { force: true });
159
193
  this.log.info('Stopped sequencer');
160
194
  }
@@ -164,18 +198,15 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
164
198
  try {
165
199
  await this.work();
166
200
  } catch (err) {
167
- this.emit('checkpoint-error', { error: err as Error });
168
- if (err instanceof SequencerTooSlowError) {
169
- // TODO(palla/mbps): Add missing states
170
- // Log as warn only if we had to abort halfway through the block proposal
171
- const logLvl = [SequencerState.INITIALIZING_CHECKPOINT, SequencerState.PROPOSER_CHECK].includes(
172
- err.proposedState,
173
- )
174
- ? ('debug' as const)
175
- : ('warn' as const);
176
- this.log[logLvl](err.message, { now: this.dateProvider.nowInSeconds() });
201
+ if (
202
+ err instanceof SequencerTooSlowError &&
203
+ [SequencerState.INITIALIZING_CHECKPOINT, SequencerState.PROPOSER_CHECK].includes(err.proposedState)
204
+ ) {
205
+ // No need to alert if we just didn't get to start in time
206
+ this.log.debug(err.message, { now: this.dateProvider.nowInSeconds() });
177
207
  } else {
178
208
  // Re-throw other errors
209
+ this.emit('checkpoint-error', { error: err as Error });
179
210
  throw err;
180
211
  }
181
212
  } finally {
@@ -199,14 +230,24 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
199
230
  @trackSpan('Sequencer.work')
200
231
  protected async work() {
201
232
  this.setState(SequencerState.SYNCHRONIZING, undefined);
202
- const { slot, ts, now, epoch } = this.epochCache.getEpochAndSlotInNextL1Slot();
233
+ const { slot, targetSlot, epoch, targetEpoch, ts, nowSeconds } = this.getSlotContextInNextL1Slot();
203
234
 
204
235
  // Check if we are synced and it's our slot, grab a publisher, check previous block invalidation, etc
205
- const checkpointProposalJob = await this.prepareCheckpointProposal(slot, ts, now);
236
+ const checkpointProposalJob = await this.prepareCheckpointProposal(
237
+ slot,
238
+ targetSlot,
239
+ epoch,
240
+ targetEpoch,
241
+ ts,
242
+ nowSeconds,
243
+ );
206
244
  if (!checkpointProposalJob) {
207
245
  return;
208
246
  }
209
247
 
248
+ // Track the job so we can await its pending L1 submission during shutdown
249
+ this.lastCheckpointProposalJob = checkpointProposalJob;
250
+
210
251
  // Execute the checkpoint proposal job
211
252
  const checkpoint = await checkpointProposalJob.execute();
212
253
 
@@ -215,51 +256,94 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
215
256
  this.lastCheckpointProposed = checkpoint;
216
257
  }
217
258
 
218
- // Log fee strategy comparison if on fisherman
259
+ // Log fee strategy comparison if on fisherman (uses target epoch since we mirror the proposer's perspective)
219
260
  if (
220
261
  this.config.fishermanMode &&
221
- (this.lastEpochForStrategyComparison === undefined || epoch > this.lastEpochForStrategyComparison)
262
+ (this.lastEpochForStrategyComparison === undefined || targetEpoch > this.lastEpochForStrategyComparison)
222
263
  ) {
223
- this.logStrategyComparison(epoch, checkpointProposalJob.getPublisher());
224
- this.lastEpochForStrategyComparison = epoch;
264
+ this.logStrategyComparison(targetEpoch, checkpointProposalJob.getPublisher());
265
+ this.lastEpochForStrategyComparison = targetEpoch;
225
266
  }
226
267
 
227
268
  return checkpoint;
228
269
  }
229
270
 
271
+ /** Returns slot and target slot from a single clock snapshot. */
272
+ protected getSlotContextInNextL1Slot(): SequencerSlotContext {
273
+ const { slot, ts, nowSeconds, epoch } = this.epochCache.getEpochAndSlotInNextL1Slot();
274
+ const targetSlot = SlotNumber(
275
+ slot + (this.epochCache.isProposerPipeliningEnabled() ? PROPOSER_PIPELINING_SLOT_OFFSET : 0),
276
+ );
277
+ return { slot, targetSlot, epoch, targetEpoch: getEpochAtSlot(targetSlot, this.l1Constants), ts, nowSeconds };
278
+ }
279
+
230
280
  /**
231
281
  * Prepares the checkpoint proposal by performing all necessary checks and setup.
232
282
  * This is the initial step in the main loop.
233
283
  * @returns CheckpointProposalJob if successful, undefined if we are not yet synced or are not the proposer.
234
284
  */
235
285
  @trackSpan('Sequencer.prepareCheckpointProposal')
236
- private async prepareCheckpointProposal(
286
+ protected async prepareCheckpointProposal(
237
287
  slot: SlotNumber,
288
+ targetSlot: SlotNumber,
289
+ epoch: EpochNumber,
290
+ targetEpoch: EpochNumber,
238
291
  ts: bigint,
239
- now: bigint,
292
+ nowSeconds: bigint,
240
293
  ): Promise<CheckpointProposalJob | undefined> {
241
- // Check we have not already processed this slot (cheapest check)
294
+ // Check we have not already processed this target slot (cheapest check)
242
295
  // We only check this if enforce timetable is set, since we want to keep processing the same slot if we are not
243
296
  // running against actual time (eg when we use sandbox-style automining)
244
297
  if (
245
298
  this.lastSlotForCheckpointProposalJob &&
246
- this.lastSlotForCheckpointProposalJob >= slot &&
299
+ this.lastSlotForCheckpointProposalJob >= targetSlot &&
247
300
  this.config.enforceTimeTable
248
301
  ) {
249
- this.log.trace(`Slot ${slot} has already been processed`);
302
+ this.log.trace(`Target slot ${targetSlot} has already been processed`);
250
303
  return undefined;
251
304
  }
252
305
 
253
- // But if we have already proposed for this slot, the we definitely have to skip it, automining or not
254
- if (this.lastCheckpointProposed && this.lastCheckpointProposed.header.slotNumber >= slot) {
255
- this.log.trace(`Slot ${slot} has already been published as checkpoint ${this.lastCheckpointProposed.number}`);
306
+ // But if we have already proposed for this slot, then we definitely have to skip it, automining or not
307
+ if (this.lastCheckpointProposed && this.lastCheckpointProposed.header.slotNumber >= targetSlot) {
308
+ this.log.trace(
309
+ `Slot ${targetSlot} has already been published as checkpoint ${this.lastCheckpointProposed.number}`,
310
+ );
311
+ return undefined;
312
+ }
313
+
314
+ // Test-only: skip proposing for explicitly paused slots. Attestation paths run in the validator
315
+ // client and are not gated by this hook, so paused proposers still attest to others' proposals.
316
+ if (this.config.pauseProposingForSlots?.some(s => s === targetSlot)) {
317
+ this.log.warn(`Skipping proposal for paused slot ${targetSlot} (test-only pauseProposingForSlots hook)`, {
318
+ targetSlot,
319
+ });
256
320
  return undefined;
257
321
  }
258
322
 
259
323
  // Check all components are synced to latest as seen by the archiver (queries all subsystems)
260
324
  const syncedTo = await this.checkSync({ ts, slot });
261
325
  if (!syncedTo) {
262
- await this.tryVoteWhenSyncFails({ slot, ts });
326
+ await this.tryVoteWhenSyncFails({ slot, targetSlot, ts });
327
+ return undefined;
328
+ }
329
+
330
+ // If escape hatch is open for the target epoch, do not start checkpoint proposal work and do not attempt invalidations.
331
+ // Still perform governance/slashing voting (as proposer) once per slot.
332
+ // When pipelining, we check the target epoch (slot+1's epoch) since that's the epoch we're building for.
333
+ const isEscapeHatchOpen = await this.epochCache.isEscapeHatchOpen(targetEpoch);
334
+
335
+ if (isEscapeHatchOpen) {
336
+ this.setState(SequencerState.PROPOSER_CHECK, slot);
337
+ const [canPropose, proposer] = await this.checkCanPropose(targetSlot);
338
+ if (canPropose) {
339
+ await this.tryVoteWhenEscapeHatchOpen({ slot, targetSlot, proposer });
340
+ } else {
341
+ this.log.trace(`Escape hatch open but we are not proposer, skipping vote-only actions`, {
342
+ slot,
343
+ epoch,
344
+ proposer,
345
+ });
346
+ }
263
347
  return undefined;
264
348
  }
265
349
 
@@ -267,18 +351,18 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
267
351
  const checkpointNumber = CheckpointNumber(syncedTo.checkpointNumber + 1);
268
352
 
269
353
  const logCtx = {
270
- now,
271
- syncedToL1Ts: syncedTo.l1Timestamp,
272
- syncedToL2Slot: getSlotAtTimestamp(syncedTo.l1Timestamp, this.l1Constants),
354
+ nowSeconds,
355
+ syncedToL2Slot: syncedTo.syncedL2Slot,
273
356
  slot,
357
+ targetSlot,
274
358
  slotTs: ts,
275
359
  checkpointNumber,
276
360
  isPendingChainValid: pick(syncedTo.pendingChainValidationStatus, 'valid', 'reason', 'invalidIndex'),
277
361
  };
278
362
 
279
- // Check that we are a proposer for the next slot
363
+ // Check that we are a proposer for the target slot.
280
364
  this.setState(SequencerState.PROPOSER_CHECK, slot);
281
- const [canPropose, proposer] = await this.checkCanPropose(slot);
365
+ const [canPropose, proposer] = await this.checkCanPropose(targetSlot);
282
366
 
283
367
  // If we are not a proposer check if we should invalidate an invalid checkpoint, and bail
284
368
  if (!canPropose) {
@@ -286,13 +370,23 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
286
370
  return undefined;
287
371
  }
288
372
 
289
- // Check that the slot is not taken by a block already (should never happen, since only us can propose for this slot)
290
- if (syncedTo.block && syncedTo.block.header.getSlot() >= slot) {
373
+ // Guard: don't exceed 1-deep pipeline. Without a proposed checkpoint, we can only build
374
+ // confirmed + 1. With a proposed checkpoint, we can build confirmed + 2.
375
+ const confirmedCkpt = syncedTo.checkpointedCheckpointNumber;
376
+ if (checkpointNumber > confirmedCkpt + 2) {
377
+ this.log.verbose(
378
+ `Skipping slot ${targetSlot}: checkpoint ${checkpointNumber} exceeds max pipeline depth (confirmed=${confirmedCkpt})`,
379
+ );
380
+ return undefined;
381
+ }
382
+
383
+ // Check that the target slot is not taken by a block already (should never happen, since only us can propose for this slot)
384
+ if (syncedTo.blockData && syncedTo.blockData.header.getSlot() >= targetSlot) {
291
385
  this.log.warn(
292
- `Cannot propose block at next L2 slot ${slot} since that slot was taken by block ${syncedTo.blockNumber}`,
293
- { ...logCtx, block: syncedTo.block.header.toInspect() },
386
+ `Cannot propose block at target slot ${targetSlot} since that slot was taken by block ${syncedTo.blockNumber}`,
387
+ { ...logCtx, block: syncedTo.blockData.header.toInspect() },
294
388
  );
295
- this.metrics.recordBlockProposalPrecheckFailed('slot_already_taken');
389
+ this.metrics.recordCheckpointPrecheckFailed('slot_already_taken');
296
390
  return undefined;
297
391
  }
298
392
 
@@ -303,83 +397,175 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
303
397
  const proposerForPublisher = this.config.fishermanMode ? undefined : proposer;
304
398
  const { attestorAddress, publisher } = await this.publisherFactory.create(proposerForPublisher);
305
399
  this.log.verbose(`Created publisher at address ${publisher.getSenderAddress()} for attestor ${attestorAddress}`);
306
- this.publisher = publisher;
307
400
 
308
- // In fisherman mode, set the actual proposer's address for simulations
309
- if (this.config.fishermanMode && proposer) {
310
- publisher.setProposerAddressForSimulation(proposer);
311
- this.log.debug(`Set proposer address ${proposer} for simulation in fisherman mode`);
401
+ const isPipelining = this.epochCache.isProposerPipeliningEnabled();
402
+
403
+ // Prepare invalidation request if the pending chain is invalid (returns undefined if no need).
404
+ // Only simulate invalidation when there's no proposed parent, since we assume the proposed parent
405
+ // will invalidate the currently invalid checkpoint on L1.
406
+ const invalidateCheckpoint =
407
+ (isPipelining && syncedTo.hasProposedCheckpoint) || syncedTo.pendingChainValidationStatus.valid
408
+ ? undefined
409
+ : await publisher.simulateInvalidateCheckpoint(syncedTo.pendingChainValidationStatus);
410
+
411
+ // Determine the correct archive and L1 state overrides for the canProposeAt check.
412
+ // The L1 contract reads archives[proposedCheckpointNumber] and compares it with the provided archive.
413
+ // When invalidating or pipelining, the local archive may differ from L1's, so we adjust accordingly.
414
+ let archiveForCheck = syncedTo.archive;
415
+
416
+ if (isPipelining && syncedTo.hasProposedCheckpoint) {
417
+ this.metrics.recordPipelineDepth(syncedTo.checkpointNumber - syncedTo.checkpointedCheckpointNumber);
418
+ this.log.verbose(
419
+ `Building on top of proposed checkpoint (pending=${syncedTo.proposedCheckpointData?.checkpointNumber}) for target slot ${targetSlot}`,
420
+ { targetSlot, parentCheckpointNumber: CheckpointNumber(checkpointNumber - 1) },
421
+ );
422
+ // Match what L1 will see at archives[pending] once the proposed parent lands: the parent's
423
+ // own archive root from the gossiped proposal. `syncedTo.archive` is the world-state-local
424
+ // view and can transiently diverge from the proposed parent (e.g. before the proposed
425
+ // parent's blocks have been applied locally); diverging here would cause the canProposeAt
426
+ // override to set archives[pending] to one value while we present another for comparison.
427
+ archiveForCheck = syncedTo.proposedCheckpointData!.archive.root;
428
+ } else if (invalidateCheckpoint) {
429
+ // After invalidation, L1 will roll back to checkpoint N-1. The archive at N-1 already
430
+ // exists on L1, so we just pass the matching archive (the lastArchive of the invalid checkpoint).
431
+ archiveForCheck = invalidateCheckpoint.lastArchive;
432
+ this.metrics.recordPipelineDepth(0);
433
+ } else {
434
+ this.metrics.recordPipelineDepth(0);
435
+ }
436
+
437
+ // Build the simulation plan: pending/proven override from pipelining or invalidation (or the
438
+ // current snapshot when neither applies, to short-circuit any pending prune in simulation),
439
+ // plus the parent checkpoint cell and fee header when pipelining.
440
+ const simulationOverridesPlan = await buildCheckpointSimulationOverridesPlan({
441
+ checkpointNumber,
442
+ proposedCheckpointData:
443
+ isPipelining && syncedTo.hasProposedCheckpoint ? syncedTo.proposedCheckpointData : undefined,
444
+ invalidateToPendingCheckpointNumber: invalidateCheckpoint?.forcePendingCheckpointNumber,
445
+ checkpointedCheckpointNumber: syncedTo.checkpointedCheckpointNumber,
446
+ rollup: this.rollupContract,
447
+ signatureContext: this.signatureContext,
448
+ log: this.log,
449
+ });
450
+
451
+ // The plan always pins both pending/proven (to short-circuit `canPruneAtTime` in simulation),
452
+ // so `provenOverride` always reflects the assumed proven checkpoint we are pinning the
453
+ // simulation to. We additionally warn when the pin is load-bearing — i.e. when a prune would
454
+ // actually fire at the target slot without it — so observers can spot "we are building
455
+ // optimistically across a pruning boundary" in the logs.
456
+ const provenOverride = simulationOverridesPlan?.chainTipsOverride?.proven;
457
+ if (provenOverride !== undefined && (await this.l2BlockSource.isPruneDueAtSlot(targetSlot))) {
458
+ this.log.warn(
459
+ `Assuming proof for epoch ending at checkpoint ${provenOverride} lands by target slot ${targetSlot}`,
460
+ { checkpointNumber, slot, targetSlot, provenOverride },
461
+ );
312
462
  }
313
463
 
314
- // Prepare invalidation request if the pending chain is invalid (returns undefined if no need)
315
- const invalidateCheckpoint = await publisher.simulateInvalidateCheckpoint(syncedTo.pendingChainValidationStatus);
464
+ this.emit('preparing-checkpoint', {
465
+ targetSlot,
466
+ checkpointNumber,
467
+ hadProposedParent: this.epochCache.isProposerPipeliningEnabled() && syncedTo.hasProposedCheckpoint,
468
+ provenOverride,
469
+ simulatedPending: simulationOverridesPlan?.chainTipsOverride?.pending,
470
+ });
316
471
 
317
- // Check with the rollup contract if we can indeed propose at the next L2 slot. This check should not fail
318
- // if all the previous checks are good, but we do it just in case.
319
- const canProposeCheck = await publisher.canProposeAtNextEthBlock(
320
- syncedTo.archive,
472
+ const canProposeCheck = await publisher.canProposeAt(
473
+ archiveForCheck,
321
474
  proposer ?? EthAddress.ZERO,
322
- invalidateCheckpoint,
475
+ simulationOverridesPlan,
323
476
  );
324
477
 
478
+ const proposeContext = {
479
+ hasProposedCheckpoint: syncedTo.hasProposedCheckpoint,
480
+ proposedCheckpointNumber: syncedTo.proposedCheckpointData?.checkpointNumber,
481
+ checkpointedCheckpointNumber: syncedTo.checkpointedCheckpointNumber,
482
+ isInvalidating: !!invalidateCheckpoint,
483
+ invalidatingCheckpointNumber: invalidateCheckpoint?.checkpointNumber,
484
+ archiveForCheck: archiveForCheck.toString(),
485
+ overridePendingCheckpointNumber: simulationOverridesPlan?.chainTipsOverride?.pending,
486
+ overrideArchive: simulationOverridesPlan?.pendingCheckpointState?.archive,
487
+ overrideFeeHeader: simulationOverridesPlan?.pendingCheckpointState?.feeHeader,
488
+ };
489
+
325
490
  if (canProposeCheck === undefined) {
326
491
  this.log.warn(
327
492
  `Cannot propose checkpoint ${checkpointNumber} at slot ${slot} due to failed rollup contract check`,
328
- logCtx,
493
+ { ...logCtx, ...proposeContext },
329
494
  );
330
495
  this.emit('proposer-rollup-check-failed', { reason: 'Rollup contract check failed', slot });
331
- this.metrics.recordBlockProposalPrecheckFailed('rollup_contract_check_failed');
496
+ this.metrics.recordCheckpointPrecheckFailed('rollup_contract_check_failed');
332
497
  return undefined;
333
498
  }
334
499
 
335
- if (canProposeCheck.slot !== slot) {
500
+ if (canProposeCheck.slot !== targetSlot) {
336
501
  this.log.warn(
337
- `Cannot propose block due to slot mismatch with rollup contract (this can be caused by a clock out of sync). Expected slot ${slot} but got ${canProposeCheck.slot}.`,
338
- { ...logCtx, rollup: canProposeCheck, expectedSlot: slot },
502
+ `Cannot propose block due to slot mismatch with rollup contract (this can be caused by a clock out of sync). Expected slot ${targetSlot} but got ${canProposeCheck.slot}.`,
503
+ { ...logCtx, ...proposeContext, rollup: canProposeCheck, expectedSlot: targetSlot },
339
504
  );
340
505
  this.emit('proposer-rollup-check-failed', { reason: 'Slot mismatch', slot });
341
- this.metrics.recordBlockProposalPrecheckFailed('slot_mismatch');
506
+ this.metrics.recordCheckpointPrecheckFailed('slot_mismatch');
342
507
  return undefined;
343
508
  }
344
509
 
345
510
  if (canProposeCheck.checkpointNumber !== checkpointNumber) {
346
511
  this.log.warn(
347
512
  `Cannot propose due to block mismatch with rollup contract (this can be caused by a pending archiver sync). Expected checkpoint ${checkpointNumber} but got ${canProposeCheck.checkpointNumber}.`,
348
- { ...logCtx, rollup: canProposeCheck, expectedSlot: slot },
513
+ { ...logCtx, ...proposeContext, rollup: canProposeCheck, expectedSlot: slot },
349
514
  );
350
515
  this.emit('proposer-rollup-check-failed', { reason: 'Block mismatch', slot });
351
- this.metrics.recordBlockProposalPrecheckFailed('block_number_mismatch');
516
+ this.metrics.recordCheckpointPrecheckFailed('block_number_mismatch');
352
517
  return undefined;
353
518
  }
354
519
 
355
- this.lastSlotForCheckpointProposalJob = slot;
356
- this.log.info(`Preparing checkpoint proposal ${checkpointNumber} at slot ${slot}`, { ...logCtx, proposer });
520
+ this.lastSlotForCheckpointProposalJob = targetSlot;
521
+
522
+ await this.p2pClient.prepareForSlot(targetSlot);
523
+ this.log.info(
524
+ `Preparing checkpoint proposal ${checkpointNumber} for target slot ${targetSlot} during wall-clock slot ${slot}`,
525
+ {
526
+ ...logCtx,
527
+ ...proposeContext,
528
+ proposer,
529
+ pipeliningEnabled: this.epochCache.isProposerPipeliningEnabled(),
530
+ },
531
+ );
357
532
 
358
533
  // Create and return the checkpoint proposal job
359
534
  return this.createCheckpointProposalJob(
360
535
  slot,
536
+ targetSlot,
537
+ targetEpoch,
361
538
  checkpointNumber,
362
539
  syncedTo.blockNumber,
540
+ syncedTo.checkpointedCheckpointNumber,
363
541
  proposer,
364
542
  publisher,
365
543
  attestorAddress,
366
544
  invalidateCheckpoint,
545
+ syncedTo.proposedCheckpointData,
367
546
  );
368
547
  }
369
548
 
370
549
  protected createCheckpointProposalJob(
371
550
  slot: SlotNumber,
551
+ targetSlot: SlotNumber,
552
+ targetEpoch: EpochNumber,
372
553
  checkpointNumber: CheckpointNumber,
373
554
  syncedToBlockNumber: BlockNumber,
555
+ checkpointedCheckpointNumber: CheckpointNumber,
374
556
  proposer: EthAddress | undefined,
375
557
  publisher: SequencerPublisher,
376
558
  attestorAddress: EthAddress,
377
559
  invalidateCheckpoint: InvalidateCheckpointRequest | undefined,
560
+ proposedCheckpointData?: ProposedCheckpointData,
378
561
  ): CheckpointProposalJob {
379
562
  return new CheckpointProposalJob(
380
563
  slot,
564
+ targetSlot,
565
+ targetEpoch,
381
566
  checkpointNumber,
382
567
  syncedToBlockNumber,
568
+ checkpointedCheckpointNumber,
383
569
  proposer,
384
570
  publisher,
385
571
  attestorAddress,
@@ -389,22 +575,33 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
389
575
  this.p2pClient,
390
576
  this.worldState,
391
577
  this.l1ToL2MessageSource,
578
+ this.l2BlockSource,
392
579
  this.checkpointsBuilder,
393
580
  this.l2BlockSource,
394
581
  this.l1Constants,
582
+ this.signatureContext,
395
583
  this.config,
396
584
  this.timetable,
397
585
  this.slasherClient,
398
586
  this.epochCache,
399
587
  this.dateProvider,
400
588
  this.metrics,
589
+ this.checkpointProposalJobMetrics.createRecorder(),
401
590
  this,
402
591
  this.setState.bind(this),
403
- this.log,
404
592
  this.tracer,
593
+ this.log.getBindings(),
594
+ proposedCheckpointData,
405
595
  );
406
596
  }
407
597
 
598
+ /**
599
+ * Returns the current sequencer state.
600
+ */
601
+ public getState(): SequencerState {
602
+ return this.state;
603
+ }
604
+
408
605
  /**
409
606
  * Internal helper for setting the sequencer state and checks if we have enough time left in the slot to transition to the new state.
410
607
  * @param proposedState - The new state to transition to.
@@ -430,19 +627,35 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
430
627
  this.timetable.assertTimeLeft(proposedState, secondsIntoSlot);
431
628
  }
432
629
 
630
+ const oldState = this.state;
631
+ const oldStateSlotNumber = this.stateSlotNumber;
632
+ const stateChanged = proposedState !== oldState;
633
+ const transitionAtMs = performance.now();
634
+ const stateDurationMs = transitionAtMs - this.stateEnteredAtMs;
635
+
433
636
  const boringStates = [SequencerState.IDLE, SequencerState.SYNCHRONIZING];
434
637
  const logLevel =
435
- boringStates.includes(proposedState) && boringStates.includes(this.state)
436
- ? ('trace' as const)
437
- : ('debug' as const);
438
- this.log[logLevel](`Transitioning from ${this.state} to ${proposedState}`, { slotNumber, secondsIntoSlot });
638
+ boringStates.includes(proposedState) && boringStates.includes(oldState) ? ('trace' as const) : ('debug' as const);
639
+ this.stateLog[logLevel](`Transitioning from ${oldState} to ${proposedState}`, {
640
+ oldState,
641
+ newState: proposedState,
642
+ slotNumber,
643
+ stateSlotNumber: oldStateSlotNumber,
644
+ secondsIntoSlot,
645
+ ...(stateChanged && { stateDurationMs: Math.ceil(stateDurationMs) }),
646
+ });
439
647
 
440
648
  this.emit('state-changed', {
441
- oldState: this.state,
649
+ oldState,
442
650
  newState: proposedState,
443
651
  secondsIntoSlot,
444
652
  slot: slotNumber,
445
653
  });
654
+ if (stateChanged) {
655
+ this.metrics.recordStateDuration(stateDurationMs, oldState);
656
+ this.stateEnteredAtMs = transitionAtMs;
657
+ this.stateSlotNumber = slotNumber;
658
+ }
446
659
  this.state = proposedState;
447
660
  }
448
661
 
@@ -451,16 +664,15 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
451
664
  * We don't check against the previous block submitted since it may have been reorg'd out.
452
665
  */
453
666
  protected async checkSync(args: { ts: bigint; slot: SlotNumber }): Promise<SequencerSyncCheckResult | undefined> {
454
- // Check that the archiver and dependencies have synced to the previous L1 slot at least
455
- // TODO(#14766): Archiver reports L1 timestamp based on L1 blocks seen, which means that a missed L1 block will
456
- // cause the archiver L1 timestamp to fall behind, and cause this sequencer to start processing one L1 slot later.
457
- const l1Timestamp = await this.l2BlockSource.getL1Timestamp();
458
- const { slot, ts } = args;
459
- if (l1Timestamp === undefined || l1Timestamp + BigInt(this.l1Constants.ethereumSlotDuration) < ts) {
667
+ // Check that the archiver has fully synced the L2 slot before the one we want to propose in.
668
+ // The archiver reports sync progress via L1 block timestamps and synced checkpoint slots.
669
+ // See getSyncedL2SlotNumber for how missed L1 blocks are handled.
670
+ const syncedL2Slot = await this.l2BlockSource.getSyncedL2SlotNumber();
671
+ const { slot } = args;
672
+ if (syncedL2Slot === undefined || syncedL2Slot + 1 < slot) {
460
673
  this.log.debug(`Cannot propose block at next L2 slot ${slot} due to pending sync from L1`, {
461
674
  slot,
462
- ts,
463
- l1Timestamp,
675
+ syncedL2Slot,
464
676
  });
465
677
  return undefined;
466
678
  }
@@ -470,54 +682,114 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
470
682
  number: syncSummary.latestBlockNumber,
471
683
  hash: syncSummary.latestBlockHash,
472
684
  })),
473
- this.l2BlockSource.getL2Tips().then(t => t.proposed),
685
+ this.l2BlockSource
686
+ .getL2Tips()
687
+ .then(t => ({ proposed: t.proposed, checkpointed: t.checkpointed, proposedCheckpoint: t.proposedCheckpoint })),
474
688
  this.p2pClient.getStatus().then(p2p => p2p.syncedToL2Block),
475
- this.l1ToL2MessageSource.getL2Tips().then(t => t.proposed),
689
+ this.l1ToL2MessageSource.getL2Tips().then(t => ({ proposed: t.proposed, checkpointed: t.checkpointed })),
476
690
  this.l2BlockSource.getPendingChainValidationStatus(),
691
+ this.l2BlockSource.getProposedCheckpointData(),
477
692
  ] as const);
478
693
 
479
- const [worldState, l2BlockSource, p2p, l1ToL2MessageSource, pendingChainValidationStatus] = syncedBlocks;
694
+ const [worldState, l2Tips, p2p, l1ToL2MessageSourceTips, pendingChainValidationStatus, proposedCheckpointData] =
695
+ syncedBlocks;
480
696
 
481
- // Handle zero as a special case, since the block hash won't match across services if we're changing the prefilled data for the genesis block,
482
- // as the world state can compute the new genesis block hash, but other components use the hardcoded constant.
483
- // TODO(palla/mbps): Fix the above. All components should be able to handle dynamic genesis block hashes.
484
697
  const result =
485
- (l2BlockSource.number === 0 && worldState.number === 0 && p2p.number === 0 && l1ToL2MessageSource.number === 0) ||
486
- (worldState.hash === l2BlockSource.hash &&
487
- p2p.hash === l2BlockSource.hash &&
488
- l1ToL2MessageSource.hash === l2BlockSource.hash);
698
+ worldState.hash === l2Tips.proposed.hash &&
699
+ p2p.hash === l2Tips.proposed.hash &&
700
+ l1ToL2MessageSourceTips.proposed.hash === l2Tips.proposed.hash &&
701
+ l1ToL2MessageSourceTips.checkpointed.block.hash === l2Tips.checkpointed.block.hash &&
702
+ l1ToL2MessageSourceTips.checkpointed.checkpoint.hash === l2Tips.checkpointed.checkpoint.hash;
489
703
 
490
704
  if (!result) {
491
- this.log.debug(`Sequencer sync check failed`, { worldState, l2BlockSource, p2p, l1ToL2MessageSource });
705
+ this.log.debug(`Sequencer sync check failed`, {
706
+ worldState,
707
+ l2BlockSource: l2Tips.proposed,
708
+ p2p,
709
+ l1ToL2MessageSourceTips,
710
+ });
492
711
  return undefined;
493
712
  }
494
713
 
495
- // Special case for genesis state
496
714
  const blockNumber = worldState.number;
497
- if (blockNumber < INITIAL_L2_BLOCK_NUM) {
498
- const archive = new Fr((await this.worldState.getCommitted().getTreeInfo(MerkleTreeId.ARCHIVE)).root);
499
- return {
500
- checkpointNumber: CheckpointNumber.ZERO,
501
- blockNumber: BlockNumber.ZERO,
502
- archive,
503
- l1Timestamp,
504
- pendingChainValidationStatus,
505
- };
506
- }
507
-
508
- const block = await this.l2BlockSource.getL2BlockNew(blockNumber);
509
- if (!block) {
510
- // this shouldn't really happen because a moment ago we checked that all components were in sync
511
- this.log.error(`Failed to get L2 block ${blockNumber} from the archiver with all components in sync`);
715
+ const blockData = await this.l2BlockSource.getBlockData({ number: blockNumber });
716
+ if (!blockData) {
717
+ this.log.warn(`Sequencer sync check failed: failed to get L2 block data ${blockNumber} from the archiver`, {
718
+ blockNumber,
719
+ l2Tips,
720
+ syncedL2Slot,
721
+ ...args,
722
+ });
723
+ return undefined;
724
+ }
725
+
726
+ // Refuse to build a checkpoint on top of a proposed block whose enclosing checkpoint was never
727
+ // proposed. Under pipelining we may have received and reexecuted such a block locally — advancing
728
+ // our world-state tip past the checkpointed tip while the proposing node never published the
729
+ // matching proposed checkpoint (e.g. it crashed before assembling it). Building on this orphan block
730
+ // would fork the chain off a tip no other node can follow. The archiver prunes these orphan blocks
731
+ // once their build slot ends; this guard is the correctness barrier during the grace window before.
732
+ if (
733
+ blockData.checkpointNumber > l2Tips.checkpointed.checkpoint.number &&
734
+ (l2Tips.proposedCheckpoint.checkpoint.number !== blockData.checkpointNumber ||
735
+ proposedCheckpointData?.checkpointNumber !== blockData.checkpointNumber)
736
+ ) {
737
+ this.log.warn(`Sequencer sync check failed: proposed block has no matching proposed checkpoint`, {
738
+ blockCheckpointNumber: blockData.checkpointNumber,
739
+ checkpointedCheckpointNumber: l2Tips.checkpointed.checkpoint.number,
740
+ proposedCheckpointTipNumber: l2Tips.proposedCheckpoint.checkpoint.number,
741
+ proposedCheckpointDataNumber: proposedCheckpointData?.checkpointNumber,
742
+ blockNumber: blockData.header.getBlockNumber(),
743
+ blockSlot: blockData.header.getSlot(),
744
+ syncedL2Slot,
745
+ ...args,
746
+ });
747
+ return undefined;
748
+ }
749
+
750
+ const hasProposedCheckpoint = l2Tips.proposedCheckpoint.checkpoint.number > l2Tips.checkpointed.checkpoint.number;
751
+
752
+ // The l2Tips and proposedCheckpointData reads above come from independent archiver snapshots
753
+ // (a JS-side tips cache vs. a direct store read on `#proposedCheckpoints`). A concurrent archiver
754
+ // write that mutates both can be observed split, leaving us with `hasProposedCheckpoint=true` but
755
+ // no proposedCheckpointData (or one whose number doesn't match the tip). Refuse to proceed in that
756
+ // window — the next checkSync tick will see a coherent snapshot.
757
+ if (
758
+ hasProposedCheckpoint &&
759
+ (!proposedCheckpointData ||
760
+ proposedCheckpointData.checkpointNumber !== l2Tips.proposedCheckpoint.checkpoint.number)
761
+ ) {
762
+ this.log.warn(`Sequencer sync check failed: inconsistent proposed-checkpoint state`, {
763
+ proposedCheckpointTipNumber: l2Tips.proposedCheckpoint.checkpoint.number,
764
+ checkpointedTipNumber: l2Tips.checkpointed.checkpoint.number,
765
+ proposedCheckpointDataNumber: proposedCheckpointData?.checkpointNumber,
766
+ syncedL2Slot,
767
+ ...args,
768
+ });
769
+ return undefined;
770
+ }
771
+
772
+ // Check that the proposed checkpoint is indeed the parent of the checkpoint we'll be building
773
+ // The checkpoint number to build is derived as blockData.checkpointNumber + 1
774
+ if (proposedCheckpointData && proposedCheckpointData.checkpointNumber !== blockData.checkpointNumber) {
775
+ this.log.warn(`Sequencer sync check failed: proposed checkpoint number mismatch`, {
776
+ proposedCheckpointNumber: proposedCheckpointData.checkpointNumber,
777
+ blockCheckpointNumber: blockData.checkpointNumber,
778
+ syncedL2Slot,
779
+ ...args,
780
+ });
512
781
  return undefined;
513
782
  }
514
783
 
515
784
  return {
516
- block,
517
- blockNumber: block.number,
518
- checkpointNumber: block.checkpointNumber,
519
- archive: block.archive.root,
520
- l1Timestamp,
785
+ blockData,
786
+ blockNumber: blockData.header.getBlockNumber(),
787
+ checkpointNumber: blockData.checkpointNumber,
788
+ checkpointedCheckpointNumber: l2Tips.checkpointed.checkpoint.number,
789
+ archive: blockData.archive.root,
790
+ hasProposedCheckpoint,
791
+ proposedCheckpointData,
792
+ syncedL2Slot,
521
793
  pendingChainValidationStatus,
522
794
  };
523
795
  }
@@ -526,17 +798,20 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
526
798
  * Checks if we are the proposer for the next slot.
527
799
  * @returns True if we can propose, and the proposer address (undefined if anyone can propose)
528
800
  */
529
- protected async checkCanPropose(slot: SlotNumber): Promise<[boolean, EthAddress | undefined]> {
801
+ protected async checkCanPropose(targetSlot: SlotNumber): Promise<[boolean, EthAddress | undefined]> {
530
802
  let proposer: EthAddress | undefined;
531
803
 
532
804
  try {
533
- proposer = await this.epochCache.getProposerAttesterAddressInSlot(slot);
805
+ proposer = await this.epochCache.getProposerAttesterAddressInSlot(targetSlot);
534
806
  } catch (e) {
535
807
  if (e instanceof NoCommitteeError) {
536
- this.log.warn(`Cannot propose at next L2 slot ${slot} since the committee does not exist on L1`);
808
+ if (this.lastSlotForNoCommitteeWarning !== targetSlot) {
809
+ this.lastSlotForNoCommitteeWarning = targetSlot;
810
+ this.log.warn(`Cannot propose at target slot ${targetSlot} since the committee does not exist on L1`);
811
+ }
537
812
  return [false, undefined];
538
813
  }
539
- this.log.error(`Error getting proposer for slot ${slot}`, e);
814
+ this.log.error(`Error getting proposer for target slot ${targetSlot}`, e);
540
815
  return [false, undefined];
541
816
  }
542
817
 
@@ -553,10 +828,18 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
553
828
  const weAreProposer = validatorAddresses.some(addr => addr.equals(proposer));
554
829
 
555
830
  if (!weAreProposer) {
556
- this.log.debug(`Cannot propose at slot ${slot} since we are not a proposer`, { validatorAddresses, proposer });
831
+ this.log.debug(`Cannot propose at target slot ${targetSlot} since we are not a proposer`, {
832
+ targetSlot,
833
+ validatorAddresses,
834
+ proposer,
835
+ });
557
836
  return [false, proposer];
558
837
  }
559
838
 
839
+ this.log.info(`We are the proposer for pipeline slot ${targetSlot}`, {
840
+ targetSlot,
841
+ proposer,
842
+ });
560
843
  return [true, proposer];
561
844
  }
562
845
 
@@ -565,11 +848,11 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
565
848
  * This allows the sequencer to participate in governance/slashing votes even when it cannot build blocks.
566
849
  */
567
850
  @trackSpan('Seqeuencer.tryVoteWhenSyncFails', ({ slot }) => ({ [Attributes.SLOT_NUMBER]: slot }))
568
- protected async tryVoteWhenSyncFails(args: { slot: SlotNumber; ts: bigint }): Promise<void> {
569
- const { slot } = args;
851
+ protected async tryVoteWhenSyncFails(args: { slot: SlotNumber; targetSlot: SlotNumber; ts: bigint }): Promise<void> {
852
+ const { slot, targetSlot } = args;
570
853
 
571
854
  // Prevent duplicate attempts in the same slot
572
- if (this.lastSlotForVoteWhenSyncFailed === slot) {
855
+ if (this.lastSlotForFallbackVote === slot) {
573
856
  this.log.trace(`Already attempted to vote in slot ${slot} (skipping)`);
574
857
  return;
575
858
  }
@@ -594,14 +877,14 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
594
877
  });
595
878
 
596
879
  // Check if we're a proposer or proposal is open
597
- const [canPropose, proposer] = await this.checkCanPropose(slot);
880
+ const [canPropose, proposer] = await this.checkCanPropose(targetSlot);
598
881
  if (!canPropose) {
599
882
  this.log.trace(`Cannot vote in slot ${slot} since we are not a proposer`, { slot, proposer });
600
883
  return;
601
884
  }
602
885
 
603
886
  // Mark this slot as attempted
604
- this.lastSlotForVoteWhenSyncFailed = slot;
887
+ this.lastSlotForFallbackVote = slot;
605
888
 
606
889
  // Get a publisher for voting
607
890
  const { attestorAddress, publisher } = await this.publisherFactory.create(proposer);
@@ -611,9 +894,9 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
611
894
  slot,
612
895
  });
613
896
 
614
- // Enqueue governance and slashing votes
897
+ // Enqueue governance and slashing votes (voter uses the target slot for L1 submission)
615
898
  const voter = new CheckpointVoter(
616
- slot,
899
+ targetSlot,
617
900
  publisher,
618
901
  attestorAddress,
619
902
  this.validatorClient,
@@ -632,11 +915,82 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
632
915
  }
633
916
 
634
917
  this.log.info(`Voting in slot ${slot} despite sync failure`, { slot });
635
- await publisher.sendRequests();
918
+ // Votes are EIP-712-signed for `targetSlot` (the pipelined slot in which the multicall is
919
+ // expected to mine). Delay submission to the start of `targetSlot` so the tx mines in the
920
+ // slot the votes were signed for. We fire-and-forget so we don't block the sequencer's
921
+ // work loop while waiting for the target slot to start.
922
+ void publisher.sendRequestsAt(targetSlot).catch(err => {
923
+ this.log.error(`Failed to publish votes despite sync failure for slot ${slot}`, err, { slot });
924
+ });
636
925
  }
637
926
 
638
927
  /**
639
- * Considers invalidating a checkpoint if the pending chain is invalid. Depends on how long the invalid checkpoint
928
+ * Tries to vote on slashing actions and governance proposals when escape hatch is open.
929
+ * This allows the sequencer to participate in voting without performing checkpoint proposal work.
930
+ */
931
+ @trackSpan('Sequencer.tryVoteWhenEscapeHatchOpen', ({ slot }) => ({ [Attributes.SLOT_NUMBER]: slot }))
932
+ protected async tryVoteWhenEscapeHatchOpen(args: {
933
+ slot: SlotNumber;
934
+ targetSlot: SlotNumber;
935
+ proposer: EthAddress | undefined;
936
+ }): Promise<void> {
937
+ const { slot, targetSlot, proposer } = args;
938
+
939
+ // Prevent duplicate attempts in the same slot
940
+ if (this.lastSlotForFallbackVote === slot) {
941
+ this.log.trace(`Already attempted to vote in slot ${slot} (escape hatch open, skipping)`);
942
+ return;
943
+ }
944
+
945
+ // Mark this slot as attempted
946
+ this.lastSlotForFallbackVote = slot;
947
+
948
+ const { attestorAddress, publisher } = await this.publisherFactory.create(proposer);
949
+
950
+ this.log.debug(`Escape hatch open for slot ${slot}, attempting vote-only actions`, {
951
+ slot,
952
+ targetSlot,
953
+ attestorAddress,
954
+ });
955
+
956
+ // Under proposer pipelining, the multicall is expected to mine in `targetSlot` (slot + 1).
957
+ // Governance and slashing votes are EIP-712-signed against the slot they will mine in, and the
958
+ // L1 contract checks `msg.sender == getCurrentProposer()` using the mining slot. So we must
959
+ // sign for `targetSlot` and delay submission to the start of `targetSlot`. When pipelining is
960
+ // disabled `targetSlot == slot` and `sendRequestsAt` resolves with no extra sleep.
961
+ const voter = new CheckpointVoter(
962
+ targetSlot,
963
+ publisher,
964
+ attestorAddress,
965
+ this.validatorClient,
966
+ this.slasherClient,
967
+ this.l1Constants,
968
+ this.config,
969
+ this.metrics,
970
+ this.log,
971
+ );
972
+
973
+ const votesPromises = voter.enqueueVotes();
974
+ const votes = await Promise.all(votesPromises);
975
+
976
+ if (votes.every(p => !p)) {
977
+ this.log.debug(`No votes to enqueue for slot ${slot} (escape hatch open)`);
978
+ return;
979
+ }
980
+
981
+ this.log.info(`Voting in slot ${slot} (escape hatch open)`, { slot, targetSlot });
982
+ // Votes are EIP-712-signed for `targetSlot`. Delay submission to the start of `targetSlot` so
983
+ // the multicall mines in the slot the votes were signed for; otherwise the L1 contract reads
984
+ // `signaler = getCurrentProposer()` against the wrong slot and signature verification fails
985
+ // silently inside Multicall3. Fire-and-forget so we don't block the sequencer's work loop while
986
+ // waiting for the target slot to start, mirroring tryVoteWhenSyncFails.
987
+ void publisher.sendRequestsAt(targetSlot).catch(err => {
988
+ this.log.error(`Failed to publish escape-hatch votes for slot ${slot}`, err, { slot, targetSlot });
989
+ });
990
+ }
991
+
992
+ /**
993
+ * Considers invalidating a block if the pending chain is invalid. Depends on how long the invalid block
640
994
  * has been there without being invalidated and whether the sequencer is in the committee or not. We always
641
995
  * have the proposer try to invalidate, but if they fail, the sequencers in the committee are expected to try,
642
996
  * and if they fail, any sequencer will try as well.
@@ -645,7 +999,7 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
645
999
  syncedTo: SequencerSyncCheckResult,
646
1000
  currentSlot: SlotNumber,
647
1001
  ): Promise<void> {
648
- const { pendingChainValidationStatus, l1Timestamp } = syncedTo;
1002
+ const { pendingChainValidationStatus, syncedL2Slot } = syncedTo;
649
1003
  if (pendingChainValidationStatus.valid) {
650
1004
  return;
651
1005
  }
@@ -660,7 +1014,7 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
660
1014
 
661
1015
  const logData = {
662
1016
  invalidL1Timestamp: invalidCheckpointTimestamp,
663
- l1Timestamp,
1017
+ syncedL2Slot,
664
1018
  invalidCheckpoint: pendingChainValidationStatus.checkpoint,
665
1019
  secondsBeforeInvalidatingBlockAsCommitteeMember,
666
1020
  secondsBeforeInvalidatingBlockAsNonCommitteeMember,
@@ -788,6 +1142,11 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
788
1142
  return this.validatorClient?.getValidatorAddresses();
789
1143
  }
790
1144
 
1145
+ /** Updates the publisher factory's node keystore adapter after a keystore reload. */
1146
+ public updatePublisherNodeKeyStore(adapter: NodeKeystoreAdapter): void {
1147
+ this.publisherFactory.updateNodeKeyStore(adapter);
1148
+ }
1149
+
791
1150
  public getConfig() {
792
1151
  return this.config;
793
1152
  }
@@ -798,10 +1157,13 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
798
1157
  }
799
1158
 
800
1159
  type SequencerSyncCheckResult = {
801
- block?: L2BlockNew;
1160
+ blockData?: BlockData;
802
1161
  checkpointNumber: CheckpointNumber;
1162
+ checkpointedCheckpointNumber: CheckpointNumber;
803
1163
  blockNumber: BlockNumber;
804
1164
  archive: Fr;
805
- l1Timestamp: bigint;
1165
+ hasProposedCheckpoint: boolean;
1166
+ proposedCheckpointData?: ProposedCheckpointData;
1167
+ syncedL2Slot: SlotNumber;
806
1168
  pendingChainValidationStatus: ValidateCheckpointResult;
807
1169
  };