@aztec/sequencer-client 0.0.1-commit.b33fc05d0 → 0.0.1-commit.b3d3157a
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +281 -21
- package/dest/client/sequencer-client.d.ts +8 -14
- package/dest/client/sequencer-client.d.ts.map +1 -1
- package/dest/client/sequencer-client.js +33 -83
- package/dest/config.d.ts +5 -1
- package/dest/config.d.ts.map +1 -1
- package/dest/config.js +42 -15
- package/dest/global_variable_builder/fee_predictor.d.ts +37 -0
- package/dest/global_variable_builder/fee_predictor.d.ts.map +1 -0
- package/dest/global_variable_builder/fee_predictor.js +128 -0
- package/dest/global_variable_builder/fee_provider.d.ts +21 -0
- package/dest/global_variable_builder/fee_provider.d.ts.map +1 -0
- package/dest/global_variable_builder/fee_provider.js +58 -0
- package/dest/global_variable_builder/global_builder.d.ts +14 -14
- package/dest/global_variable_builder/global_builder.d.ts.map +1 -1
- package/dest/global_variable_builder/global_builder.js +16 -50
- package/dest/global_variable_builder/index.d.ts +4 -2
- package/dest/global_variable_builder/index.d.ts.map +1 -1
- package/dest/global_variable_builder/index.js +2 -0
- package/dest/index.d.ts +2 -2
- package/dest/index.d.ts.map +1 -1
- package/dest/index.js +1 -1
- package/dest/publisher/config.d.ts +15 -3
- package/dest/publisher/config.d.ts.map +1 -1
- package/dest/publisher/config.js +19 -4
- package/dest/publisher/l1_tx_failed_store/failed_tx_store.d.ts +3 -4
- package/dest/publisher/l1_tx_failed_store/failed_tx_store.d.ts.map +1 -1
- package/dest/publisher/sequencer-bundle-simulator.d.ts +96 -0
- package/dest/publisher/sequencer-bundle-simulator.d.ts.map +1 -0
- package/dest/publisher/sequencer-bundle-simulator.js +198 -0
- package/dest/publisher/sequencer-publisher-factory.d.ts +3 -5
- package/dest/publisher/sequencer-publisher-factory.d.ts.map +1 -1
- package/dest/publisher/sequencer-publisher-factory.js +2 -3
- package/dest/publisher/sequencer-publisher.d.ts +73 -65
- package/dest/publisher/sequencer-publisher.d.ts.map +1 -1
- package/dest/publisher/sequencer-publisher.js +318 -523
- package/dest/sequencer/automine/automine_factory.d.ts +54 -0
- package/dest/sequencer/automine/automine_factory.d.ts.map +1 -0
- package/dest/sequencer/automine/automine_factory.js +84 -0
- package/dest/sequencer/automine/automine_sequencer.d.ts +151 -0
- package/dest/sequencer/automine/automine_sequencer.d.ts.map +1 -0
- package/dest/sequencer/automine/automine_sequencer.js +509 -0
- package/dest/sequencer/chain_state_overrides.d.ts +61 -0
- package/dest/sequencer/chain_state_overrides.d.ts.map +1 -0
- package/dest/sequencer/chain_state_overrides.js +98 -0
- package/dest/sequencer/checkpoint_proposal_job.d.ts +60 -10
- package/dest/sequencer/checkpoint_proposal_job.d.ts.map +1 -1
- package/dest/sequencer/checkpoint_proposal_job.js +748 -200
- package/dest/sequencer/checkpoint_proposal_job_metrics.d.ts +34 -0
- package/dest/sequencer/checkpoint_proposal_job_metrics.d.ts.map +1 -0
- package/dest/sequencer/checkpoint_proposal_job_metrics.js +72 -0
- package/dest/sequencer/checkpoint_voter.d.ts +1 -2
- package/dest/sequencer/checkpoint_voter.d.ts.map +1 -1
- package/dest/sequencer/checkpoint_voter.js +2 -5
- package/dest/sequencer/events.d.ts +48 -2
- package/dest/sequencer/events.d.ts.map +1 -1
- package/dest/sequencer/index.d.ts +3 -1
- package/dest/sequencer/index.d.ts.map +1 -1
- package/dest/sequencer/index.js +2 -0
- package/dest/sequencer/metrics.d.ts +13 -10
- package/dest/sequencer/metrics.d.ts.map +1 -1
- package/dest/sequencer/metrics.js +45 -20
- package/dest/sequencer/sequencer.d.ts +63 -13
- package/dest/sequencer/sequencer.d.ts.map +1 -1
- package/dest/sequencer/sequencer.js +349 -107
- package/dest/sequencer/timetable.d.ts +11 -1
- package/dest/sequencer/timetable.d.ts.map +1 -1
- package/dest/sequencer/timetable.js +41 -36
- package/dest/sequencer/types.d.ts +2 -2
- package/dest/sequencer/types.d.ts.map +1 -1
- package/dest/test/mock_checkpoint_builder.d.ts +4 -4
- package/dest/test/mock_checkpoint_builder.d.ts.map +1 -1
- package/dest/test/utils.d.ts +1 -1
- package/dest/test/utils.d.ts.map +1 -1
- package/dest/test/utils.js +7 -6
- package/package.json +27 -28
- package/src/client/sequencer-client.ts +53 -114
- package/src/config.ts +48 -12
- package/src/global_variable_builder/README.md +44 -0
- package/src/global_variable_builder/fee_predictor.ts +172 -0
- package/src/global_variable_builder/fee_provider.ts +75 -0
- package/src/global_variable_builder/global_builder.ts +25 -62
- package/src/global_variable_builder/index.ts +3 -1
- package/src/index.ts +10 -1
- package/src/publisher/config.ts +40 -6
- package/src/publisher/l1_tx_failed_store/failed_tx_store.ts +3 -1
- package/src/publisher/sequencer-bundle-simulator.ts +254 -0
- package/src/publisher/sequencer-publisher-factory.ts +3 -6
- package/src/publisher/sequencer-publisher.ts +377 -568
- package/src/sequencer/README.md +162 -450
- package/src/sequencer/automine/README.md +46 -0
- package/src/sequencer/automine/automine_factory.ts +145 -0
- package/src/sequencer/automine/automine_sequencer.ts +595 -0
- package/src/sequencer/chain_state_overrides.ts +162 -0
- package/src/sequencer/checkpoint_proposal_job.ts +894 -220
- package/src/sequencer/checkpoint_proposal_job_metrics.ts +128 -0
- package/src/sequencer/checkpoint_voter.ts +1 -12
- package/src/sequencer/events.ts +51 -2
- package/src/sequencer/index.ts +2 -0
- package/src/sequencer/metrics.ts +57 -24
- package/src/sequencer/sequencer.ts +429 -121
- package/src/sequencer/timetable.ts +49 -44
- package/src/sequencer/types.ts +1 -1
- package/src/test/mock_checkpoint_builder.ts +3 -3
- package/src/test/utils.ts +28 -10
|
@@ -1,20 +1,26 @@
|
|
|
1
1
|
import { getKzg } from '@aztec/blob-lib';
|
|
2
|
-
import {
|
|
3
|
-
import type { EpochCache } from '@aztec/epoch-cache';
|
|
2
|
+
import { type EpochCache, PROPOSER_PIPELINING_SLOT_OFFSET } from '@aztec/epoch-cache';
|
|
4
3
|
import { NoCommitteeError, type RollupContract } from '@aztec/ethereum/contracts';
|
|
5
4
|
import { BlockNumber, CheckpointNumber, EpochNumber, SlotNumber } from '@aztec/foundation/branded-types';
|
|
6
5
|
import { merge, omit, pick } from '@aztec/foundation/collection';
|
|
7
6
|
import { Fr } from '@aztec/foundation/curves/bn254';
|
|
8
7
|
import { EthAddress } from '@aztec/foundation/eth-address';
|
|
9
|
-
import { createLogger } from '@aztec/foundation/log';
|
|
8
|
+
import { type Logger, createLogger } from '@aztec/foundation/log';
|
|
10
9
|
import { RunningPromise } from '@aztec/foundation/running-promise';
|
|
11
10
|
import type { DateProvider } from '@aztec/foundation/timer';
|
|
12
11
|
import type { TypedEventEmitter } from '@aztec/foundation/types';
|
|
13
12
|
import type { P2P } from '@aztec/p2p';
|
|
14
13
|
import type { SlasherClientInterface } from '@aztec/slasher';
|
|
15
|
-
import type {
|
|
16
|
-
|
|
17
|
-
|
|
14
|
+
import type {
|
|
15
|
+
BlockData,
|
|
16
|
+
L2BlockSink,
|
|
17
|
+
L2BlockSource,
|
|
18
|
+
ProposedCheckpointSink,
|
|
19
|
+
ValidateCheckpointResult,
|
|
20
|
+
} from '@aztec/stdlib/block';
|
|
21
|
+
import type { Checkpoint, ProposedCheckpointData } from '@aztec/stdlib/checkpoint';
|
|
22
|
+
import type { ChainConfig } from '@aztec/stdlib/config';
|
|
23
|
+
import { getEpochAtSlot, getSlotStartBuildTimestamp, getTimestampForSlot } from '@aztec/stdlib/epoch-helpers';
|
|
18
24
|
import {
|
|
19
25
|
type ResolvedSequencerConfig,
|
|
20
26
|
type SequencerConfig,
|
|
@@ -22,8 +28,9 @@ import {
|
|
|
22
28
|
type WorldStateSynchronizer,
|
|
23
29
|
} from '@aztec/stdlib/interfaces/server';
|
|
24
30
|
import type { L1ToL2MessageSource } from '@aztec/stdlib/messaging';
|
|
31
|
+
import type { CoordinationSignatureContext } from '@aztec/stdlib/p2p';
|
|
25
32
|
import { pickFromSchema } from '@aztec/stdlib/schemas';
|
|
26
|
-
import {
|
|
33
|
+
import { MIN_EXECUTION_TIME } from '@aztec/stdlib/timetable';
|
|
27
34
|
import { Attributes, type TelemetryClient, type Tracer, getTelemetryClient, trackSpan } from '@aztec/telemetry-client';
|
|
28
35
|
import { FullNodeCheckpointsBuilder, NodeKeystoreAdapter, type ValidatorClient } from '@aztec/validator-client';
|
|
29
36
|
|
|
@@ -33,7 +40,9 @@ import { DefaultSequencerConfig } from '../config.js';
|
|
|
33
40
|
import type { GlobalVariableBuilder } from '../global_variable_builder/global_builder.js';
|
|
34
41
|
import type { SequencerPublisherFactory } from '../publisher/sequencer-publisher-factory.js';
|
|
35
42
|
import type { InvalidateCheckpointRequest, SequencerPublisher } from '../publisher/sequencer-publisher.js';
|
|
43
|
+
import { buildCheckpointSimulationOverridesPlan } from './chain_state_overrides.js';
|
|
36
44
|
import { CheckpointProposalJob } from './checkpoint_proposal_job.js';
|
|
45
|
+
import { CheckpointProposalJobMetrics } from './checkpoint_proposal_job_metrics.js';
|
|
37
46
|
import { CheckpointVoter } from './checkpoint_voter.js';
|
|
38
47
|
import { SequencerInterruptedError, SequencerTooSlowError } from './errors.js';
|
|
39
48
|
import type { SequencerEvents } from './events.js';
|
|
@@ -44,6 +53,16 @@ import { SequencerState } from './utils.js';
|
|
|
44
53
|
|
|
45
54
|
export { SequencerState };
|
|
46
55
|
|
|
56
|
+
/** Slot snapshot used to prepare a checkpoint proposal. */
|
|
57
|
+
type SequencerSlotContext = {
|
|
58
|
+
slot: SlotNumber;
|
|
59
|
+
targetSlot: SlotNumber;
|
|
60
|
+
epoch: EpochNumber;
|
|
61
|
+
targetEpoch: EpochNumber;
|
|
62
|
+
ts: bigint;
|
|
63
|
+
nowSeconds: bigint;
|
|
64
|
+
};
|
|
65
|
+
|
|
47
66
|
/**
|
|
48
67
|
* Sequencer client
|
|
49
68
|
* - Checks whether it is elected as proposer for the next slot
|
|
@@ -55,7 +74,11 @@ export { SequencerState };
|
|
|
55
74
|
export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<SequencerEvents>) {
|
|
56
75
|
private runningPromise?: RunningPromise;
|
|
57
76
|
private state = SequencerState.STOPPED;
|
|
77
|
+
private stateSlotNumber: SlotNumber | undefined;
|
|
78
|
+
private stateEnteredAtMs = performance.now();
|
|
58
79
|
private metrics: SequencerMetrics;
|
|
80
|
+
private checkpointProposalJobMetrics: CheckpointProposalJobMetrics;
|
|
81
|
+
private readonly stateLog: Logger;
|
|
59
82
|
|
|
60
83
|
/** The last slot for which we attempted to perform our voting duties with degraded block production */
|
|
61
84
|
private lastSlotForFallbackVote: SlotNumber | undefined;
|
|
@@ -72,11 +95,15 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
72
95
|
/** The last epoch for which we logged strategy comparison in fisherman mode. */
|
|
73
96
|
private lastEpochForStrategyComparison: EpochNumber | undefined;
|
|
74
97
|
|
|
98
|
+
/** The last checkpoint proposal job, tracked so we can await its pending L1 submission during shutdown. */
|
|
99
|
+
protected lastCheckpointProposalJob: CheckpointProposalJob | undefined;
|
|
100
|
+
|
|
75
101
|
/** The maximum number of seconds that the sequencer can be into a slot to transition to a particular state. */
|
|
76
102
|
protected timetable!: SequencerTimetable;
|
|
77
103
|
|
|
78
104
|
/** Config for the sequencer */
|
|
79
105
|
protected config: ResolvedSequencerConfig = DefaultSequencerConfig;
|
|
106
|
+
private readonly signatureContext: CoordinationSignatureContext;
|
|
80
107
|
|
|
81
108
|
constructor(
|
|
82
109
|
protected publisherFactory: SequencerPublisherFactory,
|
|
@@ -85,25 +112,31 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
85
112
|
protected p2pClient: P2P,
|
|
86
113
|
protected worldState: WorldStateSynchronizer,
|
|
87
114
|
protected slasherClient: SlasherClientInterface | undefined,
|
|
88
|
-
protected l2BlockSource: L2BlockSource & L2BlockSink,
|
|
115
|
+
protected l2BlockSource: L2BlockSource & L2BlockSink & ProposedCheckpointSink,
|
|
89
116
|
protected l1ToL2MessageSource: L1ToL2MessageSource,
|
|
90
117
|
protected checkpointsBuilder: FullNodeCheckpointsBuilder,
|
|
91
118
|
protected l1Constants: SequencerRollupConstants,
|
|
92
119
|
protected dateProvider: DateProvider,
|
|
93
120
|
protected epochCache: EpochCache,
|
|
94
121
|
protected rollupContract: RollupContract,
|
|
95
|
-
config: SequencerConfig,
|
|
122
|
+
config: SequencerConfig & Pick<ChainConfig, 'l1ChainId' | 'rollupAddress'>,
|
|
96
123
|
protected telemetry: TelemetryClient = getTelemetryClient(),
|
|
97
124
|
protected log = createLogger('sequencer'),
|
|
98
125
|
) {
|
|
99
126
|
super();
|
|
127
|
+
this.stateLog = log.createChild('state');
|
|
100
128
|
|
|
101
129
|
// Add [FISHERMAN] prefix to logger if in fisherman mode
|
|
102
130
|
if (config.fishermanMode) {
|
|
103
131
|
this.log = log.createChild('[FISHERMAN]');
|
|
104
132
|
}
|
|
105
133
|
|
|
134
|
+
this.signatureContext = {
|
|
135
|
+
chainId: config.l1ChainId,
|
|
136
|
+
rollupAddress: config.rollupAddress,
|
|
137
|
+
};
|
|
106
138
|
this.metrics = new SequencerMetrics(telemetry, this.rollupContract, 'Sequencer');
|
|
139
|
+
this.checkpointProposalJobMetrics = new CheckpointProposalJobMetrics(telemetry);
|
|
107
140
|
this.updateConfig(config);
|
|
108
141
|
}
|
|
109
142
|
|
|
@@ -112,12 +145,13 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
112
145
|
const filteredConfig = pickFromSchema(config, SequencerConfigSchema);
|
|
113
146
|
this.log.info(`Updated sequencer config`, omit(filteredConfig, 'txPublicSetupAllowListExtend'));
|
|
114
147
|
this.config = merge(this.config, filteredConfig);
|
|
148
|
+
const p2pPropagationTime = this.config.attestationPropagationTime;
|
|
115
149
|
this.timetable = new SequencerTimetable(
|
|
116
150
|
{
|
|
117
151
|
ethereumSlotDuration: this.l1Constants.ethereumSlotDuration,
|
|
118
152
|
aztecSlotDuration: this.aztecSlotDuration,
|
|
119
153
|
l1PublishingTime: this.l1PublishingTime,
|
|
120
|
-
p2pPropagationTime
|
|
154
|
+
p2pPropagationTime,
|
|
121
155
|
blockDurationMs: this.config.blockDurationMs,
|
|
122
156
|
enforce: this.config.enforceTimeTable,
|
|
123
157
|
},
|
|
@@ -143,12 +177,19 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
143
177
|
this.log.info('Started sequencer');
|
|
144
178
|
}
|
|
145
179
|
|
|
180
|
+
/** Triggers an immediate run of the sequencer, bypassing the polling interval. */
|
|
181
|
+
public trigger() {
|
|
182
|
+
return this.runningPromise?.trigger();
|
|
183
|
+
}
|
|
184
|
+
|
|
146
185
|
/** Stops the sequencer from building blocks and moves to STOPPED state. */
|
|
147
186
|
public async stop(): Promise<void> {
|
|
148
187
|
this.log.info(`Stopping sequencer`);
|
|
149
188
|
this.setState(SequencerState.STOPPING, undefined, { force: true });
|
|
150
|
-
this.
|
|
189
|
+
this.lastCheckpointProposalJob?.interrupt();
|
|
190
|
+
await this.publisherFactory.stopAll();
|
|
151
191
|
await this.runningPromise?.stop();
|
|
192
|
+
await this.lastCheckpointProposalJob?.awaitPendingSubmission();
|
|
152
193
|
this.setState(SequencerState.STOPPED, undefined, { force: true });
|
|
153
194
|
this.log.info('Stopped sequencer');
|
|
154
195
|
}
|
|
@@ -158,17 +199,15 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
158
199
|
try {
|
|
159
200
|
await this.work();
|
|
160
201
|
} catch (err) {
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
)
|
|
167
|
-
? ('debug' as const)
|
|
168
|
-
: ('warn' as const);
|
|
169
|
-
this.log[logLvl](err.message, { now: this.dateProvider.nowInSeconds() });
|
|
202
|
+
if (
|
|
203
|
+
err instanceof SequencerTooSlowError &&
|
|
204
|
+
[SequencerState.INITIALIZING_CHECKPOINT, SequencerState.PROPOSER_CHECK].includes(err.proposedState)
|
|
205
|
+
) {
|
|
206
|
+
// No need to alert if we just didn't get to start in time
|
|
207
|
+
this.log.debug(err.message, { now: this.dateProvider.nowInSeconds() });
|
|
170
208
|
} else {
|
|
171
209
|
// Re-throw other errors
|
|
210
|
+
this.emit('checkpoint-error', { error: err as Error });
|
|
172
211
|
throw err;
|
|
173
212
|
}
|
|
174
213
|
} finally {
|
|
@@ -192,14 +231,24 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
192
231
|
@trackSpan('Sequencer.work')
|
|
193
232
|
protected async work() {
|
|
194
233
|
this.setState(SequencerState.SYNCHRONIZING, undefined);
|
|
195
|
-
const { slot,
|
|
234
|
+
const { slot, targetSlot, epoch, targetEpoch, ts, nowSeconds } = this.getSlotContextInNextL1Slot();
|
|
196
235
|
|
|
197
236
|
// Check if we are synced and it's our slot, grab a publisher, check previous block invalidation, etc
|
|
198
|
-
const checkpointProposalJob = await this.prepareCheckpointProposal(
|
|
237
|
+
const checkpointProposalJob = await this.prepareCheckpointProposal(
|
|
238
|
+
slot,
|
|
239
|
+
targetSlot,
|
|
240
|
+
epoch,
|
|
241
|
+
targetEpoch,
|
|
242
|
+
ts,
|
|
243
|
+
nowSeconds,
|
|
244
|
+
);
|
|
199
245
|
if (!checkpointProposalJob) {
|
|
200
246
|
return;
|
|
201
247
|
}
|
|
202
248
|
|
|
249
|
+
// Track the job so we can await its pending L1 submission during shutdown
|
|
250
|
+
this.lastCheckpointProposalJob = checkpointProposalJob;
|
|
251
|
+
|
|
203
252
|
// Execute the checkpoint proposal job
|
|
204
253
|
const checkpoint = await checkpointProposalJob.execute();
|
|
205
254
|
|
|
@@ -208,64 +257,85 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
208
257
|
this.lastCheckpointProposed = checkpoint;
|
|
209
258
|
}
|
|
210
259
|
|
|
211
|
-
// Log fee strategy comparison if on fisherman
|
|
260
|
+
// Log fee strategy comparison if on fisherman (uses target epoch since we mirror the proposer's perspective)
|
|
212
261
|
if (
|
|
213
262
|
this.config.fishermanMode &&
|
|
214
|
-
(this.lastEpochForStrategyComparison === undefined ||
|
|
263
|
+
(this.lastEpochForStrategyComparison === undefined || targetEpoch > this.lastEpochForStrategyComparison)
|
|
215
264
|
) {
|
|
216
|
-
this.logStrategyComparison(
|
|
217
|
-
this.lastEpochForStrategyComparison =
|
|
265
|
+
this.logStrategyComparison(targetEpoch, checkpointProposalJob.getPublisher());
|
|
266
|
+
this.lastEpochForStrategyComparison = targetEpoch;
|
|
218
267
|
}
|
|
219
268
|
|
|
220
269
|
return checkpoint;
|
|
221
270
|
}
|
|
222
271
|
|
|
272
|
+
/** Returns slot and target slot from a single clock snapshot. */
|
|
273
|
+
protected getSlotContextInNextL1Slot(): SequencerSlotContext {
|
|
274
|
+
const { slot, ts, nowSeconds, epoch } = this.epochCache.getEpochAndSlotInNextL1Slot();
|
|
275
|
+
const targetSlot = SlotNumber(slot + PROPOSER_PIPELINING_SLOT_OFFSET);
|
|
276
|
+
return { slot, targetSlot, epoch, targetEpoch: getEpochAtSlot(targetSlot, this.l1Constants), ts, nowSeconds };
|
|
277
|
+
}
|
|
278
|
+
|
|
223
279
|
/**
|
|
224
280
|
* Prepares the checkpoint proposal by performing all necessary checks and setup.
|
|
225
281
|
* This is the initial step in the main loop.
|
|
226
282
|
* @returns CheckpointProposalJob if successful, undefined if we are not yet synced or are not the proposer.
|
|
227
283
|
*/
|
|
228
284
|
@trackSpan('Sequencer.prepareCheckpointProposal')
|
|
229
|
-
|
|
230
|
-
epoch: EpochNumber,
|
|
285
|
+
protected async prepareCheckpointProposal(
|
|
231
286
|
slot: SlotNumber,
|
|
287
|
+
targetSlot: SlotNumber,
|
|
288
|
+
epoch: EpochNumber,
|
|
289
|
+
targetEpoch: EpochNumber,
|
|
232
290
|
ts: bigint,
|
|
233
|
-
|
|
291
|
+
nowSeconds: bigint,
|
|
234
292
|
): Promise<CheckpointProposalJob | undefined> {
|
|
235
|
-
// Check we have not already processed this slot (cheapest check)
|
|
293
|
+
// Check we have not already processed this target slot (cheapest check)
|
|
236
294
|
// We only check this if enforce timetable is set, since we want to keep processing the same slot if we are not
|
|
237
295
|
// running against actual time (eg when we use sandbox-style automining)
|
|
238
296
|
if (
|
|
239
297
|
this.lastSlotForCheckpointProposalJob &&
|
|
240
|
-
this.lastSlotForCheckpointProposalJob >=
|
|
298
|
+
this.lastSlotForCheckpointProposalJob >= targetSlot &&
|
|
241
299
|
this.config.enforceTimeTable
|
|
242
300
|
) {
|
|
243
|
-
this.log.trace(`
|
|
301
|
+
this.log.trace(`Target slot ${targetSlot} has already been processed`);
|
|
244
302
|
return undefined;
|
|
245
303
|
}
|
|
246
304
|
|
|
247
|
-
// But if we have already proposed for this slot,
|
|
248
|
-
if (this.lastCheckpointProposed && this.lastCheckpointProposed.header.slotNumber >=
|
|
249
|
-
this.log.trace(
|
|
305
|
+
// But if we have already proposed for this slot, then we definitely have to skip it, automining or not
|
|
306
|
+
if (this.lastCheckpointProposed && this.lastCheckpointProposed.header.slotNumber >= targetSlot) {
|
|
307
|
+
this.log.trace(
|
|
308
|
+
`Slot ${targetSlot} has already been published as checkpoint ${this.lastCheckpointProposed.number}`,
|
|
309
|
+
);
|
|
310
|
+
return undefined;
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
// Test-only: skip proposing for explicitly paused slots. Attestation paths run in the validator
|
|
314
|
+
// client and are not gated by this hook, so paused proposers still attest to others' proposals.
|
|
315
|
+
if (this.config.pauseProposingForSlots?.some(s => s === targetSlot)) {
|
|
316
|
+
this.log.warn(`Skipping proposal for paused slot ${targetSlot} (test-only pauseProposingForSlots hook)`, {
|
|
317
|
+
targetSlot,
|
|
318
|
+
});
|
|
250
319
|
return undefined;
|
|
251
320
|
}
|
|
252
321
|
|
|
253
322
|
// Check all components are synced to latest as seen by the archiver (queries all subsystems)
|
|
254
323
|
const syncedTo = await this.checkSync({ ts, slot });
|
|
255
324
|
if (!syncedTo) {
|
|
256
|
-
await this.tryVoteWhenSyncFails({ slot, ts });
|
|
325
|
+
await this.tryVoteWhenSyncFails({ slot, targetSlot, ts });
|
|
257
326
|
return undefined;
|
|
258
327
|
}
|
|
259
328
|
|
|
260
|
-
// If escape hatch is open for
|
|
329
|
+
// If escape hatch is open for the target epoch, do not start checkpoint proposal work and do not attempt invalidations.
|
|
261
330
|
// Still perform governance/slashing voting (as proposer) once per slot.
|
|
262
|
-
|
|
331
|
+
// When pipelining, we check the target epoch (slot+1's epoch) since that's the epoch we're building for.
|
|
332
|
+
const isEscapeHatchOpen = await this.epochCache.isEscapeHatchOpen(targetEpoch);
|
|
263
333
|
|
|
264
334
|
if (isEscapeHatchOpen) {
|
|
265
335
|
this.setState(SequencerState.PROPOSER_CHECK, slot);
|
|
266
|
-
const [canPropose, proposer] = await this.checkCanPropose(
|
|
336
|
+
const [canPropose, proposer] = await this.checkCanPropose(targetSlot);
|
|
267
337
|
if (canPropose) {
|
|
268
|
-
await this.tryVoteWhenEscapeHatchOpen({ slot, proposer });
|
|
338
|
+
await this.tryVoteWhenEscapeHatchOpen({ slot, targetSlot, proposer });
|
|
269
339
|
} else {
|
|
270
340
|
this.log.trace(`Escape hatch open but we are not proposer, skipping vote-only actions`, {
|
|
271
341
|
slot,
|
|
@@ -280,17 +350,18 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
280
350
|
const checkpointNumber = CheckpointNumber(syncedTo.checkpointNumber + 1);
|
|
281
351
|
|
|
282
352
|
const logCtx = {
|
|
283
|
-
|
|
353
|
+
nowSeconds,
|
|
284
354
|
syncedToL2Slot: syncedTo.syncedL2Slot,
|
|
285
355
|
slot,
|
|
356
|
+
targetSlot,
|
|
286
357
|
slotTs: ts,
|
|
287
358
|
checkpointNumber,
|
|
288
359
|
isPendingChainValid: pick(syncedTo.pendingChainValidationStatus, 'valid', 'reason', 'invalidIndex'),
|
|
289
360
|
};
|
|
290
361
|
|
|
291
|
-
// Check that we are a proposer for the
|
|
362
|
+
// Check that we are a proposer for the target slot.
|
|
292
363
|
this.setState(SequencerState.PROPOSER_CHECK, slot);
|
|
293
|
-
const [canPropose, proposer] = await this.checkCanPropose(
|
|
364
|
+
const [canPropose, proposer] = await this.checkCanPropose(targetSlot);
|
|
294
365
|
|
|
295
366
|
// If we are not a proposer check if we should invalidate an invalid checkpoint, and bail
|
|
296
367
|
if (!canPropose) {
|
|
@@ -298,10 +369,20 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
298
369
|
return undefined;
|
|
299
370
|
}
|
|
300
371
|
|
|
301
|
-
//
|
|
302
|
-
|
|
372
|
+
// Guard: don't exceed 1-deep pipeline. Without a proposed checkpoint, we can only build
|
|
373
|
+
// confirmed + 1. With a proposed checkpoint, we can build confirmed + 2.
|
|
374
|
+
const confirmedCkpt = syncedTo.checkpointedCheckpointNumber;
|
|
375
|
+
if (checkpointNumber > confirmedCkpt + 2) {
|
|
376
|
+
this.log.verbose(
|
|
377
|
+
`Skipping slot ${targetSlot}: checkpoint ${checkpointNumber} exceeds max pipeline depth (confirmed=${confirmedCkpt})`,
|
|
378
|
+
);
|
|
379
|
+
return undefined;
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
// Check that the target slot is not taken by a block already (should never happen, since only us can propose for this slot)
|
|
383
|
+
if (syncedTo.blockData && syncedTo.blockData.header.getSlot() >= targetSlot) {
|
|
303
384
|
this.log.warn(
|
|
304
|
-
`Cannot propose block at
|
|
385
|
+
`Cannot propose block at target slot ${targetSlot} since that slot was taken by block ${syncedTo.blockNumber}`,
|
|
305
386
|
{ ...logCtx, block: syncedTo.blockData.header.toInspect() },
|
|
306
387
|
);
|
|
307
388
|
this.metrics.recordCheckpointPrecheckFailed('slot_already_taken');
|
|
@@ -316,37 +397,106 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
316
397
|
const { attestorAddress, publisher } = await this.publisherFactory.create(proposerForPublisher);
|
|
317
398
|
this.log.verbose(`Created publisher at address ${publisher.getSenderAddress()} for attestor ${attestorAddress}`);
|
|
318
399
|
|
|
319
|
-
//
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
400
|
+
// Prepare invalidation request if the pending chain is invalid (returns undefined if no need).
|
|
401
|
+
// Only simulate invalidation when there's no proposed parent, since we assume the proposed parent
|
|
402
|
+
// will invalidate the currently invalid checkpoint on L1.
|
|
403
|
+
const invalidateCheckpoint =
|
|
404
|
+
syncedTo.hasProposedCheckpoint || syncedTo.pendingChainValidationStatus.valid
|
|
405
|
+
? undefined
|
|
406
|
+
: await publisher.simulateInvalidateCheckpoint(syncedTo.pendingChainValidationStatus);
|
|
407
|
+
|
|
408
|
+
// Determine the correct archive and L1 state overrides for the canProposeAt check.
|
|
409
|
+
// The L1 contract reads archives[proposedCheckpointNumber] and compares it with the provided archive.
|
|
410
|
+
// When invalidating or pipelining, the local archive may differ from L1's, so we adjust accordingly.
|
|
411
|
+
let archiveForCheck = syncedTo.archive;
|
|
412
|
+
|
|
413
|
+
if (syncedTo.hasProposedCheckpoint) {
|
|
414
|
+
this.metrics.recordPipelineDepth(syncedTo.checkpointNumber - syncedTo.checkpointedCheckpointNumber);
|
|
415
|
+
this.log.verbose(
|
|
416
|
+
`Building on top of proposed checkpoint (pending=${syncedTo.proposedCheckpointData?.checkpointNumber}) for target slot ${targetSlot}`,
|
|
417
|
+
{ targetSlot, parentCheckpointNumber: CheckpointNumber(checkpointNumber - 1) },
|
|
418
|
+
);
|
|
419
|
+
// Match what L1 will see at archives[pending] once the proposed parent lands: the parent's
|
|
420
|
+
// own archive root from the gossiped proposal. `syncedTo.archive` is the world-state-local
|
|
421
|
+
// view and can transiently diverge from the proposed parent (e.g. before the proposed
|
|
422
|
+
// parent's blocks have been applied locally); diverging here would cause the canProposeAt
|
|
423
|
+
// override to set archives[pending] to one value while we present another for comparison.
|
|
424
|
+
archiveForCheck = syncedTo.proposedCheckpointData!.archive.root;
|
|
425
|
+
} else if (invalidateCheckpoint) {
|
|
426
|
+
// After invalidation, L1 will roll back to checkpoint N-1. The archive at N-1 already
|
|
427
|
+
// exists on L1, so we just pass the matching archive (the lastArchive of the invalid checkpoint).
|
|
428
|
+
archiveForCheck = invalidateCheckpoint.lastArchive;
|
|
429
|
+
this.metrics.recordPipelineDepth(0);
|
|
430
|
+
} else {
|
|
431
|
+
this.metrics.recordPipelineDepth(0);
|
|
323
432
|
}
|
|
324
433
|
|
|
325
|
-
//
|
|
326
|
-
|
|
434
|
+
// Build the simulation plan: pending/proven override from pipelining or invalidation (or the
|
|
435
|
+
// current snapshot when neither applies, to short-circuit any pending prune in simulation),
|
|
436
|
+
// plus the parent checkpoint cell and fee header when pipelining.
|
|
437
|
+
const simulationOverridesPlan = await buildCheckpointSimulationOverridesPlan({
|
|
438
|
+
checkpointNumber,
|
|
439
|
+
proposedCheckpointData: syncedTo.hasProposedCheckpoint ? syncedTo.proposedCheckpointData : undefined,
|
|
440
|
+
invalidateToPendingCheckpointNumber: invalidateCheckpoint?.forcePendingCheckpointNumber,
|
|
441
|
+
checkpointedCheckpointNumber: syncedTo.checkpointedCheckpointNumber,
|
|
442
|
+
rollup: this.rollupContract,
|
|
443
|
+
signatureContext: this.signatureContext,
|
|
444
|
+
log: this.log,
|
|
445
|
+
});
|
|
446
|
+
|
|
447
|
+
// The plan always pins both pending/proven (to short-circuit `canPruneAtTime` in simulation),
|
|
448
|
+
// so `provenOverride` always reflects the assumed proven checkpoint we are pinning the
|
|
449
|
+
// simulation to. We additionally warn when the pin is load-bearing — i.e. when a prune would
|
|
450
|
+
// actually fire at the target slot without it — so observers can spot "we are building
|
|
451
|
+
// optimistically across a pruning boundary" in the logs.
|
|
452
|
+
const provenOverride = simulationOverridesPlan?.chainTipsOverride?.proven;
|
|
453
|
+
if (provenOverride !== undefined && (await this.l2BlockSource.isPruneDueAtSlot(targetSlot))) {
|
|
454
|
+
this.log.warn(
|
|
455
|
+
`Assuming proof for epoch ending at checkpoint ${provenOverride} lands by target slot ${targetSlot}`,
|
|
456
|
+
{ checkpointNumber, slot, targetSlot, provenOverride },
|
|
457
|
+
);
|
|
458
|
+
}
|
|
459
|
+
|
|
460
|
+
this.emit('preparing-checkpoint', {
|
|
461
|
+
targetSlot,
|
|
462
|
+
checkpointNumber,
|
|
463
|
+
hadProposedParent: syncedTo.hasProposedCheckpoint,
|
|
464
|
+
provenOverride,
|
|
465
|
+
simulatedPending: simulationOverridesPlan?.chainTipsOverride?.pending,
|
|
466
|
+
});
|
|
327
467
|
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
const canProposeCheck = await publisher.canProposeAtNextEthBlock(
|
|
331
|
-
syncedTo.archive,
|
|
468
|
+
const canProposeCheck = await publisher.canProposeAt(
|
|
469
|
+
archiveForCheck,
|
|
332
470
|
proposer ?? EthAddress.ZERO,
|
|
333
|
-
|
|
471
|
+
simulationOverridesPlan,
|
|
334
472
|
);
|
|
335
473
|
|
|
474
|
+
const proposeContext = {
|
|
475
|
+
hasProposedCheckpoint: syncedTo.hasProposedCheckpoint,
|
|
476
|
+
proposedCheckpointNumber: syncedTo.proposedCheckpointData?.checkpointNumber,
|
|
477
|
+
checkpointedCheckpointNumber: syncedTo.checkpointedCheckpointNumber,
|
|
478
|
+
isInvalidating: !!invalidateCheckpoint,
|
|
479
|
+
invalidatingCheckpointNumber: invalidateCheckpoint?.checkpointNumber,
|
|
480
|
+
archiveForCheck: archiveForCheck.toString(),
|
|
481
|
+
overridePendingCheckpointNumber: simulationOverridesPlan?.chainTipsOverride?.pending,
|
|
482
|
+
overrideArchive: simulationOverridesPlan?.pendingCheckpointState?.archive,
|
|
483
|
+
overrideFeeHeader: simulationOverridesPlan?.pendingCheckpointState?.feeHeader,
|
|
484
|
+
};
|
|
485
|
+
|
|
336
486
|
if (canProposeCheck === undefined) {
|
|
337
487
|
this.log.warn(
|
|
338
488
|
`Cannot propose checkpoint ${checkpointNumber} at slot ${slot} due to failed rollup contract check`,
|
|
339
|
-
logCtx,
|
|
489
|
+
{ ...logCtx, ...proposeContext },
|
|
340
490
|
);
|
|
341
491
|
this.emit('proposer-rollup-check-failed', { reason: 'Rollup contract check failed', slot });
|
|
342
492
|
this.metrics.recordCheckpointPrecheckFailed('rollup_contract_check_failed');
|
|
343
493
|
return undefined;
|
|
344
494
|
}
|
|
345
495
|
|
|
346
|
-
if (canProposeCheck.slot !==
|
|
496
|
+
if (canProposeCheck.slot !== targetSlot) {
|
|
347
497
|
this.log.warn(
|
|
348
|
-
`Cannot propose block due to slot mismatch with rollup contract (this can be caused by a clock out of sync). Expected slot ${
|
|
349
|
-
{ ...logCtx, rollup: canProposeCheck, expectedSlot:
|
|
498
|
+
`Cannot propose block due to slot mismatch with rollup contract (this can be caused by a clock out of sync). Expected slot ${targetSlot} but got ${canProposeCheck.slot}.`,
|
|
499
|
+
{ ...logCtx, ...proposeContext, rollup: canProposeCheck, expectedSlot: targetSlot },
|
|
350
500
|
);
|
|
351
501
|
this.emit('proposer-rollup-check-failed', { reason: 'Slot mismatch', slot });
|
|
352
502
|
this.metrics.recordCheckpointPrecheckFailed('slot_mismatch');
|
|
@@ -356,45 +506,61 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
356
506
|
if (canProposeCheck.checkpointNumber !== checkpointNumber) {
|
|
357
507
|
this.log.warn(
|
|
358
508
|
`Cannot propose due to block mismatch with rollup contract (this can be caused by a pending archiver sync). Expected checkpoint ${checkpointNumber} but got ${canProposeCheck.checkpointNumber}.`,
|
|
359
|
-
{ ...logCtx, rollup: canProposeCheck, expectedSlot: slot },
|
|
509
|
+
{ ...logCtx, ...proposeContext, rollup: canProposeCheck, expectedSlot: slot },
|
|
360
510
|
);
|
|
361
511
|
this.emit('proposer-rollup-check-failed', { reason: 'Block mismatch', slot });
|
|
362
512
|
this.metrics.recordCheckpointPrecheckFailed('block_number_mismatch');
|
|
363
513
|
return undefined;
|
|
364
514
|
}
|
|
365
515
|
|
|
366
|
-
this.lastSlotForCheckpointProposalJob =
|
|
367
|
-
|
|
368
|
-
this.
|
|
516
|
+
this.lastSlotForCheckpointProposalJob = targetSlot;
|
|
517
|
+
|
|
518
|
+
await this.p2pClient.prepareForSlot(targetSlot);
|
|
519
|
+
this.log.info(
|
|
520
|
+
`Preparing checkpoint proposal ${checkpointNumber} for target slot ${targetSlot} during wall-clock slot ${slot}`,
|
|
521
|
+
{
|
|
522
|
+
...logCtx,
|
|
523
|
+
...proposeContext,
|
|
524
|
+
proposer,
|
|
525
|
+
},
|
|
526
|
+
);
|
|
369
527
|
|
|
370
528
|
// Create and return the checkpoint proposal job
|
|
371
529
|
return this.createCheckpointProposalJob(
|
|
372
|
-
epoch,
|
|
373
530
|
slot,
|
|
531
|
+
targetSlot,
|
|
532
|
+
targetEpoch,
|
|
374
533
|
checkpointNumber,
|
|
375
534
|
syncedTo.blockNumber,
|
|
535
|
+
syncedTo.checkpointedCheckpointNumber,
|
|
376
536
|
proposer,
|
|
377
537
|
publisher,
|
|
378
538
|
attestorAddress,
|
|
379
539
|
invalidateCheckpoint,
|
|
540
|
+
syncedTo.proposedCheckpointData,
|
|
380
541
|
);
|
|
381
542
|
}
|
|
382
543
|
|
|
383
544
|
protected createCheckpointProposalJob(
|
|
384
|
-
epoch: EpochNumber,
|
|
385
545
|
slot: SlotNumber,
|
|
546
|
+
targetSlot: SlotNumber,
|
|
547
|
+
targetEpoch: EpochNumber,
|
|
386
548
|
checkpointNumber: CheckpointNumber,
|
|
387
549
|
syncedToBlockNumber: BlockNumber,
|
|
550
|
+
checkpointedCheckpointNumber: CheckpointNumber,
|
|
388
551
|
proposer: EthAddress | undefined,
|
|
389
552
|
publisher: SequencerPublisher,
|
|
390
553
|
attestorAddress: EthAddress,
|
|
391
554
|
invalidateCheckpoint: InvalidateCheckpointRequest | undefined,
|
|
555
|
+
proposedCheckpointData?: ProposedCheckpointData,
|
|
392
556
|
): CheckpointProposalJob {
|
|
393
557
|
return new CheckpointProposalJob(
|
|
394
|
-
epoch,
|
|
395
558
|
slot,
|
|
559
|
+
targetSlot,
|
|
560
|
+
targetEpoch,
|
|
396
561
|
checkpointNumber,
|
|
397
562
|
syncedToBlockNumber,
|
|
563
|
+
checkpointedCheckpointNumber,
|
|
398
564
|
proposer,
|
|
399
565
|
publisher,
|
|
400
566
|
attestorAddress,
|
|
@@ -408,16 +574,19 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
408
574
|
this.checkpointsBuilder,
|
|
409
575
|
this.l2BlockSource,
|
|
410
576
|
this.l1Constants,
|
|
577
|
+
this.signatureContext,
|
|
411
578
|
this.config,
|
|
412
579
|
this.timetable,
|
|
413
580
|
this.slasherClient,
|
|
414
581
|
this.epochCache,
|
|
415
582
|
this.dateProvider,
|
|
416
583
|
this.metrics,
|
|
584
|
+
this.checkpointProposalJobMetrics.createRecorder(),
|
|
417
585
|
this,
|
|
418
|
-
this.setState
|
|
586
|
+
(state, slot, timeReferenceSlot) => this.setState(state, slot, { timeReferenceSlot }),
|
|
419
587
|
this.tracer,
|
|
420
588
|
this.log.getBindings(),
|
|
589
|
+
proposedCheckpointData,
|
|
421
590
|
);
|
|
422
591
|
}
|
|
423
592
|
|
|
@@ -431,13 +600,14 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
431
600
|
/**
|
|
432
601
|
* Internal helper for setting the sequencer state and checks if we have enough time left in the slot to transition to the new state.
|
|
433
602
|
* @param proposedState - The new state to transition to.
|
|
434
|
-
* @param slotNumber - The
|
|
603
|
+
* @param slotNumber - The logical slot associated with the new state.
|
|
435
604
|
* @param force - Whether to force the transition even if the sequencer is stopped.
|
|
605
|
+
* @param timeReferenceSlot - The slot to use for deadline checks when it differs from the logical slot.
|
|
436
606
|
*/
|
|
437
607
|
protected setState(
|
|
438
608
|
proposedState: SequencerState,
|
|
439
609
|
slotNumber: SlotNumber | undefined,
|
|
440
|
-
opts: { force?: boolean } = {},
|
|
610
|
+
opts: { force?: boolean; timeReferenceSlot?: SlotNumber } = {},
|
|
441
611
|
): void {
|
|
442
612
|
if (this.state === SequencerState.STOPPING && proposedState !== SequencerState.STOPPED && !opts.force) {
|
|
443
613
|
this.log.warn(`Cannot set sequencer to ${proposedState} as it is stopping.`);
|
|
@@ -448,24 +618,43 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
448
618
|
return;
|
|
449
619
|
}
|
|
450
620
|
let secondsIntoSlot = undefined;
|
|
451
|
-
|
|
452
|
-
|
|
621
|
+
const timeReferenceSlot = opts.timeReferenceSlot ?? slotNumber;
|
|
622
|
+
if (timeReferenceSlot !== undefined) {
|
|
623
|
+
secondsIntoSlot = this.getSecondsIntoSlot(timeReferenceSlot);
|
|
453
624
|
this.timetable.assertTimeLeft(proposedState, secondsIntoSlot);
|
|
454
625
|
}
|
|
455
626
|
|
|
627
|
+
const oldState = this.state;
|
|
628
|
+
const oldStateSlotNumber = this.stateSlotNumber;
|
|
629
|
+
const stateChanged = proposedState !== oldState;
|
|
630
|
+
const transitionAtMs = performance.now();
|
|
631
|
+
const stateDurationMs = transitionAtMs - this.stateEnteredAtMs;
|
|
632
|
+
|
|
456
633
|
const boringStates = [SequencerState.IDLE, SequencerState.SYNCHRONIZING];
|
|
457
634
|
const logLevel =
|
|
458
|
-
boringStates.includes(proposedState) && boringStates.includes(
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
635
|
+
boringStates.includes(proposedState) && boringStates.includes(oldState) ? ('trace' as const) : ('debug' as const);
|
|
636
|
+
this.stateLog[logLevel](`Transitioning from ${oldState} to ${proposedState}`, {
|
|
637
|
+
oldState,
|
|
638
|
+
newState: proposedState,
|
|
639
|
+
slotNumber,
|
|
640
|
+
timeReferenceSlot,
|
|
641
|
+
stateSlotNumber: oldStateSlotNumber,
|
|
642
|
+
secondsIntoSlot,
|
|
643
|
+
...(stateChanged && { stateDurationMs: Math.ceil(stateDurationMs) }),
|
|
644
|
+
});
|
|
462
645
|
|
|
463
646
|
this.emit('state-changed', {
|
|
464
|
-
oldState
|
|
647
|
+
oldState,
|
|
465
648
|
newState: proposedState,
|
|
466
649
|
secondsIntoSlot,
|
|
467
650
|
slot: slotNumber,
|
|
651
|
+
timeReferenceSlot,
|
|
468
652
|
});
|
|
653
|
+
if (stateChanged) {
|
|
654
|
+
this.metrics.recordStateDuration(stateDurationMs, oldState);
|
|
655
|
+
this.stateEnteredAtMs = transitionAtMs;
|
|
656
|
+
this.stateSlotNumber = slotNumber;
|
|
657
|
+
}
|
|
469
658
|
this.state = proposedState;
|
|
470
659
|
}
|
|
471
660
|
|
|
@@ -475,8 +664,8 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
475
664
|
*/
|
|
476
665
|
protected async checkSync(args: { ts: bigint; slot: SlotNumber }): Promise<SequencerSyncCheckResult | undefined> {
|
|
477
666
|
// Check that the archiver has fully synced the L2 slot before the one we want to propose in.
|
|
478
|
-
//
|
|
479
|
-
//
|
|
667
|
+
// The archiver reports sync progress via L1 block timestamps and synced checkpoint slots.
|
|
668
|
+
// See getSyncedL2SlotNumber for how missed L1 blocks are handled.
|
|
480
669
|
const syncedL2Slot = await this.l2BlockSource.getSyncedL2SlotNumber();
|
|
481
670
|
const { slot } = args;
|
|
482
671
|
if (syncedL2Slot === undefined || syncedL2Slot + 1 < slot) {
|
|
@@ -492,45 +681,113 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
492
681
|
number: syncSummary.latestBlockNumber,
|
|
493
682
|
hash: syncSummary.latestBlockHash,
|
|
494
683
|
})),
|
|
495
|
-
this.l2BlockSource
|
|
684
|
+
this.l2BlockSource
|
|
685
|
+
.getL2Tips()
|
|
686
|
+
.then(t => ({ proposed: t.proposed, checkpointed: t.checkpointed, proposedCheckpoint: t.proposedCheckpoint })),
|
|
496
687
|
this.p2pClient.getStatus().then(p2p => p2p.syncedToL2Block),
|
|
497
|
-
this.l1ToL2MessageSource.getL2Tips().then(t => t.proposed),
|
|
688
|
+
this.l1ToL2MessageSource.getL2Tips().then(t => ({ proposed: t.proposed, checkpointed: t.checkpointed })),
|
|
498
689
|
this.l2BlockSource.getPendingChainValidationStatus(),
|
|
690
|
+
this.l2BlockSource.getProposedCheckpointData(),
|
|
499
691
|
] as const);
|
|
500
692
|
|
|
501
|
-
const [worldState,
|
|
693
|
+
const [worldState, l2Tips, p2p, l1ToL2MessageSourceTips, pendingChainValidationStatus, proposedCheckpointData] =
|
|
694
|
+
syncedBlocks;
|
|
502
695
|
|
|
503
|
-
// Handle zero as a special case, since the block hash won't match across services if we're changing the prefilled data for the genesis block,
|
|
504
|
-
// as the world state can compute the new genesis block hash, but other components use the hardcoded constant.
|
|
505
|
-
// TODO(palla/mbps): Fix the above. All components should be able to handle dynamic genesis block hashes.
|
|
506
696
|
const result =
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
697
|
+
worldState.hash === l2Tips.proposed.hash &&
|
|
698
|
+
p2p.hash === l2Tips.proposed.hash &&
|
|
699
|
+
l1ToL2MessageSourceTips.proposed.hash === l2Tips.proposed.hash &&
|
|
700
|
+
l1ToL2MessageSourceTips.checkpointed.block.hash === l2Tips.checkpointed.block.hash &&
|
|
701
|
+
l1ToL2MessageSourceTips.checkpointed.checkpoint.hash === l2Tips.checkpointed.checkpoint.hash;
|
|
511
702
|
|
|
512
703
|
if (!result) {
|
|
513
|
-
this.log.debug(`Sequencer sync check failed`, {
|
|
704
|
+
this.log.debug(`Sequencer sync check failed`, {
|
|
705
|
+
worldState,
|
|
706
|
+
l2BlockSource: l2Tips.proposed,
|
|
707
|
+
p2p,
|
|
708
|
+
l1ToL2MessageSourceTips,
|
|
709
|
+
});
|
|
514
710
|
return undefined;
|
|
515
711
|
}
|
|
516
712
|
|
|
517
|
-
// Special case for genesis state
|
|
518
713
|
const blockNumber = worldState.number;
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
714
|
+
const blockData = await this.l2BlockSource.getBlockData({ number: blockNumber });
|
|
715
|
+
if (!blockData) {
|
|
716
|
+
this.log.warn(`Sequencer sync check failed: failed to get L2 block data ${blockNumber} from the archiver`, {
|
|
717
|
+
blockNumber,
|
|
718
|
+
l2Tips,
|
|
719
|
+
syncedL2Slot,
|
|
720
|
+
...args,
|
|
721
|
+
});
|
|
722
|
+
return undefined;
|
|
723
|
+
}
|
|
724
|
+
|
|
725
|
+
// Refuse to build a checkpoint on top of a proposed block whose enclosing checkpoint was never
|
|
726
|
+
// proposed. Under pipelining we may have received and reexecuted such a block locally — advancing
|
|
727
|
+
// our world-state tip past the checkpointed tip — while the proposing node never published the
|
|
728
|
+
// matching proposed checkpoint (e.g. it crashed before assembling it). Building on this orphan block
|
|
729
|
+
// would fork the chain off a tip no other node can follow. The archiver prunes these orphan blocks
|
|
730
|
+
// once their build slot ends; this guard is the correctness barrier during the grace window before.
|
|
731
|
+
if (
|
|
732
|
+
blockData.checkpointNumber > l2Tips.checkpointed.checkpoint.number &&
|
|
733
|
+
(l2Tips.proposedCheckpoint.checkpoint.number !== blockData.checkpointNumber ||
|
|
734
|
+
proposedCheckpointData?.checkpointNumber !== blockData.checkpointNumber)
|
|
735
|
+
) {
|
|
736
|
+
const logCtx = {
|
|
737
|
+
blockCheckpointNumber: blockData.checkpointNumber,
|
|
738
|
+
checkpointedCheckpointNumber: l2Tips.checkpointed.checkpoint.number,
|
|
739
|
+
proposedCheckpointTipNumber: l2Tips.proposedCheckpoint.checkpoint.number,
|
|
740
|
+
proposedCheckpointDataNumber: proposedCheckpointData?.checkpointNumber,
|
|
741
|
+
blockNumber: blockData.header.getBlockNumber(),
|
|
742
|
+
blockSlot: blockData.header.getSlot(),
|
|
525
743
|
syncedL2Slot,
|
|
526
|
-
|
|
744
|
+
...args,
|
|
527
745
|
};
|
|
746
|
+
|
|
747
|
+
// Under pipelining the block proposal for a checkpoint leads its checkpoint proposal by up to one
|
|
748
|
+
// slot, so a world-state tip sitting in an as-yet-unproposed checkpoint is the expected steady state
|
|
749
|
+
// until that checkpoint is due. Only treat it as abnormal — and warn — once the checkpoint is overdue
|
|
750
|
+
// by the same deadline the archiver uses to prune the orphan block (see pruneOrphanProposedBlocks).
|
|
751
|
+
// Before then this is normal pipelining and we wait it out quietly.
|
|
752
|
+
if (this.isProposedCheckpointOverdue(blockData.header.getSlot())) {
|
|
753
|
+
this.log.warn(`Sequencer sync check failed: proposed block has no matching proposed checkpoint`, logCtx);
|
|
754
|
+
} else {
|
|
755
|
+
this.log.debug(`Waiting for proposed checkpoint to catch up with reexecuted block`, logCtx);
|
|
756
|
+
}
|
|
757
|
+
return undefined;
|
|
528
758
|
}
|
|
529
759
|
|
|
530
|
-
const
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
760
|
+
const hasProposedCheckpoint = l2Tips.proposedCheckpoint.checkpoint.number > l2Tips.checkpointed.checkpoint.number;
|
|
761
|
+
|
|
762
|
+
// The l2Tips and proposedCheckpointData reads above come from independent archiver snapshots
|
|
763
|
+
// (a JS-side tips cache vs. a direct store read on `#proposedCheckpoints`). A concurrent archiver
|
|
764
|
+
// write that mutates both can be observed split, leaving us with `hasProposedCheckpoint=true` but
|
|
765
|
+
// no proposedCheckpointData (or one whose number doesn't match the tip). Refuse to proceed in that
|
|
766
|
+
// window — the next checkSync tick will see a coherent snapshot.
|
|
767
|
+
if (
|
|
768
|
+
hasProposedCheckpoint &&
|
|
769
|
+
(!proposedCheckpointData ||
|
|
770
|
+
proposedCheckpointData.checkpointNumber !== l2Tips.proposedCheckpoint.checkpoint.number)
|
|
771
|
+
) {
|
|
772
|
+
this.log.warn(`Sequencer sync check failed: inconsistent proposed-checkpoint state`, {
|
|
773
|
+
proposedCheckpointTipNumber: l2Tips.proposedCheckpoint.checkpoint.number,
|
|
774
|
+
checkpointedTipNumber: l2Tips.checkpointed.checkpoint.number,
|
|
775
|
+
proposedCheckpointDataNumber: proposedCheckpointData?.checkpointNumber,
|
|
776
|
+
syncedL2Slot,
|
|
777
|
+
...args,
|
|
778
|
+
});
|
|
779
|
+
return undefined;
|
|
780
|
+
}
|
|
781
|
+
|
|
782
|
+
// Check that the proposed checkpoint is indeed the parent of the checkpoint we'll be building
|
|
783
|
+
// The checkpoint number to build is derived as blockData.checkpointNumber + 1
|
|
784
|
+
if (proposedCheckpointData && proposedCheckpointData.checkpointNumber !== blockData.checkpointNumber) {
|
|
785
|
+
this.log.warn(`Sequencer sync check failed: proposed checkpoint number mismatch`, {
|
|
786
|
+
proposedCheckpointNumber: proposedCheckpointData.checkpointNumber,
|
|
787
|
+
blockCheckpointNumber: blockData.checkpointNumber,
|
|
788
|
+
syncedL2Slot,
|
|
789
|
+
...args,
|
|
790
|
+
});
|
|
534
791
|
return undefined;
|
|
535
792
|
}
|
|
536
793
|
|
|
@@ -538,30 +795,48 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
538
795
|
blockData,
|
|
539
796
|
blockNumber: blockData.header.getBlockNumber(),
|
|
540
797
|
checkpointNumber: blockData.checkpointNumber,
|
|
798
|
+
checkpointedCheckpointNumber: l2Tips.checkpointed.checkpoint.number,
|
|
541
799
|
archive: blockData.archive.root,
|
|
800
|
+
hasProposedCheckpoint,
|
|
801
|
+
proposedCheckpointData,
|
|
542
802
|
syncedL2Slot,
|
|
543
803
|
pendingChainValidationStatus,
|
|
544
804
|
};
|
|
545
805
|
}
|
|
546
806
|
|
|
807
|
+
/**
|
|
808
|
+
* Whether the enclosing checkpoint of a reexecuted block is overdue: past the deadline by which a
|
|
809
|
+
* well-behaved proposer should have published it. Mirrors the archiver's orphan-prune deadline (the
|
|
810
|
+
* start of the slot after the block's build slot, plus a grace period) so the sequencer only warns
|
|
811
|
+
* about a missing proposed checkpoint once the archiver itself would prune the orphan block. The grace
|
|
812
|
+
* is derived from the block build duration the same way the archiver defaults it at node wiring.
|
|
813
|
+
*/
|
|
814
|
+
private isProposedCheckpointOverdue(blockSlot: SlotNumber): boolean {
|
|
815
|
+
const expectedBySlot = SlotNumber(Number(blockSlot) - PROPOSER_PIPELINING_SLOT_OFFSET + 1);
|
|
816
|
+
const graceSeconds =
|
|
817
|
+
this.config.blockDurationMs !== undefined ? Math.ceil(this.config.blockDurationMs / 1000) : MIN_EXECUTION_TIME;
|
|
818
|
+
const expectedByTime = getTimestampForSlot(expectedBySlot, this.l1Constants) + BigInt(graceSeconds);
|
|
819
|
+
return BigInt(this.dateProvider.nowInSeconds()) >= expectedByTime;
|
|
820
|
+
}
|
|
821
|
+
|
|
547
822
|
/**
|
|
548
823
|
* Checks if we are the proposer for the next slot.
|
|
549
824
|
* @returns True if we can propose, and the proposer address (undefined if anyone can propose)
|
|
550
825
|
*/
|
|
551
|
-
protected async checkCanPropose(
|
|
826
|
+
protected async checkCanPropose(targetSlot: SlotNumber): Promise<[boolean, EthAddress | undefined]> {
|
|
552
827
|
let proposer: EthAddress | undefined;
|
|
553
828
|
|
|
554
829
|
try {
|
|
555
|
-
proposer = await this.epochCache.getProposerAttesterAddressInSlot(
|
|
830
|
+
proposer = await this.epochCache.getProposerAttesterAddressInSlot(targetSlot);
|
|
556
831
|
} catch (e) {
|
|
557
832
|
if (e instanceof NoCommitteeError) {
|
|
558
|
-
if (this.lastSlotForNoCommitteeWarning !==
|
|
559
|
-
this.lastSlotForNoCommitteeWarning =
|
|
560
|
-
this.log.warn(`Cannot propose at
|
|
833
|
+
if (this.lastSlotForNoCommitteeWarning !== targetSlot) {
|
|
834
|
+
this.lastSlotForNoCommitteeWarning = targetSlot;
|
|
835
|
+
this.log.warn(`Cannot propose at target slot ${targetSlot} since the committee does not exist on L1`);
|
|
561
836
|
}
|
|
562
837
|
return [false, undefined];
|
|
563
838
|
}
|
|
564
|
-
this.log.error(`Error getting proposer for slot ${
|
|
839
|
+
this.log.error(`Error getting proposer for target slot ${targetSlot}`, e);
|
|
565
840
|
return [false, undefined];
|
|
566
841
|
}
|
|
567
842
|
|
|
@@ -578,10 +853,18 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
578
853
|
const weAreProposer = validatorAddresses.some(addr => addr.equals(proposer));
|
|
579
854
|
|
|
580
855
|
if (!weAreProposer) {
|
|
581
|
-
this.log.debug(`Cannot propose at slot ${
|
|
856
|
+
this.log.debug(`Cannot propose at target slot ${targetSlot} since we are not a proposer`, {
|
|
857
|
+
targetSlot,
|
|
858
|
+
validatorAddresses,
|
|
859
|
+
proposer,
|
|
860
|
+
});
|
|
582
861
|
return [false, proposer];
|
|
583
862
|
}
|
|
584
863
|
|
|
864
|
+
this.log.info(`We are the proposer for pipeline slot ${targetSlot}`, {
|
|
865
|
+
targetSlot,
|
|
866
|
+
proposer,
|
|
867
|
+
});
|
|
585
868
|
return [true, proposer];
|
|
586
869
|
}
|
|
587
870
|
|
|
@@ -590,8 +873,8 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
590
873
|
* This allows the sequencer to participate in governance/slashing votes even when it cannot build blocks.
|
|
591
874
|
*/
|
|
592
875
|
@trackSpan('Seqeuencer.tryVoteWhenSyncFails', ({ slot }) => ({ [Attributes.SLOT_NUMBER]: slot }))
|
|
593
|
-
protected async tryVoteWhenSyncFails(args: { slot: SlotNumber; ts: bigint }): Promise<void> {
|
|
594
|
-
const { slot } = args;
|
|
876
|
+
protected async tryVoteWhenSyncFails(args: { slot: SlotNumber; targetSlot: SlotNumber; ts: bigint }): Promise<void> {
|
|
877
|
+
const { slot, targetSlot } = args;
|
|
595
878
|
|
|
596
879
|
// Prevent duplicate attempts in the same slot
|
|
597
880
|
if (this.lastSlotForFallbackVote === slot) {
|
|
@@ -619,7 +902,7 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
619
902
|
});
|
|
620
903
|
|
|
621
904
|
// Check if we're a proposer or proposal is open
|
|
622
|
-
const [canPropose, proposer] = await this.checkCanPropose(
|
|
905
|
+
const [canPropose, proposer] = await this.checkCanPropose(targetSlot);
|
|
623
906
|
if (!canPropose) {
|
|
624
907
|
this.log.trace(`Cannot vote in slot ${slot} since we are not a proposer`, { slot, proposer });
|
|
625
908
|
return;
|
|
@@ -636,9 +919,9 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
636
919
|
slot,
|
|
637
920
|
});
|
|
638
921
|
|
|
639
|
-
// Enqueue governance and slashing votes
|
|
922
|
+
// Enqueue governance and slashing votes (voter uses the target slot for L1 submission)
|
|
640
923
|
const voter = new CheckpointVoter(
|
|
641
|
-
|
|
924
|
+
targetSlot,
|
|
642
925
|
publisher,
|
|
643
926
|
attestorAddress,
|
|
644
927
|
this.validatorClient,
|
|
@@ -657,7 +940,13 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
657
940
|
}
|
|
658
941
|
|
|
659
942
|
this.log.info(`Voting in slot ${slot} despite sync failure`, { slot });
|
|
660
|
-
|
|
943
|
+
// Votes are EIP-712-signed for `targetSlot` (the pipelined slot in which the multicall is
|
|
944
|
+
// expected to mine). Delay submission to the start of `targetSlot` so the tx mines in the
|
|
945
|
+
// slot the votes were signed for. We fire-and-forget so we don't block the sequencer's
|
|
946
|
+
// work loop while waiting for the target slot to start.
|
|
947
|
+
void publisher.sendRequestsAt(targetSlot).catch(err => {
|
|
948
|
+
this.log.error(`Failed to publish votes despite sync failure for slot ${slot}`, err, { slot });
|
|
949
|
+
});
|
|
661
950
|
}
|
|
662
951
|
|
|
663
952
|
/**
|
|
@@ -667,9 +956,10 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
667
956
|
@trackSpan('Sequencer.tryVoteWhenEscapeHatchOpen', ({ slot }) => ({ [Attributes.SLOT_NUMBER]: slot }))
|
|
668
957
|
protected async tryVoteWhenEscapeHatchOpen(args: {
|
|
669
958
|
slot: SlotNumber;
|
|
959
|
+
targetSlot: SlotNumber;
|
|
670
960
|
proposer: EthAddress | undefined;
|
|
671
961
|
}): Promise<void> {
|
|
672
|
-
const { slot, proposer } = args;
|
|
962
|
+
const { slot, targetSlot, proposer } = args;
|
|
673
963
|
|
|
674
964
|
// Prevent duplicate attempts in the same slot
|
|
675
965
|
if (this.lastSlotForFallbackVote === slot) {
|
|
@@ -682,10 +972,18 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
682
972
|
|
|
683
973
|
const { attestorAddress, publisher } = await this.publisherFactory.create(proposer);
|
|
684
974
|
|
|
685
|
-
this.log.debug(`Escape hatch open for slot ${slot}, attempting vote-only actions`, {
|
|
975
|
+
this.log.debug(`Escape hatch open for slot ${slot}, attempting vote-only actions`, {
|
|
976
|
+
slot,
|
|
977
|
+
targetSlot,
|
|
978
|
+
attestorAddress,
|
|
979
|
+
});
|
|
686
980
|
|
|
981
|
+
// Under proposer pipelining, the multicall is expected to mine in `targetSlot` (slot + 1).
|
|
982
|
+
// Governance and slashing votes are EIP-712-signed against the slot they will mine in, and the
|
|
983
|
+
// L1 contract checks `msg.sender == getCurrentProposer()` using the mining slot. So we must
|
|
984
|
+
// sign for `targetSlot` and delay submission to the start of `targetSlot`.
|
|
687
985
|
const voter = new CheckpointVoter(
|
|
688
|
-
|
|
986
|
+
targetSlot,
|
|
689
987
|
publisher,
|
|
690
988
|
attestorAddress,
|
|
691
989
|
this.validatorClient,
|
|
@@ -704,8 +1002,15 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
704
1002
|
return;
|
|
705
1003
|
}
|
|
706
1004
|
|
|
707
|
-
this.log.info(`Voting in slot ${slot} (escape hatch open)`, { slot });
|
|
708
|
-
|
|
1005
|
+
this.log.info(`Voting in slot ${slot} (escape hatch open)`, { slot, targetSlot });
|
|
1006
|
+
// Votes are EIP-712-signed for `targetSlot`. Delay submission to the start of `targetSlot` so
|
|
1007
|
+
// the multicall mines in the slot the votes were signed for; otherwise the L1 contract reads
|
|
1008
|
+
// `signaler = getCurrentProposer()` against the wrong slot and signature verification fails
|
|
1009
|
+
// silently inside Multicall3. Fire-and-forget so we don't block the sequencer's work loop while
|
|
1010
|
+
// waiting for the target slot to start, mirroring tryVoteWhenSyncFails.
|
|
1011
|
+
void publisher.sendRequestsAt(targetSlot).catch(err => {
|
|
1012
|
+
this.log.error(`Failed to publish escape-hatch votes for slot ${slot}`, err, { slot, targetSlot });
|
|
1013
|
+
});
|
|
709
1014
|
}
|
|
710
1015
|
|
|
711
1016
|
/**
|
|
@@ -878,8 +1183,11 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
878
1183
|
type SequencerSyncCheckResult = {
|
|
879
1184
|
blockData?: BlockData;
|
|
880
1185
|
checkpointNumber: CheckpointNumber;
|
|
1186
|
+
checkpointedCheckpointNumber: CheckpointNumber;
|
|
881
1187
|
blockNumber: BlockNumber;
|
|
882
1188
|
archive: Fr;
|
|
1189
|
+
hasProposedCheckpoint: boolean;
|
|
1190
|
+
proposedCheckpointData?: ProposedCheckpointData;
|
|
883
1191
|
syncedL2Slot: SlotNumber;
|
|
884
1192
|
pendingChainValidationStatus: ValidateCheckpointResult;
|
|
885
1193
|
};
|