@aztec/sequencer-client 0.0.1-commit.2448fdb → 0.0.1-commit.2606882
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +281 -21
- package/dest/client/sequencer-client.d.ts +4 -3
- package/dest/client/sequencer-client.d.ts.map +1 -1
- package/dest/client/sequencer-client.js +11 -11
- package/dest/config.d.ts +4 -1
- package/dest/config.d.ts.map +1 -1
- package/dest/config.js +35 -14
- package/dest/global_variable_builder/fee_predictor.d.ts +37 -0
- package/dest/global_variable_builder/fee_predictor.d.ts.map +1 -0
- package/dest/global_variable_builder/fee_predictor.js +128 -0
- package/dest/global_variable_builder/fee_provider.d.ts +21 -0
- package/dest/global_variable_builder/fee_provider.d.ts.map +1 -0
- package/dest/global_variable_builder/fee_provider.js +58 -0
- package/dest/global_variable_builder/global_builder.d.ts +4 -10
- package/dest/global_variable_builder/global_builder.d.ts.map +1 -1
- package/dest/global_variable_builder/global_builder.js +5 -40
- package/dest/global_variable_builder/index.d.ts +3 -1
- package/dest/global_variable_builder/index.d.ts.map +1 -1
- package/dest/global_variable_builder/index.js +2 -0
- package/dest/index.d.ts +2 -2
- package/dest/index.d.ts.map +1 -1
- package/dest/index.js +1 -1
- package/dest/publisher/config.d.ts +19 -3
- package/dest/publisher/config.d.ts.map +1 -1
- package/dest/publisher/config.js +25 -5
- package/dest/publisher/index.d.ts +2 -1
- package/dest/publisher/index.d.ts.map +1 -1
- package/dest/publisher/l1_tx_failed_store/factory.d.ts +11 -0
- package/dest/publisher/l1_tx_failed_store/factory.d.ts.map +1 -0
- package/dest/publisher/l1_tx_failed_store/factory.js +22 -0
- package/dest/publisher/l1_tx_failed_store/failed_tx_store.d.ts +58 -0
- package/dest/publisher/l1_tx_failed_store/failed_tx_store.d.ts.map +1 -0
- package/dest/publisher/l1_tx_failed_store/failed_tx_store.js +1 -0
- package/dest/publisher/l1_tx_failed_store/file_store_failed_tx_store.d.ts +15 -0
- package/dest/publisher/l1_tx_failed_store/file_store_failed_tx_store.d.ts.map +1 -0
- package/dest/publisher/l1_tx_failed_store/file_store_failed_tx_store.js +34 -0
- package/dest/publisher/l1_tx_failed_store/index.d.ts +4 -0
- package/dest/publisher/l1_tx_failed_store/index.d.ts.map +1 -0
- package/dest/publisher/l1_tx_failed_store/index.js +2 -0
- package/dest/publisher/sequencer-bundle-simulator.d.ts +96 -0
- package/dest/publisher/sequencer-bundle-simulator.d.ts.map +1 -0
- package/dest/publisher/sequencer-bundle-simulator.js +198 -0
- package/dest/publisher/sequencer-publisher-factory.d.ts +3 -5
- package/dest/publisher/sequencer-publisher-factory.d.ts.map +1 -1
- package/dest/publisher/sequencer-publisher-factory.js +16 -3
- package/dest/publisher/sequencer-publisher.d.ts +77 -66
- package/dest/publisher/sequencer-publisher.d.ts.map +1 -1
- package/dest/publisher/sequencer-publisher.js +410 -398
- package/dest/sequencer/automine/automine_factory.d.ts +54 -0
- package/dest/sequencer/automine/automine_factory.d.ts.map +1 -0
- package/dest/sequencer/automine/automine_factory.js +84 -0
- package/dest/sequencer/automine/automine_sequencer.d.ts +151 -0
- package/dest/sequencer/automine/automine_sequencer.d.ts.map +1 -0
- package/dest/sequencer/automine/automine_sequencer.js +509 -0
- package/dest/sequencer/chain_state_overrides.d.ts +61 -0
- package/dest/sequencer/chain_state_overrides.d.ts.map +1 -0
- package/dest/sequencer/chain_state_overrides.js +98 -0
- package/dest/sequencer/checkpoint_proposal_job.d.ts +58 -10
- package/dest/sequencer/checkpoint_proposal_job.d.ts.map +1 -1
- package/dest/sequencer/checkpoint_proposal_job.js +736 -191
- package/dest/sequencer/checkpoint_proposal_job_metrics.d.ts +34 -0
- package/dest/sequencer/checkpoint_proposal_job_metrics.d.ts.map +1 -0
- package/dest/sequencer/checkpoint_proposal_job_metrics.js +72 -0
- package/dest/sequencer/events.d.ts +48 -2
- package/dest/sequencer/events.d.ts.map +1 -1
- package/dest/sequencer/index.d.ts +3 -1
- package/dest/sequencer/index.d.ts.map +1 -1
- package/dest/sequencer/index.js +2 -0
- package/dest/sequencer/metrics.d.ts +13 -10
- package/dest/sequencer/metrics.d.ts.map +1 -1
- package/dest/sequencer/metrics.js +45 -20
- package/dest/sequencer/sequencer.d.ts +59 -12
- package/dest/sequencer/sequencer.d.ts.map +1 -1
- package/dest/sequencer/sequencer.js +344 -105
- package/dest/sequencer/timetable.d.ts +11 -1
- package/dest/sequencer/timetable.d.ts.map +1 -1
- package/dest/sequencer/timetable.js +41 -36
- package/dest/sequencer/types.d.ts +2 -2
- package/dest/sequencer/types.d.ts.map +1 -1
- package/dest/test/mock_checkpoint_builder.d.ts +1 -1
- package/dest/test/mock_checkpoint_builder.d.ts.map +1 -1
- package/dest/test/mock_checkpoint_builder.js +0 -2
- package/dest/test/utils.d.ts +1 -1
- package/dest/test/utils.d.ts.map +1 -1
- package/dest/test/utils.js +7 -6
- package/package.json +27 -28
- package/src/client/sequencer-client.ts +19 -20
- package/src/config.ts +40 -11
- package/src/global_variable_builder/README.md +44 -0
- package/src/global_variable_builder/fee_predictor.ts +172 -0
- package/src/global_variable_builder/fee_provider.ts +75 -0
- package/src/global_variable_builder/global_builder.ts +10 -46
- package/src/global_variable_builder/index.ts +2 -0
- package/src/index.ts +10 -1
- package/src/publisher/config.ts +49 -6
- package/src/publisher/index.ts +3 -0
- package/src/publisher/l1_tx_failed_store/factory.ts +32 -0
- package/src/publisher/l1_tx_failed_store/failed_tx_store.ts +57 -0
- package/src/publisher/l1_tx_failed_store/file_store_failed_tx_store.ts +46 -0
- package/src/publisher/l1_tx_failed_store/index.ts +3 -0
- package/src/publisher/sequencer-bundle-simulator.ts +254 -0
- package/src/publisher/sequencer-publisher-factory.ts +18 -6
- package/src/publisher/sequencer-publisher.ts +485 -474
- package/src/sequencer/README.md +162 -450
- package/src/sequencer/automine/README.md +46 -0
- package/src/sequencer/automine/automine_factory.ts +145 -0
- package/src/sequencer/automine/automine_sequencer.ts +595 -0
- package/src/sequencer/chain_state_overrides.ts +162 -0
- package/src/sequencer/checkpoint_proposal_job.ts +857 -201
- package/src/sequencer/checkpoint_proposal_job_metrics.ts +128 -0
- package/src/sequencer/events.ts +51 -2
- package/src/sequencer/index.ts +2 -0
- package/src/sequencer/metrics.ts +57 -24
- package/src/sequencer/sequencer.ts +421 -118
- package/src/sequencer/timetable.ts +49 -44
- package/src/sequencer/types.ts +1 -1
- package/src/test/mock_checkpoint_builder.ts +0 -2
- package/src/test/utils.ts +28 -10
|
@@ -1,20 +1,26 @@
|
|
|
1
1
|
import { getKzg } from '@aztec/blob-lib';
|
|
2
|
-
import {
|
|
3
|
-
import type { EpochCache } from '@aztec/epoch-cache';
|
|
2
|
+
import { type EpochCache, PROPOSER_PIPELINING_SLOT_OFFSET } from '@aztec/epoch-cache';
|
|
4
3
|
import { NoCommitteeError, type RollupContract } from '@aztec/ethereum/contracts';
|
|
5
4
|
import { BlockNumber, CheckpointNumber, EpochNumber, SlotNumber } from '@aztec/foundation/branded-types';
|
|
6
5
|
import { merge, omit, pick } from '@aztec/foundation/collection';
|
|
7
6
|
import { Fr } from '@aztec/foundation/curves/bn254';
|
|
8
7
|
import { EthAddress } from '@aztec/foundation/eth-address';
|
|
9
|
-
import { createLogger } from '@aztec/foundation/log';
|
|
8
|
+
import { type Logger, createLogger } from '@aztec/foundation/log';
|
|
10
9
|
import { RunningPromise } from '@aztec/foundation/running-promise';
|
|
11
10
|
import type { DateProvider } from '@aztec/foundation/timer';
|
|
12
11
|
import type { TypedEventEmitter } from '@aztec/foundation/types';
|
|
13
12
|
import type { P2P } from '@aztec/p2p';
|
|
14
13
|
import type { SlasherClientInterface } from '@aztec/slasher';
|
|
15
|
-
import type {
|
|
16
|
-
|
|
17
|
-
|
|
14
|
+
import type {
|
|
15
|
+
BlockData,
|
|
16
|
+
L2BlockSink,
|
|
17
|
+
L2BlockSource,
|
|
18
|
+
ProposedCheckpointSink,
|
|
19
|
+
ValidateCheckpointResult,
|
|
20
|
+
} from '@aztec/stdlib/block';
|
|
21
|
+
import type { Checkpoint, ProposedCheckpointData } from '@aztec/stdlib/checkpoint';
|
|
22
|
+
import type { ChainConfig } from '@aztec/stdlib/config';
|
|
23
|
+
import { getEpochAtSlot, getSlotStartBuildTimestamp, getTimestampForSlot } from '@aztec/stdlib/epoch-helpers';
|
|
18
24
|
import {
|
|
19
25
|
type ResolvedSequencerConfig,
|
|
20
26
|
type SequencerConfig,
|
|
@@ -22,8 +28,9 @@ import {
|
|
|
22
28
|
type WorldStateSynchronizer,
|
|
23
29
|
} from '@aztec/stdlib/interfaces/server';
|
|
24
30
|
import type { L1ToL2MessageSource } from '@aztec/stdlib/messaging';
|
|
31
|
+
import type { CoordinationSignatureContext } from '@aztec/stdlib/p2p';
|
|
25
32
|
import { pickFromSchema } from '@aztec/stdlib/schemas';
|
|
26
|
-
import {
|
|
33
|
+
import { MIN_EXECUTION_TIME } from '@aztec/stdlib/timetable';
|
|
27
34
|
import { Attributes, type TelemetryClient, type Tracer, getTelemetryClient, trackSpan } from '@aztec/telemetry-client';
|
|
28
35
|
import { FullNodeCheckpointsBuilder, NodeKeystoreAdapter, type ValidatorClient } from '@aztec/validator-client';
|
|
29
36
|
|
|
@@ -33,7 +40,9 @@ import { DefaultSequencerConfig } from '../config.js';
|
|
|
33
40
|
import type { GlobalVariableBuilder } from '../global_variable_builder/global_builder.js';
|
|
34
41
|
import type { SequencerPublisherFactory } from '../publisher/sequencer-publisher-factory.js';
|
|
35
42
|
import type { InvalidateCheckpointRequest, SequencerPublisher } from '../publisher/sequencer-publisher.js';
|
|
43
|
+
import { buildCheckpointSimulationOverridesPlan } from './chain_state_overrides.js';
|
|
36
44
|
import { CheckpointProposalJob } from './checkpoint_proposal_job.js';
|
|
45
|
+
import { CheckpointProposalJobMetrics } from './checkpoint_proposal_job_metrics.js';
|
|
37
46
|
import { CheckpointVoter } from './checkpoint_voter.js';
|
|
38
47
|
import { SequencerInterruptedError, SequencerTooSlowError } from './errors.js';
|
|
39
48
|
import type { SequencerEvents } from './events.js';
|
|
@@ -44,6 +53,16 @@ import { SequencerState } from './utils.js';
|
|
|
44
53
|
|
|
45
54
|
export { SequencerState };
|
|
46
55
|
|
|
56
|
+
/** Slot snapshot used to prepare a checkpoint proposal. */
|
|
57
|
+
type SequencerSlotContext = {
|
|
58
|
+
slot: SlotNumber;
|
|
59
|
+
targetSlot: SlotNumber;
|
|
60
|
+
epoch: EpochNumber;
|
|
61
|
+
targetEpoch: EpochNumber;
|
|
62
|
+
ts: bigint;
|
|
63
|
+
nowSeconds: bigint;
|
|
64
|
+
};
|
|
65
|
+
|
|
47
66
|
/**
|
|
48
67
|
* Sequencer client
|
|
49
68
|
* - Checks whether it is elected as proposer for the next slot
|
|
@@ -55,7 +74,11 @@ export { SequencerState };
|
|
|
55
74
|
export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<SequencerEvents>) {
|
|
56
75
|
private runningPromise?: RunningPromise;
|
|
57
76
|
private state = SequencerState.STOPPED;
|
|
77
|
+
private stateSlotNumber: SlotNumber | undefined;
|
|
78
|
+
private stateEnteredAtMs = performance.now();
|
|
58
79
|
private metrics: SequencerMetrics;
|
|
80
|
+
private checkpointProposalJobMetrics: CheckpointProposalJobMetrics;
|
|
81
|
+
private readonly stateLog: Logger;
|
|
59
82
|
|
|
60
83
|
/** The last slot for which we attempted to perform our voting duties with degraded block production */
|
|
61
84
|
private lastSlotForFallbackVote: SlotNumber | undefined;
|
|
@@ -72,11 +95,15 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
72
95
|
/** The last epoch for which we logged strategy comparison in fisherman mode. */
|
|
73
96
|
private lastEpochForStrategyComparison: EpochNumber | undefined;
|
|
74
97
|
|
|
98
|
+
/** The last checkpoint proposal job, tracked so we can await its pending L1 submission during shutdown. */
|
|
99
|
+
protected lastCheckpointProposalJob: CheckpointProposalJob | undefined;
|
|
100
|
+
|
|
75
101
|
/** The maximum number of seconds that the sequencer can be into a slot to transition to a particular state. */
|
|
76
102
|
protected timetable!: SequencerTimetable;
|
|
77
103
|
|
|
78
104
|
/** Config for the sequencer */
|
|
79
105
|
protected config: ResolvedSequencerConfig = DefaultSequencerConfig;
|
|
106
|
+
private readonly signatureContext: CoordinationSignatureContext;
|
|
80
107
|
|
|
81
108
|
constructor(
|
|
82
109
|
protected publisherFactory: SequencerPublisherFactory,
|
|
@@ -85,25 +112,31 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
85
112
|
protected p2pClient: P2P,
|
|
86
113
|
protected worldState: WorldStateSynchronizer,
|
|
87
114
|
protected slasherClient: SlasherClientInterface | undefined,
|
|
88
|
-
protected l2BlockSource: L2BlockSource & L2BlockSink,
|
|
115
|
+
protected l2BlockSource: L2BlockSource & L2BlockSink & ProposedCheckpointSink,
|
|
89
116
|
protected l1ToL2MessageSource: L1ToL2MessageSource,
|
|
90
117
|
protected checkpointsBuilder: FullNodeCheckpointsBuilder,
|
|
91
118
|
protected l1Constants: SequencerRollupConstants,
|
|
92
119
|
protected dateProvider: DateProvider,
|
|
93
120
|
protected epochCache: EpochCache,
|
|
94
121
|
protected rollupContract: RollupContract,
|
|
95
|
-
config: SequencerConfig,
|
|
122
|
+
config: SequencerConfig & Pick<ChainConfig, 'l1ChainId' | 'rollupAddress'>,
|
|
96
123
|
protected telemetry: TelemetryClient = getTelemetryClient(),
|
|
97
124
|
protected log = createLogger('sequencer'),
|
|
98
125
|
) {
|
|
99
126
|
super();
|
|
127
|
+
this.stateLog = log.createChild('state');
|
|
100
128
|
|
|
101
129
|
// Add [FISHERMAN] prefix to logger if in fisherman mode
|
|
102
130
|
if (config.fishermanMode) {
|
|
103
131
|
this.log = log.createChild('[FISHERMAN]');
|
|
104
132
|
}
|
|
105
133
|
|
|
134
|
+
this.signatureContext = {
|
|
135
|
+
chainId: config.l1ChainId,
|
|
136
|
+
rollupAddress: config.rollupAddress,
|
|
137
|
+
};
|
|
106
138
|
this.metrics = new SequencerMetrics(telemetry, this.rollupContract, 'Sequencer');
|
|
139
|
+
this.checkpointProposalJobMetrics = new CheckpointProposalJobMetrics(telemetry);
|
|
107
140
|
this.updateConfig(config);
|
|
108
141
|
}
|
|
109
142
|
|
|
@@ -112,12 +145,13 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
112
145
|
const filteredConfig = pickFromSchema(config, SequencerConfigSchema);
|
|
113
146
|
this.log.info(`Updated sequencer config`, omit(filteredConfig, 'txPublicSetupAllowListExtend'));
|
|
114
147
|
this.config = merge(this.config, filteredConfig);
|
|
148
|
+
const p2pPropagationTime = this.config.attestationPropagationTime;
|
|
115
149
|
this.timetable = new SequencerTimetable(
|
|
116
150
|
{
|
|
117
151
|
ethereumSlotDuration: this.l1Constants.ethereumSlotDuration,
|
|
118
152
|
aztecSlotDuration: this.aztecSlotDuration,
|
|
119
153
|
l1PublishingTime: this.l1PublishingTime,
|
|
120
|
-
p2pPropagationTime
|
|
154
|
+
p2pPropagationTime,
|
|
121
155
|
blockDurationMs: this.config.blockDurationMs,
|
|
122
156
|
enforce: this.config.enforceTimeTable,
|
|
123
157
|
},
|
|
@@ -152,8 +186,10 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
152
186
|
public async stop(): Promise<void> {
|
|
153
187
|
this.log.info(`Stopping sequencer`);
|
|
154
188
|
this.setState(SequencerState.STOPPING, undefined, { force: true });
|
|
155
|
-
this.
|
|
189
|
+
this.lastCheckpointProposalJob?.interrupt();
|
|
190
|
+
await this.publisherFactory.stopAll();
|
|
156
191
|
await this.runningPromise?.stop();
|
|
192
|
+
await this.lastCheckpointProposalJob?.awaitPendingSubmission();
|
|
157
193
|
this.setState(SequencerState.STOPPED, undefined, { force: true });
|
|
158
194
|
this.log.info('Stopped sequencer');
|
|
159
195
|
}
|
|
@@ -163,17 +199,15 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
163
199
|
try {
|
|
164
200
|
await this.work();
|
|
165
201
|
} catch (err) {
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
)
|
|
172
|
-
? ('debug' as const)
|
|
173
|
-
: ('warn' as const);
|
|
174
|
-
this.log[logLvl](err.message, { now: this.dateProvider.nowInSeconds() });
|
|
202
|
+
if (
|
|
203
|
+
err instanceof SequencerTooSlowError &&
|
|
204
|
+
[SequencerState.INITIALIZING_CHECKPOINT, SequencerState.PROPOSER_CHECK].includes(err.proposedState)
|
|
205
|
+
) {
|
|
206
|
+
// No need to alert if we just didn't get to start in time
|
|
207
|
+
this.log.debug(err.message, { now: this.dateProvider.nowInSeconds() });
|
|
175
208
|
} else {
|
|
176
209
|
// Re-throw other errors
|
|
210
|
+
this.emit('checkpoint-error', { error: err as Error });
|
|
177
211
|
throw err;
|
|
178
212
|
}
|
|
179
213
|
} finally {
|
|
@@ -197,14 +231,24 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
197
231
|
@trackSpan('Sequencer.work')
|
|
198
232
|
protected async work() {
|
|
199
233
|
this.setState(SequencerState.SYNCHRONIZING, undefined);
|
|
200
|
-
const { slot,
|
|
234
|
+
const { slot, targetSlot, epoch, targetEpoch, ts, nowSeconds } = this.getSlotContextInNextL1Slot();
|
|
201
235
|
|
|
202
236
|
// Check if we are synced and it's our slot, grab a publisher, check previous block invalidation, etc
|
|
203
|
-
const checkpointProposalJob = await this.prepareCheckpointProposal(
|
|
237
|
+
const checkpointProposalJob = await this.prepareCheckpointProposal(
|
|
238
|
+
slot,
|
|
239
|
+
targetSlot,
|
|
240
|
+
epoch,
|
|
241
|
+
targetEpoch,
|
|
242
|
+
ts,
|
|
243
|
+
nowSeconds,
|
|
244
|
+
);
|
|
204
245
|
if (!checkpointProposalJob) {
|
|
205
246
|
return;
|
|
206
247
|
}
|
|
207
248
|
|
|
249
|
+
// Track the job so we can await its pending L1 submission during shutdown
|
|
250
|
+
this.lastCheckpointProposalJob = checkpointProposalJob;
|
|
251
|
+
|
|
208
252
|
// Execute the checkpoint proposal job
|
|
209
253
|
const checkpoint = await checkpointProposalJob.execute();
|
|
210
254
|
|
|
@@ -213,64 +257,85 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
213
257
|
this.lastCheckpointProposed = checkpoint;
|
|
214
258
|
}
|
|
215
259
|
|
|
216
|
-
// Log fee strategy comparison if on fisherman
|
|
260
|
+
// Log fee strategy comparison if on fisherman (uses target epoch since we mirror the proposer's perspective)
|
|
217
261
|
if (
|
|
218
262
|
this.config.fishermanMode &&
|
|
219
|
-
(this.lastEpochForStrategyComparison === undefined ||
|
|
263
|
+
(this.lastEpochForStrategyComparison === undefined || targetEpoch > this.lastEpochForStrategyComparison)
|
|
220
264
|
) {
|
|
221
|
-
this.logStrategyComparison(
|
|
222
|
-
this.lastEpochForStrategyComparison =
|
|
265
|
+
this.logStrategyComparison(targetEpoch, checkpointProposalJob.getPublisher());
|
|
266
|
+
this.lastEpochForStrategyComparison = targetEpoch;
|
|
223
267
|
}
|
|
224
268
|
|
|
225
269
|
return checkpoint;
|
|
226
270
|
}
|
|
227
271
|
|
|
272
|
+
/** Returns slot and target slot from a single clock snapshot. */
|
|
273
|
+
protected getSlotContextInNextL1Slot(): SequencerSlotContext {
|
|
274
|
+
const { slot, ts, nowSeconds, epoch } = this.epochCache.getEpochAndSlotInNextL1Slot();
|
|
275
|
+
const targetSlot = SlotNumber(slot + PROPOSER_PIPELINING_SLOT_OFFSET);
|
|
276
|
+
return { slot, targetSlot, epoch, targetEpoch: getEpochAtSlot(targetSlot, this.l1Constants), ts, nowSeconds };
|
|
277
|
+
}
|
|
278
|
+
|
|
228
279
|
/**
|
|
229
280
|
* Prepares the checkpoint proposal by performing all necessary checks and setup.
|
|
230
281
|
* This is the initial step in the main loop.
|
|
231
282
|
* @returns CheckpointProposalJob if successful, undefined if we are not yet synced or are not the proposer.
|
|
232
283
|
*/
|
|
233
284
|
@trackSpan('Sequencer.prepareCheckpointProposal')
|
|
234
|
-
|
|
235
|
-
epoch: EpochNumber,
|
|
285
|
+
protected async prepareCheckpointProposal(
|
|
236
286
|
slot: SlotNumber,
|
|
287
|
+
targetSlot: SlotNumber,
|
|
288
|
+
epoch: EpochNumber,
|
|
289
|
+
targetEpoch: EpochNumber,
|
|
237
290
|
ts: bigint,
|
|
238
|
-
|
|
291
|
+
nowSeconds: bigint,
|
|
239
292
|
): Promise<CheckpointProposalJob | undefined> {
|
|
240
|
-
// Check we have not already processed this slot (cheapest check)
|
|
293
|
+
// Check we have not already processed this target slot (cheapest check)
|
|
241
294
|
// We only check this if enforce timetable is set, since we want to keep processing the same slot if we are not
|
|
242
295
|
// running against actual time (eg when we use sandbox-style automining)
|
|
243
296
|
if (
|
|
244
297
|
this.lastSlotForCheckpointProposalJob &&
|
|
245
|
-
this.lastSlotForCheckpointProposalJob >=
|
|
298
|
+
this.lastSlotForCheckpointProposalJob >= targetSlot &&
|
|
246
299
|
this.config.enforceTimeTable
|
|
247
300
|
) {
|
|
248
|
-
this.log.trace(`
|
|
301
|
+
this.log.trace(`Target slot ${targetSlot} has already been processed`);
|
|
302
|
+
return undefined;
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
// But if we have already proposed for this slot, then we definitely have to skip it, automining or not
|
|
306
|
+
if (this.lastCheckpointProposed && this.lastCheckpointProposed.header.slotNumber >= targetSlot) {
|
|
307
|
+
this.log.trace(
|
|
308
|
+
`Slot ${targetSlot} has already been published as checkpoint ${this.lastCheckpointProposed.number}`,
|
|
309
|
+
);
|
|
249
310
|
return undefined;
|
|
250
311
|
}
|
|
251
312
|
|
|
252
|
-
//
|
|
253
|
-
|
|
254
|
-
|
|
313
|
+
// Test-only: skip proposing for explicitly paused slots. Attestation paths run in the validator
|
|
314
|
+
// client and are not gated by this hook, so paused proposers still attest to others' proposals.
|
|
315
|
+
if (this.config.pauseProposingForSlots?.some(s => s === targetSlot)) {
|
|
316
|
+
this.log.warn(`Skipping proposal for paused slot ${targetSlot} (test-only pauseProposingForSlots hook)`, {
|
|
317
|
+
targetSlot,
|
|
318
|
+
});
|
|
255
319
|
return undefined;
|
|
256
320
|
}
|
|
257
321
|
|
|
258
322
|
// Check all components are synced to latest as seen by the archiver (queries all subsystems)
|
|
259
323
|
const syncedTo = await this.checkSync({ ts, slot });
|
|
260
324
|
if (!syncedTo) {
|
|
261
|
-
await this.tryVoteWhenSyncFails({ slot, ts });
|
|
325
|
+
await this.tryVoteWhenSyncFails({ slot, targetSlot, ts });
|
|
262
326
|
return undefined;
|
|
263
327
|
}
|
|
264
328
|
|
|
265
|
-
// If escape hatch is open for
|
|
329
|
+
// If escape hatch is open for the target epoch, do not start checkpoint proposal work and do not attempt invalidations.
|
|
266
330
|
// Still perform governance/slashing voting (as proposer) once per slot.
|
|
267
|
-
|
|
331
|
+
// When pipelining, we check the target epoch (slot+1's epoch) since that's the epoch we're building for.
|
|
332
|
+
const isEscapeHatchOpen = await this.epochCache.isEscapeHatchOpen(targetEpoch);
|
|
268
333
|
|
|
269
334
|
if (isEscapeHatchOpen) {
|
|
270
335
|
this.setState(SequencerState.PROPOSER_CHECK, slot);
|
|
271
|
-
const [canPropose, proposer] = await this.checkCanPropose(
|
|
336
|
+
const [canPropose, proposer] = await this.checkCanPropose(targetSlot);
|
|
272
337
|
if (canPropose) {
|
|
273
|
-
await this.tryVoteWhenEscapeHatchOpen({ slot, proposer });
|
|
338
|
+
await this.tryVoteWhenEscapeHatchOpen({ slot, targetSlot, proposer });
|
|
274
339
|
} else {
|
|
275
340
|
this.log.trace(`Escape hatch open but we are not proposer, skipping vote-only actions`, {
|
|
276
341
|
slot,
|
|
@@ -285,17 +350,18 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
285
350
|
const checkpointNumber = CheckpointNumber(syncedTo.checkpointNumber + 1);
|
|
286
351
|
|
|
287
352
|
const logCtx = {
|
|
288
|
-
|
|
353
|
+
nowSeconds,
|
|
289
354
|
syncedToL2Slot: syncedTo.syncedL2Slot,
|
|
290
355
|
slot,
|
|
356
|
+
targetSlot,
|
|
291
357
|
slotTs: ts,
|
|
292
358
|
checkpointNumber,
|
|
293
359
|
isPendingChainValid: pick(syncedTo.pendingChainValidationStatus, 'valid', 'reason', 'invalidIndex'),
|
|
294
360
|
};
|
|
295
361
|
|
|
296
|
-
// Check that we are a proposer for the
|
|
362
|
+
// Check that we are a proposer for the target slot.
|
|
297
363
|
this.setState(SequencerState.PROPOSER_CHECK, slot);
|
|
298
|
-
const [canPropose, proposer] = await this.checkCanPropose(
|
|
364
|
+
const [canPropose, proposer] = await this.checkCanPropose(targetSlot);
|
|
299
365
|
|
|
300
366
|
// If we are not a proposer check if we should invalidate an invalid checkpoint, and bail
|
|
301
367
|
if (!canPropose) {
|
|
@@ -303,10 +369,20 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
303
369
|
return undefined;
|
|
304
370
|
}
|
|
305
371
|
|
|
306
|
-
//
|
|
307
|
-
|
|
372
|
+
// Guard: don't exceed 1-deep pipeline. Without a proposed checkpoint, we can only build
|
|
373
|
+
// confirmed + 1. With a proposed checkpoint, we can build confirmed + 2.
|
|
374
|
+
const confirmedCkpt = syncedTo.checkpointedCheckpointNumber;
|
|
375
|
+
if (checkpointNumber > confirmedCkpt + 2) {
|
|
376
|
+
this.log.verbose(
|
|
377
|
+
`Skipping slot ${targetSlot}: checkpoint ${checkpointNumber} exceeds max pipeline depth (confirmed=${confirmedCkpt})`,
|
|
378
|
+
);
|
|
379
|
+
return undefined;
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
// Check that the target slot is not taken by a block already (should never happen, since only us can propose for this slot)
|
|
383
|
+
if (syncedTo.blockData && syncedTo.blockData.header.getSlot() >= targetSlot) {
|
|
308
384
|
this.log.warn(
|
|
309
|
-
`Cannot propose block at
|
|
385
|
+
`Cannot propose block at target slot ${targetSlot} since that slot was taken by block ${syncedTo.blockNumber}`,
|
|
310
386
|
{ ...logCtx, block: syncedTo.blockData.header.toInspect() },
|
|
311
387
|
);
|
|
312
388
|
this.metrics.recordCheckpointPrecheckFailed('slot_already_taken');
|
|
@@ -321,37 +397,106 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
321
397
|
const { attestorAddress, publisher } = await this.publisherFactory.create(proposerForPublisher);
|
|
322
398
|
this.log.verbose(`Created publisher at address ${publisher.getSenderAddress()} for attestor ${attestorAddress}`);
|
|
323
399
|
|
|
324
|
-
//
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
400
|
+
// Prepare invalidation request if the pending chain is invalid (returns undefined if no need).
|
|
401
|
+
// Only simulate invalidation when there's no proposed parent, since we assume the proposed parent
|
|
402
|
+
// will invalidate the currently invalid checkpoint on L1.
|
|
403
|
+
const invalidateCheckpoint =
|
|
404
|
+
syncedTo.hasProposedCheckpoint || syncedTo.pendingChainValidationStatus.valid
|
|
405
|
+
? undefined
|
|
406
|
+
: await publisher.simulateInvalidateCheckpoint(syncedTo.pendingChainValidationStatus);
|
|
407
|
+
|
|
408
|
+
// Determine the correct archive and L1 state overrides for the canProposeAt check.
|
|
409
|
+
// The L1 contract reads archives[proposedCheckpointNumber] and compares it with the provided archive.
|
|
410
|
+
// When invalidating or pipelining, the local archive may differ from L1's, so we adjust accordingly.
|
|
411
|
+
let archiveForCheck = syncedTo.archive;
|
|
412
|
+
|
|
413
|
+
if (syncedTo.hasProposedCheckpoint) {
|
|
414
|
+
this.metrics.recordPipelineDepth(syncedTo.checkpointNumber - syncedTo.checkpointedCheckpointNumber);
|
|
415
|
+
this.log.verbose(
|
|
416
|
+
`Building on top of proposed checkpoint (pending=${syncedTo.proposedCheckpointData?.checkpointNumber}) for target slot ${targetSlot}`,
|
|
417
|
+
{ targetSlot, parentCheckpointNumber: CheckpointNumber(checkpointNumber - 1) },
|
|
418
|
+
);
|
|
419
|
+
// Match what L1 will see at archives[pending] once the proposed parent lands: the parent's
|
|
420
|
+
// own archive root from the gossiped proposal. `syncedTo.archive` is the world-state-local
|
|
421
|
+
// view and can transiently diverge from the proposed parent (e.g. before the proposed
|
|
422
|
+
// parent's blocks have been applied locally); diverging here would cause the canProposeAt
|
|
423
|
+
// override to set archives[pending] to one value while we present another for comparison.
|
|
424
|
+
archiveForCheck = syncedTo.proposedCheckpointData!.archive.root;
|
|
425
|
+
} else if (invalidateCheckpoint) {
|
|
426
|
+
// After invalidation, L1 will roll back to checkpoint N-1. The archive at N-1 already
|
|
427
|
+
// exists on L1, so we just pass the matching archive (the lastArchive of the invalid checkpoint).
|
|
428
|
+
archiveForCheck = invalidateCheckpoint.lastArchive;
|
|
429
|
+
this.metrics.recordPipelineDepth(0);
|
|
430
|
+
} else {
|
|
431
|
+
this.metrics.recordPipelineDepth(0);
|
|
432
|
+
}
|
|
433
|
+
|
|
434
|
+
// Build the simulation plan: pending/proven override from pipelining or invalidation (or the
|
|
435
|
+
// current snapshot when neither applies, to short-circuit any pending prune in simulation),
|
|
436
|
+
// plus the parent checkpoint cell and fee header when pipelining.
|
|
437
|
+
const simulationOverridesPlan = await buildCheckpointSimulationOverridesPlan({
|
|
438
|
+
checkpointNumber,
|
|
439
|
+
proposedCheckpointData: syncedTo.hasProposedCheckpoint ? syncedTo.proposedCheckpointData : undefined,
|
|
440
|
+
invalidateToPendingCheckpointNumber: invalidateCheckpoint?.forcePendingCheckpointNumber,
|
|
441
|
+
checkpointedCheckpointNumber: syncedTo.checkpointedCheckpointNumber,
|
|
442
|
+
rollup: this.rollupContract,
|
|
443
|
+
signatureContext: this.signatureContext,
|
|
444
|
+
log: this.log,
|
|
445
|
+
});
|
|
446
|
+
|
|
447
|
+
// The plan always pins both pending/proven (to short-circuit `canPruneAtTime` in simulation),
|
|
448
|
+
// so `provenOverride` always reflects the assumed proven checkpoint we are pinning the
|
|
449
|
+
// simulation to. We additionally warn when the pin is load-bearing — i.e. when a prune would
|
|
450
|
+
// actually fire at the target slot without it — so observers can spot "we are building
|
|
451
|
+
// optimistically across a pruning boundary" in the logs.
|
|
452
|
+
const provenOverride = simulationOverridesPlan?.chainTipsOverride?.proven;
|
|
453
|
+
if (provenOverride !== undefined && (await this.l2BlockSource.isPruneDueAtSlot(targetSlot))) {
|
|
454
|
+
this.log.warn(
|
|
455
|
+
`Assuming proof for epoch ending at checkpoint ${provenOverride} lands by target slot ${targetSlot}`,
|
|
456
|
+
{ checkpointNumber, slot, targetSlot, provenOverride },
|
|
457
|
+
);
|
|
328
458
|
}
|
|
329
459
|
|
|
330
|
-
|
|
331
|
-
|
|
460
|
+
this.emit('preparing-checkpoint', {
|
|
461
|
+
targetSlot,
|
|
462
|
+
checkpointNumber,
|
|
463
|
+
hadProposedParent: syncedTo.hasProposedCheckpoint,
|
|
464
|
+
provenOverride,
|
|
465
|
+
simulatedPending: simulationOverridesPlan?.chainTipsOverride?.pending,
|
|
466
|
+
});
|
|
332
467
|
|
|
333
|
-
// Check with the rollup contract if we can indeed propose at the next L2 slot. This check should not fail
|
|
334
|
-
// if all the previous checks are good, but we do it just in case.
|
|
335
468
|
const canProposeCheck = await publisher.canProposeAt(
|
|
336
|
-
|
|
469
|
+
archiveForCheck,
|
|
337
470
|
proposer ?? EthAddress.ZERO,
|
|
338
|
-
|
|
471
|
+
simulationOverridesPlan,
|
|
339
472
|
);
|
|
340
473
|
|
|
474
|
+
const proposeContext = {
|
|
475
|
+
hasProposedCheckpoint: syncedTo.hasProposedCheckpoint,
|
|
476
|
+
proposedCheckpointNumber: syncedTo.proposedCheckpointData?.checkpointNumber,
|
|
477
|
+
checkpointedCheckpointNumber: syncedTo.checkpointedCheckpointNumber,
|
|
478
|
+
isInvalidating: !!invalidateCheckpoint,
|
|
479
|
+
invalidatingCheckpointNumber: invalidateCheckpoint?.checkpointNumber,
|
|
480
|
+
archiveForCheck: archiveForCheck.toString(),
|
|
481
|
+
overridePendingCheckpointNumber: simulationOverridesPlan?.chainTipsOverride?.pending,
|
|
482
|
+
overrideArchive: simulationOverridesPlan?.pendingCheckpointState?.archive,
|
|
483
|
+
overrideFeeHeader: simulationOverridesPlan?.pendingCheckpointState?.feeHeader,
|
|
484
|
+
};
|
|
485
|
+
|
|
341
486
|
if (canProposeCheck === undefined) {
|
|
342
487
|
this.log.warn(
|
|
343
488
|
`Cannot propose checkpoint ${checkpointNumber} at slot ${slot} due to failed rollup contract check`,
|
|
344
|
-
logCtx,
|
|
489
|
+
{ ...logCtx, ...proposeContext },
|
|
345
490
|
);
|
|
346
491
|
this.emit('proposer-rollup-check-failed', { reason: 'Rollup contract check failed', slot });
|
|
347
492
|
this.metrics.recordCheckpointPrecheckFailed('rollup_contract_check_failed');
|
|
348
493
|
return undefined;
|
|
349
494
|
}
|
|
350
495
|
|
|
351
|
-
if (canProposeCheck.slot !==
|
|
496
|
+
if (canProposeCheck.slot !== targetSlot) {
|
|
352
497
|
this.log.warn(
|
|
353
|
-
`Cannot propose block due to slot mismatch with rollup contract (this can be caused by a clock out of sync). Expected slot ${
|
|
354
|
-
{ ...logCtx, rollup: canProposeCheck, expectedSlot:
|
|
498
|
+
`Cannot propose block due to slot mismatch with rollup contract (this can be caused by a clock out of sync). Expected slot ${targetSlot} but got ${canProposeCheck.slot}.`,
|
|
499
|
+
{ ...logCtx, ...proposeContext, rollup: canProposeCheck, expectedSlot: targetSlot },
|
|
355
500
|
);
|
|
356
501
|
this.emit('proposer-rollup-check-failed', { reason: 'Slot mismatch', slot });
|
|
357
502
|
this.metrics.recordCheckpointPrecheckFailed('slot_mismatch');
|
|
@@ -361,45 +506,61 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
361
506
|
if (canProposeCheck.checkpointNumber !== checkpointNumber) {
|
|
362
507
|
this.log.warn(
|
|
363
508
|
`Cannot propose due to block mismatch with rollup contract (this can be caused by a pending archiver sync). Expected checkpoint ${checkpointNumber} but got ${canProposeCheck.checkpointNumber}.`,
|
|
364
|
-
{ ...logCtx, rollup: canProposeCheck, expectedSlot: slot },
|
|
509
|
+
{ ...logCtx, ...proposeContext, rollup: canProposeCheck, expectedSlot: slot },
|
|
365
510
|
);
|
|
366
511
|
this.emit('proposer-rollup-check-failed', { reason: 'Block mismatch', slot });
|
|
367
512
|
this.metrics.recordCheckpointPrecheckFailed('block_number_mismatch');
|
|
368
513
|
return undefined;
|
|
369
514
|
}
|
|
370
515
|
|
|
371
|
-
this.lastSlotForCheckpointProposalJob =
|
|
372
|
-
|
|
373
|
-
this.
|
|
516
|
+
this.lastSlotForCheckpointProposalJob = targetSlot;
|
|
517
|
+
|
|
518
|
+
await this.p2pClient.prepareForSlot(targetSlot);
|
|
519
|
+
this.log.info(
|
|
520
|
+
`Preparing checkpoint proposal ${checkpointNumber} for target slot ${targetSlot} during wall-clock slot ${slot}`,
|
|
521
|
+
{
|
|
522
|
+
...logCtx,
|
|
523
|
+
...proposeContext,
|
|
524
|
+
proposer,
|
|
525
|
+
},
|
|
526
|
+
);
|
|
374
527
|
|
|
375
528
|
// Create and return the checkpoint proposal job
|
|
376
529
|
return this.createCheckpointProposalJob(
|
|
377
|
-
epoch,
|
|
378
530
|
slot,
|
|
531
|
+
targetSlot,
|
|
532
|
+
targetEpoch,
|
|
379
533
|
checkpointNumber,
|
|
380
534
|
syncedTo.blockNumber,
|
|
535
|
+
syncedTo.checkpointedCheckpointNumber,
|
|
381
536
|
proposer,
|
|
382
537
|
publisher,
|
|
383
538
|
attestorAddress,
|
|
384
539
|
invalidateCheckpoint,
|
|
540
|
+
syncedTo.proposedCheckpointData,
|
|
385
541
|
);
|
|
386
542
|
}
|
|
387
543
|
|
|
388
544
|
protected createCheckpointProposalJob(
|
|
389
|
-
epoch: EpochNumber,
|
|
390
545
|
slot: SlotNumber,
|
|
546
|
+
targetSlot: SlotNumber,
|
|
547
|
+
targetEpoch: EpochNumber,
|
|
391
548
|
checkpointNumber: CheckpointNumber,
|
|
392
549
|
syncedToBlockNumber: BlockNumber,
|
|
550
|
+
checkpointedCheckpointNumber: CheckpointNumber,
|
|
393
551
|
proposer: EthAddress | undefined,
|
|
394
552
|
publisher: SequencerPublisher,
|
|
395
553
|
attestorAddress: EthAddress,
|
|
396
554
|
invalidateCheckpoint: InvalidateCheckpointRequest | undefined,
|
|
555
|
+
proposedCheckpointData?: ProposedCheckpointData,
|
|
397
556
|
): CheckpointProposalJob {
|
|
398
557
|
return new CheckpointProposalJob(
|
|
399
|
-
epoch,
|
|
400
558
|
slot,
|
|
559
|
+
targetSlot,
|
|
560
|
+
targetEpoch,
|
|
401
561
|
checkpointNumber,
|
|
402
562
|
syncedToBlockNumber,
|
|
563
|
+
checkpointedCheckpointNumber,
|
|
403
564
|
proposer,
|
|
404
565
|
publisher,
|
|
405
566
|
attestorAddress,
|
|
@@ -413,16 +574,19 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
413
574
|
this.checkpointsBuilder,
|
|
414
575
|
this.l2BlockSource,
|
|
415
576
|
this.l1Constants,
|
|
577
|
+
this.signatureContext,
|
|
416
578
|
this.config,
|
|
417
579
|
this.timetable,
|
|
418
580
|
this.slasherClient,
|
|
419
581
|
this.epochCache,
|
|
420
582
|
this.dateProvider,
|
|
421
583
|
this.metrics,
|
|
584
|
+
this.checkpointProposalJobMetrics.createRecorder(),
|
|
422
585
|
this,
|
|
423
|
-
this.setState
|
|
586
|
+
(state, slot, timeReferenceSlot) => this.setState(state, slot, { timeReferenceSlot }),
|
|
424
587
|
this.tracer,
|
|
425
588
|
this.log.getBindings(),
|
|
589
|
+
proposedCheckpointData,
|
|
426
590
|
);
|
|
427
591
|
}
|
|
428
592
|
|
|
@@ -436,13 +600,14 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
436
600
|
/**
|
|
437
601
|
* Internal helper for setting the sequencer state and checks if we have enough time left in the slot to transition to the new state.
|
|
438
602
|
* @param proposedState - The new state to transition to.
|
|
439
|
-
* @param slotNumber - The
|
|
603
|
+
* @param slotNumber - The logical slot associated with the new state.
|
|
440
604
|
* @param force - Whether to force the transition even if the sequencer is stopped.
|
|
605
|
+
* @param timeReferenceSlot - The slot to use for deadline checks when it differs from the logical slot.
|
|
441
606
|
*/
|
|
442
607
|
protected setState(
|
|
443
608
|
proposedState: SequencerState,
|
|
444
609
|
slotNumber: SlotNumber | undefined,
|
|
445
|
-
opts: { force?: boolean } = {},
|
|
610
|
+
opts: { force?: boolean; timeReferenceSlot?: SlotNumber } = {},
|
|
446
611
|
): void {
|
|
447
612
|
if (this.state === SequencerState.STOPPING && proposedState !== SequencerState.STOPPED && !opts.force) {
|
|
448
613
|
this.log.warn(`Cannot set sequencer to ${proposedState} as it is stopping.`);
|
|
@@ -453,24 +618,43 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
453
618
|
return;
|
|
454
619
|
}
|
|
455
620
|
let secondsIntoSlot = undefined;
|
|
456
|
-
|
|
457
|
-
|
|
621
|
+
const timeReferenceSlot = opts.timeReferenceSlot ?? slotNumber;
|
|
622
|
+
if (timeReferenceSlot !== undefined) {
|
|
623
|
+
secondsIntoSlot = this.getSecondsIntoSlot(timeReferenceSlot);
|
|
458
624
|
this.timetable.assertTimeLeft(proposedState, secondsIntoSlot);
|
|
459
625
|
}
|
|
460
626
|
|
|
627
|
+
const oldState = this.state;
|
|
628
|
+
const oldStateSlotNumber = this.stateSlotNumber;
|
|
629
|
+
const stateChanged = proposedState !== oldState;
|
|
630
|
+
const transitionAtMs = performance.now();
|
|
631
|
+
const stateDurationMs = transitionAtMs - this.stateEnteredAtMs;
|
|
632
|
+
|
|
461
633
|
const boringStates = [SequencerState.IDLE, SequencerState.SYNCHRONIZING];
|
|
462
634
|
const logLevel =
|
|
463
|
-
boringStates.includes(proposedState) && boringStates.includes(
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
635
|
+
boringStates.includes(proposedState) && boringStates.includes(oldState) ? ('trace' as const) : ('debug' as const);
|
|
636
|
+
this.stateLog[logLevel](`Transitioning from ${oldState} to ${proposedState}`, {
|
|
637
|
+
oldState,
|
|
638
|
+
newState: proposedState,
|
|
639
|
+
slotNumber,
|
|
640
|
+
timeReferenceSlot,
|
|
641
|
+
stateSlotNumber: oldStateSlotNumber,
|
|
642
|
+
secondsIntoSlot,
|
|
643
|
+
...(stateChanged && { stateDurationMs: Math.ceil(stateDurationMs) }),
|
|
644
|
+
});
|
|
467
645
|
|
|
468
646
|
this.emit('state-changed', {
|
|
469
|
-
oldState
|
|
647
|
+
oldState,
|
|
470
648
|
newState: proposedState,
|
|
471
649
|
secondsIntoSlot,
|
|
472
650
|
slot: slotNumber,
|
|
651
|
+
timeReferenceSlot,
|
|
473
652
|
});
|
|
653
|
+
if (stateChanged) {
|
|
654
|
+
this.metrics.recordStateDuration(stateDurationMs, oldState);
|
|
655
|
+
this.stateEnteredAtMs = transitionAtMs;
|
|
656
|
+
this.stateSlotNumber = slotNumber;
|
|
657
|
+
}
|
|
474
658
|
this.state = proposedState;
|
|
475
659
|
}
|
|
476
660
|
|
|
@@ -497,45 +681,113 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
497
681
|
number: syncSummary.latestBlockNumber,
|
|
498
682
|
hash: syncSummary.latestBlockHash,
|
|
499
683
|
})),
|
|
500
|
-
this.l2BlockSource
|
|
684
|
+
this.l2BlockSource
|
|
685
|
+
.getL2Tips()
|
|
686
|
+
.then(t => ({ proposed: t.proposed, checkpointed: t.checkpointed, proposedCheckpoint: t.proposedCheckpoint })),
|
|
501
687
|
this.p2pClient.getStatus().then(p2p => p2p.syncedToL2Block),
|
|
502
|
-
this.l1ToL2MessageSource.getL2Tips().then(t => t.proposed),
|
|
688
|
+
this.l1ToL2MessageSource.getL2Tips().then(t => ({ proposed: t.proposed, checkpointed: t.checkpointed })),
|
|
503
689
|
this.l2BlockSource.getPendingChainValidationStatus(),
|
|
690
|
+
this.l2BlockSource.getProposedCheckpointData(),
|
|
504
691
|
] as const);
|
|
505
692
|
|
|
506
|
-
const [worldState,
|
|
693
|
+
const [worldState, l2Tips, p2p, l1ToL2MessageSourceTips, pendingChainValidationStatus, proposedCheckpointData] =
|
|
694
|
+
syncedBlocks;
|
|
507
695
|
|
|
508
|
-
// Handle zero as a special case, since the block hash won't match across services if we're changing the prefilled data for the genesis block,
|
|
509
|
-
// as the world state can compute the new genesis block hash, but other components use the hardcoded constant.
|
|
510
|
-
// TODO(palla/mbps): Fix the above. All components should be able to handle dynamic genesis block hashes.
|
|
511
696
|
const result =
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
697
|
+
worldState.hash === l2Tips.proposed.hash &&
|
|
698
|
+
p2p.hash === l2Tips.proposed.hash &&
|
|
699
|
+
l1ToL2MessageSourceTips.proposed.hash === l2Tips.proposed.hash &&
|
|
700
|
+
l1ToL2MessageSourceTips.checkpointed.block.hash === l2Tips.checkpointed.block.hash &&
|
|
701
|
+
l1ToL2MessageSourceTips.checkpointed.checkpoint.hash === l2Tips.checkpointed.checkpoint.hash;
|
|
516
702
|
|
|
517
703
|
if (!result) {
|
|
518
|
-
this.log.debug(`Sequencer sync check failed`, {
|
|
704
|
+
this.log.debug(`Sequencer sync check failed`, {
|
|
705
|
+
worldState,
|
|
706
|
+
l2BlockSource: l2Tips.proposed,
|
|
707
|
+
p2p,
|
|
708
|
+
l1ToL2MessageSourceTips,
|
|
709
|
+
});
|
|
519
710
|
return undefined;
|
|
520
711
|
}
|
|
521
712
|
|
|
522
|
-
// Special case for genesis state
|
|
523
713
|
const blockNumber = worldState.number;
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
714
|
+
const blockData = await this.l2BlockSource.getBlockData({ number: blockNumber });
|
|
715
|
+
if (!blockData) {
|
|
716
|
+
this.log.warn(`Sequencer sync check failed: failed to get L2 block data ${blockNumber} from the archiver`, {
|
|
717
|
+
blockNumber,
|
|
718
|
+
l2Tips,
|
|
719
|
+
syncedL2Slot,
|
|
720
|
+
...args,
|
|
721
|
+
});
|
|
722
|
+
return undefined;
|
|
723
|
+
}
|
|
724
|
+
|
|
725
|
+
// Refuse to build a checkpoint on top of a proposed block whose enclosing checkpoint was never
|
|
726
|
+
// proposed. Under pipelining we may have received and reexecuted such a block locally — advancing
|
|
727
|
+
// our world-state tip past the checkpointed tip — while the proposing node never published the
|
|
728
|
+
// matching proposed checkpoint (e.g. it crashed before assembling it). Building on this orphan block
|
|
729
|
+
// would fork the chain off a tip no other node can follow. The archiver prunes these orphan blocks
|
|
730
|
+
// once their build slot ends; this guard is the correctness barrier during the grace window before.
|
|
731
|
+
if (
|
|
732
|
+
blockData.checkpointNumber > l2Tips.checkpointed.checkpoint.number &&
|
|
733
|
+
(l2Tips.proposedCheckpoint.checkpoint.number !== blockData.checkpointNumber ||
|
|
734
|
+
proposedCheckpointData?.checkpointNumber !== blockData.checkpointNumber)
|
|
735
|
+
) {
|
|
736
|
+
const logCtx = {
|
|
737
|
+
blockCheckpointNumber: blockData.checkpointNumber,
|
|
738
|
+
checkpointedCheckpointNumber: l2Tips.checkpointed.checkpoint.number,
|
|
739
|
+
proposedCheckpointTipNumber: l2Tips.proposedCheckpoint.checkpoint.number,
|
|
740
|
+
proposedCheckpointDataNumber: proposedCheckpointData?.checkpointNumber,
|
|
741
|
+
blockNumber: blockData.header.getBlockNumber(),
|
|
742
|
+
blockSlot: blockData.header.getSlot(),
|
|
530
743
|
syncedL2Slot,
|
|
531
|
-
|
|
744
|
+
...args,
|
|
532
745
|
};
|
|
746
|
+
|
|
747
|
+
// Under pipelining the block proposal for a checkpoint leads its checkpoint proposal by up to one
|
|
748
|
+
// slot, so a world-state tip sitting in an as-yet-unproposed checkpoint is the expected steady state
|
|
749
|
+
// until that checkpoint is due. Only treat it as abnormal — and warn — once the checkpoint is overdue
|
|
750
|
+
// by the same deadline the archiver uses to prune the orphan block (see pruneOrphanProposedBlocks).
|
|
751
|
+
// Before then this is normal pipelining and we wait it out quietly.
|
|
752
|
+
if (this.isProposedCheckpointOverdue(blockData.header.getSlot())) {
|
|
753
|
+
this.log.warn(`Sequencer sync check failed: proposed block has no matching proposed checkpoint`, logCtx);
|
|
754
|
+
} else {
|
|
755
|
+
this.log.debug(`Waiting for proposed checkpoint to catch up with reexecuted block`, logCtx);
|
|
756
|
+
}
|
|
757
|
+
return undefined;
|
|
533
758
|
}
|
|
534
759
|
|
|
535
|
-
const
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
760
|
+
const hasProposedCheckpoint = l2Tips.proposedCheckpoint.checkpoint.number > l2Tips.checkpointed.checkpoint.number;
|
|
761
|
+
|
|
762
|
+
// The l2Tips and proposedCheckpointData reads above come from independent archiver snapshots
|
|
763
|
+
// (a JS-side tips cache vs. a direct store read on `#proposedCheckpoints`). A concurrent archiver
|
|
764
|
+
// write that mutates both can be observed split, leaving us with `hasProposedCheckpoint=true` but
|
|
765
|
+
// no proposedCheckpointData (or one whose number doesn't match the tip). Refuse to proceed in that
|
|
766
|
+
// window — the next checkSync tick will see a coherent snapshot.
|
|
767
|
+
if (
|
|
768
|
+
hasProposedCheckpoint &&
|
|
769
|
+
(!proposedCheckpointData ||
|
|
770
|
+
proposedCheckpointData.checkpointNumber !== l2Tips.proposedCheckpoint.checkpoint.number)
|
|
771
|
+
) {
|
|
772
|
+
this.log.warn(`Sequencer sync check failed: inconsistent proposed-checkpoint state`, {
|
|
773
|
+
proposedCheckpointTipNumber: l2Tips.proposedCheckpoint.checkpoint.number,
|
|
774
|
+
checkpointedTipNumber: l2Tips.checkpointed.checkpoint.number,
|
|
775
|
+
proposedCheckpointDataNumber: proposedCheckpointData?.checkpointNumber,
|
|
776
|
+
syncedL2Slot,
|
|
777
|
+
...args,
|
|
778
|
+
});
|
|
779
|
+
return undefined;
|
|
780
|
+
}
|
|
781
|
+
|
|
782
|
+
// Check that the proposed checkpoint is indeed the parent of the checkpoint we'll be building
|
|
783
|
+
// The checkpoint number to build is derived as blockData.checkpointNumber + 1
|
|
784
|
+
if (proposedCheckpointData && proposedCheckpointData.checkpointNumber !== blockData.checkpointNumber) {
|
|
785
|
+
this.log.warn(`Sequencer sync check failed: proposed checkpoint number mismatch`, {
|
|
786
|
+
proposedCheckpointNumber: proposedCheckpointData.checkpointNumber,
|
|
787
|
+
blockCheckpointNumber: blockData.checkpointNumber,
|
|
788
|
+
syncedL2Slot,
|
|
789
|
+
...args,
|
|
790
|
+
});
|
|
539
791
|
return undefined;
|
|
540
792
|
}
|
|
541
793
|
|
|
@@ -543,30 +795,48 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
543
795
|
blockData,
|
|
544
796
|
blockNumber: blockData.header.getBlockNumber(),
|
|
545
797
|
checkpointNumber: blockData.checkpointNumber,
|
|
798
|
+
checkpointedCheckpointNumber: l2Tips.checkpointed.checkpoint.number,
|
|
546
799
|
archive: blockData.archive.root,
|
|
800
|
+
hasProposedCheckpoint,
|
|
801
|
+
proposedCheckpointData,
|
|
547
802
|
syncedL2Slot,
|
|
548
803
|
pendingChainValidationStatus,
|
|
549
804
|
};
|
|
550
805
|
}
|
|
551
806
|
|
|
807
|
+
/**
|
|
808
|
+
* Whether the enclosing checkpoint of a reexecuted block is overdue: past the deadline by which a
|
|
809
|
+
* well-behaved proposer should have published it. Mirrors the archiver's orphan-prune deadline (the
|
|
810
|
+
* start of the slot after the block's build slot, plus a grace period) so the sequencer only warns
|
|
811
|
+
* about a missing proposed checkpoint once the archiver itself would prune the orphan block. The grace
|
|
812
|
+
* is derived from the block build duration the same way the archiver defaults it at node wiring.
|
|
813
|
+
*/
|
|
814
|
+
private isProposedCheckpointOverdue(blockSlot: SlotNumber): boolean {
|
|
815
|
+
const expectedBySlot = SlotNumber(Number(blockSlot) - PROPOSER_PIPELINING_SLOT_OFFSET + 1);
|
|
816
|
+
const graceSeconds =
|
|
817
|
+
this.config.blockDurationMs !== undefined ? Math.ceil(this.config.blockDurationMs / 1000) : MIN_EXECUTION_TIME;
|
|
818
|
+
const expectedByTime = getTimestampForSlot(expectedBySlot, this.l1Constants) + BigInt(graceSeconds);
|
|
819
|
+
return BigInt(this.dateProvider.nowInSeconds()) >= expectedByTime;
|
|
820
|
+
}
|
|
821
|
+
|
|
552
822
|
/**
|
|
553
823
|
* Checks if we are the proposer for the next slot.
|
|
554
824
|
* @returns True if we can propose, and the proposer address (undefined if anyone can propose)
|
|
555
825
|
*/
|
|
556
|
-
protected async checkCanPropose(
|
|
826
|
+
protected async checkCanPropose(targetSlot: SlotNumber): Promise<[boolean, EthAddress | undefined]> {
|
|
557
827
|
let proposer: EthAddress | undefined;
|
|
558
828
|
|
|
559
829
|
try {
|
|
560
|
-
proposer = await this.epochCache.getProposerAttesterAddressInSlot(
|
|
830
|
+
proposer = await this.epochCache.getProposerAttesterAddressInSlot(targetSlot);
|
|
561
831
|
} catch (e) {
|
|
562
832
|
if (e instanceof NoCommitteeError) {
|
|
563
|
-
if (this.lastSlotForNoCommitteeWarning !==
|
|
564
|
-
this.lastSlotForNoCommitteeWarning =
|
|
565
|
-
this.log.warn(`Cannot propose at
|
|
833
|
+
if (this.lastSlotForNoCommitteeWarning !== targetSlot) {
|
|
834
|
+
this.lastSlotForNoCommitteeWarning = targetSlot;
|
|
835
|
+
this.log.warn(`Cannot propose at target slot ${targetSlot} since the committee does not exist on L1`);
|
|
566
836
|
}
|
|
567
837
|
return [false, undefined];
|
|
568
838
|
}
|
|
569
|
-
this.log.error(`Error getting proposer for slot ${
|
|
839
|
+
this.log.error(`Error getting proposer for target slot ${targetSlot}`, e);
|
|
570
840
|
return [false, undefined];
|
|
571
841
|
}
|
|
572
842
|
|
|
@@ -583,10 +853,18 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
583
853
|
const weAreProposer = validatorAddresses.some(addr => addr.equals(proposer));
|
|
584
854
|
|
|
585
855
|
if (!weAreProposer) {
|
|
586
|
-
this.log.debug(`Cannot propose at slot ${
|
|
856
|
+
this.log.debug(`Cannot propose at target slot ${targetSlot} since we are not a proposer`, {
|
|
857
|
+
targetSlot,
|
|
858
|
+
validatorAddresses,
|
|
859
|
+
proposer,
|
|
860
|
+
});
|
|
587
861
|
return [false, proposer];
|
|
588
862
|
}
|
|
589
863
|
|
|
864
|
+
this.log.info(`We are the proposer for pipeline slot ${targetSlot}`, {
|
|
865
|
+
targetSlot,
|
|
866
|
+
proposer,
|
|
867
|
+
});
|
|
590
868
|
return [true, proposer];
|
|
591
869
|
}
|
|
592
870
|
|
|
@@ -595,8 +873,8 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
595
873
|
* This allows the sequencer to participate in governance/slashing votes even when it cannot build blocks.
|
|
596
874
|
*/
|
|
597
875
|
@trackSpan('Seqeuencer.tryVoteWhenSyncFails', ({ slot }) => ({ [Attributes.SLOT_NUMBER]: slot }))
|
|
598
|
-
protected async tryVoteWhenSyncFails(args: { slot: SlotNumber; ts: bigint }): Promise<void> {
|
|
599
|
-
const { slot } = args;
|
|
876
|
+
protected async tryVoteWhenSyncFails(args: { slot: SlotNumber; targetSlot: SlotNumber; ts: bigint }): Promise<void> {
|
|
877
|
+
const { slot, targetSlot } = args;
|
|
600
878
|
|
|
601
879
|
// Prevent duplicate attempts in the same slot
|
|
602
880
|
if (this.lastSlotForFallbackVote === slot) {
|
|
@@ -624,7 +902,7 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
624
902
|
});
|
|
625
903
|
|
|
626
904
|
// Check if we're a proposer or proposal is open
|
|
627
|
-
const [canPropose, proposer] = await this.checkCanPropose(
|
|
905
|
+
const [canPropose, proposer] = await this.checkCanPropose(targetSlot);
|
|
628
906
|
if (!canPropose) {
|
|
629
907
|
this.log.trace(`Cannot vote in slot ${slot} since we are not a proposer`, { slot, proposer });
|
|
630
908
|
return;
|
|
@@ -641,9 +919,9 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
641
919
|
slot,
|
|
642
920
|
});
|
|
643
921
|
|
|
644
|
-
// Enqueue governance and slashing votes
|
|
922
|
+
// Enqueue governance and slashing votes (voter uses the target slot for L1 submission)
|
|
645
923
|
const voter = new CheckpointVoter(
|
|
646
|
-
|
|
924
|
+
targetSlot,
|
|
647
925
|
publisher,
|
|
648
926
|
attestorAddress,
|
|
649
927
|
this.validatorClient,
|
|
@@ -662,7 +940,13 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
662
940
|
}
|
|
663
941
|
|
|
664
942
|
this.log.info(`Voting in slot ${slot} despite sync failure`, { slot });
|
|
665
|
-
|
|
943
|
+
// Votes are EIP-712-signed for `targetSlot` (the pipelined slot in which the multicall is
|
|
944
|
+
// expected to mine). Delay submission to the start of `targetSlot` so the tx mines in the
|
|
945
|
+
// slot the votes were signed for. We fire-and-forget so we don't block the sequencer's
|
|
946
|
+
// work loop while waiting for the target slot to start.
|
|
947
|
+
void publisher.sendRequestsAt(targetSlot).catch(err => {
|
|
948
|
+
this.log.error(`Failed to publish votes despite sync failure for slot ${slot}`, err, { slot });
|
|
949
|
+
});
|
|
666
950
|
}
|
|
667
951
|
|
|
668
952
|
/**
|
|
@@ -672,9 +956,10 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
672
956
|
@trackSpan('Sequencer.tryVoteWhenEscapeHatchOpen', ({ slot }) => ({ [Attributes.SLOT_NUMBER]: slot }))
|
|
673
957
|
protected async tryVoteWhenEscapeHatchOpen(args: {
|
|
674
958
|
slot: SlotNumber;
|
|
959
|
+
targetSlot: SlotNumber;
|
|
675
960
|
proposer: EthAddress | undefined;
|
|
676
961
|
}): Promise<void> {
|
|
677
|
-
const { slot, proposer } = args;
|
|
962
|
+
const { slot, targetSlot, proposer } = args;
|
|
678
963
|
|
|
679
964
|
// Prevent duplicate attempts in the same slot
|
|
680
965
|
if (this.lastSlotForFallbackVote === slot) {
|
|
@@ -687,10 +972,18 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
687
972
|
|
|
688
973
|
const { attestorAddress, publisher } = await this.publisherFactory.create(proposer);
|
|
689
974
|
|
|
690
|
-
this.log.debug(`Escape hatch open for slot ${slot}, attempting vote-only actions`, {
|
|
975
|
+
this.log.debug(`Escape hatch open for slot ${slot}, attempting vote-only actions`, {
|
|
976
|
+
slot,
|
|
977
|
+
targetSlot,
|
|
978
|
+
attestorAddress,
|
|
979
|
+
});
|
|
691
980
|
|
|
981
|
+
// Under proposer pipelining, the multicall is expected to mine in `targetSlot` (slot + 1).
|
|
982
|
+
// Governance and slashing votes are EIP-712-signed against the slot they will mine in, and the
|
|
983
|
+
// L1 contract checks `msg.sender == getCurrentProposer()` using the mining slot. So we must
|
|
984
|
+
// sign for `targetSlot` and delay submission to the start of `targetSlot`.
|
|
692
985
|
const voter = new CheckpointVoter(
|
|
693
|
-
|
|
986
|
+
targetSlot,
|
|
694
987
|
publisher,
|
|
695
988
|
attestorAddress,
|
|
696
989
|
this.validatorClient,
|
|
@@ -709,8 +1002,15 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
709
1002
|
return;
|
|
710
1003
|
}
|
|
711
1004
|
|
|
712
|
-
this.log.info(`Voting in slot ${slot} (escape hatch open)`, { slot });
|
|
713
|
-
|
|
1005
|
+
this.log.info(`Voting in slot ${slot} (escape hatch open)`, { slot, targetSlot });
|
|
1006
|
+
// Votes are EIP-712-signed for `targetSlot`. Delay submission to the start of `targetSlot` so
|
|
1007
|
+
// the multicall mines in the slot the votes were signed for; otherwise the L1 contract reads
|
|
1008
|
+
// `signaler = getCurrentProposer()` against the wrong slot and signature verification fails
|
|
1009
|
+
// silently inside Multicall3. Fire-and-forget so we don't block the sequencer's work loop while
|
|
1010
|
+
// waiting for the target slot to start, mirroring tryVoteWhenSyncFails.
|
|
1011
|
+
void publisher.sendRequestsAt(targetSlot).catch(err => {
|
|
1012
|
+
this.log.error(`Failed to publish escape-hatch votes for slot ${slot}`, err, { slot, targetSlot });
|
|
1013
|
+
});
|
|
714
1014
|
}
|
|
715
1015
|
|
|
716
1016
|
/**
|
|
@@ -883,8 +1183,11 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
883
1183
|
type SequencerSyncCheckResult = {
|
|
884
1184
|
blockData?: BlockData;
|
|
885
1185
|
checkpointNumber: CheckpointNumber;
|
|
1186
|
+
checkpointedCheckpointNumber: CheckpointNumber;
|
|
886
1187
|
blockNumber: BlockNumber;
|
|
887
1188
|
archive: Fr;
|
|
1189
|
+
hasProposedCheckpoint: boolean;
|
|
1190
|
+
proposedCheckpointData?: ProposedCheckpointData;
|
|
888
1191
|
syncedL2Slot: SlotNumber;
|
|
889
1192
|
pendingChainValidationStatus: ValidateCheckpointResult;
|
|
890
1193
|
};
|