@aztec/sequencer-client 0.0.1-commit.e3c1de76 → 0.0.1-commit.e588bc7e5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dest/client/sequencer-client.d.ts +15 -7
- package/dest/client/sequencer-client.d.ts.map +1 -1
- package/dest/client/sequencer-client.js +60 -26
- package/dest/config.d.ts +26 -7
- package/dest/config.d.ts.map +1 -1
- package/dest/config.js +47 -28
- package/dest/global_variable_builder/global_builder.d.ts +15 -11
- package/dest/global_variable_builder/global_builder.d.ts.map +1 -1
- package/dest/global_variable_builder/global_builder.js +29 -25
- package/dest/global_variable_builder/index.d.ts +2 -2
- package/dest/global_variable_builder/index.d.ts.map +1 -1
- package/dest/publisher/config.d.ts +47 -17
- package/dest/publisher/config.d.ts.map +1 -1
- package/dest/publisher/config.js +121 -42
- package/dest/publisher/index.d.ts +2 -1
- package/dest/publisher/index.d.ts.map +1 -1
- package/dest/publisher/l1_tx_failed_store/factory.d.ts +11 -0
- package/dest/publisher/l1_tx_failed_store/factory.d.ts.map +1 -0
- package/dest/publisher/l1_tx_failed_store/factory.js +22 -0
- package/dest/publisher/l1_tx_failed_store/failed_tx_store.d.ts +59 -0
- package/dest/publisher/l1_tx_failed_store/failed_tx_store.d.ts.map +1 -0
- package/dest/publisher/l1_tx_failed_store/failed_tx_store.js +1 -0
- package/dest/publisher/l1_tx_failed_store/file_store_failed_tx_store.d.ts +15 -0
- package/dest/publisher/l1_tx_failed_store/file_store_failed_tx_store.d.ts.map +1 -0
- package/dest/publisher/l1_tx_failed_store/file_store_failed_tx_store.js +34 -0
- package/dest/publisher/l1_tx_failed_store/index.d.ts +4 -0
- package/dest/publisher/l1_tx_failed_store/index.d.ts.map +1 -0
- package/dest/publisher/l1_tx_failed_store/index.js +2 -0
- package/dest/publisher/sequencer-publisher-factory.d.ts +11 -3
- package/dest/publisher/sequencer-publisher-factory.d.ts.map +1 -1
- package/dest/publisher/sequencer-publisher-factory.js +27 -2
- package/dest/publisher/sequencer-publisher.d.ts +76 -30
- package/dest/publisher/sequencer-publisher.d.ts.map +1 -1
- package/dest/publisher/sequencer-publisher.js +396 -71
- package/dest/sequencer/checkpoint_proposal_job.d.ts +39 -8
- package/dest/sequencer/checkpoint_proposal_job.d.ts.map +1 -1
- package/dest/sequencer/checkpoint_proposal_job.js +368 -196
- package/dest/sequencer/checkpoint_voter.d.ts +1 -2
- package/dest/sequencer/checkpoint_voter.d.ts.map +1 -1
- package/dest/sequencer/checkpoint_voter.js +2 -5
- package/dest/sequencer/events.d.ts +2 -1
- package/dest/sequencer/events.d.ts.map +1 -1
- package/dest/sequencer/metrics.d.ts +21 -5
- package/dest/sequencer/metrics.d.ts.map +1 -1
- package/dest/sequencer/metrics.js +97 -15
- package/dest/sequencer/sequencer.d.ts +42 -17
- package/dest/sequencer/sequencer.d.ts.map +1 -1
- package/dest/sequencer/sequencer.js +147 -89
- package/dest/sequencer/timetable.d.ts +4 -6
- package/dest/sequencer/timetable.d.ts.map +1 -1
- package/dest/sequencer/timetable.js +7 -11
- package/dest/sequencer/types.d.ts +2 -2
- package/dest/sequencer/types.d.ts.map +1 -1
- package/dest/test/index.d.ts +3 -5
- package/dest/test/index.d.ts.map +1 -1
- package/dest/test/mock_checkpoint_builder.d.ts +11 -11
- package/dest/test/mock_checkpoint_builder.d.ts.map +1 -1
- package/dest/test/mock_checkpoint_builder.js +45 -34
- package/dest/test/utils.d.ts +3 -3
- package/dest/test/utils.d.ts.map +1 -1
- package/dest/test/utils.js +5 -4
- package/package.json +27 -28
- package/src/client/sequencer-client.ts +76 -23
- package/src/config.ts +65 -38
- package/src/global_variable_builder/global_builder.ts +38 -27
- package/src/global_variable_builder/index.ts +1 -1
- package/src/publisher/config.ts +153 -43
- package/src/publisher/index.ts +3 -0
- package/src/publisher/l1_tx_failed_store/factory.ts +32 -0
- package/src/publisher/l1_tx_failed_store/failed_tx_store.ts +55 -0
- package/src/publisher/l1_tx_failed_store/file_store_failed_tx_store.ts +46 -0
- package/src/publisher/l1_tx_failed_store/index.ts +3 -0
- package/src/publisher/sequencer-publisher-factory.ts +38 -6
- package/src/publisher/sequencer-publisher.ts +442 -95
- package/src/sequencer/checkpoint_proposal_job.ts +481 -202
- package/src/sequencer/checkpoint_voter.ts +1 -12
- package/src/sequencer/events.ts +1 -1
- package/src/sequencer/metrics.ts +106 -18
- package/src/sequencer/sequencer.ts +212 -105
- package/src/sequencer/timetable.ts +13 -12
- package/src/sequencer/types.ts +1 -1
- package/src/test/index.ts +2 -4
- package/src/test/mock_checkpoint_builder.ts +63 -49
- package/src/test/utils.ts +5 -2
|
@@ -12,9 +12,9 @@ import type { DateProvider } from '@aztec/foundation/timer';
|
|
|
12
12
|
import type { TypedEventEmitter } from '@aztec/foundation/types';
|
|
13
13
|
import type { P2P } from '@aztec/p2p';
|
|
14
14
|
import type { SlasherClientInterface } from '@aztec/slasher';
|
|
15
|
-
import type {
|
|
16
|
-
import type { Checkpoint } from '@aztec/stdlib/checkpoint';
|
|
17
|
-
import {
|
|
15
|
+
import type { BlockData, L2BlockSink, L2BlockSource, ValidateCheckpointResult } from '@aztec/stdlib/block';
|
|
16
|
+
import type { Checkpoint, ProposedCheckpointData } from '@aztec/stdlib/checkpoint';
|
|
17
|
+
import { getSlotStartBuildTimestamp } from '@aztec/stdlib/epoch-helpers';
|
|
18
18
|
import {
|
|
19
19
|
type ResolvedSequencerConfig,
|
|
20
20
|
type SequencerConfig,
|
|
@@ -25,7 +25,7 @@ import type { L1ToL2MessageSource } from '@aztec/stdlib/messaging';
|
|
|
25
25
|
import { pickFromSchema } from '@aztec/stdlib/schemas';
|
|
26
26
|
import { MerkleTreeId } from '@aztec/stdlib/trees';
|
|
27
27
|
import { Attributes, type TelemetryClient, type Tracer, getTelemetryClient, trackSpan } from '@aztec/telemetry-client';
|
|
28
|
-
import { FullNodeCheckpointsBuilder, type ValidatorClient } from '@aztec/validator-client';
|
|
28
|
+
import { FullNodeCheckpointsBuilder, NodeKeystoreAdapter, type ValidatorClient } from '@aztec/validator-client';
|
|
29
29
|
|
|
30
30
|
import EventEmitter from 'node:events';
|
|
31
31
|
|
|
@@ -60,6 +60,9 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
60
60
|
/** The last slot for which we attempted to perform our voting duties with degraded block production */
|
|
61
61
|
private lastSlotForFallbackVote: SlotNumber | undefined;
|
|
62
62
|
|
|
63
|
+
/** The last slot for which we logged "no committee" warning, to avoid spam */
|
|
64
|
+
private lastSlotForNoCommitteeWarning: SlotNumber | undefined;
|
|
65
|
+
|
|
63
66
|
/** The last slot for which we triggered a checkpoint proposal job, to prevent duplicate attempts. */
|
|
64
67
|
private lastSlotForCheckpointProposalJob: SlotNumber | undefined;
|
|
65
68
|
|
|
@@ -69,17 +72,12 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
69
72
|
/** The last epoch for which we logged strategy comparison in fisherman mode. */
|
|
70
73
|
private lastEpochForStrategyComparison: EpochNumber | undefined;
|
|
71
74
|
|
|
75
|
+
/** The last checkpoint proposal job, tracked so we can await its pending L1 submission during shutdown. */
|
|
76
|
+
private lastCheckpointProposalJob: CheckpointProposalJob | undefined;
|
|
77
|
+
|
|
72
78
|
/** The maximum number of seconds that the sequencer can be into a slot to transition to a particular state. */
|
|
73
79
|
protected timetable!: SequencerTimetable;
|
|
74
80
|
|
|
75
|
-
// This shouldn't be here as this gets re-created each time we build/propose a block.
|
|
76
|
-
// But we have a number of tests that abuse/rely on this class having a permanent publisher.
|
|
77
|
-
// As long as those tests only configure a single publisher they will continue to work.
|
|
78
|
-
// This will get re-assigned every time the sequencer goes to build a new block to a publisher that is valid
|
|
79
|
-
// for the block proposer.
|
|
80
|
-
// TODO(palla/mbps): Remove this field and fix tests
|
|
81
|
-
protected publisher: SequencerPublisher | undefined;
|
|
82
|
-
|
|
83
81
|
/** Config for the sequencer */
|
|
84
82
|
protected config: ResolvedSequencerConfig = DefaultSequencerConfig;
|
|
85
83
|
|
|
@@ -115,7 +113,7 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
115
113
|
/** Updates sequencer config by the defined values and updates the timetable */
|
|
116
114
|
public updateConfig(config: Partial<SequencerConfig>) {
|
|
117
115
|
const filteredConfig = pickFromSchema(config, SequencerConfigSchema);
|
|
118
|
-
this.log.info(`Updated sequencer config`, omit(filteredConfig, '
|
|
116
|
+
this.log.info(`Updated sequencer config`, omit(filteredConfig, 'txPublicSetupAllowListExtend'));
|
|
119
117
|
this.config = merge(this.config, filteredConfig);
|
|
120
118
|
this.timetable = new SequencerTimetable(
|
|
121
119
|
{
|
|
@@ -131,10 +129,9 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
131
129
|
);
|
|
132
130
|
}
|
|
133
131
|
|
|
134
|
-
/** Initializes the sequencer (precomputes tables
|
|
135
|
-
public
|
|
132
|
+
/** Initializes the sequencer (precomputes tables). Takes about 3s. */
|
|
133
|
+
public init() {
|
|
136
134
|
getKzg();
|
|
137
|
-
this.publisher = (await this.publisherFactory.create(undefined)).publisher;
|
|
138
135
|
}
|
|
139
136
|
|
|
140
137
|
/** Starts the sequencer and moves to IDLE state. */
|
|
@@ -153,8 +150,9 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
153
150
|
public async stop(): Promise<void> {
|
|
154
151
|
this.log.info(`Stopping sequencer`);
|
|
155
152
|
this.setState(SequencerState.STOPPING, undefined, { force: true });
|
|
156
|
-
this.
|
|
153
|
+
await this.publisherFactory.stopAll();
|
|
157
154
|
await this.runningPromise?.stop();
|
|
155
|
+
await this.lastCheckpointProposalJob?.awaitPendingSubmission();
|
|
158
156
|
this.setState(SequencerState.STOPPED, undefined, { force: true });
|
|
159
157
|
this.log.info('Stopped sequencer');
|
|
160
158
|
}
|
|
@@ -166,7 +164,6 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
166
164
|
} catch (err) {
|
|
167
165
|
this.emit('checkpoint-error', { error: err as Error });
|
|
168
166
|
if (err instanceof SequencerTooSlowError) {
|
|
169
|
-
// TODO(palla/mbps): Add missing states
|
|
170
167
|
// Log as warn only if we had to abort halfway through the block proposal
|
|
171
168
|
const logLvl = [SequencerState.INITIALIZING_CHECKPOINT, SequencerState.PROPOSER_CHECK].includes(
|
|
172
169
|
err.proposedState,
|
|
@@ -199,14 +196,25 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
199
196
|
@trackSpan('Sequencer.work')
|
|
200
197
|
protected async work() {
|
|
201
198
|
this.setState(SequencerState.SYNCHRONIZING, undefined);
|
|
202
|
-
const { slot, ts,
|
|
199
|
+
const { slot, ts, nowSeconds, epoch } = this.epochCache.getEpochAndSlotInNextL1Slot();
|
|
200
|
+
const { slot: targetSlot, epoch: targetEpoch } = this.epochCache.getTargetEpochAndSlotInNextL1Slot();
|
|
203
201
|
|
|
204
202
|
// Check if we are synced and it's our slot, grab a publisher, check previous block invalidation, etc
|
|
205
|
-
const checkpointProposalJob = await this.prepareCheckpointProposal(
|
|
203
|
+
const checkpointProposalJob = await this.prepareCheckpointProposal(
|
|
204
|
+
slot,
|
|
205
|
+
targetSlot,
|
|
206
|
+
epoch,
|
|
207
|
+
targetEpoch,
|
|
208
|
+
ts,
|
|
209
|
+
nowSeconds,
|
|
210
|
+
);
|
|
206
211
|
if (!checkpointProposalJob) {
|
|
207
212
|
return;
|
|
208
213
|
}
|
|
209
214
|
|
|
215
|
+
// Track the job so we can await its pending L1 submission during shutdown
|
|
216
|
+
this.lastCheckpointProposalJob = checkpointProposalJob;
|
|
217
|
+
|
|
210
218
|
// Execute the checkpoint proposal job
|
|
211
219
|
const checkpoint = await checkpointProposalJob.execute();
|
|
212
220
|
|
|
@@ -215,13 +223,13 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
215
223
|
this.lastCheckpointProposed = checkpoint;
|
|
216
224
|
}
|
|
217
225
|
|
|
218
|
-
// Log fee strategy comparison if on fisherman
|
|
226
|
+
// Log fee strategy comparison if on fisherman (uses target epoch since we mirror the proposer's perspective)
|
|
219
227
|
if (
|
|
220
228
|
this.config.fishermanMode &&
|
|
221
|
-
(this.lastEpochForStrategyComparison === undefined ||
|
|
229
|
+
(this.lastEpochForStrategyComparison === undefined || targetEpoch > this.lastEpochForStrategyComparison)
|
|
222
230
|
) {
|
|
223
|
-
this.logStrategyComparison(
|
|
224
|
-
this.lastEpochForStrategyComparison =
|
|
231
|
+
this.logStrategyComparison(targetEpoch, checkpointProposalJob.getPublisher());
|
|
232
|
+
this.lastEpochForStrategyComparison = targetEpoch;
|
|
225
233
|
}
|
|
226
234
|
|
|
227
235
|
return checkpoint;
|
|
@@ -233,44 +241,49 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
233
241
|
* @returns CheckpointProposalJob if successful, undefined if we are not yet synced or are not the proposer.
|
|
234
242
|
*/
|
|
235
243
|
@trackSpan('Sequencer.prepareCheckpointProposal')
|
|
236
|
-
|
|
237
|
-
epoch: EpochNumber,
|
|
244
|
+
protected async prepareCheckpointProposal(
|
|
238
245
|
slot: SlotNumber,
|
|
246
|
+
targetSlot: SlotNumber,
|
|
247
|
+
epoch: EpochNumber,
|
|
248
|
+
targetEpoch: EpochNumber,
|
|
239
249
|
ts: bigint,
|
|
240
|
-
|
|
250
|
+
nowSeconds: bigint,
|
|
241
251
|
): Promise<CheckpointProposalJob | undefined> {
|
|
242
|
-
// Check we have not already processed this slot (cheapest check)
|
|
252
|
+
// Check we have not already processed this target slot (cheapest check)
|
|
243
253
|
// We only check this if enforce timetable is set, since we want to keep processing the same slot if we are not
|
|
244
254
|
// running against actual time (eg when we use sandbox-style automining)
|
|
245
255
|
if (
|
|
246
256
|
this.lastSlotForCheckpointProposalJob &&
|
|
247
|
-
this.lastSlotForCheckpointProposalJob >=
|
|
257
|
+
this.lastSlotForCheckpointProposalJob >= targetSlot &&
|
|
248
258
|
this.config.enforceTimeTable
|
|
249
259
|
) {
|
|
250
|
-
this.log.trace(`
|
|
260
|
+
this.log.trace(`Target slot ${targetSlot} has already been processed`);
|
|
251
261
|
return undefined;
|
|
252
262
|
}
|
|
253
263
|
|
|
254
|
-
// But if we have already proposed for this slot,
|
|
255
|
-
if (this.lastCheckpointProposed && this.lastCheckpointProposed.header.slotNumber >=
|
|
256
|
-
this.log.trace(
|
|
264
|
+
// But if we have already proposed for this slot, then we definitely have to skip it, automining or not
|
|
265
|
+
if (this.lastCheckpointProposed && this.lastCheckpointProposed.header.slotNumber >= targetSlot) {
|
|
266
|
+
this.log.trace(
|
|
267
|
+
`Slot ${targetSlot} has already been published as checkpoint ${this.lastCheckpointProposed.number}`,
|
|
268
|
+
);
|
|
257
269
|
return undefined;
|
|
258
270
|
}
|
|
259
271
|
|
|
260
272
|
// Check all components are synced to latest as seen by the archiver (queries all subsystems)
|
|
261
273
|
const syncedTo = await this.checkSync({ ts, slot });
|
|
262
274
|
if (!syncedTo) {
|
|
263
|
-
await this.tryVoteWhenSyncFails({ slot, ts });
|
|
275
|
+
await this.tryVoteWhenSyncFails({ slot, targetSlot, ts });
|
|
264
276
|
return undefined;
|
|
265
277
|
}
|
|
266
278
|
|
|
267
|
-
// If escape hatch is open for
|
|
279
|
+
// If escape hatch is open for the target epoch, do not start checkpoint proposal work and do not attempt invalidations.
|
|
268
280
|
// Still perform governance/slashing voting (as proposer) once per slot.
|
|
269
|
-
|
|
281
|
+
// When pipelining, we check the target epoch (slot+1's epoch) since that's the epoch we're building for.
|
|
282
|
+
const isEscapeHatchOpen = await this.epochCache.isEscapeHatchOpen(targetEpoch);
|
|
270
283
|
|
|
271
284
|
if (isEscapeHatchOpen) {
|
|
272
285
|
this.setState(SequencerState.PROPOSER_CHECK, slot);
|
|
273
|
-
const [canPropose, proposer] = await this.checkCanPropose(
|
|
286
|
+
const [canPropose, proposer] = await this.checkCanPropose(targetSlot);
|
|
274
287
|
if (canPropose) {
|
|
275
288
|
await this.tryVoteWhenEscapeHatchOpen({ slot, proposer });
|
|
276
289
|
} else {
|
|
@@ -286,19 +299,29 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
286
299
|
// Next checkpoint follows from the last synced one
|
|
287
300
|
const checkpointNumber = CheckpointNumber(syncedTo.checkpointNumber + 1);
|
|
288
301
|
|
|
302
|
+
// Guard: don't exceed 1-deep pipeline. Without a proposed checkpoint, we can only build
|
|
303
|
+
// confirmed + 1. With a proposed checkpoint, we can build confirmed + 2.
|
|
304
|
+
const confirmedCkpt = syncedTo.checkpointedCheckpointNumber;
|
|
305
|
+
if (checkpointNumber > confirmedCkpt + 2) {
|
|
306
|
+
this.log.warn(
|
|
307
|
+
`Skipping slot ${targetSlot}: checkpoint ${checkpointNumber} exceeds max pipeline depth (confirmed=${confirmedCkpt})`,
|
|
308
|
+
);
|
|
309
|
+
return undefined;
|
|
310
|
+
}
|
|
311
|
+
|
|
289
312
|
const logCtx = {
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
syncedToL2Slot: getSlotAtTimestamp(syncedTo.l1Timestamp, this.l1Constants),
|
|
313
|
+
nowSeconds,
|
|
314
|
+
syncedToL2Slot: syncedTo.syncedL2Slot,
|
|
293
315
|
slot,
|
|
316
|
+
targetSlot,
|
|
294
317
|
slotTs: ts,
|
|
295
318
|
checkpointNumber,
|
|
296
319
|
isPendingChainValid: pick(syncedTo.pendingChainValidationStatus, 'valid', 'reason', 'invalidIndex'),
|
|
297
320
|
};
|
|
298
321
|
|
|
299
|
-
// Check that we are a proposer for the
|
|
322
|
+
// Check that we are a proposer for the target slot.
|
|
300
323
|
this.setState(SequencerState.PROPOSER_CHECK, slot);
|
|
301
|
-
const [canPropose, proposer] = await this.checkCanPropose(
|
|
324
|
+
const [canPropose, proposer] = await this.checkCanPropose(targetSlot);
|
|
302
325
|
|
|
303
326
|
// If we are not a proposer check if we should invalidate an invalid checkpoint, and bail
|
|
304
327
|
if (!canPropose) {
|
|
@@ -306,13 +329,13 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
306
329
|
return undefined;
|
|
307
330
|
}
|
|
308
331
|
|
|
309
|
-
// Check that the slot is not taken by a block already (should never happen, since only us can propose for this slot)
|
|
310
|
-
if (syncedTo.
|
|
332
|
+
// Check that the target slot is not taken by a block already (should never happen, since only us can propose for this slot)
|
|
333
|
+
if (syncedTo.blockData && syncedTo.blockData.header.getSlot() >= targetSlot) {
|
|
311
334
|
this.log.warn(
|
|
312
|
-
`Cannot propose block at
|
|
313
|
-
{ ...logCtx, block: syncedTo.
|
|
335
|
+
`Cannot propose block at target slot ${targetSlot} since that slot was taken by block ${syncedTo.blockNumber}`,
|
|
336
|
+
{ ...logCtx, block: syncedTo.blockData.header.toInspect() },
|
|
314
337
|
);
|
|
315
|
-
this.metrics.
|
|
338
|
+
this.metrics.recordCheckpointPrecheckFailed('slot_already_taken');
|
|
316
339
|
return undefined;
|
|
317
340
|
}
|
|
318
341
|
|
|
@@ -323,7 +346,6 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
323
346
|
const proposerForPublisher = this.config.fishermanMode ? undefined : proposer;
|
|
324
347
|
const { attestorAddress, publisher } = await this.publisherFactory.create(proposerForPublisher);
|
|
325
348
|
this.log.verbose(`Created publisher at address ${publisher.getSenderAddress()} for attestor ${attestorAddress}`);
|
|
326
|
-
this.publisher = publisher;
|
|
327
349
|
|
|
328
350
|
// In fisherman mode, set the actual proposer's address for simulations
|
|
329
351
|
if (this.config.fishermanMode && proposer) {
|
|
@@ -332,15 +354,41 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
332
354
|
}
|
|
333
355
|
|
|
334
356
|
// Prepare invalidation request if the pending chain is invalid (returns undefined if no need)
|
|
335
|
-
|
|
357
|
+
let invalidateCheckpoint = await publisher.simulateInvalidateCheckpoint(syncedTo.pendingChainValidationStatus);
|
|
358
|
+
|
|
359
|
+
// Determine the correct archive and L1 state overrides for the canProposeAt check.
|
|
360
|
+
// The L1 contract reads archives[proposedCheckpointNumber] and compares it with the provided archive.
|
|
361
|
+
// When invalidating or pipelining, the local archive may differ from L1's, so we adjust accordingly.
|
|
362
|
+
let archiveForCheck = syncedTo.archive;
|
|
363
|
+
const l1Overrides: {
|
|
364
|
+
forcePendingCheckpointNumber?: CheckpointNumber;
|
|
365
|
+
forceArchive?: { checkpointNumber: CheckpointNumber; archive: Fr };
|
|
366
|
+
} = {};
|
|
367
|
+
|
|
368
|
+
if (this.epochCache.isProposerPipeliningEnabled() && syncedTo.hasProposedCheckpoint) {
|
|
369
|
+
// Parent checkpoint hasn't landed on L1 yet. Override both the proposed checkpoint number
|
|
370
|
+
// and the archive at that checkpoint so L1 simulation sees the correct chain tip.
|
|
371
|
+
const parentCheckpointNumber = CheckpointNumber(checkpointNumber - 1);
|
|
372
|
+
l1Overrides.forcePendingCheckpointNumber = parentCheckpointNumber;
|
|
373
|
+
l1Overrides.forceArchive = { checkpointNumber: parentCheckpointNumber, archive: syncedTo.archive };
|
|
374
|
+
this.metrics.recordPipelineDepth(1);
|
|
375
|
+
|
|
376
|
+
this.log.verbose(
|
|
377
|
+
`Building on top of proposed checkpoint (pending=${syncedTo.proposedCheckpointData?.checkpointNumber})`,
|
|
378
|
+
);
|
|
379
|
+
// Clear the invalidation - the proposed checkpoint should handle it.
|
|
380
|
+
invalidateCheckpoint = undefined;
|
|
381
|
+
} else if (invalidateCheckpoint) {
|
|
382
|
+
// After invalidation, L1 will roll back to checkpoint N-1. The archive at N-1 already
|
|
383
|
+
// exists on L1, so we just pass the matching archive (the lastArchive of the invalid checkpoint).
|
|
384
|
+
archiveForCheck = invalidateCheckpoint.lastArchive;
|
|
385
|
+
l1Overrides.forcePendingCheckpointNumber = invalidateCheckpoint.forcePendingCheckpointNumber;
|
|
386
|
+
this.metrics.recordPipelineDepth(0);
|
|
387
|
+
} else {
|
|
388
|
+
this.metrics.recordPipelineDepth(0);
|
|
389
|
+
}
|
|
336
390
|
|
|
337
|
-
|
|
338
|
-
// if all the previous checks are good, but we do it just in case.
|
|
339
|
-
const canProposeCheck = await publisher.canProposeAtNextEthBlock(
|
|
340
|
-
syncedTo.archive,
|
|
341
|
-
proposer ?? EthAddress.ZERO,
|
|
342
|
-
invalidateCheckpoint,
|
|
343
|
-
);
|
|
391
|
+
const canProposeCheck = await publisher.canProposeAt(archiveForCheck, proposer ?? EthAddress.ZERO, l1Overrides);
|
|
344
392
|
|
|
345
393
|
if (canProposeCheck === undefined) {
|
|
346
394
|
this.log.warn(
|
|
@@ -348,17 +396,17 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
348
396
|
logCtx,
|
|
349
397
|
);
|
|
350
398
|
this.emit('proposer-rollup-check-failed', { reason: 'Rollup contract check failed', slot });
|
|
351
|
-
this.metrics.
|
|
399
|
+
this.metrics.recordCheckpointPrecheckFailed('rollup_contract_check_failed');
|
|
352
400
|
return undefined;
|
|
353
401
|
}
|
|
354
402
|
|
|
355
|
-
if (canProposeCheck.slot !==
|
|
403
|
+
if (canProposeCheck.slot !== targetSlot) {
|
|
356
404
|
this.log.warn(
|
|
357
|
-
`Cannot propose block due to slot mismatch with rollup contract (this can be caused by a clock out of sync). Expected slot ${
|
|
358
|
-
{ ...logCtx, rollup: canProposeCheck, expectedSlot:
|
|
405
|
+
`Cannot propose block due to slot mismatch with rollup contract (this can be caused by a clock out of sync). Expected slot ${targetSlot} but got ${canProposeCheck.slot}.`,
|
|
406
|
+
{ ...logCtx, rollup: canProposeCheck, expectedSlot: targetSlot },
|
|
359
407
|
);
|
|
360
408
|
this.emit('proposer-rollup-check-failed', { reason: 'Slot mismatch', slot });
|
|
361
|
-
this.metrics.
|
|
409
|
+
this.metrics.recordCheckpointPrecheckFailed('slot_mismatch');
|
|
362
410
|
return undefined;
|
|
363
411
|
}
|
|
364
412
|
|
|
@@ -368,39 +416,53 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
368
416
|
{ ...logCtx, rollup: canProposeCheck, expectedSlot: slot },
|
|
369
417
|
);
|
|
370
418
|
this.emit('proposer-rollup-check-failed', { reason: 'Block mismatch', slot });
|
|
371
|
-
this.metrics.
|
|
419
|
+
this.metrics.recordCheckpointPrecheckFailed('block_number_mismatch');
|
|
372
420
|
return undefined;
|
|
373
421
|
}
|
|
374
422
|
|
|
375
|
-
this.lastSlotForCheckpointProposalJob =
|
|
376
|
-
|
|
423
|
+
this.lastSlotForCheckpointProposalJob = targetSlot;
|
|
424
|
+
|
|
425
|
+
await this.p2pClient.prepareForSlot(targetSlot);
|
|
426
|
+
this.log.info(
|
|
427
|
+
`Preparing checkpoint proposal ${checkpointNumber} for target slot ${targetSlot} during wall-clock slot ${slot}`,
|
|
428
|
+
{
|
|
429
|
+
...logCtx,
|
|
430
|
+
proposer,
|
|
431
|
+
pipeliningEnabled: this.epochCache.isProposerPipeliningEnabled(),
|
|
432
|
+
},
|
|
433
|
+
);
|
|
377
434
|
|
|
378
435
|
// Create and return the checkpoint proposal job
|
|
379
436
|
return this.createCheckpointProposalJob(
|
|
380
|
-
epoch,
|
|
381
437
|
slot,
|
|
438
|
+
targetSlot,
|
|
439
|
+
targetEpoch,
|
|
382
440
|
checkpointNumber,
|
|
383
441
|
syncedTo.blockNumber,
|
|
384
442
|
proposer,
|
|
385
443
|
publisher,
|
|
386
444
|
attestorAddress,
|
|
387
445
|
invalidateCheckpoint,
|
|
446
|
+
syncedTo.proposedCheckpointData,
|
|
388
447
|
);
|
|
389
448
|
}
|
|
390
449
|
|
|
391
450
|
protected createCheckpointProposalJob(
|
|
392
|
-
epoch: EpochNumber,
|
|
393
451
|
slot: SlotNumber,
|
|
452
|
+
targetSlot: SlotNumber,
|
|
453
|
+
targetEpoch: EpochNumber,
|
|
394
454
|
checkpointNumber: CheckpointNumber,
|
|
395
455
|
syncedToBlockNumber: BlockNumber,
|
|
396
456
|
proposer: EthAddress | undefined,
|
|
397
457
|
publisher: SequencerPublisher,
|
|
398
458
|
attestorAddress: EthAddress,
|
|
399
459
|
invalidateCheckpoint: InvalidateCheckpointRequest | undefined,
|
|
460
|
+
proposedCheckpointData?: ProposedCheckpointData,
|
|
400
461
|
): CheckpointProposalJob {
|
|
401
462
|
return new CheckpointProposalJob(
|
|
402
|
-
epoch,
|
|
403
463
|
slot,
|
|
464
|
+
targetSlot,
|
|
465
|
+
targetEpoch,
|
|
404
466
|
checkpointNumber,
|
|
405
467
|
syncedToBlockNumber,
|
|
406
468
|
proposer,
|
|
@@ -426,9 +488,17 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
426
488
|
this.setState.bind(this),
|
|
427
489
|
this.tracer,
|
|
428
490
|
this.log.getBindings(),
|
|
491
|
+
proposedCheckpointData,
|
|
429
492
|
);
|
|
430
493
|
}
|
|
431
494
|
|
|
495
|
+
/**
|
|
496
|
+
* Returns the current sequencer state.
|
|
497
|
+
*/
|
|
498
|
+
public getState(): SequencerState {
|
|
499
|
+
return this.state;
|
|
500
|
+
}
|
|
501
|
+
|
|
432
502
|
/**
|
|
433
503
|
* Internal helper for setting the sequencer state and checks if we have enough time left in the slot to transition to the new state.
|
|
434
504
|
* @param proposedState - The new state to transition to.
|
|
@@ -475,16 +545,15 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
475
545
|
* We don't check against the previous block submitted since it may have been reorg'd out.
|
|
476
546
|
*/
|
|
477
547
|
protected async checkSync(args: { ts: bigint; slot: SlotNumber }): Promise<SequencerSyncCheckResult | undefined> {
|
|
478
|
-
// Check that the archiver
|
|
479
|
-
//
|
|
480
|
-
//
|
|
481
|
-
const
|
|
482
|
-
const { slot
|
|
483
|
-
if (
|
|
548
|
+
// Check that the archiver has fully synced the L2 slot before the one we want to propose in.
|
|
549
|
+
// The archiver reports sync progress via L1 block timestamps and synced checkpoint slots.
|
|
550
|
+
// See getSyncedL2SlotNumber for how missed L1 blocks are handled.
|
|
551
|
+
const syncedL2Slot = await this.l2BlockSource.getSyncedL2SlotNumber();
|
|
552
|
+
const { slot } = args;
|
|
553
|
+
if (syncedL2Slot === undefined || syncedL2Slot + 1 < slot) {
|
|
484
554
|
this.log.debug(`Cannot propose block at next L2 slot ${slot} due to pending sync from L1`, {
|
|
485
555
|
slot,
|
|
486
|
-
|
|
487
|
-
l1Timestamp,
|
|
556
|
+
syncedL2Slot,
|
|
488
557
|
});
|
|
489
558
|
return undefined;
|
|
490
559
|
}
|
|
@@ -494,25 +563,37 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
494
563
|
number: syncSummary.latestBlockNumber,
|
|
495
564
|
hash: syncSummary.latestBlockHash,
|
|
496
565
|
})),
|
|
497
|
-
this.l2BlockSource
|
|
566
|
+
this.l2BlockSource
|
|
567
|
+
.getL2Tips()
|
|
568
|
+
.then(t => ({ proposed: t.proposed, checkpointed: t.checkpointed, proposedCheckpoint: t.proposedCheckpoint })),
|
|
498
569
|
this.p2pClient.getStatus().then(p2p => p2p.syncedToL2Block),
|
|
499
570
|
this.l1ToL2MessageSource.getL2Tips().then(t => t.proposed),
|
|
500
571
|
this.l2BlockSource.getPendingChainValidationStatus(),
|
|
572
|
+
this.l2BlockSource.getProposedCheckpointOnly(),
|
|
501
573
|
] as const);
|
|
502
574
|
|
|
503
|
-
const [worldState,
|
|
575
|
+
const [worldState, l2Tips, p2p, l1ToL2MessageSource, pendingChainValidationStatus, proposedCheckpointData] =
|
|
576
|
+
syncedBlocks;
|
|
504
577
|
|
|
505
578
|
// Handle zero as a special case, since the block hash won't match across services if we're changing the prefilled data for the genesis block,
|
|
506
579
|
// as the world state can compute the new genesis block hash, but other components use the hardcoded constant.
|
|
507
580
|
// TODO(palla/mbps): Fix the above. All components should be able to handle dynamic genesis block hashes.
|
|
508
581
|
const result =
|
|
509
|
-
(
|
|
510
|
-
|
|
511
|
-
p2p.
|
|
512
|
-
l1ToL2MessageSource.
|
|
582
|
+
(l2Tips.proposed.number === 0 &&
|
|
583
|
+
worldState.number === 0 &&
|
|
584
|
+
p2p.number === 0 &&
|
|
585
|
+
l1ToL2MessageSource.number === 0) ||
|
|
586
|
+
(worldState.hash === l2Tips.proposed.hash &&
|
|
587
|
+
p2p.hash === l2Tips.proposed.hash &&
|
|
588
|
+
l1ToL2MessageSource.hash === l2Tips.proposed.hash);
|
|
513
589
|
|
|
514
590
|
if (!result) {
|
|
515
|
-
this.log.debug(`Sequencer sync check failed`, {
|
|
591
|
+
this.log.debug(`Sequencer sync check failed`, {
|
|
592
|
+
worldState,
|
|
593
|
+
l2BlockSource: l2Tips.proposed,
|
|
594
|
+
p2p,
|
|
595
|
+
l1ToL2MessageSource,
|
|
596
|
+
});
|
|
516
597
|
return undefined;
|
|
517
598
|
}
|
|
518
599
|
|
|
@@ -522,26 +603,33 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
522
603
|
const archive = new Fr((await this.worldState.getCommitted().getTreeInfo(MerkleTreeId.ARCHIVE)).root);
|
|
523
604
|
return {
|
|
524
605
|
checkpointNumber: CheckpointNumber.ZERO,
|
|
606
|
+
checkpointedCheckpointNumber: CheckpointNumber.ZERO,
|
|
525
607
|
blockNumber: BlockNumber.ZERO,
|
|
526
608
|
archive,
|
|
527
|
-
|
|
609
|
+
hasProposedCheckpoint: false,
|
|
610
|
+
syncedL2Slot,
|
|
528
611
|
pendingChainValidationStatus,
|
|
529
612
|
};
|
|
530
613
|
}
|
|
531
614
|
|
|
532
|
-
const
|
|
533
|
-
if (!
|
|
615
|
+
const blockData = await this.l2BlockSource.getBlockData(blockNumber);
|
|
616
|
+
if (!blockData) {
|
|
534
617
|
// this shouldn't really happen because a moment ago we checked that all components were in sync
|
|
535
|
-
this.log.error(`Failed to get L2 block ${blockNumber} from the archiver with all components in sync`);
|
|
618
|
+
this.log.error(`Failed to get L2 block data ${blockNumber} from the archiver with all components in sync`);
|
|
536
619
|
return undefined;
|
|
537
620
|
}
|
|
538
621
|
|
|
622
|
+
const hasProposedCheckpoint = l2Tips.proposedCheckpoint.checkpoint.number > l2Tips.checkpointed.checkpoint.number;
|
|
623
|
+
|
|
539
624
|
return {
|
|
540
|
-
|
|
541
|
-
blockNumber:
|
|
542
|
-
checkpointNumber:
|
|
543
|
-
|
|
544
|
-
|
|
625
|
+
blockData,
|
|
626
|
+
blockNumber: blockData.header.getBlockNumber(),
|
|
627
|
+
checkpointNumber: blockData.checkpointNumber,
|
|
628
|
+
checkpointedCheckpointNumber: l2Tips.checkpointed.checkpoint.number,
|
|
629
|
+
archive: blockData.archive.root,
|
|
630
|
+
hasProposedCheckpoint,
|
|
631
|
+
proposedCheckpointData,
|
|
632
|
+
syncedL2Slot,
|
|
545
633
|
pendingChainValidationStatus,
|
|
546
634
|
};
|
|
547
635
|
}
|
|
@@ -550,17 +638,20 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
550
638
|
* Checks if we are the proposer for the next slot.
|
|
551
639
|
* @returns True if we can propose, and the proposer address (undefined if anyone can propose)
|
|
552
640
|
*/
|
|
553
|
-
protected async checkCanPropose(
|
|
641
|
+
protected async checkCanPropose(targetSlot: SlotNumber): Promise<[boolean, EthAddress | undefined]> {
|
|
554
642
|
let proposer: EthAddress | undefined;
|
|
555
643
|
|
|
556
644
|
try {
|
|
557
|
-
proposer = await this.epochCache.getProposerAttesterAddressInSlot(
|
|
645
|
+
proposer = await this.epochCache.getProposerAttesterAddressInSlot(targetSlot);
|
|
558
646
|
} catch (e) {
|
|
559
647
|
if (e instanceof NoCommitteeError) {
|
|
560
|
-
this.
|
|
648
|
+
if (this.lastSlotForNoCommitteeWarning !== targetSlot) {
|
|
649
|
+
this.lastSlotForNoCommitteeWarning = targetSlot;
|
|
650
|
+
this.log.warn(`Cannot propose at target slot ${targetSlot} since the committee does not exist on L1`);
|
|
651
|
+
}
|
|
561
652
|
return [false, undefined];
|
|
562
653
|
}
|
|
563
|
-
this.log.error(`Error getting proposer for slot ${
|
|
654
|
+
this.log.error(`Error getting proposer for target slot ${targetSlot}`, e);
|
|
564
655
|
return [false, undefined];
|
|
565
656
|
}
|
|
566
657
|
|
|
@@ -577,10 +668,18 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
577
668
|
const weAreProposer = validatorAddresses.some(addr => addr.equals(proposer));
|
|
578
669
|
|
|
579
670
|
if (!weAreProposer) {
|
|
580
|
-
this.log.debug(`Cannot propose at slot ${
|
|
671
|
+
this.log.debug(`Cannot propose at target slot ${targetSlot} since we are not a proposer`, {
|
|
672
|
+
targetSlot,
|
|
673
|
+
validatorAddresses,
|
|
674
|
+
proposer,
|
|
675
|
+
});
|
|
581
676
|
return [false, proposer];
|
|
582
677
|
}
|
|
583
678
|
|
|
679
|
+
this.log.info(`We are the proposer for pipeline slot ${targetSlot}`, {
|
|
680
|
+
targetSlot,
|
|
681
|
+
proposer,
|
|
682
|
+
});
|
|
584
683
|
return [true, proposer];
|
|
585
684
|
}
|
|
586
685
|
|
|
@@ -589,8 +688,8 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
589
688
|
* This allows the sequencer to participate in governance/slashing votes even when it cannot build blocks.
|
|
590
689
|
*/
|
|
591
690
|
@trackSpan('Seqeuencer.tryVoteWhenSyncFails', ({ slot }) => ({ [Attributes.SLOT_NUMBER]: slot }))
|
|
592
|
-
protected async tryVoteWhenSyncFails(args: { slot: SlotNumber; ts: bigint }): Promise<void> {
|
|
593
|
-
const { slot } = args;
|
|
691
|
+
protected async tryVoteWhenSyncFails(args: { slot: SlotNumber; targetSlot: SlotNumber; ts: bigint }): Promise<void> {
|
|
692
|
+
const { slot, targetSlot } = args;
|
|
594
693
|
|
|
595
694
|
// Prevent duplicate attempts in the same slot
|
|
596
695
|
if (this.lastSlotForFallbackVote === slot) {
|
|
@@ -618,7 +717,7 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
618
717
|
});
|
|
619
718
|
|
|
620
719
|
// Check if we're a proposer or proposal is open
|
|
621
|
-
const [canPropose, proposer] = await this.checkCanPropose(
|
|
720
|
+
const [canPropose, proposer] = await this.checkCanPropose(targetSlot);
|
|
622
721
|
if (!canPropose) {
|
|
623
722
|
this.log.trace(`Cannot vote in slot ${slot} since we are not a proposer`, { slot, proposer });
|
|
624
723
|
return;
|
|
@@ -635,9 +734,9 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
635
734
|
slot,
|
|
636
735
|
});
|
|
637
736
|
|
|
638
|
-
// Enqueue governance and slashing votes
|
|
737
|
+
// Enqueue governance and slashing votes (voter uses the target slot for L1 submission)
|
|
639
738
|
const voter = new CheckpointVoter(
|
|
640
|
-
|
|
739
|
+
targetSlot,
|
|
641
740
|
publisher,
|
|
642
741
|
attestorAddress,
|
|
643
742
|
this.validatorClient,
|
|
@@ -717,7 +816,7 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
717
816
|
syncedTo: SequencerSyncCheckResult,
|
|
718
817
|
currentSlot: SlotNumber,
|
|
719
818
|
): Promise<void> {
|
|
720
|
-
const { pendingChainValidationStatus,
|
|
819
|
+
const { pendingChainValidationStatus, syncedL2Slot } = syncedTo;
|
|
721
820
|
if (pendingChainValidationStatus.valid) {
|
|
722
821
|
return;
|
|
723
822
|
}
|
|
@@ -732,7 +831,7 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
732
831
|
|
|
733
832
|
const logData = {
|
|
734
833
|
invalidL1Timestamp: invalidCheckpointTimestamp,
|
|
735
|
-
|
|
834
|
+
syncedL2Slot,
|
|
736
835
|
invalidCheckpoint: pendingChainValidationStatus.checkpoint,
|
|
737
836
|
secondsBeforeInvalidatingBlockAsCommitteeMember,
|
|
738
837
|
secondsBeforeInvalidatingBlockAsNonCommitteeMember,
|
|
@@ -860,6 +959,11 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
860
959
|
return this.validatorClient?.getValidatorAddresses();
|
|
861
960
|
}
|
|
862
961
|
|
|
962
|
+
/** Updates the publisher factory's node keystore adapter after a keystore reload. */
|
|
963
|
+
public updatePublisherNodeKeyStore(adapter: NodeKeystoreAdapter): void {
|
|
964
|
+
this.publisherFactory.updateNodeKeyStore(adapter);
|
|
965
|
+
}
|
|
966
|
+
|
|
863
967
|
public getConfig() {
|
|
864
968
|
return this.config;
|
|
865
969
|
}
|
|
@@ -870,10 +974,13 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
870
974
|
}
|
|
871
975
|
|
|
872
976
|
type SequencerSyncCheckResult = {
|
|
873
|
-
|
|
977
|
+
blockData?: BlockData;
|
|
874
978
|
checkpointNumber: CheckpointNumber;
|
|
979
|
+
checkpointedCheckpointNumber: CheckpointNumber;
|
|
875
980
|
blockNumber: BlockNumber;
|
|
876
981
|
archive: Fr;
|
|
877
|
-
|
|
982
|
+
hasProposedCheckpoint: boolean;
|
|
983
|
+
proposedCheckpointData?: ProposedCheckpointData;
|
|
984
|
+
syncedL2Slot: SlotNumber;
|
|
878
985
|
pendingChainValidationStatus: ValidateCheckpointResult;
|
|
879
986
|
};
|