@aztec/sequencer-client 0.0.1-commit.4ad48494d → 0.0.1-commit.4d3c002
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dest/client/sequencer-client.d.ts +15 -7
- package/dest/client/sequencer-client.d.ts.map +1 -1
- package/dest/client/sequencer-client.js +60 -26
- package/dest/config.d.ts +26 -6
- package/dest/config.d.ts.map +1 -1
- package/dest/config.js +44 -21
- package/dest/global_variable_builder/global_builder.d.ts +15 -11
- package/dest/global_variable_builder/global_builder.d.ts.map +1 -1
- package/dest/global_variable_builder/global_builder.js +29 -25
- package/dest/global_variable_builder/index.d.ts +2 -2
- package/dest/global_variable_builder/index.d.ts.map +1 -1
- package/dest/publisher/config.d.ts +47 -17
- package/dest/publisher/config.d.ts.map +1 -1
- package/dest/publisher/config.js +121 -42
- package/dest/publisher/index.d.ts +2 -1
- package/dest/publisher/index.d.ts.map +1 -1
- package/dest/publisher/l1_tx_failed_store/factory.d.ts +11 -0
- package/dest/publisher/l1_tx_failed_store/factory.d.ts.map +1 -0
- package/dest/publisher/l1_tx_failed_store/factory.js +22 -0
- package/dest/publisher/l1_tx_failed_store/failed_tx_store.d.ts +59 -0
- package/dest/publisher/l1_tx_failed_store/failed_tx_store.d.ts.map +1 -0
- package/dest/publisher/l1_tx_failed_store/failed_tx_store.js +1 -0
- package/dest/publisher/l1_tx_failed_store/file_store_failed_tx_store.d.ts +15 -0
- package/dest/publisher/l1_tx_failed_store/file_store_failed_tx_store.d.ts.map +1 -0
- package/dest/publisher/l1_tx_failed_store/file_store_failed_tx_store.js +34 -0
- package/dest/publisher/l1_tx_failed_store/index.d.ts +4 -0
- package/dest/publisher/l1_tx_failed_store/index.d.ts.map +1 -0
- package/dest/publisher/l1_tx_failed_store/index.js +2 -0
- package/dest/publisher/sequencer-publisher-factory.d.ts +11 -3
- package/dest/publisher/sequencer-publisher-factory.d.ts.map +1 -1
- package/dest/publisher/sequencer-publisher-factory.js +27 -2
- package/dest/publisher/sequencer-publisher.d.ts +76 -30
- package/dest/publisher/sequencer-publisher.d.ts.map +1 -1
- package/dest/publisher/sequencer-publisher.js +396 -71
- package/dest/sequencer/checkpoint_proposal_job.d.ts +33 -8
- package/dest/sequencer/checkpoint_proposal_job.d.ts.map +1 -1
- package/dest/sequencer/checkpoint_proposal_job.js +347 -170
- package/dest/sequencer/checkpoint_voter.d.ts +1 -2
- package/dest/sequencer/checkpoint_voter.d.ts.map +1 -1
- package/dest/sequencer/checkpoint_voter.js +2 -5
- package/dest/sequencer/events.d.ts +2 -1
- package/dest/sequencer/events.d.ts.map +1 -1
- package/dest/sequencer/metrics.d.ts +18 -5
- package/dest/sequencer/metrics.d.ts.map +1 -1
- package/dest/sequencer/metrics.js +72 -15
- package/dest/sequencer/sequencer.d.ts +40 -17
- package/dest/sequencer/sequencer.d.ts.map +1 -1
- package/dest/sequencer/sequencer.js +145 -92
- package/dest/sequencer/timetable.d.ts +4 -3
- package/dest/sequencer/timetable.d.ts.map +1 -1
- package/dest/sequencer/timetable.js +6 -7
- package/dest/sequencer/types.d.ts +2 -2
- package/dest/sequencer/types.d.ts.map +1 -1
- package/dest/test/index.d.ts +3 -5
- package/dest/test/index.d.ts.map +1 -1
- package/dest/test/mock_checkpoint_builder.d.ts +11 -11
- package/dest/test/mock_checkpoint_builder.d.ts.map +1 -1
- package/dest/test/mock_checkpoint_builder.js +45 -34
- package/dest/test/utils.d.ts +3 -3
- package/dest/test/utils.d.ts.map +1 -1
- package/dest/test/utils.js +4 -4
- package/package.json +27 -28
- package/src/client/sequencer-client.ts +76 -23
- package/src/config.ts +56 -27
- package/src/global_variable_builder/global_builder.ts +38 -27
- package/src/global_variable_builder/index.ts +1 -1
- package/src/publisher/config.ts +153 -43
- package/src/publisher/index.ts +3 -0
- package/src/publisher/l1_tx_failed_store/factory.ts +32 -0
- package/src/publisher/l1_tx_failed_store/failed_tx_store.ts +55 -0
- package/src/publisher/l1_tx_failed_store/file_store_failed_tx_store.ts +46 -0
- package/src/publisher/l1_tx_failed_store/index.ts +3 -0
- package/src/publisher/sequencer-publisher-factory.ts +38 -6
- package/src/publisher/sequencer-publisher.ts +442 -95
- package/src/sequencer/checkpoint_proposal_job.ts +456 -176
- package/src/sequencer/checkpoint_voter.ts +1 -12
- package/src/sequencer/events.ts +1 -1
- package/src/sequencer/metrics.ts +82 -18
- package/src/sequencer/sequencer.ts +208 -108
- package/src/sequencer/timetable.ts +7 -7
- package/src/sequencer/types.ts +1 -1
- package/src/test/index.ts +2 -4
- package/src/test/mock_checkpoint_builder.ts +63 -49
- package/src/test/utils.ts +4 -2
|
@@ -12,9 +12,9 @@ import type { DateProvider } from '@aztec/foundation/timer';
|
|
|
12
12
|
import type { TypedEventEmitter } from '@aztec/foundation/types';
|
|
13
13
|
import type { P2P } from '@aztec/p2p';
|
|
14
14
|
import type { SlasherClientInterface } from '@aztec/slasher';
|
|
15
|
-
import type {
|
|
16
|
-
import type { Checkpoint } from '@aztec/stdlib/checkpoint';
|
|
17
|
-
import {
|
|
15
|
+
import type { BlockData, L2BlockSink, L2BlockSource, ValidateCheckpointResult } from '@aztec/stdlib/block';
|
|
16
|
+
import type { Checkpoint, ProposedCheckpointData } from '@aztec/stdlib/checkpoint';
|
|
17
|
+
import { getSlotStartBuildTimestamp } from '@aztec/stdlib/epoch-helpers';
|
|
18
18
|
import {
|
|
19
19
|
type ResolvedSequencerConfig,
|
|
20
20
|
type SequencerConfig,
|
|
@@ -25,7 +25,7 @@ import type { L1ToL2MessageSource } from '@aztec/stdlib/messaging';
|
|
|
25
25
|
import { pickFromSchema } from '@aztec/stdlib/schemas';
|
|
26
26
|
import { MerkleTreeId } from '@aztec/stdlib/trees';
|
|
27
27
|
import { Attributes, type TelemetryClient, type Tracer, getTelemetryClient, trackSpan } from '@aztec/telemetry-client';
|
|
28
|
-
import { FullNodeCheckpointsBuilder, type ValidatorClient } from '@aztec/validator-client';
|
|
28
|
+
import { FullNodeCheckpointsBuilder, NodeKeystoreAdapter, type ValidatorClient } from '@aztec/validator-client';
|
|
29
29
|
|
|
30
30
|
import EventEmitter from 'node:events';
|
|
31
31
|
|
|
@@ -72,17 +72,12 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
72
72
|
/** The last epoch for which we logged strategy comparison in fisherman mode. */
|
|
73
73
|
private lastEpochForStrategyComparison: EpochNumber | undefined;
|
|
74
74
|
|
|
75
|
+
/** The last checkpoint proposal job, tracked so we can await its pending L1 submission during shutdown. */
|
|
76
|
+
private lastCheckpointProposalJob: CheckpointProposalJob | undefined;
|
|
77
|
+
|
|
75
78
|
/** The maximum number of seconds that the sequencer can be into a slot to transition to a particular state. */
|
|
76
79
|
protected timetable!: SequencerTimetable;
|
|
77
80
|
|
|
78
|
-
// This shouldn't be here as this gets re-created each time we build/propose a block.
|
|
79
|
-
// But we have a number of tests that abuse/rely on this class having a permanent publisher.
|
|
80
|
-
// As long as those tests only configure a single publisher they will continue to work.
|
|
81
|
-
// This will get re-assigned every time the sequencer goes to build a new block to a publisher that is valid
|
|
82
|
-
// for the block proposer.
|
|
83
|
-
// TODO(palla/mbps): Remove this field and fix tests
|
|
84
|
-
protected publisher: SequencerPublisher | undefined;
|
|
85
|
-
|
|
86
81
|
/** Config for the sequencer */
|
|
87
82
|
protected config: ResolvedSequencerConfig = DefaultSequencerConfig;
|
|
88
83
|
|
|
@@ -118,7 +113,7 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
118
113
|
/** Updates sequencer config by the defined values and updates the timetable */
|
|
119
114
|
public updateConfig(config: Partial<SequencerConfig>) {
|
|
120
115
|
const filteredConfig = pickFromSchema(config, SequencerConfigSchema);
|
|
121
|
-
this.log.info(`Updated sequencer config`, omit(filteredConfig, '
|
|
116
|
+
this.log.info(`Updated sequencer config`, omit(filteredConfig, 'txPublicSetupAllowListExtend'));
|
|
122
117
|
this.config = merge(this.config, filteredConfig);
|
|
123
118
|
this.timetable = new SequencerTimetable(
|
|
124
119
|
{
|
|
@@ -134,10 +129,9 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
134
129
|
);
|
|
135
130
|
}
|
|
136
131
|
|
|
137
|
-
/** Initializes the sequencer (precomputes tables
|
|
138
|
-
public
|
|
132
|
+
/** Initializes the sequencer (precomputes tables). Takes about 3s. */
|
|
133
|
+
public init() {
|
|
139
134
|
getKzg();
|
|
140
|
-
this.publisher = (await this.publisherFactory.create(undefined)).publisher;
|
|
141
135
|
}
|
|
142
136
|
|
|
143
137
|
/** Starts the sequencer and moves to IDLE state. */
|
|
@@ -156,8 +150,9 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
156
150
|
public async stop(): Promise<void> {
|
|
157
151
|
this.log.info(`Stopping sequencer`);
|
|
158
152
|
this.setState(SequencerState.STOPPING, undefined, { force: true });
|
|
159
|
-
this.
|
|
153
|
+
await this.publisherFactory.stopAll();
|
|
160
154
|
await this.runningPromise?.stop();
|
|
155
|
+
await this.lastCheckpointProposalJob?.awaitPendingSubmission();
|
|
161
156
|
this.setState(SequencerState.STOPPED, undefined, { force: true });
|
|
162
157
|
this.log.info('Stopped sequencer');
|
|
163
158
|
}
|
|
@@ -169,7 +164,6 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
169
164
|
} catch (err) {
|
|
170
165
|
this.emit('checkpoint-error', { error: err as Error });
|
|
171
166
|
if (err instanceof SequencerTooSlowError) {
|
|
172
|
-
// TODO(palla/mbps): Add missing states
|
|
173
167
|
// Log as warn only if we had to abort halfway through the block proposal
|
|
174
168
|
const logLvl = [SequencerState.INITIALIZING_CHECKPOINT, SequencerState.PROPOSER_CHECK].includes(
|
|
175
169
|
err.proposedState,
|
|
@@ -202,14 +196,25 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
202
196
|
@trackSpan('Sequencer.work')
|
|
203
197
|
protected async work() {
|
|
204
198
|
this.setState(SequencerState.SYNCHRONIZING, undefined);
|
|
205
|
-
const { slot, ts,
|
|
199
|
+
const { slot, ts, nowSeconds, epoch } = this.epochCache.getEpochAndSlotInNextL1Slot();
|
|
200
|
+
const { slot: targetSlot, epoch: targetEpoch } = this.epochCache.getTargetEpochAndSlotInNextL1Slot();
|
|
206
201
|
|
|
207
202
|
// Check if we are synced and it's our slot, grab a publisher, check previous block invalidation, etc
|
|
208
|
-
const checkpointProposalJob = await this.prepareCheckpointProposal(
|
|
203
|
+
const checkpointProposalJob = await this.prepareCheckpointProposal(
|
|
204
|
+
slot,
|
|
205
|
+
targetSlot,
|
|
206
|
+
epoch,
|
|
207
|
+
targetEpoch,
|
|
208
|
+
ts,
|
|
209
|
+
nowSeconds,
|
|
210
|
+
);
|
|
209
211
|
if (!checkpointProposalJob) {
|
|
210
212
|
return;
|
|
211
213
|
}
|
|
212
214
|
|
|
215
|
+
// Track the job so we can await its pending L1 submission during shutdown
|
|
216
|
+
this.lastCheckpointProposalJob = checkpointProposalJob;
|
|
217
|
+
|
|
213
218
|
// Execute the checkpoint proposal job
|
|
214
219
|
const checkpoint = await checkpointProposalJob.execute();
|
|
215
220
|
|
|
@@ -218,13 +223,13 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
218
223
|
this.lastCheckpointProposed = checkpoint;
|
|
219
224
|
}
|
|
220
225
|
|
|
221
|
-
// Log fee strategy comparison if on fisherman
|
|
226
|
+
// Log fee strategy comparison if on fisherman (uses target epoch since we mirror the proposer's perspective)
|
|
222
227
|
if (
|
|
223
228
|
this.config.fishermanMode &&
|
|
224
|
-
(this.lastEpochForStrategyComparison === undefined ||
|
|
229
|
+
(this.lastEpochForStrategyComparison === undefined || targetEpoch > this.lastEpochForStrategyComparison)
|
|
225
230
|
) {
|
|
226
|
-
this.logStrategyComparison(
|
|
227
|
-
this.lastEpochForStrategyComparison =
|
|
231
|
+
this.logStrategyComparison(targetEpoch, checkpointProposalJob.getPublisher());
|
|
232
|
+
this.lastEpochForStrategyComparison = targetEpoch;
|
|
228
233
|
}
|
|
229
234
|
|
|
230
235
|
return checkpoint;
|
|
@@ -236,44 +241,49 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
236
241
|
* @returns CheckpointProposalJob if successful, undefined if we are not yet synced or are not the proposer.
|
|
237
242
|
*/
|
|
238
243
|
@trackSpan('Sequencer.prepareCheckpointProposal')
|
|
239
|
-
|
|
240
|
-
epoch: EpochNumber,
|
|
244
|
+
protected async prepareCheckpointProposal(
|
|
241
245
|
slot: SlotNumber,
|
|
246
|
+
targetSlot: SlotNumber,
|
|
247
|
+
epoch: EpochNumber,
|
|
248
|
+
targetEpoch: EpochNumber,
|
|
242
249
|
ts: bigint,
|
|
243
|
-
|
|
250
|
+
nowSeconds: bigint,
|
|
244
251
|
): Promise<CheckpointProposalJob | undefined> {
|
|
245
|
-
// Check we have not already processed this slot (cheapest check)
|
|
252
|
+
// Check we have not already processed this target slot (cheapest check)
|
|
246
253
|
// We only check this if enforce timetable is set, since we want to keep processing the same slot if we are not
|
|
247
254
|
// running against actual time (eg when we use sandbox-style automining)
|
|
248
255
|
if (
|
|
249
256
|
this.lastSlotForCheckpointProposalJob &&
|
|
250
|
-
this.lastSlotForCheckpointProposalJob >=
|
|
257
|
+
this.lastSlotForCheckpointProposalJob >= targetSlot &&
|
|
251
258
|
this.config.enforceTimeTable
|
|
252
259
|
) {
|
|
253
|
-
this.log.trace(`
|
|
260
|
+
this.log.trace(`Target slot ${targetSlot} has already been processed`);
|
|
254
261
|
return undefined;
|
|
255
262
|
}
|
|
256
263
|
|
|
257
|
-
// But if we have already proposed for this slot,
|
|
258
|
-
if (this.lastCheckpointProposed && this.lastCheckpointProposed.header.slotNumber >=
|
|
259
|
-
this.log.trace(
|
|
264
|
+
// But if we have already proposed for this slot, then we definitely have to skip it, automining or not
|
|
265
|
+
if (this.lastCheckpointProposed && this.lastCheckpointProposed.header.slotNumber >= targetSlot) {
|
|
266
|
+
this.log.trace(
|
|
267
|
+
`Slot ${targetSlot} has already been published as checkpoint ${this.lastCheckpointProposed.number}`,
|
|
268
|
+
);
|
|
260
269
|
return undefined;
|
|
261
270
|
}
|
|
262
271
|
|
|
263
272
|
// Check all components are synced to latest as seen by the archiver (queries all subsystems)
|
|
264
273
|
const syncedTo = await this.checkSync({ ts, slot });
|
|
265
274
|
if (!syncedTo) {
|
|
266
|
-
await this.tryVoteWhenSyncFails({ slot, ts });
|
|
275
|
+
await this.tryVoteWhenSyncFails({ slot, targetSlot, ts });
|
|
267
276
|
return undefined;
|
|
268
277
|
}
|
|
269
278
|
|
|
270
|
-
// If escape hatch is open for
|
|
279
|
+
// If escape hatch is open for the target epoch, do not start checkpoint proposal work and do not attempt invalidations.
|
|
271
280
|
// Still perform governance/slashing voting (as proposer) once per slot.
|
|
272
|
-
|
|
281
|
+
// When pipelining, we check the target epoch (slot+1's epoch) since that's the epoch we're building for.
|
|
282
|
+
const isEscapeHatchOpen = await this.epochCache.isEscapeHatchOpen(targetEpoch);
|
|
273
283
|
|
|
274
284
|
if (isEscapeHatchOpen) {
|
|
275
285
|
this.setState(SequencerState.PROPOSER_CHECK, slot);
|
|
276
|
-
const [canPropose, proposer] = await this.checkCanPropose(
|
|
286
|
+
const [canPropose, proposer] = await this.checkCanPropose(targetSlot);
|
|
277
287
|
if (canPropose) {
|
|
278
288
|
await this.tryVoteWhenEscapeHatchOpen({ slot, proposer });
|
|
279
289
|
} else {
|
|
@@ -289,19 +299,29 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
289
299
|
// Next checkpoint follows from the last synced one
|
|
290
300
|
const checkpointNumber = CheckpointNumber(syncedTo.checkpointNumber + 1);
|
|
291
301
|
|
|
302
|
+
// Guard: don't exceed 1-deep pipeline. Without a proposed checkpoint, we can only build
|
|
303
|
+
// confirmed + 1. With a proposed checkpoint, we can build confirmed + 2.
|
|
304
|
+
const confirmedCkpt = syncedTo.checkpointedCheckpointNumber;
|
|
305
|
+
if (checkpointNumber > confirmedCkpt + 2) {
|
|
306
|
+
this.log.warn(
|
|
307
|
+
`Skipping slot ${targetSlot}: checkpoint ${checkpointNumber} exceeds max pipeline depth (confirmed=${confirmedCkpt})`,
|
|
308
|
+
);
|
|
309
|
+
return undefined;
|
|
310
|
+
}
|
|
311
|
+
|
|
292
312
|
const logCtx = {
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
syncedToL2Slot: getSlotAtTimestamp(syncedTo.l1Timestamp, this.l1Constants),
|
|
313
|
+
nowSeconds,
|
|
314
|
+
syncedToL2Slot: syncedTo.syncedL2Slot,
|
|
296
315
|
slot,
|
|
316
|
+
targetSlot,
|
|
297
317
|
slotTs: ts,
|
|
298
318
|
checkpointNumber,
|
|
299
319
|
isPendingChainValid: pick(syncedTo.pendingChainValidationStatus, 'valid', 'reason', 'invalidIndex'),
|
|
300
320
|
};
|
|
301
321
|
|
|
302
|
-
// Check that we are a proposer for the
|
|
322
|
+
// Check that we are a proposer for the target slot.
|
|
303
323
|
this.setState(SequencerState.PROPOSER_CHECK, slot);
|
|
304
|
-
const [canPropose, proposer] = await this.checkCanPropose(
|
|
324
|
+
const [canPropose, proposer] = await this.checkCanPropose(targetSlot);
|
|
305
325
|
|
|
306
326
|
// If we are not a proposer check if we should invalidate an invalid checkpoint, and bail
|
|
307
327
|
if (!canPropose) {
|
|
@@ -309,13 +329,13 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
309
329
|
return undefined;
|
|
310
330
|
}
|
|
311
331
|
|
|
312
|
-
// Check that the slot is not taken by a block already (should never happen, since only us can propose for this slot)
|
|
313
|
-
if (syncedTo.
|
|
332
|
+
// Check that the target slot is not taken by a block already (should never happen, since only us can propose for this slot)
|
|
333
|
+
if (syncedTo.blockData && syncedTo.blockData.header.getSlot() >= targetSlot) {
|
|
314
334
|
this.log.warn(
|
|
315
|
-
`Cannot propose block at
|
|
316
|
-
{ ...logCtx, block: syncedTo.
|
|
335
|
+
`Cannot propose block at target slot ${targetSlot} since that slot was taken by block ${syncedTo.blockNumber}`,
|
|
336
|
+
{ ...logCtx, block: syncedTo.blockData.header.toInspect() },
|
|
317
337
|
);
|
|
318
|
-
this.metrics.
|
|
338
|
+
this.metrics.recordCheckpointPrecheckFailed('slot_already_taken');
|
|
319
339
|
return undefined;
|
|
320
340
|
}
|
|
321
341
|
|
|
@@ -326,7 +346,6 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
326
346
|
const proposerForPublisher = this.config.fishermanMode ? undefined : proposer;
|
|
327
347
|
const { attestorAddress, publisher } = await this.publisherFactory.create(proposerForPublisher);
|
|
328
348
|
this.log.verbose(`Created publisher at address ${publisher.getSenderAddress()} for attestor ${attestorAddress}`);
|
|
329
|
-
this.publisher = publisher;
|
|
330
349
|
|
|
331
350
|
// In fisherman mode, set the actual proposer's address for simulations
|
|
332
351
|
if (this.config.fishermanMode && proposer) {
|
|
@@ -335,15 +354,41 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
335
354
|
}
|
|
336
355
|
|
|
337
356
|
// Prepare invalidation request if the pending chain is invalid (returns undefined if no need)
|
|
338
|
-
|
|
357
|
+
let invalidateCheckpoint = await publisher.simulateInvalidateCheckpoint(syncedTo.pendingChainValidationStatus);
|
|
358
|
+
|
|
359
|
+
// Determine the correct archive and L1 state overrides for the canProposeAt check.
|
|
360
|
+
// The L1 contract reads archives[proposedCheckpointNumber] and compares it with the provided archive.
|
|
361
|
+
// When invalidating or pipelining, the local archive may differ from L1's, so we adjust accordingly.
|
|
362
|
+
let archiveForCheck = syncedTo.archive;
|
|
363
|
+
const l1Overrides: {
|
|
364
|
+
forcePendingCheckpointNumber?: CheckpointNumber;
|
|
365
|
+
forceArchive?: { checkpointNumber: CheckpointNumber; archive: Fr };
|
|
366
|
+
} = {};
|
|
367
|
+
|
|
368
|
+
if (this.epochCache.isProposerPipeliningEnabled() && syncedTo.hasProposedCheckpoint) {
|
|
369
|
+
// Parent checkpoint hasn't landed on L1 yet. Override both the proposed checkpoint number
|
|
370
|
+
// and the archive at that checkpoint so L1 simulation sees the correct chain tip.
|
|
371
|
+
const parentCheckpointNumber = CheckpointNumber(checkpointNumber - 1);
|
|
372
|
+
l1Overrides.forcePendingCheckpointNumber = parentCheckpointNumber;
|
|
373
|
+
l1Overrides.forceArchive = { checkpointNumber: parentCheckpointNumber, archive: syncedTo.archive };
|
|
374
|
+
this.metrics.recordPipelineDepth(1);
|
|
375
|
+
|
|
376
|
+
this.log.verbose(
|
|
377
|
+
`Building on top of proposed checkpoint (pending=${syncedTo.proposedCheckpointData?.checkpointNumber})`,
|
|
378
|
+
);
|
|
379
|
+
// Clear the invalidation - the proposed checkpoint should handle it.
|
|
380
|
+
invalidateCheckpoint = undefined;
|
|
381
|
+
} else if (invalidateCheckpoint) {
|
|
382
|
+
// After invalidation, L1 will roll back to checkpoint N-1. The archive at N-1 already
|
|
383
|
+
// exists on L1, so we just pass the matching archive (the lastArchive of the invalid checkpoint).
|
|
384
|
+
archiveForCheck = invalidateCheckpoint.lastArchive;
|
|
385
|
+
l1Overrides.forcePendingCheckpointNumber = invalidateCheckpoint.forcePendingCheckpointNumber;
|
|
386
|
+
this.metrics.recordPipelineDepth(0);
|
|
387
|
+
} else {
|
|
388
|
+
this.metrics.recordPipelineDepth(0);
|
|
389
|
+
}
|
|
339
390
|
|
|
340
|
-
|
|
341
|
-
// if all the previous checks are good, but we do it just in case.
|
|
342
|
-
const canProposeCheck = await publisher.canProposeAtNextEthBlock(
|
|
343
|
-
syncedTo.archive,
|
|
344
|
-
proposer ?? EthAddress.ZERO,
|
|
345
|
-
invalidateCheckpoint,
|
|
346
|
-
);
|
|
391
|
+
const canProposeCheck = await publisher.canProposeAt(archiveForCheck, proposer ?? EthAddress.ZERO, l1Overrides);
|
|
347
392
|
|
|
348
393
|
if (canProposeCheck === undefined) {
|
|
349
394
|
this.log.warn(
|
|
@@ -351,17 +396,17 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
351
396
|
logCtx,
|
|
352
397
|
);
|
|
353
398
|
this.emit('proposer-rollup-check-failed', { reason: 'Rollup contract check failed', slot });
|
|
354
|
-
this.metrics.
|
|
399
|
+
this.metrics.recordCheckpointPrecheckFailed('rollup_contract_check_failed');
|
|
355
400
|
return undefined;
|
|
356
401
|
}
|
|
357
402
|
|
|
358
|
-
if (canProposeCheck.slot !==
|
|
403
|
+
if (canProposeCheck.slot !== targetSlot) {
|
|
359
404
|
this.log.warn(
|
|
360
|
-
`Cannot propose block due to slot mismatch with rollup contract (this can be caused by a clock out of sync). Expected slot ${
|
|
361
|
-
{ ...logCtx, rollup: canProposeCheck, expectedSlot:
|
|
405
|
+
`Cannot propose block due to slot mismatch with rollup contract (this can be caused by a clock out of sync). Expected slot ${targetSlot} but got ${canProposeCheck.slot}.`,
|
|
406
|
+
{ ...logCtx, rollup: canProposeCheck, expectedSlot: targetSlot },
|
|
362
407
|
);
|
|
363
408
|
this.emit('proposer-rollup-check-failed', { reason: 'Slot mismatch', slot });
|
|
364
|
-
this.metrics.
|
|
409
|
+
this.metrics.recordCheckpointPrecheckFailed('slot_mismatch');
|
|
365
410
|
return undefined;
|
|
366
411
|
}
|
|
367
412
|
|
|
@@ -371,40 +416,53 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
371
416
|
{ ...logCtx, rollup: canProposeCheck, expectedSlot: slot },
|
|
372
417
|
);
|
|
373
418
|
this.emit('proposer-rollup-check-failed', { reason: 'Block mismatch', slot });
|
|
374
|
-
this.metrics.
|
|
419
|
+
this.metrics.recordCheckpointPrecheckFailed('block_number_mismatch');
|
|
375
420
|
return undefined;
|
|
376
421
|
}
|
|
377
422
|
|
|
378
|
-
this.lastSlotForCheckpointProposalJob =
|
|
379
|
-
|
|
380
|
-
this.
|
|
423
|
+
this.lastSlotForCheckpointProposalJob = targetSlot;
|
|
424
|
+
|
|
425
|
+
await this.p2pClient.prepareForSlot(targetSlot);
|
|
426
|
+
this.log.info(
|
|
427
|
+
`Preparing checkpoint proposal ${checkpointNumber} for target slot ${targetSlot} during wall-clock slot ${slot}`,
|
|
428
|
+
{
|
|
429
|
+
...logCtx,
|
|
430
|
+
proposer,
|
|
431
|
+
pipeliningEnabled: this.epochCache.isProposerPipeliningEnabled(),
|
|
432
|
+
},
|
|
433
|
+
);
|
|
381
434
|
|
|
382
435
|
// Create and return the checkpoint proposal job
|
|
383
436
|
return this.createCheckpointProposalJob(
|
|
384
|
-
epoch,
|
|
385
437
|
slot,
|
|
438
|
+
targetSlot,
|
|
439
|
+
targetEpoch,
|
|
386
440
|
checkpointNumber,
|
|
387
441
|
syncedTo.blockNumber,
|
|
388
442
|
proposer,
|
|
389
443
|
publisher,
|
|
390
444
|
attestorAddress,
|
|
391
445
|
invalidateCheckpoint,
|
|
446
|
+
syncedTo.proposedCheckpointData,
|
|
392
447
|
);
|
|
393
448
|
}
|
|
394
449
|
|
|
395
450
|
protected createCheckpointProposalJob(
|
|
396
|
-
epoch: EpochNumber,
|
|
397
451
|
slot: SlotNumber,
|
|
452
|
+
targetSlot: SlotNumber,
|
|
453
|
+
targetEpoch: EpochNumber,
|
|
398
454
|
checkpointNumber: CheckpointNumber,
|
|
399
455
|
syncedToBlockNumber: BlockNumber,
|
|
400
456
|
proposer: EthAddress | undefined,
|
|
401
457
|
publisher: SequencerPublisher,
|
|
402
458
|
attestorAddress: EthAddress,
|
|
403
459
|
invalidateCheckpoint: InvalidateCheckpointRequest | undefined,
|
|
460
|
+
proposedCheckpointData?: ProposedCheckpointData,
|
|
404
461
|
): CheckpointProposalJob {
|
|
405
462
|
return new CheckpointProposalJob(
|
|
406
|
-
epoch,
|
|
407
463
|
slot,
|
|
464
|
+
targetSlot,
|
|
465
|
+
targetEpoch,
|
|
408
466
|
checkpointNumber,
|
|
409
467
|
syncedToBlockNumber,
|
|
410
468
|
proposer,
|
|
@@ -430,9 +488,17 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
430
488
|
this.setState.bind(this),
|
|
431
489
|
this.tracer,
|
|
432
490
|
this.log.getBindings(),
|
|
491
|
+
proposedCheckpointData,
|
|
433
492
|
);
|
|
434
493
|
}
|
|
435
494
|
|
|
495
|
+
/**
|
|
496
|
+
* Returns the current sequencer state.
|
|
497
|
+
*/
|
|
498
|
+
public getState(): SequencerState {
|
|
499
|
+
return this.state;
|
|
500
|
+
}
|
|
501
|
+
|
|
436
502
|
/**
|
|
437
503
|
* Internal helper for setting the sequencer state and checks if we have enough time left in the slot to transition to the new state.
|
|
438
504
|
* @param proposedState - The new state to transition to.
|
|
@@ -479,16 +545,15 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
479
545
|
* We don't check against the previous block submitted since it may have been reorg'd out.
|
|
480
546
|
*/
|
|
481
547
|
protected async checkSync(args: { ts: bigint; slot: SlotNumber }): Promise<SequencerSyncCheckResult | undefined> {
|
|
482
|
-
// Check that the archiver
|
|
483
|
-
//
|
|
484
|
-
//
|
|
485
|
-
const
|
|
486
|
-
const { slot
|
|
487
|
-
if (
|
|
548
|
+
// Check that the archiver has fully synced the L2 slot before the one we want to propose in.
|
|
549
|
+
// The archiver reports sync progress via L1 block timestamps and synced checkpoint slots.
|
|
550
|
+
// See getSyncedL2SlotNumber for how missed L1 blocks are handled.
|
|
551
|
+
const syncedL2Slot = await this.l2BlockSource.getSyncedL2SlotNumber();
|
|
552
|
+
const { slot } = args;
|
|
553
|
+
if (syncedL2Slot === undefined || syncedL2Slot + 1 < slot) {
|
|
488
554
|
this.log.debug(`Cannot propose block at next L2 slot ${slot} due to pending sync from L1`, {
|
|
489
555
|
slot,
|
|
490
|
-
|
|
491
|
-
l1Timestamp,
|
|
556
|
+
syncedL2Slot,
|
|
492
557
|
});
|
|
493
558
|
return undefined;
|
|
494
559
|
}
|
|
@@ -498,25 +563,37 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
498
563
|
number: syncSummary.latestBlockNumber,
|
|
499
564
|
hash: syncSummary.latestBlockHash,
|
|
500
565
|
})),
|
|
501
|
-
this.l2BlockSource
|
|
566
|
+
this.l2BlockSource
|
|
567
|
+
.getL2Tips()
|
|
568
|
+
.then(t => ({ proposed: t.proposed, checkpointed: t.checkpointed, proposedCheckpoint: t.proposedCheckpoint })),
|
|
502
569
|
this.p2pClient.getStatus().then(p2p => p2p.syncedToL2Block),
|
|
503
570
|
this.l1ToL2MessageSource.getL2Tips().then(t => t.proposed),
|
|
504
571
|
this.l2BlockSource.getPendingChainValidationStatus(),
|
|
572
|
+
this.l2BlockSource.getProposedCheckpointOnly(),
|
|
505
573
|
] as const);
|
|
506
574
|
|
|
507
|
-
const [worldState,
|
|
575
|
+
const [worldState, l2Tips, p2p, l1ToL2MessageSource, pendingChainValidationStatus, proposedCheckpointData] =
|
|
576
|
+
syncedBlocks;
|
|
508
577
|
|
|
509
578
|
// Handle zero as a special case, since the block hash won't match across services if we're changing the prefilled data for the genesis block,
|
|
510
579
|
// as the world state can compute the new genesis block hash, but other components use the hardcoded constant.
|
|
511
580
|
// TODO(palla/mbps): Fix the above. All components should be able to handle dynamic genesis block hashes.
|
|
512
581
|
const result =
|
|
513
|
-
(
|
|
514
|
-
|
|
515
|
-
p2p.
|
|
516
|
-
l1ToL2MessageSource.
|
|
582
|
+
(l2Tips.proposed.number === 0 &&
|
|
583
|
+
worldState.number === 0 &&
|
|
584
|
+
p2p.number === 0 &&
|
|
585
|
+
l1ToL2MessageSource.number === 0) ||
|
|
586
|
+
(worldState.hash === l2Tips.proposed.hash &&
|
|
587
|
+
p2p.hash === l2Tips.proposed.hash &&
|
|
588
|
+
l1ToL2MessageSource.hash === l2Tips.proposed.hash);
|
|
517
589
|
|
|
518
590
|
if (!result) {
|
|
519
|
-
this.log.debug(`Sequencer sync check failed`, {
|
|
591
|
+
this.log.debug(`Sequencer sync check failed`, {
|
|
592
|
+
worldState,
|
|
593
|
+
l2BlockSource: l2Tips.proposed,
|
|
594
|
+
p2p,
|
|
595
|
+
l1ToL2MessageSource,
|
|
596
|
+
});
|
|
520
597
|
return undefined;
|
|
521
598
|
}
|
|
522
599
|
|
|
@@ -526,26 +603,33 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
526
603
|
const archive = new Fr((await this.worldState.getCommitted().getTreeInfo(MerkleTreeId.ARCHIVE)).root);
|
|
527
604
|
return {
|
|
528
605
|
checkpointNumber: CheckpointNumber.ZERO,
|
|
606
|
+
checkpointedCheckpointNumber: CheckpointNumber.ZERO,
|
|
529
607
|
blockNumber: BlockNumber.ZERO,
|
|
530
608
|
archive,
|
|
531
|
-
|
|
609
|
+
hasProposedCheckpoint: false,
|
|
610
|
+
syncedL2Slot,
|
|
532
611
|
pendingChainValidationStatus,
|
|
533
612
|
};
|
|
534
613
|
}
|
|
535
614
|
|
|
536
|
-
const
|
|
537
|
-
if (!
|
|
615
|
+
const blockData = await this.l2BlockSource.getBlockData(blockNumber);
|
|
616
|
+
if (!blockData) {
|
|
538
617
|
// this shouldn't really happen because a moment ago we checked that all components were in sync
|
|
539
|
-
this.log.error(`Failed to get L2 block ${blockNumber} from the archiver with all components in sync`);
|
|
618
|
+
this.log.error(`Failed to get L2 block data ${blockNumber} from the archiver with all components in sync`);
|
|
540
619
|
return undefined;
|
|
541
620
|
}
|
|
542
621
|
|
|
622
|
+
const hasProposedCheckpoint = l2Tips.proposedCheckpoint.checkpoint.number > l2Tips.checkpointed.checkpoint.number;
|
|
623
|
+
|
|
543
624
|
return {
|
|
544
|
-
|
|
545
|
-
blockNumber:
|
|
546
|
-
checkpointNumber:
|
|
547
|
-
|
|
548
|
-
|
|
625
|
+
blockData,
|
|
626
|
+
blockNumber: blockData.header.getBlockNumber(),
|
|
627
|
+
checkpointNumber: blockData.checkpointNumber,
|
|
628
|
+
checkpointedCheckpointNumber: l2Tips.checkpointed.checkpoint.number,
|
|
629
|
+
archive: blockData.archive.root,
|
|
630
|
+
hasProposedCheckpoint,
|
|
631
|
+
proposedCheckpointData,
|
|
632
|
+
syncedL2Slot,
|
|
549
633
|
pendingChainValidationStatus,
|
|
550
634
|
};
|
|
551
635
|
}
|
|
@@ -554,20 +638,20 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
554
638
|
* Checks if we are the proposer for the next slot.
|
|
555
639
|
* @returns True if we can propose, and the proposer address (undefined if anyone can propose)
|
|
556
640
|
*/
|
|
557
|
-
protected async checkCanPropose(
|
|
641
|
+
protected async checkCanPropose(targetSlot: SlotNumber): Promise<[boolean, EthAddress | undefined]> {
|
|
558
642
|
let proposer: EthAddress | undefined;
|
|
559
643
|
|
|
560
644
|
try {
|
|
561
|
-
proposer = await this.epochCache.getProposerAttesterAddressInSlot(
|
|
645
|
+
proposer = await this.epochCache.getProposerAttesterAddressInSlot(targetSlot);
|
|
562
646
|
} catch (e) {
|
|
563
647
|
if (e instanceof NoCommitteeError) {
|
|
564
|
-
if (this.lastSlotForNoCommitteeWarning !==
|
|
565
|
-
this.lastSlotForNoCommitteeWarning =
|
|
566
|
-
this.log.warn(`Cannot propose at
|
|
648
|
+
if (this.lastSlotForNoCommitteeWarning !== targetSlot) {
|
|
649
|
+
this.lastSlotForNoCommitteeWarning = targetSlot;
|
|
650
|
+
this.log.warn(`Cannot propose at target slot ${targetSlot} since the committee does not exist on L1`);
|
|
567
651
|
}
|
|
568
652
|
return [false, undefined];
|
|
569
653
|
}
|
|
570
|
-
this.log.error(`Error getting proposer for slot ${
|
|
654
|
+
this.log.error(`Error getting proposer for target slot ${targetSlot}`, e);
|
|
571
655
|
return [false, undefined];
|
|
572
656
|
}
|
|
573
657
|
|
|
@@ -584,10 +668,18 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
584
668
|
const weAreProposer = validatorAddresses.some(addr => addr.equals(proposer));
|
|
585
669
|
|
|
586
670
|
if (!weAreProposer) {
|
|
587
|
-
this.log.debug(`Cannot propose at slot ${
|
|
671
|
+
this.log.debug(`Cannot propose at target slot ${targetSlot} since we are not a proposer`, {
|
|
672
|
+
targetSlot,
|
|
673
|
+
validatorAddresses,
|
|
674
|
+
proposer,
|
|
675
|
+
});
|
|
588
676
|
return [false, proposer];
|
|
589
677
|
}
|
|
590
678
|
|
|
679
|
+
this.log.info(`We are the proposer for pipeline slot ${targetSlot}`, {
|
|
680
|
+
targetSlot,
|
|
681
|
+
proposer,
|
|
682
|
+
});
|
|
591
683
|
return [true, proposer];
|
|
592
684
|
}
|
|
593
685
|
|
|
@@ -596,8 +688,8 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
596
688
|
* This allows the sequencer to participate in governance/slashing votes even when it cannot build blocks.
|
|
597
689
|
*/
|
|
598
690
|
@trackSpan('Seqeuencer.tryVoteWhenSyncFails', ({ slot }) => ({ [Attributes.SLOT_NUMBER]: slot }))
|
|
599
|
-
protected async tryVoteWhenSyncFails(args: { slot: SlotNumber; ts: bigint }): Promise<void> {
|
|
600
|
-
const { slot } = args;
|
|
691
|
+
protected async tryVoteWhenSyncFails(args: { slot: SlotNumber; targetSlot: SlotNumber; ts: bigint }): Promise<void> {
|
|
692
|
+
const { slot, targetSlot } = args;
|
|
601
693
|
|
|
602
694
|
// Prevent duplicate attempts in the same slot
|
|
603
695
|
if (this.lastSlotForFallbackVote === slot) {
|
|
@@ -625,7 +717,7 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
625
717
|
});
|
|
626
718
|
|
|
627
719
|
// Check if we're a proposer or proposal is open
|
|
628
|
-
const [canPropose, proposer] = await this.checkCanPropose(
|
|
720
|
+
const [canPropose, proposer] = await this.checkCanPropose(targetSlot);
|
|
629
721
|
if (!canPropose) {
|
|
630
722
|
this.log.trace(`Cannot vote in slot ${slot} since we are not a proposer`, { slot, proposer });
|
|
631
723
|
return;
|
|
@@ -642,9 +734,9 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
642
734
|
slot,
|
|
643
735
|
});
|
|
644
736
|
|
|
645
|
-
// Enqueue governance and slashing votes
|
|
737
|
+
// Enqueue governance and slashing votes (voter uses the target slot for L1 submission)
|
|
646
738
|
const voter = new CheckpointVoter(
|
|
647
|
-
|
|
739
|
+
targetSlot,
|
|
648
740
|
publisher,
|
|
649
741
|
attestorAddress,
|
|
650
742
|
this.validatorClient,
|
|
@@ -724,7 +816,7 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
724
816
|
syncedTo: SequencerSyncCheckResult,
|
|
725
817
|
currentSlot: SlotNumber,
|
|
726
818
|
): Promise<void> {
|
|
727
|
-
const { pendingChainValidationStatus,
|
|
819
|
+
const { pendingChainValidationStatus, syncedL2Slot } = syncedTo;
|
|
728
820
|
if (pendingChainValidationStatus.valid) {
|
|
729
821
|
return;
|
|
730
822
|
}
|
|
@@ -739,7 +831,7 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
739
831
|
|
|
740
832
|
const logData = {
|
|
741
833
|
invalidL1Timestamp: invalidCheckpointTimestamp,
|
|
742
|
-
|
|
834
|
+
syncedL2Slot,
|
|
743
835
|
invalidCheckpoint: pendingChainValidationStatus.checkpoint,
|
|
744
836
|
secondsBeforeInvalidatingBlockAsCommitteeMember,
|
|
745
837
|
secondsBeforeInvalidatingBlockAsNonCommitteeMember,
|
|
@@ -867,6 +959,11 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
867
959
|
return this.validatorClient?.getValidatorAddresses();
|
|
868
960
|
}
|
|
869
961
|
|
|
962
|
+
/** Updates the publisher factory's node keystore adapter after a keystore reload. */
|
|
963
|
+
public updatePublisherNodeKeyStore(adapter: NodeKeystoreAdapter): void {
|
|
964
|
+
this.publisherFactory.updateNodeKeyStore(adapter);
|
|
965
|
+
}
|
|
966
|
+
|
|
870
967
|
public getConfig() {
|
|
871
968
|
return this.config;
|
|
872
969
|
}
|
|
@@ -877,10 +974,13 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
877
974
|
}
|
|
878
975
|
|
|
879
976
|
type SequencerSyncCheckResult = {
|
|
880
|
-
|
|
977
|
+
blockData?: BlockData;
|
|
881
978
|
checkpointNumber: CheckpointNumber;
|
|
979
|
+
checkpointedCheckpointNumber: CheckpointNumber;
|
|
882
980
|
blockNumber: BlockNumber;
|
|
883
981
|
archive: Fr;
|
|
884
|
-
|
|
982
|
+
hasProposedCheckpoint: boolean;
|
|
983
|
+
proposedCheckpointData?: ProposedCheckpointData;
|
|
984
|
+
syncedL2Slot: SlotNumber;
|
|
885
985
|
pendingChainValidationStatus: ValidateCheckpointResult;
|
|
886
986
|
};
|