@aztec/sequencer-client 0.0.1-commit.e2b2873ed → 0.0.1-commit.e304674f1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dest/client/sequencer-client.d.ts +15 -7
- package/dest/client/sequencer-client.d.ts.map +1 -1
- package/dest/client/sequencer-client.js +60 -30
- package/dest/config.d.ts +26 -6
- package/dest/config.d.ts.map +1 -1
- package/dest/config.js +44 -21
- package/dest/global_variable_builder/global_builder.d.ts +15 -11
- package/dest/global_variable_builder/global_builder.d.ts.map +1 -1
- package/dest/global_variable_builder/global_builder.js +29 -25
- package/dest/global_variable_builder/index.d.ts +2 -2
- package/dest/global_variable_builder/index.d.ts.map +1 -1
- package/dest/publisher/config.d.ts +47 -17
- package/dest/publisher/config.d.ts.map +1 -1
- package/dest/publisher/config.js +121 -42
- package/dest/publisher/index.d.ts +2 -1
- package/dest/publisher/index.d.ts.map +1 -1
- package/dest/publisher/l1_tx_failed_store/factory.d.ts +11 -0
- package/dest/publisher/l1_tx_failed_store/factory.d.ts.map +1 -0
- package/dest/publisher/l1_tx_failed_store/factory.js +22 -0
- package/dest/publisher/l1_tx_failed_store/failed_tx_store.d.ts +59 -0
- package/dest/publisher/l1_tx_failed_store/failed_tx_store.d.ts.map +1 -0
- package/dest/publisher/l1_tx_failed_store/failed_tx_store.js +1 -0
- package/dest/publisher/l1_tx_failed_store/file_store_failed_tx_store.d.ts +15 -0
- package/dest/publisher/l1_tx_failed_store/file_store_failed_tx_store.d.ts.map +1 -0
- package/dest/publisher/l1_tx_failed_store/file_store_failed_tx_store.js +34 -0
- package/dest/publisher/l1_tx_failed_store/index.d.ts +4 -0
- package/dest/publisher/l1_tx_failed_store/index.d.ts.map +1 -0
- package/dest/publisher/l1_tx_failed_store/index.js +2 -0
- package/dest/publisher/sequencer-publisher-factory.d.ts +11 -5
- package/dest/publisher/sequencer-publisher-factory.d.ts.map +1 -1
- package/dest/publisher/sequencer-publisher-factory.js +27 -3
- package/dest/publisher/sequencer-publisher.d.ts +82 -37
- package/dest/publisher/sequencer-publisher.d.ts.map +1 -1
- package/dest/publisher/sequencer-publisher.js +430 -118
- package/dest/sequencer/checkpoint_proposal_job.d.ts +36 -9
- package/dest/sequencer/checkpoint_proposal_job.d.ts.map +1 -1
- package/dest/sequencer/checkpoint_proposal_job.js +361 -192
- package/dest/sequencer/checkpoint_voter.d.ts +1 -2
- package/dest/sequencer/checkpoint_voter.d.ts.map +1 -1
- package/dest/sequencer/checkpoint_voter.js +2 -5
- package/dest/sequencer/events.d.ts +2 -1
- package/dest/sequencer/events.d.ts.map +1 -1
- package/dest/sequencer/metrics.d.ts +21 -5
- package/dest/sequencer/metrics.d.ts.map +1 -1
- package/dest/sequencer/metrics.js +97 -15
- package/dest/sequencer/sequencer.d.ts +40 -17
- package/dest/sequencer/sequencer.d.ts.map +1 -1
- package/dest/sequencer/sequencer.js +152 -95
- package/dest/sequencer/timetable.d.ts +7 -3
- package/dest/sequencer/timetable.d.ts.map +1 -1
- package/dest/sequencer/timetable.js +21 -12
- package/dest/sequencer/types.d.ts +2 -2
- package/dest/sequencer/types.d.ts.map +1 -1
- package/dest/test/index.d.ts +3 -5
- package/dest/test/index.d.ts.map +1 -1
- package/dest/test/mock_checkpoint_builder.d.ts +11 -11
- package/dest/test/mock_checkpoint_builder.d.ts.map +1 -1
- package/dest/test/mock_checkpoint_builder.js +45 -34
- package/dest/test/utils.d.ts +3 -3
- package/dest/test/utils.d.ts.map +1 -1
- package/dest/test/utils.js +5 -4
- package/package.json +27 -28
- package/src/client/sequencer-client.ts +76 -30
- package/src/config.ts +56 -27
- package/src/global_variable_builder/global_builder.ts +38 -27
- package/src/global_variable_builder/index.ts +1 -1
- package/src/publisher/config.ts +153 -43
- package/src/publisher/index.ts +3 -0
- package/src/publisher/l1_tx_failed_store/factory.ts +32 -0
- package/src/publisher/l1_tx_failed_store/failed_tx_store.ts +55 -0
- package/src/publisher/l1_tx_failed_store/file_store_failed_tx_store.ts +46 -0
- package/src/publisher/l1_tx_failed_store/index.ts +3 -0
- package/src/publisher/sequencer-publisher-factory.ts +38 -9
- package/src/publisher/sequencer-publisher.ts +503 -168
- package/src/sequencer/README.md +81 -12
- package/src/sequencer/checkpoint_proposal_job.ts +471 -201
- package/src/sequencer/checkpoint_voter.ts +1 -12
- package/src/sequencer/events.ts +1 -1
- package/src/sequencer/metrics.ts +106 -18
- package/src/sequencer/sequencer.ts +216 -109
- package/src/sequencer/timetable.ts +26 -15
- package/src/sequencer/types.ts +1 -1
- package/src/test/index.ts +2 -4
- package/src/test/mock_checkpoint_builder.ts +63 -49
- package/src/test/utils.ts +5 -2
|
@@ -12,9 +12,9 @@ import type { DateProvider } from '@aztec/foundation/timer';
|
|
|
12
12
|
import type { TypedEventEmitter } from '@aztec/foundation/types';
|
|
13
13
|
import type { P2P } from '@aztec/p2p';
|
|
14
14
|
import type { SlasherClientInterface } from '@aztec/slasher';
|
|
15
|
-
import type {
|
|
16
|
-
import type { Checkpoint } from '@aztec/stdlib/checkpoint';
|
|
17
|
-
import {
|
|
15
|
+
import type { BlockData, L2BlockSink, L2BlockSource, ValidateCheckpointResult } from '@aztec/stdlib/block';
|
|
16
|
+
import type { Checkpoint, ProposedCheckpointData } from '@aztec/stdlib/checkpoint';
|
|
17
|
+
import { getSlotStartBuildTimestamp } from '@aztec/stdlib/epoch-helpers';
|
|
18
18
|
import {
|
|
19
19
|
type ResolvedSequencerConfig,
|
|
20
20
|
type SequencerConfig,
|
|
@@ -25,7 +25,7 @@ import type { L1ToL2MessageSource } from '@aztec/stdlib/messaging';
|
|
|
25
25
|
import { pickFromSchema } from '@aztec/stdlib/schemas';
|
|
26
26
|
import { MerkleTreeId } from '@aztec/stdlib/trees';
|
|
27
27
|
import { Attributes, type TelemetryClient, type Tracer, getTelemetryClient, trackSpan } from '@aztec/telemetry-client';
|
|
28
|
-
import { FullNodeCheckpointsBuilder, type ValidatorClient } from '@aztec/validator-client';
|
|
28
|
+
import { FullNodeCheckpointsBuilder, NodeKeystoreAdapter, type ValidatorClient } from '@aztec/validator-client';
|
|
29
29
|
|
|
30
30
|
import EventEmitter from 'node:events';
|
|
31
31
|
|
|
@@ -72,17 +72,12 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
72
72
|
/** The last epoch for which we logged strategy comparison in fisherman mode. */
|
|
73
73
|
private lastEpochForStrategyComparison: EpochNumber | undefined;
|
|
74
74
|
|
|
75
|
+
/** The last checkpoint proposal job, tracked so we can await its pending L1 submission during shutdown. */
|
|
76
|
+
private lastCheckpointProposalJob: CheckpointProposalJob | undefined;
|
|
77
|
+
|
|
75
78
|
/** The maximum number of seconds that the sequencer can be into a slot to transition to a particular state. */
|
|
76
79
|
protected timetable!: SequencerTimetable;
|
|
77
80
|
|
|
78
|
-
// This shouldn't be here as this gets re-created each time we build/propose a block.
|
|
79
|
-
// But we have a number of tests that abuse/rely on this class having a permanent publisher.
|
|
80
|
-
// As long as those tests only configure a single publisher they will continue to work.
|
|
81
|
-
// This will get re-assigned every time the sequencer goes to build a new block to a publisher that is valid
|
|
82
|
-
// for the block proposer.
|
|
83
|
-
// TODO(palla/mbps): Remove this field and fix tests
|
|
84
|
-
protected publisher: SequencerPublisher | undefined;
|
|
85
|
-
|
|
86
81
|
/** Config for the sequencer */
|
|
87
82
|
protected config: ResolvedSequencerConfig = DefaultSequencerConfig;
|
|
88
83
|
|
|
@@ -118,7 +113,7 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
118
113
|
/** Updates sequencer config by the defined values and updates the timetable */
|
|
119
114
|
public updateConfig(config: Partial<SequencerConfig>) {
|
|
120
115
|
const filteredConfig = pickFromSchema(config, SequencerConfigSchema);
|
|
121
|
-
this.log.info(`Updated sequencer config`, omit(filteredConfig, '
|
|
116
|
+
this.log.info(`Updated sequencer config`, omit(filteredConfig, 'txPublicSetupAllowListExtend'));
|
|
122
117
|
this.config = merge(this.config, filteredConfig);
|
|
123
118
|
this.timetable = new SequencerTimetable(
|
|
124
119
|
{
|
|
@@ -128,16 +123,16 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
128
123
|
p2pPropagationTime: this.config.attestationPropagationTime,
|
|
129
124
|
blockDurationMs: this.config.blockDurationMs,
|
|
130
125
|
enforce: this.config.enforceTimeTable,
|
|
126
|
+
pipelining: this.epochCache.isProposerPipeliningEnabled(),
|
|
131
127
|
},
|
|
132
128
|
this.metrics,
|
|
133
129
|
this.log,
|
|
134
130
|
);
|
|
135
131
|
}
|
|
136
132
|
|
|
137
|
-
/** Initializes the sequencer (precomputes tables
|
|
138
|
-
public
|
|
133
|
+
/** Initializes the sequencer (precomputes tables). Takes about 3s. */
|
|
134
|
+
public init() {
|
|
139
135
|
getKzg();
|
|
140
|
-
this.publisher = (await this.publisherFactory.create(undefined)).publisher;
|
|
141
136
|
}
|
|
142
137
|
|
|
143
138
|
/** Starts the sequencer and moves to IDLE state. */
|
|
@@ -156,8 +151,9 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
156
151
|
public async stop(): Promise<void> {
|
|
157
152
|
this.log.info(`Stopping sequencer`);
|
|
158
153
|
this.setState(SequencerState.STOPPING, undefined, { force: true });
|
|
159
|
-
this.
|
|
154
|
+
await this.publisherFactory.stopAll();
|
|
160
155
|
await this.runningPromise?.stop();
|
|
156
|
+
await this.lastCheckpointProposalJob?.awaitPendingSubmission();
|
|
161
157
|
this.setState(SequencerState.STOPPED, undefined, { force: true });
|
|
162
158
|
this.log.info('Stopped sequencer');
|
|
163
159
|
}
|
|
@@ -169,7 +165,6 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
169
165
|
} catch (err) {
|
|
170
166
|
this.emit('checkpoint-error', { error: err as Error });
|
|
171
167
|
if (err instanceof SequencerTooSlowError) {
|
|
172
|
-
// TODO(palla/mbps): Add missing states
|
|
173
168
|
// Log as warn only if we had to abort halfway through the block proposal
|
|
174
169
|
const logLvl = [SequencerState.INITIALIZING_CHECKPOINT, SequencerState.PROPOSER_CHECK].includes(
|
|
175
170
|
err.proposedState,
|
|
@@ -202,14 +197,25 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
202
197
|
@trackSpan('Sequencer.work')
|
|
203
198
|
protected async work() {
|
|
204
199
|
this.setState(SequencerState.SYNCHRONIZING, undefined);
|
|
205
|
-
const { slot, ts,
|
|
200
|
+
const { slot, ts, nowSeconds, epoch } = this.epochCache.getEpochAndSlotInNextL1Slot();
|
|
201
|
+
const { slot: targetSlot, epoch: targetEpoch } = this.epochCache.getTargetEpochAndSlotInNextL1Slot();
|
|
206
202
|
|
|
207
203
|
// Check if we are synced and it's our slot, grab a publisher, check previous block invalidation, etc
|
|
208
|
-
const checkpointProposalJob = await this.prepareCheckpointProposal(
|
|
204
|
+
const checkpointProposalJob = await this.prepareCheckpointProposal(
|
|
205
|
+
slot,
|
|
206
|
+
targetSlot,
|
|
207
|
+
epoch,
|
|
208
|
+
targetEpoch,
|
|
209
|
+
ts,
|
|
210
|
+
nowSeconds,
|
|
211
|
+
);
|
|
209
212
|
if (!checkpointProposalJob) {
|
|
210
213
|
return;
|
|
211
214
|
}
|
|
212
215
|
|
|
216
|
+
// Track the job so we can await its pending L1 submission during shutdown
|
|
217
|
+
this.lastCheckpointProposalJob = checkpointProposalJob;
|
|
218
|
+
|
|
213
219
|
// Execute the checkpoint proposal job
|
|
214
220
|
const checkpoint = await checkpointProposalJob.execute();
|
|
215
221
|
|
|
@@ -218,13 +224,13 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
218
224
|
this.lastCheckpointProposed = checkpoint;
|
|
219
225
|
}
|
|
220
226
|
|
|
221
|
-
// Log fee strategy comparison if on fisherman
|
|
227
|
+
// Log fee strategy comparison if on fisherman (uses target epoch since we mirror the proposer's perspective)
|
|
222
228
|
if (
|
|
223
229
|
this.config.fishermanMode &&
|
|
224
|
-
(this.lastEpochForStrategyComparison === undefined ||
|
|
230
|
+
(this.lastEpochForStrategyComparison === undefined || targetEpoch > this.lastEpochForStrategyComparison)
|
|
225
231
|
) {
|
|
226
|
-
this.logStrategyComparison(
|
|
227
|
-
this.lastEpochForStrategyComparison =
|
|
232
|
+
this.logStrategyComparison(targetEpoch, checkpointProposalJob.getPublisher());
|
|
233
|
+
this.lastEpochForStrategyComparison = targetEpoch;
|
|
228
234
|
}
|
|
229
235
|
|
|
230
236
|
return checkpoint;
|
|
@@ -236,44 +242,49 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
236
242
|
* @returns CheckpointProposalJob if successful, undefined if we are not yet synced or are not the proposer.
|
|
237
243
|
*/
|
|
238
244
|
@trackSpan('Sequencer.prepareCheckpointProposal')
|
|
239
|
-
|
|
240
|
-
epoch: EpochNumber,
|
|
245
|
+
protected async prepareCheckpointProposal(
|
|
241
246
|
slot: SlotNumber,
|
|
247
|
+
targetSlot: SlotNumber,
|
|
248
|
+
epoch: EpochNumber,
|
|
249
|
+
targetEpoch: EpochNumber,
|
|
242
250
|
ts: bigint,
|
|
243
|
-
|
|
251
|
+
nowSeconds: bigint,
|
|
244
252
|
): Promise<CheckpointProposalJob | undefined> {
|
|
245
|
-
// Check we have not already processed this slot (cheapest check)
|
|
253
|
+
// Check we have not already processed this target slot (cheapest check)
|
|
246
254
|
// We only check this if enforce timetable is set, since we want to keep processing the same slot if we are not
|
|
247
255
|
// running against actual time (eg when we use sandbox-style automining)
|
|
248
256
|
if (
|
|
249
257
|
this.lastSlotForCheckpointProposalJob &&
|
|
250
|
-
this.lastSlotForCheckpointProposalJob >=
|
|
258
|
+
this.lastSlotForCheckpointProposalJob >= targetSlot &&
|
|
251
259
|
this.config.enforceTimeTable
|
|
252
260
|
) {
|
|
253
|
-
this.log.trace(`
|
|
261
|
+
this.log.trace(`Target slot ${targetSlot} has already been processed`);
|
|
254
262
|
return undefined;
|
|
255
263
|
}
|
|
256
264
|
|
|
257
|
-
// But if we have already proposed for this slot,
|
|
258
|
-
if (this.lastCheckpointProposed && this.lastCheckpointProposed.header.slotNumber >=
|
|
259
|
-
this.log.trace(
|
|
265
|
+
// But if we have already proposed for this slot, then we definitely have to skip it, automining or not
|
|
266
|
+
if (this.lastCheckpointProposed && this.lastCheckpointProposed.header.slotNumber >= targetSlot) {
|
|
267
|
+
this.log.trace(
|
|
268
|
+
`Slot ${targetSlot} has already been published as checkpoint ${this.lastCheckpointProposed.number}`,
|
|
269
|
+
);
|
|
260
270
|
return undefined;
|
|
261
271
|
}
|
|
262
272
|
|
|
263
273
|
// Check all components are synced to latest as seen by the archiver (queries all subsystems)
|
|
264
274
|
const syncedTo = await this.checkSync({ ts, slot });
|
|
265
275
|
if (!syncedTo) {
|
|
266
|
-
await this.tryVoteWhenSyncFails({ slot, ts });
|
|
276
|
+
await this.tryVoteWhenSyncFails({ slot, targetSlot, ts });
|
|
267
277
|
return undefined;
|
|
268
278
|
}
|
|
269
279
|
|
|
270
|
-
// If escape hatch is open for
|
|
280
|
+
// If escape hatch is open for the target epoch, do not start checkpoint proposal work and do not attempt invalidations.
|
|
271
281
|
// Still perform governance/slashing voting (as proposer) once per slot.
|
|
272
|
-
|
|
282
|
+
// When pipelining, we check the target epoch (slot+1's epoch) since that's the epoch we're building for.
|
|
283
|
+
const isEscapeHatchOpen = await this.epochCache.isEscapeHatchOpen(targetEpoch);
|
|
273
284
|
|
|
274
285
|
if (isEscapeHatchOpen) {
|
|
275
286
|
this.setState(SequencerState.PROPOSER_CHECK, slot);
|
|
276
|
-
const [canPropose, proposer] = await this.checkCanPropose(
|
|
287
|
+
const [canPropose, proposer] = await this.checkCanPropose(targetSlot);
|
|
277
288
|
if (canPropose) {
|
|
278
289
|
await this.tryVoteWhenEscapeHatchOpen({ slot, proposer });
|
|
279
290
|
} else {
|
|
@@ -290,18 +301,18 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
290
301
|
const checkpointNumber = CheckpointNumber(syncedTo.checkpointNumber + 1);
|
|
291
302
|
|
|
292
303
|
const logCtx = {
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
syncedToL2Slot: getSlotAtTimestamp(syncedTo.l1Timestamp, this.l1Constants),
|
|
304
|
+
nowSeconds,
|
|
305
|
+
syncedToL2Slot: syncedTo.syncedL2Slot,
|
|
296
306
|
slot,
|
|
307
|
+
targetSlot,
|
|
297
308
|
slotTs: ts,
|
|
298
309
|
checkpointNumber,
|
|
299
310
|
isPendingChainValid: pick(syncedTo.pendingChainValidationStatus, 'valid', 'reason', 'invalidIndex'),
|
|
300
311
|
};
|
|
301
312
|
|
|
302
|
-
// Check that we are a proposer for the
|
|
313
|
+
// Check that we are a proposer for the target slot.
|
|
303
314
|
this.setState(SequencerState.PROPOSER_CHECK, slot);
|
|
304
|
-
const [canPropose, proposer] = await this.checkCanPropose(
|
|
315
|
+
const [canPropose, proposer] = await this.checkCanPropose(targetSlot);
|
|
305
316
|
|
|
306
317
|
// If we are not a proposer check if we should invalidate an invalid checkpoint, and bail
|
|
307
318
|
if (!canPropose) {
|
|
@@ -309,13 +320,23 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
309
320
|
return undefined;
|
|
310
321
|
}
|
|
311
322
|
|
|
312
|
-
//
|
|
313
|
-
|
|
323
|
+
// Guard: don't exceed 1-deep pipeline. Without a proposed checkpoint, we can only build
|
|
324
|
+
// confirmed + 1. With a proposed checkpoint, we can build confirmed + 2.
|
|
325
|
+
const confirmedCkpt = syncedTo.checkpointedCheckpointNumber;
|
|
326
|
+
if (checkpointNumber > confirmedCkpt + 2) {
|
|
327
|
+
this.log.verbose(
|
|
328
|
+
`Skipping slot ${targetSlot}: checkpoint ${checkpointNumber} exceeds max pipeline depth (confirmed=${confirmedCkpt})`,
|
|
329
|
+
);
|
|
330
|
+
return undefined;
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
// Check that the target slot is not taken by a block already (should never happen, since only us can propose for this slot)
|
|
334
|
+
if (syncedTo.blockData && syncedTo.blockData.header.getSlot() >= targetSlot) {
|
|
314
335
|
this.log.warn(
|
|
315
|
-
`Cannot propose block at
|
|
316
|
-
{ ...logCtx, block: syncedTo.
|
|
336
|
+
`Cannot propose block at target slot ${targetSlot} since that slot was taken by block ${syncedTo.blockNumber}`,
|
|
337
|
+
{ ...logCtx, block: syncedTo.blockData.header.toInspect() },
|
|
317
338
|
);
|
|
318
|
-
this.metrics.
|
|
339
|
+
this.metrics.recordCheckpointPrecheckFailed('slot_already_taken');
|
|
319
340
|
return undefined;
|
|
320
341
|
}
|
|
321
342
|
|
|
@@ -326,7 +347,6 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
326
347
|
const proposerForPublisher = this.config.fishermanMode ? undefined : proposer;
|
|
327
348
|
const { attestorAddress, publisher } = await this.publisherFactory.create(proposerForPublisher);
|
|
328
349
|
this.log.verbose(`Created publisher at address ${publisher.getSenderAddress()} for attestor ${attestorAddress}`);
|
|
329
|
-
this.publisher = publisher;
|
|
330
350
|
|
|
331
351
|
// In fisherman mode, set the actual proposer's address for simulations
|
|
332
352
|
if (this.config.fishermanMode && proposer) {
|
|
@@ -335,15 +355,41 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
335
355
|
}
|
|
336
356
|
|
|
337
357
|
// Prepare invalidation request if the pending chain is invalid (returns undefined if no need)
|
|
338
|
-
|
|
358
|
+
let invalidateCheckpoint = await publisher.simulateInvalidateCheckpoint(syncedTo.pendingChainValidationStatus);
|
|
359
|
+
|
|
360
|
+
// Determine the correct archive and L1 state overrides for the canProposeAt check.
|
|
361
|
+
// The L1 contract reads archives[proposedCheckpointNumber] and compares it with the provided archive.
|
|
362
|
+
// When invalidating or pipelining, the local archive may differ from L1's, so we adjust accordingly.
|
|
363
|
+
let archiveForCheck = syncedTo.archive;
|
|
364
|
+
const l1Overrides: {
|
|
365
|
+
forcePendingCheckpointNumber?: CheckpointNumber;
|
|
366
|
+
forceArchive?: { checkpointNumber: CheckpointNumber; archive: Fr };
|
|
367
|
+
} = {};
|
|
368
|
+
|
|
369
|
+
if (this.epochCache.isProposerPipeliningEnabled() && syncedTo.hasProposedCheckpoint) {
|
|
370
|
+
// Parent checkpoint hasn't landed on L1 yet. Override both the proposed checkpoint number
|
|
371
|
+
// and the archive at that checkpoint so L1 simulation sees the correct chain tip.
|
|
372
|
+
const parentCheckpointNumber = CheckpointNumber(checkpointNumber - 1);
|
|
373
|
+
l1Overrides.forcePendingCheckpointNumber = parentCheckpointNumber;
|
|
374
|
+
l1Overrides.forceArchive = { checkpointNumber: parentCheckpointNumber, archive: syncedTo.archive };
|
|
375
|
+
this.metrics.recordPipelineDepth(1);
|
|
376
|
+
|
|
377
|
+
this.log.verbose(
|
|
378
|
+
`Building on top of proposed checkpoint (pending=${syncedTo.proposedCheckpointData?.checkpointNumber})`,
|
|
379
|
+
);
|
|
380
|
+
// Clear the invalidation - the proposed checkpoint should handle it.
|
|
381
|
+
invalidateCheckpoint = undefined;
|
|
382
|
+
} else if (invalidateCheckpoint) {
|
|
383
|
+
// After invalidation, L1 will roll back to checkpoint N-1. The archive at N-1 already
|
|
384
|
+
// exists on L1, so we just pass the matching archive (the lastArchive of the invalid checkpoint).
|
|
385
|
+
archiveForCheck = invalidateCheckpoint.lastArchive;
|
|
386
|
+
l1Overrides.forcePendingCheckpointNumber = invalidateCheckpoint.forcePendingCheckpointNumber;
|
|
387
|
+
this.metrics.recordPipelineDepth(0);
|
|
388
|
+
} else {
|
|
389
|
+
this.metrics.recordPipelineDepth(0);
|
|
390
|
+
}
|
|
339
391
|
|
|
340
|
-
|
|
341
|
-
// if all the previous checks are good, but we do it just in case.
|
|
342
|
-
const canProposeCheck = await publisher.canProposeAtNextEthBlock(
|
|
343
|
-
syncedTo.archive,
|
|
344
|
-
proposer ?? EthAddress.ZERO,
|
|
345
|
-
invalidateCheckpoint,
|
|
346
|
-
);
|
|
392
|
+
const canProposeCheck = await publisher.canProposeAt(archiveForCheck, proposer ?? EthAddress.ZERO, l1Overrides);
|
|
347
393
|
|
|
348
394
|
if (canProposeCheck === undefined) {
|
|
349
395
|
this.log.warn(
|
|
@@ -351,17 +397,17 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
351
397
|
logCtx,
|
|
352
398
|
);
|
|
353
399
|
this.emit('proposer-rollup-check-failed', { reason: 'Rollup contract check failed', slot });
|
|
354
|
-
this.metrics.
|
|
400
|
+
this.metrics.recordCheckpointPrecheckFailed('rollup_contract_check_failed');
|
|
355
401
|
return undefined;
|
|
356
402
|
}
|
|
357
403
|
|
|
358
|
-
if (canProposeCheck.slot !==
|
|
404
|
+
if (canProposeCheck.slot !== targetSlot) {
|
|
359
405
|
this.log.warn(
|
|
360
|
-
`Cannot propose block due to slot mismatch with rollup contract (this can be caused by a clock out of sync). Expected slot ${
|
|
361
|
-
{ ...logCtx, rollup: canProposeCheck, expectedSlot:
|
|
406
|
+
`Cannot propose block due to slot mismatch with rollup contract (this can be caused by a clock out of sync). Expected slot ${targetSlot} but got ${canProposeCheck.slot}.`,
|
|
407
|
+
{ ...logCtx, rollup: canProposeCheck, expectedSlot: targetSlot },
|
|
362
408
|
);
|
|
363
409
|
this.emit('proposer-rollup-check-failed', { reason: 'Slot mismatch', slot });
|
|
364
|
-
this.metrics.
|
|
410
|
+
this.metrics.recordCheckpointPrecheckFailed('slot_mismatch');
|
|
365
411
|
return undefined;
|
|
366
412
|
}
|
|
367
413
|
|
|
@@ -371,40 +417,53 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
371
417
|
{ ...logCtx, rollup: canProposeCheck, expectedSlot: slot },
|
|
372
418
|
);
|
|
373
419
|
this.emit('proposer-rollup-check-failed', { reason: 'Block mismatch', slot });
|
|
374
|
-
this.metrics.
|
|
420
|
+
this.metrics.recordCheckpointPrecheckFailed('block_number_mismatch');
|
|
375
421
|
return undefined;
|
|
376
422
|
}
|
|
377
423
|
|
|
378
|
-
this.lastSlotForCheckpointProposalJob =
|
|
379
|
-
|
|
380
|
-
this.
|
|
424
|
+
this.lastSlotForCheckpointProposalJob = targetSlot;
|
|
425
|
+
|
|
426
|
+
await this.p2pClient.prepareForSlot(targetSlot);
|
|
427
|
+
this.log.info(
|
|
428
|
+
`Preparing checkpoint proposal ${checkpointNumber} for target slot ${targetSlot} during wall-clock slot ${slot}`,
|
|
429
|
+
{
|
|
430
|
+
...logCtx,
|
|
431
|
+
proposer,
|
|
432
|
+
pipeliningEnabled: this.epochCache.isProposerPipeliningEnabled(),
|
|
433
|
+
},
|
|
434
|
+
);
|
|
381
435
|
|
|
382
436
|
// Create and return the checkpoint proposal job
|
|
383
437
|
return this.createCheckpointProposalJob(
|
|
384
|
-
epoch,
|
|
385
438
|
slot,
|
|
439
|
+
targetSlot,
|
|
440
|
+
targetEpoch,
|
|
386
441
|
checkpointNumber,
|
|
387
442
|
syncedTo.blockNumber,
|
|
388
443
|
proposer,
|
|
389
444
|
publisher,
|
|
390
445
|
attestorAddress,
|
|
391
446
|
invalidateCheckpoint,
|
|
447
|
+
syncedTo.proposedCheckpointData,
|
|
392
448
|
);
|
|
393
449
|
}
|
|
394
450
|
|
|
395
451
|
protected createCheckpointProposalJob(
|
|
396
|
-
epoch: EpochNumber,
|
|
397
452
|
slot: SlotNumber,
|
|
453
|
+
targetSlot: SlotNumber,
|
|
454
|
+
targetEpoch: EpochNumber,
|
|
398
455
|
checkpointNumber: CheckpointNumber,
|
|
399
456
|
syncedToBlockNumber: BlockNumber,
|
|
400
457
|
proposer: EthAddress | undefined,
|
|
401
458
|
publisher: SequencerPublisher,
|
|
402
459
|
attestorAddress: EthAddress,
|
|
403
460
|
invalidateCheckpoint: InvalidateCheckpointRequest | undefined,
|
|
461
|
+
proposedCheckpointData?: ProposedCheckpointData,
|
|
404
462
|
): CheckpointProposalJob {
|
|
405
463
|
return new CheckpointProposalJob(
|
|
406
|
-
epoch,
|
|
407
464
|
slot,
|
|
465
|
+
targetSlot,
|
|
466
|
+
targetEpoch,
|
|
408
467
|
checkpointNumber,
|
|
409
468
|
syncedToBlockNumber,
|
|
410
469
|
proposer,
|
|
@@ -430,9 +489,17 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
430
489
|
this.setState.bind(this),
|
|
431
490
|
this.tracer,
|
|
432
491
|
this.log.getBindings(),
|
|
492
|
+
proposedCheckpointData,
|
|
433
493
|
);
|
|
434
494
|
}
|
|
435
495
|
|
|
496
|
+
/**
|
|
497
|
+
* Returns the current sequencer state.
|
|
498
|
+
*/
|
|
499
|
+
public getState(): SequencerState {
|
|
500
|
+
return this.state;
|
|
501
|
+
}
|
|
502
|
+
|
|
436
503
|
/**
|
|
437
504
|
* Internal helper for setting the sequencer state and checks if we have enough time left in the slot to transition to the new state.
|
|
438
505
|
* @param proposedState - The new state to transition to.
|
|
@@ -479,16 +546,15 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
479
546
|
* We don't check against the previous block submitted since it may have been reorg'd out.
|
|
480
547
|
*/
|
|
481
548
|
protected async checkSync(args: { ts: bigint; slot: SlotNumber }): Promise<SequencerSyncCheckResult | undefined> {
|
|
482
|
-
// Check that the archiver
|
|
483
|
-
//
|
|
484
|
-
//
|
|
485
|
-
const
|
|
486
|
-
const { slot
|
|
487
|
-
if (
|
|
549
|
+
// Check that the archiver has fully synced the L2 slot before the one we want to propose in.
|
|
550
|
+
// The archiver reports sync progress via L1 block timestamps and synced checkpoint slots.
|
|
551
|
+
// See getSyncedL2SlotNumber for how missed L1 blocks are handled.
|
|
552
|
+
const syncedL2Slot = await this.l2BlockSource.getSyncedL2SlotNumber();
|
|
553
|
+
const { slot } = args;
|
|
554
|
+
if (syncedL2Slot === undefined || syncedL2Slot + 1 < slot) {
|
|
488
555
|
this.log.debug(`Cannot propose block at next L2 slot ${slot} due to pending sync from L1`, {
|
|
489
556
|
slot,
|
|
490
|
-
|
|
491
|
-
l1Timestamp,
|
|
557
|
+
syncedL2Slot,
|
|
492
558
|
});
|
|
493
559
|
return undefined;
|
|
494
560
|
}
|
|
@@ -498,25 +564,43 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
498
564
|
number: syncSummary.latestBlockNumber,
|
|
499
565
|
hash: syncSummary.latestBlockHash,
|
|
500
566
|
})),
|
|
501
|
-
this.l2BlockSource
|
|
567
|
+
this.l2BlockSource
|
|
568
|
+
.getL2Tips()
|
|
569
|
+
.then(t => ({ proposed: t.proposed, checkpointed: t.checkpointed, proposedCheckpoint: t.proposedCheckpoint })),
|
|
502
570
|
this.p2pClient.getStatus().then(p2p => p2p.syncedToL2Block),
|
|
503
|
-
this.l1ToL2MessageSource.getL2Tips().then(t => t.proposed),
|
|
571
|
+
this.l1ToL2MessageSource.getL2Tips().then(t => ({ proposed: t.proposed, checkpointed: t.checkpointed })),
|
|
504
572
|
this.l2BlockSource.getPendingChainValidationStatus(),
|
|
573
|
+
this.l2BlockSource.getProposedCheckpointOnly(),
|
|
505
574
|
] as const);
|
|
506
575
|
|
|
507
|
-
const [worldState,
|
|
576
|
+
const [worldState, l2Tips, p2p, l1ToL2MessageSourceTips, pendingChainValidationStatus, proposedCheckpointData] =
|
|
577
|
+
syncedBlocks;
|
|
508
578
|
|
|
509
579
|
// Handle zero as a special case, since the block hash won't match across services if we're changing the prefilled data for the genesis block,
|
|
510
580
|
// as the world state can compute the new genesis block hash, but other components use the hardcoded constant.
|
|
511
581
|
// TODO(palla/mbps): Fix the above. All components should be able to handle dynamic genesis block hashes.
|
|
512
582
|
const result =
|
|
513
|
-
(
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
583
|
+
(l2Tips.proposed.number === 0 &&
|
|
584
|
+
l2Tips.checkpointed.block.number === 0 &&
|
|
585
|
+
l2Tips.checkpointed.checkpoint.number === 0 &&
|
|
586
|
+
worldState.number === 0 &&
|
|
587
|
+
p2p.number === 0 &&
|
|
588
|
+
l1ToL2MessageSourceTips.proposed.number === 0 &&
|
|
589
|
+
l1ToL2MessageSourceTips.checkpointed.block.number === 0 &&
|
|
590
|
+
l1ToL2MessageSourceTips.checkpointed.checkpoint.number === 0) ||
|
|
591
|
+
(worldState.hash === l2Tips.proposed.hash &&
|
|
592
|
+
p2p.hash === l2Tips.proposed.hash &&
|
|
593
|
+
l1ToL2MessageSourceTips.proposed.hash === l2Tips.proposed.hash &&
|
|
594
|
+
l1ToL2MessageSourceTips.checkpointed.block.hash === l2Tips.checkpointed.block.hash &&
|
|
595
|
+
l1ToL2MessageSourceTips.checkpointed.checkpoint.hash === l2Tips.checkpointed.checkpoint.hash);
|
|
517
596
|
|
|
518
597
|
if (!result) {
|
|
519
|
-
this.log.debug(`Sequencer sync check failed`, {
|
|
598
|
+
this.log.debug(`Sequencer sync check failed`, {
|
|
599
|
+
worldState,
|
|
600
|
+
l2BlockSource: l2Tips.proposed,
|
|
601
|
+
p2p,
|
|
602
|
+
l1ToL2MessageSourceTips,
|
|
603
|
+
});
|
|
520
604
|
return undefined;
|
|
521
605
|
}
|
|
522
606
|
|
|
@@ -526,26 +610,33 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
526
610
|
const archive = new Fr((await this.worldState.getCommitted().getTreeInfo(MerkleTreeId.ARCHIVE)).root);
|
|
527
611
|
return {
|
|
528
612
|
checkpointNumber: CheckpointNumber.ZERO,
|
|
613
|
+
checkpointedCheckpointNumber: CheckpointNumber.ZERO,
|
|
529
614
|
blockNumber: BlockNumber.ZERO,
|
|
530
615
|
archive,
|
|
531
|
-
|
|
616
|
+
hasProposedCheckpoint: false,
|
|
617
|
+
syncedL2Slot,
|
|
532
618
|
pendingChainValidationStatus,
|
|
533
619
|
};
|
|
534
620
|
}
|
|
535
621
|
|
|
536
|
-
const
|
|
537
|
-
if (!
|
|
622
|
+
const blockData = await this.l2BlockSource.getBlockData(blockNumber);
|
|
623
|
+
if (!blockData) {
|
|
538
624
|
// this shouldn't really happen because a moment ago we checked that all components were in sync
|
|
539
|
-
this.log.error(`Failed to get L2 block ${blockNumber} from the archiver with all components in sync`);
|
|
625
|
+
this.log.error(`Failed to get L2 block data ${blockNumber} from the archiver with all components in sync`);
|
|
540
626
|
return undefined;
|
|
541
627
|
}
|
|
542
628
|
|
|
629
|
+
const hasProposedCheckpoint = l2Tips.proposedCheckpoint.checkpoint.number > l2Tips.checkpointed.checkpoint.number;
|
|
630
|
+
|
|
543
631
|
return {
|
|
544
|
-
|
|
545
|
-
blockNumber:
|
|
546
|
-
checkpointNumber:
|
|
547
|
-
|
|
548
|
-
|
|
632
|
+
blockData,
|
|
633
|
+
blockNumber: blockData.header.getBlockNumber(),
|
|
634
|
+
checkpointNumber: blockData.checkpointNumber,
|
|
635
|
+
checkpointedCheckpointNumber: l2Tips.checkpointed.checkpoint.number,
|
|
636
|
+
archive: blockData.archive.root,
|
|
637
|
+
hasProposedCheckpoint,
|
|
638
|
+
proposedCheckpointData,
|
|
639
|
+
syncedL2Slot,
|
|
549
640
|
pendingChainValidationStatus,
|
|
550
641
|
};
|
|
551
642
|
}
|
|
@@ -554,20 +645,20 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
554
645
|
* Checks if we are the proposer for the next slot.
|
|
555
646
|
* @returns True if we can propose, and the proposer address (undefined if anyone can propose)
|
|
556
647
|
*/
|
|
557
|
-
protected async checkCanPropose(
|
|
648
|
+
protected async checkCanPropose(targetSlot: SlotNumber): Promise<[boolean, EthAddress | undefined]> {
|
|
558
649
|
let proposer: EthAddress | undefined;
|
|
559
650
|
|
|
560
651
|
try {
|
|
561
|
-
proposer = await this.epochCache.getProposerAttesterAddressInSlot(
|
|
652
|
+
proposer = await this.epochCache.getProposerAttesterAddressInSlot(targetSlot);
|
|
562
653
|
} catch (e) {
|
|
563
654
|
if (e instanceof NoCommitteeError) {
|
|
564
|
-
if (this.lastSlotForNoCommitteeWarning !==
|
|
565
|
-
this.lastSlotForNoCommitteeWarning =
|
|
566
|
-
this.log.warn(`Cannot propose at
|
|
655
|
+
if (this.lastSlotForNoCommitteeWarning !== targetSlot) {
|
|
656
|
+
this.lastSlotForNoCommitteeWarning = targetSlot;
|
|
657
|
+
this.log.warn(`Cannot propose at target slot ${targetSlot} since the committee does not exist on L1`);
|
|
567
658
|
}
|
|
568
659
|
return [false, undefined];
|
|
569
660
|
}
|
|
570
|
-
this.log.error(`Error getting proposer for slot ${
|
|
661
|
+
this.log.error(`Error getting proposer for target slot ${targetSlot}`, e);
|
|
571
662
|
return [false, undefined];
|
|
572
663
|
}
|
|
573
664
|
|
|
@@ -584,10 +675,18 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
584
675
|
const weAreProposer = validatorAddresses.some(addr => addr.equals(proposer));
|
|
585
676
|
|
|
586
677
|
if (!weAreProposer) {
|
|
587
|
-
this.log.debug(`Cannot propose at slot ${
|
|
678
|
+
this.log.debug(`Cannot propose at target slot ${targetSlot} since we are not a proposer`, {
|
|
679
|
+
targetSlot,
|
|
680
|
+
validatorAddresses,
|
|
681
|
+
proposer,
|
|
682
|
+
});
|
|
588
683
|
return [false, proposer];
|
|
589
684
|
}
|
|
590
685
|
|
|
686
|
+
this.log.info(`We are the proposer for pipeline slot ${targetSlot}`, {
|
|
687
|
+
targetSlot,
|
|
688
|
+
proposer,
|
|
689
|
+
});
|
|
591
690
|
return [true, proposer];
|
|
592
691
|
}
|
|
593
692
|
|
|
@@ -596,8 +695,8 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
596
695
|
* This allows the sequencer to participate in governance/slashing votes even when it cannot build blocks.
|
|
597
696
|
*/
|
|
598
697
|
@trackSpan('Seqeuencer.tryVoteWhenSyncFails', ({ slot }) => ({ [Attributes.SLOT_NUMBER]: slot }))
|
|
599
|
-
protected async tryVoteWhenSyncFails(args: { slot: SlotNumber; ts: bigint }): Promise<void> {
|
|
600
|
-
const { slot } = args;
|
|
698
|
+
protected async tryVoteWhenSyncFails(args: { slot: SlotNumber; targetSlot: SlotNumber; ts: bigint }): Promise<void> {
|
|
699
|
+
const { slot, targetSlot } = args;
|
|
601
700
|
|
|
602
701
|
// Prevent duplicate attempts in the same slot
|
|
603
702
|
if (this.lastSlotForFallbackVote === slot) {
|
|
@@ -625,7 +724,7 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
625
724
|
});
|
|
626
725
|
|
|
627
726
|
// Check if we're a proposer or proposal is open
|
|
628
|
-
const [canPropose, proposer] = await this.checkCanPropose(
|
|
727
|
+
const [canPropose, proposer] = await this.checkCanPropose(targetSlot);
|
|
629
728
|
if (!canPropose) {
|
|
630
729
|
this.log.trace(`Cannot vote in slot ${slot} since we are not a proposer`, { slot, proposer });
|
|
631
730
|
return;
|
|
@@ -642,9 +741,9 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
642
741
|
slot,
|
|
643
742
|
});
|
|
644
743
|
|
|
645
|
-
// Enqueue governance and slashing votes
|
|
744
|
+
// Enqueue governance and slashing votes (voter uses the target slot for L1 submission)
|
|
646
745
|
const voter = new CheckpointVoter(
|
|
647
|
-
|
|
746
|
+
targetSlot,
|
|
648
747
|
publisher,
|
|
649
748
|
attestorAddress,
|
|
650
749
|
this.validatorClient,
|
|
@@ -724,7 +823,7 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
724
823
|
syncedTo: SequencerSyncCheckResult,
|
|
725
824
|
currentSlot: SlotNumber,
|
|
726
825
|
): Promise<void> {
|
|
727
|
-
const { pendingChainValidationStatus,
|
|
826
|
+
const { pendingChainValidationStatus, syncedL2Slot } = syncedTo;
|
|
728
827
|
if (pendingChainValidationStatus.valid) {
|
|
729
828
|
return;
|
|
730
829
|
}
|
|
@@ -739,7 +838,7 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
739
838
|
|
|
740
839
|
const logData = {
|
|
741
840
|
invalidL1Timestamp: invalidCheckpointTimestamp,
|
|
742
|
-
|
|
841
|
+
syncedL2Slot,
|
|
743
842
|
invalidCheckpoint: pendingChainValidationStatus.checkpoint,
|
|
744
843
|
secondsBeforeInvalidatingBlockAsCommitteeMember,
|
|
745
844
|
secondsBeforeInvalidatingBlockAsNonCommitteeMember,
|
|
@@ -867,6 +966,11 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
867
966
|
return this.validatorClient?.getValidatorAddresses();
|
|
868
967
|
}
|
|
869
968
|
|
|
969
|
+
/** Updates the publisher factory's node keystore adapter after a keystore reload. */
|
|
970
|
+
public updatePublisherNodeKeyStore(adapter: NodeKeystoreAdapter): void {
|
|
971
|
+
this.publisherFactory.updateNodeKeyStore(adapter);
|
|
972
|
+
}
|
|
973
|
+
|
|
870
974
|
public getConfig() {
|
|
871
975
|
return this.config;
|
|
872
976
|
}
|
|
@@ -877,10 +981,13 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
877
981
|
}
|
|
878
982
|
|
|
879
983
|
type SequencerSyncCheckResult = {
|
|
880
|
-
|
|
984
|
+
blockData?: BlockData;
|
|
881
985
|
checkpointNumber: CheckpointNumber;
|
|
986
|
+
checkpointedCheckpointNumber: CheckpointNumber;
|
|
882
987
|
blockNumber: BlockNumber;
|
|
883
988
|
archive: Fr;
|
|
884
|
-
|
|
989
|
+
hasProposedCheckpoint: boolean;
|
|
990
|
+
proposedCheckpointData?: ProposedCheckpointData;
|
|
991
|
+
syncedL2Slot: SlotNumber;
|
|
885
992
|
pendingChainValidationStatus: ValidateCheckpointResult;
|
|
886
993
|
};
|