@aztec/sequencer-client 0.0.1-commit.88c5703d4 → 0.0.1-commit.88e6f9396
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dest/client/sequencer-client.d.ts +1 -13
- package/dest/client/sequencer-client.d.ts.map +1 -1
- package/dest/client/sequencer-client.js +21 -67
- package/dest/config.d.ts +3 -3
- package/dest/config.d.ts.map +1 -1
- package/dest/config.js +2 -4
- package/dest/publisher/sequencer-publisher.d.ts +6 -4
- package/dest/publisher/sequencer-publisher.d.ts.map +1 -1
- package/dest/publisher/sequencer-publisher.js +13 -7
- package/dest/sequencer/checkpoint_proposal_job.d.ts +12 -4
- package/dest/sequencer/checkpoint_proposal_job.d.ts.map +1 -1
- package/dest/sequencer/checkpoint_proposal_job.js +122 -73
- package/dest/sequencer/events.d.ts +2 -1
- package/dest/sequencer/events.d.ts.map +1 -1
- package/dest/sequencer/metrics.d.ts +5 -1
- package/dest/sequencer/metrics.d.ts.map +1 -1
- package/dest/sequencer/metrics.js +11 -0
- package/dest/sequencer/sequencer.d.ts +5 -5
- package/dest/sequencer/sequencer.d.ts.map +1 -1
- package/dest/sequencer/sequencer.js +57 -45
- package/dest/test/mock_checkpoint_builder.d.ts +4 -8
- package/dest/test/mock_checkpoint_builder.d.ts.map +1 -1
- package/package.json +27 -27
- package/src/client/sequencer-client.ts +28 -86
- package/src/config.ts +4 -3
- package/src/publisher/sequencer-publisher.ts +17 -9
- package/src/sequencer/checkpoint_proposal_job.ts +164 -82
- package/src/sequencer/events.ts +1 -1
- package/src/sequencer/metrics.ts +14 -0
- package/src/sequencer/sequencer.ts +82 -51
- package/src/test/mock_checkpoint_builder.ts +3 -3
|
@@ -31,16 +31,21 @@ import {
|
|
|
31
31
|
MaliciousCommitteeAttestationsAndSigners,
|
|
32
32
|
} from '@aztec/stdlib/block';
|
|
33
33
|
import { type Checkpoint, validateCheckpoint } from '@aztec/stdlib/checkpoint';
|
|
34
|
-
import { getSlotStartBuildTimestamp } from '@aztec/stdlib/epoch-helpers';
|
|
34
|
+
import { getSlotStartBuildTimestamp, getTimestampForSlot } from '@aztec/stdlib/epoch-helpers';
|
|
35
35
|
import { Gas } from '@aztec/stdlib/gas';
|
|
36
36
|
import {
|
|
37
|
+
type BlockBuilderOptions,
|
|
37
38
|
InsufficientValidTxsError,
|
|
38
|
-
type PublicProcessorLimits,
|
|
39
39
|
type ResolvedSequencerConfig,
|
|
40
40
|
type WorldStateSynchronizer,
|
|
41
41
|
} from '@aztec/stdlib/interfaces/server';
|
|
42
42
|
import { type L1ToL2MessageSource, computeInHashFromL1ToL2Messages } from '@aztec/stdlib/messaging';
|
|
43
|
-
import type {
|
|
43
|
+
import type {
|
|
44
|
+
BlockProposal,
|
|
45
|
+
BlockProposalOptions,
|
|
46
|
+
CheckpointProposal,
|
|
47
|
+
CheckpointProposalOptions,
|
|
48
|
+
} from '@aztec/stdlib/p2p';
|
|
44
49
|
import { orderAttestations, trimAttestations } from '@aztec/stdlib/p2p';
|
|
45
50
|
import type { L2BlockBuiltStats } from '@aztec/stdlib/stats';
|
|
46
51
|
import { type FailedTx, Tx } from '@aztec/stdlib/tx';
|
|
@@ -72,8 +77,10 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
72
77
|
protected readonly log: Logger;
|
|
73
78
|
|
|
74
79
|
constructor(
|
|
75
|
-
private readonly
|
|
76
|
-
private readonly
|
|
80
|
+
private readonly slotNow: SlotNumber,
|
|
81
|
+
private readonly targetSlot: SlotNumber,
|
|
82
|
+
private readonly epochNow: EpochNumber,
|
|
83
|
+
private readonly targetEpoch: EpochNumber,
|
|
77
84
|
private readonly checkpointNumber: CheckpointNumber,
|
|
78
85
|
private readonly syncedToBlockNumber: BlockNumber,
|
|
79
86
|
// TODO(palla/mbps): Can we remove the proposer in favor of attestorAddress? Need to check fisherman-node flows.
|
|
@@ -101,7 +108,20 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
101
108
|
public readonly tracer: Tracer,
|
|
102
109
|
bindings?: LoggerBindings,
|
|
103
110
|
) {
|
|
104
|
-
this.log = createLogger('sequencer:checkpoint-proposal', {
|
|
111
|
+
this.log = createLogger('sequencer:checkpoint-proposal', {
|
|
112
|
+
...bindings,
|
|
113
|
+
instanceId: `slot-${this.slotNow}`,
|
|
114
|
+
});
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
/** The wall-clock slot during which the proposer builds. */
|
|
118
|
+
private get slot(): SlotNumber {
|
|
119
|
+
return this.slotNow;
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/** The wall-clock epoch. */
|
|
123
|
+
private get epoch(): EpochNumber {
|
|
124
|
+
return this.epochNow;
|
|
105
125
|
}
|
|
106
126
|
|
|
107
127
|
/**
|
|
@@ -114,7 +134,7 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
114
134
|
// In fisherman mode, we simulate slashing but don't actually publish to L1
|
|
115
135
|
// These are constant for the whole slot, so we only enqueue them once
|
|
116
136
|
const votesPromises = new CheckpointVoter(
|
|
117
|
-
this.
|
|
137
|
+
this.targetSlot,
|
|
118
138
|
this.publisher,
|
|
119
139
|
this.attestorAddress,
|
|
120
140
|
this.validatorClient,
|
|
@@ -141,6 +161,29 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
141
161
|
return;
|
|
142
162
|
}
|
|
143
163
|
|
|
164
|
+
// If pipelining, wait until the submission slot so L1 recognizes the pipelined proposer
|
|
165
|
+
if (this.epochCache.isProposerPipeliningEnabled()) {
|
|
166
|
+
const submissionSlotTimestamp =
|
|
167
|
+
getTimestampForSlot(this.targetSlot, this.l1Constants) - BigInt(this.l1Constants.ethereumSlotDuration);
|
|
168
|
+
this.log.info(`Waiting until submission slot ${this.targetSlot} for L1 submission`, {
|
|
169
|
+
slot: this.slot,
|
|
170
|
+
submissionSlot: this.targetSlot,
|
|
171
|
+
submissionSlotTimestamp,
|
|
172
|
+
});
|
|
173
|
+
await sleepUntil(new Date(Number(submissionSlotTimestamp) * 1000), this.dateProvider.nowAsDate());
|
|
174
|
+
|
|
175
|
+
// After waking, verify the parent checkpoint wasn't pruned during the sleep.
|
|
176
|
+
// We check L1's pending tip directly instead of canProposeAt, which also validates the proposer
|
|
177
|
+
// identity and would fail because the timestamp resolves to a different slot's proposer.
|
|
178
|
+
const l1Tips = await this.publisher.rollupContract.getTips();
|
|
179
|
+
if (l1Tips.pending < this.checkpointNumber - 1) {
|
|
180
|
+
this.log.warn(
|
|
181
|
+
`Parent checkpoint was pruned during pipelining sleep (L1 pending=${l1Tips.pending}, expected>=${this.checkpointNumber - 1}), skipping L1 submission for checkpoint ${this.checkpointNumber}`,
|
|
182
|
+
);
|
|
183
|
+
return undefined;
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
|
|
144
187
|
// Then send everything to L1
|
|
145
188
|
const l1Response = await this.publisher.sendRequests();
|
|
146
189
|
const proposedAction = l1Response?.successfulActions.find(a => a === 'propose');
|
|
@@ -159,7 +202,7 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
159
202
|
return {
|
|
160
203
|
// nullish operator needed for tests
|
|
161
204
|
[Attributes.COINBASE]: this.validatorClient.getCoinbaseForAttestor(this.attestorAddress)?.toString(),
|
|
162
|
-
[Attributes.SLOT_NUMBER]: this.
|
|
205
|
+
[Attributes.SLOT_NUMBER]: this.targetSlot,
|
|
163
206
|
};
|
|
164
207
|
})
|
|
165
208
|
private async proposeCheckpoint(): Promise<Checkpoint | undefined> {
|
|
@@ -169,8 +212,15 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
169
212
|
const feeRecipient = this.validatorClient.getFeeRecipientForAttestor(this.attestorAddress);
|
|
170
213
|
|
|
171
214
|
// Start the checkpoint
|
|
172
|
-
this.setStateFn(SequencerState.INITIALIZING_CHECKPOINT, this.
|
|
173
|
-
this.
|
|
215
|
+
this.setStateFn(SequencerState.INITIALIZING_CHECKPOINT, this.targetSlot);
|
|
216
|
+
this.log.info(`Starting checkpoint proposal`, {
|
|
217
|
+
buildSlot: this.slot,
|
|
218
|
+
submissionSlot: this.targetSlot,
|
|
219
|
+
pipelining: this.epochCache.isProposerPipeliningEnabled(),
|
|
220
|
+
proposer: this.proposer?.toString(),
|
|
221
|
+
coinbase: coinbase.toString(),
|
|
222
|
+
});
|
|
223
|
+
this.metrics.incOpenSlot(this.targetSlot, this.proposer?.toString() ?? 'unknown');
|
|
174
224
|
|
|
175
225
|
// Enqueues checkpoint invalidation (constant for the whole slot)
|
|
176
226
|
if (this.invalidateCheckpoint && !this.config.skipInvalidateBlockAsProposer) {
|
|
@@ -181,7 +231,7 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
181
231
|
const checkpointGlobalVariables = await this.globalsBuilder.buildCheckpointGlobalVariables(
|
|
182
232
|
coinbase,
|
|
183
233
|
feeRecipient,
|
|
184
|
-
this.
|
|
234
|
+
this.targetSlot,
|
|
185
235
|
);
|
|
186
236
|
|
|
187
237
|
// Collect L1 to L2 messages for the checkpoint and compute their hash
|
|
@@ -189,7 +239,7 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
189
239
|
const inHash = computeInHashFromL1ToL2Messages(l1ToL2Messages);
|
|
190
240
|
|
|
191
241
|
// Collect the out hashes of all the checkpoints before this one in the same epoch
|
|
192
|
-
const previousCheckpointOutHashes = (await this.l2BlockSource.getCheckpointsDataForEpoch(this.
|
|
242
|
+
const previousCheckpointOutHashes = (await this.l2BlockSource.getCheckpointsDataForEpoch(this.targetEpoch))
|
|
193
243
|
.filter(c => c.checkpointNumber < this.checkpointNumber)
|
|
194
244
|
.map(c => c.checkpointOutHash);
|
|
195
245
|
|
|
@@ -246,8 +296,8 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
246
296
|
}
|
|
247
297
|
|
|
248
298
|
if (blocksInCheckpoint.length === 0) {
|
|
249
|
-
this.log.warn(`No blocks were built for slot ${this.
|
|
250
|
-
this.eventEmitter.emit('checkpoint-empty', { slot: this.
|
|
299
|
+
this.log.warn(`No blocks were built for slot ${this.targetSlot}`, { slot: this.targetSlot });
|
|
300
|
+
this.eventEmitter.emit('checkpoint-empty', { slot: this.targetSlot });
|
|
251
301
|
return undefined;
|
|
252
302
|
}
|
|
253
303
|
|
|
@@ -255,17 +305,18 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
255
305
|
if (minBlocksForCheckpoint !== undefined && blocksInCheckpoint.length < minBlocksForCheckpoint) {
|
|
256
306
|
this.log.warn(
|
|
257
307
|
`Checkpoint has fewer blocks than minimum (${blocksInCheckpoint.length} < ${minBlocksForCheckpoint}), skipping proposal`,
|
|
258
|
-
{ slot: this.
|
|
308
|
+
{ slot: this.targetSlot, blocksBuilt: blocksInCheckpoint.length, minBlocksForCheckpoint },
|
|
259
309
|
);
|
|
260
310
|
return undefined;
|
|
261
311
|
}
|
|
262
312
|
|
|
263
313
|
// Assemble and broadcast the checkpoint proposal, including the last block that was not
|
|
264
314
|
// broadcasted yet, and wait to collect the committee attestations.
|
|
265
|
-
this.setStateFn(SequencerState.ASSEMBLING_CHECKPOINT, this.
|
|
315
|
+
this.setStateFn(SequencerState.ASSEMBLING_CHECKPOINT, this.targetSlot);
|
|
266
316
|
const checkpoint = await checkpointBuilder.completeCheckpoint();
|
|
267
317
|
|
|
268
|
-
// Final validation
|
|
318
|
+
// Final validation: per-block limits are only checked if the operator set them explicitly.
|
|
319
|
+
// Otherwise, checkpoint-level budgets were already enforced by the redistribution logic.
|
|
269
320
|
try {
|
|
270
321
|
validateCheckpoint(checkpoint, {
|
|
271
322
|
rollupManaLimit: this.l1Constants.rollupManaLimit,
|
|
@@ -292,10 +343,10 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
292
343
|
// Do not collect attestations nor publish to L1 in fisherman mode
|
|
293
344
|
if (this.config.fishermanMode) {
|
|
294
345
|
this.log.info(
|
|
295
|
-
`Built checkpoint for slot ${this.
|
|
346
|
+
`Built checkpoint for slot ${this.targetSlot} with ${blocksInCheckpoint.length} blocks. ` +
|
|
296
347
|
`Skipping proposal in fisherman mode.`,
|
|
297
348
|
{
|
|
298
|
-
slot: this.
|
|
349
|
+
slot: this.targetSlot,
|
|
299
350
|
checkpoint: checkpoint.header.toInspect(),
|
|
300
351
|
blocksBuilt: blocksInCheckpoint.length,
|
|
301
352
|
},
|
|
@@ -324,7 +375,7 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
324
375
|
const blockProposedAt = this.dateProvider.now();
|
|
325
376
|
await this.p2pClient.broadcastCheckpointProposal(proposal);
|
|
326
377
|
|
|
327
|
-
this.setStateFn(SequencerState.COLLECTING_ATTESTATIONS, this.
|
|
378
|
+
this.setStateFn(SequencerState.COLLECTING_ATTESTATIONS, this.targetSlot);
|
|
328
379
|
const attestations = await this.waitForAttestations(proposal);
|
|
329
380
|
const blockAttestedAt = this.dateProvider.now();
|
|
330
381
|
|
|
@@ -337,7 +388,7 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
337
388
|
attestationsSignature = await this.validatorClient.signAttestationsAndSigners(
|
|
338
389
|
attestations,
|
|
339
390
|
signer,
|
|
340
|
-
this.
|
|
391
|
+
this.targetSlot,
|
|
341
392
|
this.checkpointNumber,
|
|
342
393
|
);
|
|
343
394
|
} catch (err) {
|
|
@@ -350,10 +401,10 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
350
401
|
}
|
|
351
402
|
|
|
352
403
|
// Enqueue publishing the checkpoint to L1
|
|
353
|
-
this.setStateFn(SequencerState.PUBLISHING_CHECKPOINT, this.
|
|
404
|
+
this.setStateFn(SequencerState.PUBLISHING_CHECKPOINT, this.targetSlot);
|
|
354
405
|
const aztecSlotDuration = this.l1Constants.slotDuration;
|
|
355
|
-
const
|
|
356
|
-
const txTimeoutAt = new Date((
|
|
406
|
+
const submissionSlotStart = Number(getTimestampForSlot(this.targetSlot, this.l1Constants));
|
|
407
|
+
const txTimeoutAt = new Date((submissionSlotStart + aztecSlotDuration) * 1000);
|
|
357
408
|
|
|
358
409
|
// If we have been configured to potentially skip publishing checkpoint then roll the dice here
|
|
359
410
|
if (
|
|
@@ -416,7 +467,7 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
416
467
|
|
|
417
468
|
if (!timingInfo.canStart) {
|
|
418
469
|
this.log.debug(`Not enough time left in slot to start another block`, {
|
|
419
|
-
slot: this.
|
|
470
|
+
slot: this.targetSlot,
|
|
420
471
|
blocksBuilt,
|
|
421
472
|
secondsIntoSlot,
|
|
422
473
|
});
|
|
@@ -451,8 +502,8 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
451
502
|
} else if ('error' in buildResult) {
|
|
452
503
|
// If there was an error building the block, just exit the loop and give up the rest of the slot
|
|
453
504
|
if (!(buildResult.error instanceof SequencerInterruptedError)) {
|
|
454
|
-
this.log.warn(`Halting block building for slot ${this.
|
|
455
|
-
slot: this.
|
|
505
|
+
this.log.warn(`Halting block building for slot ${this.targetSlot}`, {
|
|
506
|
+
slot: this.targetSlot,
|
|
456
507
|
blocksBuilt,
|
|
457
508
|
error: buildResult.error,
|
|
458
509
|
});
|
|
@@ -462,18 +513,14 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
462
513
|
|
|
463
514
|
const { block, usedTxs } = buildResult;
|
|
464
515
|
blocksInCheckpoint.push(block);
|
|
465
|
-
|
|
466
|
-
// Sync the proposed block to the archiver to make it available
|
|
467
|
-
// We wait for the sync to succeed, as this helps catch consistency errors, even if it means we lose some time for block-building
|
|
468
|
-
// If this throws, we abort the entire checkpoint
|
|
469
|
-
await this.syncProposedBlockToArchiver(block);
|
|
470
|
-
|
|
471
516
|
usedTxs.forEach(tx => txHashesAlreadyIncluded.add(tx.txHash.toString()));
|
|
472
517
|
|
|
473
|
-
// If this is the last block,
|
|
518
|
+
// If this is the last block, sync it to the archiver and exit the loop
|
|
519
|
+
// so we can build the checkpoint and start collecting attestations.
|
|
474
520
|
if (timingInfo.isLastBlock) {
|
|
475
|
-
this.
|
|
476
|
-
|
|
521
|
+
await this.syncProposedBlockToArchiver(block);
|
|
522
|
+
this.log.verbose(`Completed final block ${blockNumber} for slot ${this.targetSlot}`, {
|
|
523
|
+
slot: this.targetSlot,
|
|
477
524
|
blockNumber,
|
|
478
525
|
blocksBuilt,
|
|
479
526
|
});
|
|
@@ -481,38 +528,61 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
481
528
|
break;
|
|
482
529
|
}
|
|
483
530
|
|
|
484
|
-
//
|
|
485
|
-
//
|
|
486
|
-
if
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
}
|
|
531
|
+
// Broadcast the block proposal (unless we're in fisherman mode) unless the block is the last one,
|
|
532
|
+
// in which case we'll broadcast it along with the checkpoint at the end of the loop.
|
|
533
|
+
// Note that we only send the block to the archiver if we manage to create the proposal, so if there's
|
|
534
|
+
// a HA error we don't pollute our archiver with a block that won't make it to the chain.
|
|
535
|
+
const proposal = await this.createBlockProposal(block, inHash, usedTxs, blockProposalOptions);
|
|
536
|
+
|
|
537
|
+
// Sync the proposed block to the archiver to make it available, only after we've managed to sign the proposal.
|
|
538
|
+
// We wait for the sync to succeed, as this helps catch consistency errors, even if it means we lose some time for block-building.
|
|
539
|
+
// If this throws, we abort the entire checkpoint.
|
|
540
|
+
await this.syncProposedBlockToArchiver(block);
|
|
541
|
+
|
|
542
|
+
// Once we have a signed proposal and the archiver agreed with our proposed block, then we broadcast it.
|
|
543
|
+
proposal && (await this.p2pClient.broadcastProposal(proposal));
|
|
498
544
|
|
|
499
545
|
// Wait until the next block's start time
|
|
500
546
|
await this.waitUntilNextSubslot(timingInfo.deadline);
|
|
501
547
|
}
|
|
502
548
|
|
|
503
|
-
this.log.verbose(`Block building loop completed for slot ${this.
|
|
504
|
-
slot: this.
|
|
549
|
+
this.log.verbose(`Block building loop completed for slot ${this.targetSlot}`, {
|
|
550
|
+
slot: this.targetSlot,
|
|
505
551
|
blocksBuilt: blocksInCheckpoint.length,
|
|
506
552
|
});
|
|
507
553
|
|
|
508
554
|
return { blocksInCheckpoint, blockPendingBroadcast };
|
|
509
555
|
}
|
|
510
556
|
|
|
557
|
+
/** Creates a block proposal for a given block via the validator client (unless in fisherman mode) */
|
|
558
|
+
private createBlockProposal(
|
|
559
|
+
block: L2Block,
|
|
560
|
+
inHash: Fr,
|
|
561
|
+
usedTxs: Tx[],
|
|
562
|
+
blockProposalOptions: BlockProposalOptions,
|
|
563
|
+
): Promise<BlockProposal | undefined> {
|
|
564
|
+
if (this.config.fishermanMode) {
|
|
565
|
+
this.log.info(`Skipping block proposal for block ${block.number} in fisherman mode`);
|
|
566
|
+
return Promise.resolve(undefined);
|
|
567
|
+
}
|
|
568
|
+
return this.validatorClient.createBlockProposal(
|
|
569
|
+
block.header,
|
|
570
|
+
block.indexWithinCheckpoint,
|
|
571
|
+
inHash,
|
|
572
|
+
block.archive.root,
|
|
573
|
+
usedTxs,
|
|
574
|
+
this.proposer,
|
|
575
|
+
blockProposalOptions,
|
|
576
|
+
);
|
|
577
|
+
}
|
|
578
|
+
|
|
511
579
|
/** Sleeps until it is time to produce the next block in the slot */
|
|
512
580
|
@trackSpan('CheckpointProposalJob.waitUntilNextSubslot')
|
|
513
581
|
private async waitUntilNextSubslot(nextSubslotStart: number) {
|
|
514
|
-
this.setStateFn(SequencerState.WAITING_UNTIL_NEXT_BLOCK, this.
|
|
515
|
-
this.log.verbose(`Waiting until time for the next block at ${nextSubslotStart}s into slot`, {
|
|
582
|
+
this.setStateFn(SequencerState.WAITING_UNTIL_NEXT_BLOCK, this.targetSlot);
|
|
583
|
+
this.log.verbose(`Waiting until time for the next block at ${nextSubslotStart}s into slot`, {
|
|
584
|
+
slot: this.targetSlot,
|
|
585
|
+
});
|
|
516
586
|
await this.waitUntilTimeInSlot(nextSubslotStart);
|
|
517
587
|
}
|
|
518
588
|
|
|
@@ -533,7 +603,7 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
533
603
|
opts;
|
|
534
604
|
|
|
535
605
|
this.log.verbose(
|
|
536
|
-
`Preparing block ${blockNumber} index ${indexWithinCheckpoint} at checkpoint ${this.checkpointNumber} for slot ${this.
|
|
606
|
+
`Preparing block ${blockNumber} index ${indexWithinCheckpoint} at checkpoint ${this.checkpointNumber} for slot ${this.targetSlot}`,
|
|
537
607
|
{ ...checkpointBuilder.getConstantData(), ...opts },
|
|
538
608
|
);
|
|
539
609
|
|
|
@@ -542,10 +612,10 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
542
612
|
const { availableTxs, canStartBuilding, minTxs } = await this.waitForMinTxs(opts);
|
|
543
613
|
if (!canStartBuilding) {
|
|
544
614
|
this.log.warn(
|
|
545
|
-
`Not enough txs to build block ${blockNumber} at index ${indexWithinCheckpoint} in slot ${this.
|
|
546
|
-
{ blockNumber, slot: this.
|
|
615
|
+
`Not enough txs to build block ${blockNumber} at index ${indexWithinCheckpoint} in slot ${this.targetSlot} (got ${availableTxs} txs but needs ${minTxs})`,
|
|
616
|
+
{ blockNumber, slot: this.targetSlot, indexWithinCheckpoint },
|
|
547
617
|
);
|
|
548
|
-
this.eventEmitter.emit('block-tx-count-check-failed', { minTxs, availableTxs, slot: this.
|
|
618
|
+
this.eventEmitter.emit('block-tx-count-check-failed', { minTxs, availableTxs, slot: this.targetSlot });
|
|
549
619
|
this.metrics.recordBlockProposalFailed('insufficient_txs');
|
|
550
620
|
return undefined;
|
|
551
621
|
}
|
|
@@ -558,16 +628,16 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
558
628
|
);
|
|
559
629
|
|
|
560
630
|
this.log.debug(
|
|
561
|
-
`Building block ${blockNumber} at index ${indexWithinCheckpoint} for slot ${this.
|
|
562
|
-
{ slot: this.
|
|
631
|
+
`Building block ${blockNumber} at index ${indexWithinCheckpoint} for slot ${this.targetSlot} with ${availableTxs} available txs`,
|
|
632
|
+
{ slot: this.targetSlot, blockNumber, indexWithinCheckpoint },
|
|
563
633
|
);
|
|
564
|
-
this.setStateFn(SequencerState.CREATING_BLOCK, this.
|
|
634
|
+
this.setStateFn(SequencerState.CREATING_BLOCK, this.targetSlot);
|
|
565
635
|
|
|
566
|
-
// Per-block limits
|
|
636
|
+
// Per-block limits are operator overrides (from SEQ_MAX_L2_BLOCK_GAS etc.) further capped
|
|
567
637
|
// by remaining checkpoint-level budgets inside CheckpointBuilder before each block is built.
|
|
568
638
|
// minValidTxs is passed into the builder so it can reject the block *before* updating state.
|
|
569
639
|
const minValidTxs = forceCreate ? 0 : (this.config.minValidTxsPerBlock ?? minTxs);
|
|
570
|
-
const blockBuilderOptions:
|
|
640
|
+
const blockBuilderOptions: BlockBuilderOptions = {
|
|
571
641
|
maxTransactions: this.config.maxTxsPerBlock,
|
|
572
642
|
maxBlockGas:
|
|
573
643
|
this.config.maxL2BlockGas !== undefined || this.config.maxDABlockGas !== undefined
|
|
@@ -576,6 +646,8 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
576
646
|
deadline: buildDeadline,
|
|
577
647
|
isBuildingProposal: true,
|
|
578
648
|
minValidTxs,
|
|
649
|
+
maxBlocksPerCheckpoint: this.timetable.maxNumberOfBlocks,
|
|
650
|
+
perBlockAllocationMultiplier: this.config.perBlockAllocationMultiplier,
|
|
579
651
|
};
|
|
580
652
|
|
|
581
653
|
// Actually build the block by executing txs. The builder throws InsufficientValidTxsError
|
|
@@ -594,16 +666,19 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
594
666
|
|
|
595
667
|
if (buildResult.status === 'insufficient-valid-txs') {
|
|
596
668
|
this.log.warn(
|
|
597
|
-
`Block ${blockNumber} at index ${indexWithinCheckpoint} on slot ${this.
|
|
669
|
+
`Block ${blockNumber} at index ${indexWithinCheckpoint} on slot ${this.targetSlot} has too few valid txs to be proposed`,
|
|
598
670
|
{
|
|
599
|
-
slot: this.
|
|
671
|
+
slot: this.targetSlot,
|
|
600
672
|
blockNumber,
|
|
601
673
|
numTxs: buildResult.processedCount,
|
|
602
674
|
indexWithinCheckpoint,
|
|
603
675
|
minValidTxs,
|
|
604
676
|
},
|
|
605
677
|
);
|
|
606
|
-
this.eventEmitter.emit('block-build-failed', {
|
|
678
|
+
this.eventEmitter.emit('block-build-failed', {
|
|
679
|
+
reason: `Insufficient valid txs`,
|
|
680
|
+
slot: this.targetSlot,
|
|
681
|
+
});
|
|
607
682
|
this.metrics.recordBlockProposalFailed('insufficient_valid_txs');
|
|
608
683
|
return undefined;
|
|
609
684
|
}
|
|
@@ -623,17 +698,24 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
623
698
|
const manaPerSec = block.header.totalManaUsed.toNumberUnsafe() / (blockBuildDuration / 1000);
|
|
624
699
|
|
|
625
700
|
this.log.info(
|
|
626
|
-
`Built block ${block.number} at checkpoint ${this.checkpointNumber} for slot ${this.
|
|
701
|
+
`Built block ${block.number} at checkpoint ${this.checkpointNumber} for slot ${this.targetSlot} with ${numTxs} txs`,
|
|
627
702
|
{ blockHash, txHashes, manaPerSec, ...blockStats },
|
|
628
703
|
);
|
|
629
704
|
|
|
630
|
-
this.eventEmitter.emit('block-proposed', {
|
|
705
|
+
this.eventEmitter.emit('block-proposed', {
|
|
706
|
+
blockNumber: block.number,
|
|
707
|
+
slot: this.targetSlot,
|
|
708
|
+
buildSlot: this.slotNow,
|
|
709
|
+
});
|
|
631
710
|
this.metrics.recordBuiltBlock(blockBuildDuration, block.header.totalManaUsed.toNumberUnsafe());
|
|
632
711
|
|
|
633
712
|
return { block, usedTxs };
|
|
634
713
|
} catch (err: any) {
|
|
635
|
-
this.eventEmitter.emit('block-build-failed', {
|
|
636
|
-
|
|
714
|
+
this.eventEmitter.emit('block-build-failed', {
|
|
715
|
+
reason: err.message,
|
|
716
|
+
slot: this.targetSlot,
|
|
717
|
+
});
|
|
718
|
+
this.log.error(`Error building block`, err, { blockNumber, slot: this.targetSlot });
|
|
637
719
|
this.metrics.recordBlockProposalFailed(err.name || 'unknown_error');
|
|
638
720
|
this.metrics.recordFailedBlock();
|
|
639
721
|
return { error: err };
|
|
@@ -646,7 +728,7 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
646
728
|
pendingTxs: AsyncIterable<Tx>,
|
|
647
729
|
blockNumber: BlockNumber,
|
|
648
730
|
blockTimestamp: bigint,
|
|
649
|
-
blockBuilderOptions:
|
|
731
|
+
blockBuilderOptions: BlockBuilderOptions,
|
|
650
732
|
) {
|
|
651
733
|
try {
|
|
652
734
|
const workTimer = new Timer();
|
|
@@ -693,10 +775,10 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
693
775
|
}
|
|
694
776
|
|
|
695
777
|
// Wait a bit before checking again
|
|
696
|
-
this.setStateFn(SequencerState.WAITING_FOR_TXS, this.
|
|
778
|
+
this.setStateFn(SequencerState.WAITING_FOR_TXS, this.targetSlot);
|
|
697
779
|
this.log.verbose(
|
|
698
|
-
`Waiting for enough txs to build block ${blockNumber} at index ${indexWithinCheckpoint} in slot ${this.
|
|
699
|
-
{ blockNumber, slot: this.
|
|
780
|
+
`Waiting for enough txs to build block ${blockNumber} at index ${indexWithinCheckpoint} in slot ${this.targetSlot} (have ${availableTxs} but need ${minTxs})`,
|
|
781
|
+
{ blockNumber, slot: this.targetSlot, indexWithinCheckpoint },
|
|
700
782
|
);
|
|
701
783
|
await this.waitForTxsPollingInterval();
|
|
702
784
|
availableTxs = await this.p2pClient.getPendingTxCount();
|
|
@@ -896,19 +978,19 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
896
978
|
private async handleCheckpointEndAsFisherman(checkpoint: Checkpoint | undefined) {
|
|
897
979
|
// Perform L1 fee analysis before clearing requests
|
|
898
980
|
// The callback is invoked asynchronously after the next block is mined
|
|
899
|
-
const feeAnalysis = await this.publisher.analyzeL1Fees(this.
|
|
981
|
+
const feeAnalysis = await this.publisher.analyzeL1Fees(this.targetSlot, analysis =>
|
|
900
982
|
this.metrics.recordFishermanFeeAnalysis(analysis),
|
|
901
983
|
);
|
|
902
984
|
|
|
903
985
|
if (checkpoint) {
|
|
904
|
-
this.log.info(`Validation checkpoint building SUCCEEDED for slot ${this.
|
|
986
|
+
this.log.info(`Validation checkpoint building SUCCEEDED for slot ${this.targetSlot}`, {
|
|
905
987
|
...checkpoint.toCheckpointInfo(),
|
|
906
988
|
...checkpoint.getStats(),
|
|
907
989
|
feeAnalysisId: feeAnalysis?.id,
|
|
908
990
|
});
|
|
909
991
|
} else {
|
|
910
|
-
this.log.warn(`Validation block building FAILED for slot ${this.
|
|
911
|
-
slot: this.
|
|
992
|
+
this.log.warn(`Validation block building FAILED for slot ${this.targetSlot}`, {
|
|
993
|
+
slot: this.targetSlot,
|
|
912
994
|
feeAnalysisId: feeAnalysis?.id,
|
|
913
995
|
});
|
|
914
996
|
this.metrics.recordCheckpointProposalFailed('block_build_failed');
|
|
@@ -922,15 +1004,15 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
922
1004
|
*/
|
|
923
1005
|
private handleHASigningError(err: any, errorContext: string): boolean {
|
|
924
1006
|
if (err instanceof DutyAlreadySignedError) {
|
|
925
|
-
this.log.info(`${errorContext} for slot ${this.
|
|
926
|
-
slot: this.
|
|
1007
|
+
this.log.info(`${errorContext} for slot ${this.targetSlot} already signed by another HA node, yielding`, {
|
|
1008
|
+
slot: this.targetSlot,
|
|
927
1009
|
signedByNode: err.signedByNode,
|
|
928
1010
|
});
|
|
929
1011
|
return true;
|
|
930
1012
|
}
|
|
931
1013
|
if (err instanceof SlashingProtectionError) {
|
|
932
|
-
this.log.info(`${errorContext} for slot ${this.
|
|
933
|
-
slot: this.
|
|
1014
|
+
this.log.info(`${errorContext} for slot ${this.targetSlot} blocked by slashing protection, yielding`, {
|
|
1015
|
+
slot: this.targetSlot,
|
|
934
1016
|
existingMessageHash: err.existingMessageHash,
|
|
935
1017
|
attemptedMessageHash: err.attemptedMessageHash,
|
|
936
1018
|
});
|
package/src/sequencer/events.ts
CHANGED
|
@@ -13,7 +13,7 @@ export type SequencerEvents = {
|
|
|
13
13
|
['proposer-rollup-check-failed']: (args: { reason: string; slot: SlotNumber }) => void;
|
|
14
14
|
['block-tx-count-check-failed']: (args: { minTxs: number; availableTxs: number; slot: SlotNumber }) => void;
|
|
15
15
|
['block-build-failed']: (args: { reason: string; slot: SlotNumber }) => void;
|
|
16
|
-
['block-proposed']: (args: { blockNumber: BlockNumber; slot: SlotNumber }) => void;
|
|
16
|
+
['block-proposed']: (args: { blockNumber: BlockNumber; slot: SlotNumber; buildSlot: SlotNumber }) => void;
|
|
17
17
|
['checkpoint-empty']: (args: { slot: SlotNumber }) => void;
|
|
18
18
|
['checkpoint-publish-failed']: (args: {
|
|
19
19
|
slot: SlotNumber;
|
package/src/sequencer/metrics.ts
CHANGED
|
@@ -49,6 +49,8 @@ export class SequencerMetrics {
|
|
|
49
49
|
private checkpointBlockCount: Gauge;
|
|
50
50
|
private checkpointTxCount: Gauge;
|
|
51
51
|
private checkpointTotalMana: Gauge;
|
|
52
|
+
private pipelineDepth: Gauge;
|
|
53
|
+
private pipelineDiscards: UpDownCounter;
|
|
52
54
|
|
|
53
55
|
// Fisherman fee analysis metrics
|
|
54
56
|
private fishermanWouldBeIncluded: UpDownCounter;
|
|
@@ -143,6 +145,10 @@ export class SequencerMetrics {
|
|
|
143
145
|
|
|
144
146
|
this.slashingAttempts = createUpDownCounterWithDefault(this.meter, Metrics.SEQUENCER_SLASHING_ATTEMPTS_COUNT);
|
|
145
147
|
|
|
148
|
+
this.pipelineDepth = this.meter.createGauge(Metrics.SEQUENCER_PIPELINE_DEPTH);
|
|
149
|
+
this.pipelineDiscards = createUpDownCounterWithDefault(this.meter, Metrics.SEQUENCER_PIPELINE_DISCARDS_COUNT);
|
|
150
|
+
this.pipelineDepth.record(0);
|
|
151
|
+
|
|
146
152
|
// Fisherman fee analysis metrics
|
|
147
153
|
this.fishermanWouldBeIncluded = createUpDownCounterWithDefault(
|
|
148
154
|
this.meter,
|
|
@@ -234,6 +240,14 @@ export class SequencerMetrics {
|
|
|
234
240
|
});
|
|
235
241
|
}
|
|
236
242
|
|
|
243
|
+
recordPipelineDepth(depth: number) {
|
|
244
|
+
this.pipelineDepth.record(depth);
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
recordPipelineDiscard(count = 1) {
|
|
248
|
+
this.pipelineDiscards.add(count);
|
|
249
|
+
}
|
|
250
|
+
|
|
237
251
|
incOpenSlot(slot: SlotNumber, proposer: string) {
|
|
238
252
|
// sequencer went through the loop a second time. Noop
|
|
239
253
|
if (slot === this.lastSeenSlot) {
|