@aztec/sequencer-client 0.0.1-commit.d431d1c → 0.0.1-commit.d6f2b3f94
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dest/client/sequencer-client.js +1 -1
- package/dest/config.d.ts +1 -2
- package/dest/config.d.ts.map +1 -1
- package/dest/config.js +5 -11
- package/dest/global_variable_builder/global_builder.js +2 -2
- package/dest/publisher/sequencer-publisher-metrics.d.ts +1 -1
- package/dest/publisher/sequencer-publisher-metrics.d.ts.map +1 -1
- package/dest/publisher/sequencer-publisher-metrics.js +12 -4
- package/dest/publisher/sequencer-publisher.d.ts +1 -2
- package/dest/publisher/sequencer-publisher.d.ts.map +1 -1
- package/dest/publisher/sequencer-publisher.js +39 -18
- package/dest/sequencer/checkpoint_proposal_job.d.ts +32 -9
- package/dest/sequencer/checkpoint_proposal_job.d.ts.map +1 -1
- package/dest/sequencer/checkpoint_proposal_job.js +83 -51
- package/dest/sequencer/metrics.d.ts +2 -2
- package/dest/sequencer/metrics.d.ts.map +1 -1
- package/dest/sequencer/metrics.js +27 -17
- package/dest/sequencer/sequencer.d.ts +5 -3
- package/dest/sequencer/sequencer.d.ts.map +1 -1
- package/dest/sequencer/sequencer.js +8 -3
- package/dest/sequencer/timetable.d.ts +1 -4
- package/dest/sequencer/timetable.d.ts.map +1 -1
- package/dest/sequencer/timetable.js +1 -4
- package/dest/test/mock_checkpoint_builder.d.ts +11 -8
- package/dest/test/mock_checkpoint_builder.d.ts.map +1 -1
- package/dest/test/mock_checkpoint_builder.js +18 -6
- package/dest/test/utils.d.ts +8 -8
- package/dest/test/utils.d.ts.map +1 -1
- package/dest/test/utils.js +5 -5
- package/package.json +28 -28
- package/src/client/sequencer-client.ts +1 -1
- package/src/config.ts +10 -14
- package/src/global_variable_builder/global_builder.ts +2 -2
- package/src/publisher/sequencer-publisher-metrics.ts +7 -3
- package/src/publisher/sequencer-publisher.ts +34 -18
- package/src/sequencer/checkpoint_proposal_job.ts +113 -77
- package/src/sequencer/metrics.ts +36 -18
- package/src/sequencer/sequencer.ts +12 -5
- package/src/sequencer/timetable.ts +6 -5
- package/src/test/mock_checkpoint_builder.ts +32 -18
- package/src/test/utils.ts +17 -11
|
@@ -1,22 +1,28 @@
|
|
|
1
1
|
import { NUM_CHECKPOINT_END_MARKER_FIELDS, getNumBlockEndBlobFields } from '@aztec/blob-lib/encoding';
|
|
2
2
|
import { BLOBS_PER_CHECKPOINT, FIELDS_PER_BLOB } from '@aztec/constants';
|
|
3
3
|
import type { EpochCache } from '@aztec/epoch-cache';
|
|
4
|
-
import {
|
|
4
|
+
import {
|
|
5
|
+
BlockNumber,
|
|
6
|
+
CheckpointNumber,
|
|
7
|
+
EpochNumber,
|
|
8
|
+
IndexWithinCheckpoint,
|
|
9
|
+
SlotNumber,
|
|
10
|
+
} from '@aztec/foundation/branded-types';
|
|
5
11
|
import { randomInt } from '@aztec/foundation/crypto/random';
|
|
6
12
|
import { Fr } from '@aztec/foundation/curves/bn254';
|
|
7
13
|
import { EthAddress } from '@aztec/foundation/eth-address';
|
|
8
14
|
import { Signature } from '@aztec/foundation/eth-signature';
|
|
9
15
|
import { filter } from '@aztec/foundation/iterator';
|
|
10
|
-
import type
|
|
16
|
+
import { type Logger, type LoggerBindings, createLogger } from '@aztec/foundation/log';
|
|
11
17
|
import { sleep, sleepUntil } from '@aztec/foundation/sleep';
|
|
12
18
|
import { type DateProvider, Timer } from '@aztec/foundation/timer';
|
|
13
|
-
import { type TypedEventEmitter, unfreeze } from '@aztec/foundation/types';
|
|
19
|
+
import { type TypedEventEmitter, isErrorClass, unfreeze } from '@aztec/foundation/types';
|
|
14
20
|
import type { P2P } from '@aztec/p2p';
|
|
15
21
|
import type { SlasherClientInterface } from '@aztec/slasher';
|
|
16
22
|
import {
|
|
17
23
|
CommitteeAttestation,
|
|
18
24
|
CommitteeAttestationsAndSigners,
|
|
19
|
-
|
|
25
|
+
L2Block,
|
|
20
26
|
type L2BlockSink,
|
|
21
27
|
type L2BlockSource,
|
|
22
28
|
MaliciousCommitteeAttestationsAndSigners,
|
|
@@ -24,10 +30,11 @@ import {
|
|
|
24
30
|
import type { Checkpoint } from '@aztec/stdlib/checkpoint';
|
|
25
31
|
import { getSlotStartBuildTimestamp } from '@aztec/stdlib/epoch-helpers';
|
|
26
32
|
import { Gas } from '@aztec/stdlib/gas';
|
|
27
|
-
import
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
33
|
+
import {
|
|
34
|
+
NoValidTxsError,
|
|
35
|
+
type PublicProcessorLimits,
|
|
36
|
+
type ResolvedSequencerConfig,
|
|
37
|
+
type WorldStateSynchronizer,
|
|
31
38
|
} from '@aztec/stdlib/interfaces/server';
|
|
32
39
|
import { type L1ToL2MessageSource, computeInHashFromL1ToL2Messages } from '@aztec/stdlib/messaging';
|
|
33
40
|
import type { BlockProposalOptions, CheckpointProposal, CheckpointProposalOptions } from '@aztec/stdlib/p2p';
|
|
@@ -59,6 +66,8 @@ const TXS_POLLING_MS = 500;
|
|
|
59
66
|
* the Sequencer once the check for being the proposer for the slot has succeeded.
|
|
60
67
|
*/
|
|
61
68
|
export class CheckpointProposalJob implements Traceable {
|
|
69
|
+
protected readonly log: Logger;
|
|
70
|
+
|
|
62
71
|
constructor(
|
|
63
72
|
private readonly epoch: EpochNumber,
|
|
64
73
|
private readonly slot: SlotNumber,
|
|
@@ -86,9 +95,11 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
86
95
|
private readonly metrics: SequencerMetrics,
|
|
87
96
|
private readonly eventEmitter: TypedEventEmitter<SequencerEvents>,
|
|
88
97
|
private readonly setStateFn: (state: SequencerState, slot?: SlotNumber) => void,
|
|
89
|
-
protected readonly log: Logger,
|
|
90
98
|
public readonly tracer: Tracer,
|
|
91
|
-
|
|
99
|
+
bindings?: LoggerBindings,
|
|
100
|
+
) {
|
|
101
|
+
this.log = createLogger('sequencer:checkpoint-proposal', { ...bindings, instanceId: `slot-${slot}` });
|
|
102
|
+
}
|
|
92
103
|
|
|
93
104
|
/**
|
|
94
105
|
* Executes the checkpoint proposal job.
|
|
@@ -190,6 +201,7 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
190
201
|
l1ToL2Messages,
|
|
191
202
|
previousCheckpointOutHashes,
|
|
192
203
|
fork,
|
|
204
|
+
this.log.getBindings(),
|
|
193
205
|
);
|
|
194
206
|
|
|
195
207
|
// Options for the validator client when creating block and checkpoint proposals
|
|
@@ -203,8 +215,8 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
203
215
|
broadcastInvalidCheckpointProposal: this.config.broadcastInvalidBlockProposal,
|
|
204
216
|
};
|
|
205
217
|
|
|
206
|
-
let blocksInCheckpoint:
|
|
207
|
-
let blockPendingBroadcast: { block:
|
|
218
|
+
let blocksInCheckpoint: L2Block[] = [];
|
|
219
|
+
let blockPendingBroadcast: { block: L2Block; txs: Tx[] } | undefined = undefined;
|
|
208
220
|
|
|
209
221
|
try {
|
|
210
222
|
// Main loop: build blocks for the checkpoint
|
|
@@ -220,19 +232,7 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
220
232
|
// These errors are expected in HA mode, so we yield and let another HA node handle the slot
|
|
221
233
|
// The only distinction between the 2 errors is SlashingProtectionError throws when the payload is different,
|
|
222
234
|
// which is normal for block building (may have picked different txs)
|
|
223
|
-
if (err
|
|
224
|
-
this.log.info(`Checkpoint proposal for slot ${this.slot} already signed by another HA node, yielding`, {
|
|
225
|
-
slot: this.slot,
|
|
226
|
-
signedByNode: err.signedByNode,
|
|
227
|
-
});
|
|
228
|
-
return undefined;
|
|
229
|
-
}
|
|
230
|
-
if (err instanceof SlashingProtectionError) {
|
|
231
|
-
this.log.info(`Checkpoint proposal for slot ${this.slot} blocked by slashing protection, yielding`, {
|
|
232
|
-
slot: this.slot,
|
|
233
|
-
existingMessageHash: err.existingMessageHash,
|
|
234
|
-
attemptedMessageHash: err.attemptedMessageHash,
|
|
235
|
-
});
|
|
235
|
+
if (this.handleHASigningError(err, 'Block proposal')) {
|
|
236
236
|
return undefined;
|
|
237
237
|
}
|
|
238
238
|
throw err;
|
|
@@ -301,20 +301,8 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
301
301
|
);
|
|
302
302
|
} catch (err) {
|
|
303
303
|
// We shouldn't really get here since we yield to another HA node
|
|
304
|
-
// as soon as we see these errors when creating block proposals.
|
|
305
|
-
if (err
|
|
306
|
-
this.log.info(`Attestations signature for slot ${this.slot} already signed by another HA node, yielding`, {
|
|
307
|
-
slot: this.slot,
|
|
308
|
-
signedByNode: err.signedByNode,
|
|
309
|
-
});
|
|
310
|
-
return undefined;
|
|
311
|
-
}
|
|
312
|
-
if (err instanceof SlashingProtectionError) {
|
|
313
|
-
this.log.info(`Attestations signature for slot ${this.slot} blocked by slashing protection, yielding`, {
|
|
314
|
-
slot: this.slot,
|
|
315
|
-
existingMessageHash: err.existingMessageHash,
|
|
316
|
-
attemptedMessageHash: err.attemptedMessageHash,
|
|
317
|
-
});
|
|
304
|
+
// as soon as we see these errors when creating block or checkpoint proposals.
|
|
305
|
+
if (this.handleHASigningError(err, 'Attestations signature')) {
|
|
318
306
|
return undefined;
|
|
319
307
|
}
|
|
320
308
|
throw err;
|
|
@@ -352,10 +340,10 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
352
340
|
inHash: Fr,
|
|
353
341
|
blockProposalOptions: BlockProposalOptions,
|
|
354
342
|
): Promise<{
|
|
355
|
-
blocksInCheckpoint:
|
|
356
|
-
blockPendingBroadcast: { block:
|
|
343
|
+
blocksInCheckpoint: L2Block[];
|
|
344
|
+
blockPendingBroadcast: { block: L2Block; txs: Tx[] } | undefined;
|
|
357
345
|
}> {
|
|
358
|
-
const blocksInCheckpoint:
|
|
346
|
+
const blocksInCheckpoint: L2Block[] = [];
|
|
359
347
|
const txHashesAlreadyIncluded = new Set<string>();
|
|
360
348
|
const initialBlockNumber = BlockNumber(this.syncedToBlockNumber + 1);
|
|
361
349
|
|
|
@@ -363,11 +351,11 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
363
351
|
let remainingBlobFields = BLOBS_PER_CHECKPOINT * FIELDS_PER_BLOB - NUM_CHECKPOINT_END_MARKER_FIELDS;
|
|
364
352
|
|
|
365
353
|
// Last block in the checkpoint will usually be flagged as pending broadcast, so we send it along with the checkpoint proposal
|
|
366
|
-
let blockPendingBroadcast: { block:
|
|
354
|
+
let blockPendingBroadcast: { block: L2Block; txs: Tx[] } | undefined = undefined;
|
|
367
355
|
|
|
368
356
|
while (true) {
|
|
369
357
|
const blocksBuilt = blocksInCheckpoint.length;
|
|
370
|
-
const indexWithinCheckpoint = blocksBuilt;
|
|
358
|
+
const indexWithinCheckpoint = IndexWithinCheckpoint(blocksBuilt);
|
|
371
359
|
const blockNumber = BlockNumber(initialBlockNumber + blocksBuilt);
|
|
372
360
|
|
|
373
361
|
const secondsIntoSlot = this.getSecondsIntoSlot();
|
|
@@ -397,6 +385,7 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
397
385
|
remainingBlobFields,
|
|
398
386
|
});
|
|
399
387
|
|
|
388
|
+
// TODO(palla/mbps): Review these conditions. We may want to keep trying in some scenarios.
|
|
400
389
|
if (!buildResult && timingInfo.isLastBlock) {
|
|
401
390
|
// If no block was produced due to not enough txs and this was the last subslot, exit
|
|
402
391
|
break;
|
|
@@ -428,7 +417,12 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
428
417
|
// Sync the proposed block to the archiver to make it available
|
|
429
418
|
// Note that the checkpoint builder uses its own fork so it should not need to wait for this syncing
|
|
430
419
|
// Eventually we should refactor the checkpoint builder to not need a separate long-lived fork
|
|
431
|
-
|
|
420
|
+
// Fire and forget - don't block the critical path, but log errors
|
|
421
|
+
this.syncProposedBlockToArchiver(block).catch(err => {
|
|
422
|
+
this.log.error(`Failed to sync proposed block ${block.number} to archiver`, { blockNumber: block.number, err });
|
|
423
|
+
});
|
|
424
|
+
|
|
425
|
+
usedTxs.forEach(tx => txHashesAlreadyIncluded.add(tx.txHash.toString()));
|
|
432
426
|
|
|
433
427
|
// If this is the last block, exit the loop now so we start collecting attestations
|
|
434
428
|
if (timingInfo.isLastBlock) {
|
|
@@ -478,18 +472,18 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
478
472
|
|
|
479
473
|
/** Builds a single block. Called from the main block building loop. */
|
|
480
474
|
@trackSpan('CheckpointProposalJob.buildSingleBlock')
|
|
481
|
-
|
|
475
|
+
protected async buildSingleBlock(
|
|
482
476
|
checkpointBuilder: CheckpointBuilder,
|
|
483
477
|
opts: {
|
|
484
478
|
forceCreate?: boolean;
|
|
485
479
|
blockTimestamp: bigint;
|
|
486
480
|
blockNumber: BlockNumber;
|
|
487
|
-
indexWithinCheckpoint:
|
|
481
|
+
indexWithinCheckpoint: IndexWithinCheckpoint;
|
|
488
482
|
buildDeadline: Date | undefined;
|
|
489
483
|
txHashesAlreadyIncluded: Set<string>;
|
|
490
484
|
remainingBlobFields: number;
|
|
491
485
|
},
|
|
492
|
-
): Promise<{ block:
|
|
486
|
+
): Promise<{ block: L2Block; usedTxs: Tx[]; remainingBlobFields: number } | { error: Error } | undefined> {
|
|
493
487
|
const {
|
|
494
488
|
blockTimestamp,
|
|
495
489
|
forceCreate,
|
|
@@ -545,45 +539,38 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
545
539
|
};
|
|
546
540
|
|
|
547
541
|
// Actually build the block by executing txs
|
|
548
|
-
const
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
usedTxs,
|
|
556
|
-
failedTxs,
|
|
557
|
-
usedTxBlobFields,
|
|
558
|
-
} = await checkpointBuilder.buildBlock(pendingTxs, blockNumber, blockTimestamp, blockBuilderOptions);
|
|
559
|
-
const blockBuildDuration = workTimer.ms();
|
|
542
|
+
const buildResult = await this.buildSingleBlockWithCheckpointBuilder(
|
|
543
|
+
checkpointBuilder,
|
|
544
|
+
pendingTxs,
|
|
545
|
+
blockNumber,
|
|
546
|
+
blockTimestamp,
|
|
547
|
+
blockBuilderOptions,
|
|
548
|
+
);
|
|
560
549
|
|
|
561
550
|
// If any txs failed during execution, drop them from the mempool so we don't pick them up again
|
|
562
|
-
await this.dropFailedTxsFromP2P(failedTxs);
|
|
551
|
+
await this.dropFailedTxsFromP2P(buildResult.failedTxs);
|
|
563
552
|
|
|
564
553
|
// Check if we have created a block with enough txs. If there were invalid txs in the pool, or if execution took
|
|
565
554
|
// too long, then we may not get to minTxsPerBlock after executing public functions.
|
|
566
555
|
const minValidTxs = this.config.minValidTxsPerBlock ?? minTxs;
|
|
567
|
-
|
|
556
|
+
const numTxs = buildResult.status === 'no-valid-txs' ? 0 : buildResult.numTxs;
|
|
557
|
+
if (buildResult.status === 'no-valid-txs' || (!forceCreate && numTxs < minValidTxs)) {
|
|
568
558
|
this.log.warn(
|
|
569
|
-
`Block ${blockNumber} at index ${indexWithinCheckpoint} on slot ${this.slot} has too few valid txs to be proposed
|
|
570
|
-
{ slot: this.slot, blockNumber, numTxs, indexWithinCheckpoint },
|
|
559
|
+
`Block ${blockNumber} at index ${indexWithinCheckpoint} on slot ${this.slot} has too few valid txs to be proposed`,
|
|
560
|
+
{ slot: this.slot, blockNumber, numTxs, indexWithinCheckpoint, minValidTxs, buildResult: buildResult.status },
|
|
571
561
|
);
|
|
572
|
-
this.eventEmitter.emit('block-
|
|
573
|
-
minTxs: minValidTxs,
|
|
574
|
-
availableTxs: numTxs,
|
|
575
|
-
slot: this.slot,
|
|
576
|
-
});
|
|
562
|
+
this.eventEmitter.emit('block-build-failed', { reason: `Insufficient valid txs`, slot: this.slot });
|
|
577
563
|
this.metrics.recordBlockProposalFailed('insufficient_valid_txs');
|
|
578
564
|
return undefined;
|
|
579
565
|
}
|
|
580
566
|
|
|
581
567
|
// Block creation succeeded, emit stats and metrics
|
|
568
|
+
const { publicGas, block, publicProcessorDuration, usedTxs, usedTxBlobFields, blockBuildDuration } = buildResult;
|
|
569
|
+
|
|
582
570
|
const blockStats = {
|
|
583
571
|
eventName: 'l2-block-built',
|
|
584
572
|
duration: blockBuildDuration,
|
|
585
573
|
publicProcessDuration: publicProcessorDuration,
|
|
586
|
-
rollupCircuitsDuration: blockBuildingTimer.ms(),
|
|
587
574
|
...block.getStats(),
|
|
588
575
|
} satisfies L2BlockBuiltStats;
|
|
589
576
|
|
|
@@ -609,17 +596,40 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
609
596
|
}
|
|
610
597
|
}
|
|
611
598
|
|
|
599
|
+
/** Uses the checkpoint builder to build a block, catching specific txs */
|
|
600
|
+
private async buildSingleBlockWithCheckpointBuilder(
|
|
601
|
+
checkpointBuilder: CheckpointBuilder,
|
|
602
|
+
pendingTxs: AsyncIterable<Tx>,
|
|
603
|
+
blockNumber: BlockNumber,
|
|
604
|
+
blockTimestamp: bigint,
|
|
605
|
+
blockBuilderOptions: PublicProcessorLimits,
|
|
606
|
+
) {
|
|
607
|
+
try {
|
|
608
|
+
const workTimer = new Timer();
|
|
609
|
+
const result = await checkpointBuilder.buildBlock(pendingTxs, blockNumber, blockTimestamp, blockBuilderOptions);
|
|
610
|
+
const blockBuildDuration = workTimer.ms();
|
|
611
|
+
return { ...result, blockBuildDuration, status: 'success' as const };
|
|
612
|
+
} catch (err: unknown) {
|
|
613
|
+
if (isErrorClass(err, NoValidTxsError)) {
|
|
614
|
+
return { failedTxs: err.failedTxs, status: 'no-valid-txs' as const };
|
|
615
|
+
}
|
|
616
|
+
throw err;
|
|
617
|
+
}
|
|
618
|
+
}
|
|
619
|
+
|
|
612
620
|
/** Waits until minTxs are available on the pool for building a block. */
|
|
613
621
|
@trackSpan('CheckpointProposalJob.waitForMinTxs')
|
|
614
622
|
private async waitForMinTxs(opts: {
|
|
615
623
|
forceCreate?: boolean;
|
|
616
624
|
blockNumber: BlockNumber;
|
|
617
|
-
indexWithinCheckpoint:
|
|
625
|
+
indexWithinCheckpoint: IndexWithinCheckpoint;
|
|
618
626
|
buildDeadline: Date | undefined;
|
|
619
627
|
}): Promise<{ canStartBuilding: boolean; availableTxs: number }> {
|
|
620
|
-
const minTxs = this.config.minTxsPerBlock;
|
|
621
628
|
const { indexWithinCheckpoint, blockNumber, buildDeadline, forceCreate } = opts;
|
|
622
629
|
|
|
630
|
+
// We only allow a block with 0 txs in the first block of the checkpoint
|
|
631
|
+
const minTxs = indexWithinCheckpoint > 0 && this.config.minTxsPerBlock === 0 ? 1 : this.config.minTxsPerBlock;
|
|
632
|
+
|
|
623
633
|
// Deadline is undefined if we are not enforcing the timetable, meaning we'll exit immediately when out of time
|
|
624
634
|
const startBuildingDeadline = buildDeadline
|
|
625
635
|
? new Date(buildDeadline.getTime() - this.timetable.minExecutionTime * 1000)
|
|
@@ -640,7 +650,7 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
640
650
|
`Waiting for enough txs to build block ${blockNumber} at index ${indexWithinCheckpoint} in slot ${this.slot} (have ${availableTxs} but need ${minTxs})`,
|
|
641
651
|
{ blockNumber, slot: this.slot, indexWithinCheckpoint },
|
|
642
652
|
);
|
|
643
|
-
await
|
|
653
|
+
await this.waitForTxsPollingInterval();
|
|
644
654
|
availableTxs = await this.p2pClient.getPendingTxCount();
|
|
645
655
|
}
|
|
646
656
|
|
|
@@ -681,7 +691,7 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
681
691
|
const attestationTimeAllowed = this.config.enforceTimeTable
|
|
682
692
|
? this.timetable.getMaxAllowedTime(SequencerState.PUBLISHING_CHECKPOINT)!
|
|
683
693
|
: this.l1Constants.slotDuration;
|
|
684
|
-
const attestationDeadline = new Date(this.
|
|
694
|
+
const attestationDeadline = new Date((this.getSlotStartBuildTimestamp() + attestationTimeAllowed) * 1000);
|
|
685
695
|
|
|
686
696
|
this.metrics.recordRequiredAttestations(numberOfRequiredAttestations, attestationTimeAllowed);
|
|
687
697
|
|
|
@@ -769,7 +779,7 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
769
779
|
const failedTxData = failedTxs.map(fail => fail.tx);
|
|
770
780
|
const failedTxHashes = failedTxData.map(tx => tx.getTxHash());
|
|
771
781
|
this.log.verbose(`Dropping failed txs ${failedTxHashes.join(', ')}`);
|
|
772
|
-
await this.p2pClient.
|
|
782
|
+
await this.p2pClient.handleFailedExecution(failedTxHashes);
|
|
773
783
|
}
|
|
774
784
|
|
|
775
785
|
/**
|
|
@@ -777,8 +787,7 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
777
787
|
* Gossip doesn't echo messages back to the sender, so the proposer's archiver/world-state
|
|
778
788
|
* would never receive its own block without this explicit sync.
|
|
779
789
|
*/
|
|
780
|
-
private async syncProposedBlockToArchiver(block:
|
|
781
|
-
// TODO(palla/mbps): Change default to false once block sync is stable.
|
|
790
|
+
private async syncProposedBlockToArchiver(block: L2Block): Promise<void> {
|
|
782
791
|
if (this.config.skipPushProposedBlocksToArchiver !== false) {
|
|
783
792
|
this.log.warn(`Skipping push of proposed block ${block.number} to archiver`, {
|
|
784
793
|
blockNumber: block.number,
|
|
@@ -818,6 +827,28 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
818
827
|
this.publisher.clearPendingRequests();
|
|
819
828
|
}
|
|
820
829
|
|
|
830
|
+
/**
|
|
831
|
+
* Helper to handle HA double-signing errors. Returns true if the error was handled (caller should yield).
|
|
832
|
+
*/
|
|
833
|
+
private handleHASigningError(err: any, errorContext: string): boolean {
|
|
834
|
+
if (err instanceof DutyAlreadySignedError) {
|
|
835
|
+
this.log.info(`${errorContext} for slot ${this.slot} already signed by another HA node, yielding`, {
|
|
836
|
+
slot: this.slot,
|
|
837
|
+
signedByNode: err.signedByNode,
|
|
838
|
+
});
|
|
839
|
+
return true;
|
|
840
|
+
}
|
|
841
|
+
if (err instanceof SlashingProtectionError) {
|
|
842
|
+
this.log.info(`${errorContext} for slot ${this.slot} blocked by slashing protection, yielding`, {
|
|
843
|
+
slot: this.slot,
|
|
844
|
+
existingMessageHash: err.existingMessageHash,
|
|
845
|
+
attemptedMessageHash: err.attemptedMessageHash,
|
|
846
|
+
});
|
|
847
|
+
return true;
|
|
848
|
+
}
|
|
849
|
+
return false;
|
|
850
|
+
}
|
|
851
|
+
|
|
821
852
|
/** Waits until a specific time within the current slot */
|
|
822
853
|
@trackSpan('CheckpointProposalJob.waitUntilTimeInSlot')
|
|
823
854
|
protected async waitUntilTimeInSlot(targetSecondsIntoSlot: number): Promise<void> {
|
|
@@ -826,6 +857,11 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
826
857
|
await sleepUntil(new Date(targetTimestamp * 1000), this.dateProvider.nowAsDate());
|
|
827
858
|
}
|
|
828
859
|
|
|
860
|
+
/** Waits the polling interval for transactions. Extracted for test overriding. */
|
|
861
|
+
protected async waitForTxsPollingInterval(): Promise<void> {
|
|
862
|
+
await sleep(TXS_POLLING_MS);
|
|
863
|
+
}
|
|
864
|
+
|
|
829
865
|
private getSlotStartBuildTimestamp(): number {
|
|
830
866
|
return getSlotStartBuildTimestamp(this.slot, this.l1Constants);
|
|
831
867
|
}
|
package/src/sequencer/metrics.ts
CHANGED
|
@@ -11,6 +11,7 @@ import {
|
|
|
11
11
|
type TelemetryClient,
|
|
12
12
|
type Tracer,
|
|
13
13
|
type UpDownCounter,
|
|
14
|
+
createUpDownCounterWithDefault,
|
|
14
15
|
} from '@aztec/telemetry-client';
|
|
15
16
|
|
|
16
17
|
import { type Hex, formatUnits } from 'viem';
|
|
@@ -67,7 +68,9 @@ export class SequencerMetrics {
|
|
|
67
68
|
this.meter = client.getMeter(name);
|
|
68
69
|
this.tracer = client.getTracer(name);
|
|
69
70
|
|
|
70
|
-
this.blockCounter = this.meter
|
|
71
|
+
this.blockCounter = createUpDownCounterWithDefault(this.meter, Metrics.SEQUENCER_BLOCK_COUNT, {
|
|
72
|
+
[Attributes.STATUS]: ['failed', 'built'],
|
|
73
|
+
});
|
|
71
74
|
|
|
72
75
|
this.blockBuildDuration = this.meter.createHistogram(Metrics.SEQUENCER_BLOCK_BUILD_DURATION);
|
|
73
76
|
|
|
@@ -77,23 +80,15 @@ export class SequencerMetrics {
|
|
|
77
80
|
|
|
78
81
|
this.checkpointAttestationDelay = this.meter.createHistogram(Metrics.SEQUENCER_CHECKPOINT_ATTESTATION_DELAY);
|
|
79
82
|
|
|
80
|
-
// Init gauges and counters
|
|
81
|
-
this.blockCounter.add(0, {
|
|
82
|
-
[Attributes.STATUS]: 'failed',
|
|
83
|
-
});
|
|
84
|
-
this.blockCounter.add(0, {
|
|
85
|
-
[Attributes.STATUS]: 'built',
|
|
86
|
-
});
|
|
87
|
-
|
|
88
83
|
this.rewards = this.meter.createGauge(Metrics.SEQUENCER_CURRENT_BLOCK_REWARDS);
|
|
89
84
|
|
|
90
|
-
this.slots = this.meter
|
|
85
|
+
this.slots = createUpDownCounterWithDefault(this.meter, Metrics.SEQUENCER_SLOT_COUNT);
|
|
91
86
|
|
|
92
87
|
/**
|
|
93
88
|
* NOTE: we do not track missed slots as a separate metric. That would be difficult to determine
|
|
94
89
|
* Instead, use a computed metric, `slots - filledSlots` to get the number of slots a sequencer has missed.
|
|
95
90
|
*/
|
|
96
|
-
this.filledSlots = this.meter
|
|
91
|
+
this.filledSlots = createUpDownCounterWithDefault(this.meter, Metrics.SEQUENCER_FILLED_SLOT_COUNT);
|
|
97
92
|
|
|
98
93
|
this.timeToCollectAttestations = this.meter.createGauge(Metrics.SEQUENCER_COLLECT_ATTESTATIONS_DURATION);
|
|
99
94
|
|
|
@@ -103,20 +98,41 @@ export class SequencerMetrics {
|
|
|
103
98
|
|
|
104
99
|
this.collectedAttestions = this.meter.createGauge(Metrics.SEQUENCER_COLLECTED_ATTESTATIONS_COUNT);
|
|
105
100
|
|
|
106
|
-
this.blockProposalFailed =
|
|
101
|
+
this.blockProposalFailed = createUpDownCounterWithDefault(
|
|
102
|
+
this.meter,
|
|
103
|
+
Metrics.SEQUENCER_BLOCK_PROPOSAL_FAILED_COUNT,
|
|
104
|
+
);
|
|
107
105
|
|
|
108
|
-
this.blockProposalSuccess =
|
|
106
|
+
this.blockProposalSuccess = createUpDownCounterWithDefault(
|
|
107
|
+
this.meter,
|
|
108
|
+
Metrics.SEQUENCER_BLOCK_PROPOSAL_SUCCESS_COUNT,
|
|
109
|
+
);
|
|
109
110
|
|
|
110
|
-
this.checkpointSuccess = this.meter
|
|
111
|
+
this.checkpointSuccess = createUpDownCounterWithDefault(this.meter, Metrics.SEQUENCER_CHECKPOINT_SUCCESS_COUNT);
|
|
111
112
|
|
|
112
|
-
this.blockProposalPrecheckFailed =
|
|
113
|
+
this.blockProposalPrecheckFailed = createUpDownCounterWithDefault(
|
|
114
|
+
this.meter,
|
|
113
115
|
Metrics.SEQUENCER_BLOCK_PROPOSAL_PRECHECK_FAILED_COUNT,
|
|
116
|
+
{
|
|
117
|
+
[Attributes.ERROR_TYPE]: [
|
|
118
|
+
'slot_already_taken',
|
|
119
|
+
'rollup_contract_check_failed',
|
|
120
|
+
'slot_mismatch',
|
|
121
|
+
'block_number_mismatch',
|
|
122
|
+
],
|
|
123
|
+
},
|
|
114
124
|
);
|
|
115
125
|
|
|
116
|
-
this.slashingAttempts = this.meter
|
|
126
|
+
this.slashingAttempts = createUpDownCounterWithDefault(this.meter, Metrics.SEQUENCER_SLASHING_ATTEMPTS_COUNT);
|
|
117
127
|
|
|
118
128
|
// Fisherman fee analysis metrics
|
|
119
|
-
this.fishermanWouldBeIncluded =
|
|
129
|
+
this.fishermanWouldBeIncluded = createUpDownCounterWithDefault(
|
|
130
|
+
this.meter,
|
|
131
|
+
Metrics.FISHERMAN_FEE_ANALYSIS_WOULD_BE_INCLUDED,
|
|
132
|
+
{
|
|
133
|
+
[Attributes.OK]: [true, false],
|
|
134
|
+
},
|
|
135
|
+
);
|
|
120
136
|
|
|
121
137
|
this.fishermanTimeBeforeBlock = this.meter.createHistogram(Metrics.FISHERMAN_FEE_ANALYSIS_TIME_BEFORE_BLOCK);
|
|
122
138
|
|
|
@@ -231,7 +247,9 @@ export class SequencerMetrics {
|
|
|
231
247
|
this.blockProposalSuccess.add(1);
|
|
232
248
|
}
|
|
233
249
|
|
|
234
|
-
recordBlockProposalPrecheckFailed(
|
|
250
|
+
recordBlockProposalPrecheckFailed(
|
|
251
|
+
checkType: 'slot_already_taken' | 'rollup_contract_check_failed' | 'slot_mismatch' | 'block_number_mismatch',
|
|
252
|
+
) {
|
|
235
253
|
this.blockProposalPrecheckFailed.add(1, {
|
|
236
254
|
[Attributes.ERROR_TYPE]: checkType,
|
|
237
255
|
});
|
|
@@ -12,7 +12,7 @@ import type { DateProvider } from '@aztec/foundation/timer';
|
|
|
12
12
|
import type { TypedEventEmitter } from '@aztec/foundation/types';
|
|
13
13
|
import type { P2P } from '@aztec/p2p';
|
|
14
14
|
import type { SlasherClientInterface } from '@aztec/slasher';
|
|
15
|
-
import type {
|
|
15
|
+
import type { L2Block, L2BlockSink, L2BlockSource, ValidateCheckpointResult } from '@aztec/stdlib/block';
|
|
16
16
|
import type { Checkpoint } from '@aztec/stdlib/checkpoint';
|
|
17
17
|
import { getSlotAtTimestamp, getSlotStartBuildTimestamp } from '@aztec/stdlib/epoch-helpers';
|
|
18
18
|
import {
|
|
@@ -60,6 +60,9 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
60
60
|
/** The last slot for which we attempted to perform our voting duties with degraded block production */
|
|
61
61
|
private lastSlotForFallbackVote: SlotNumber | undefined;
|
|
62
62
|
|
|
63
|
+
/** The last slot for which we logged "no committee" warning, to avoid spam */
|
|
64
|
+
private lastSlotForNoCommitteeWarning: SlotNumber | undefined;
|
|
65
|
+
|
|
63
66
|
/** The last slot for which we triggered a checkpoint proposal job, to prevent duplicate attempts. */
|
|
64
67
|
private lastSlotForCheckpointProposalJob: SlotNumber | undefined;
|
|
65
68
|
|
|
@@ -373,6 +376,7 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
373
376
|
}
|
|
374
377
|
|
|
375
378
|
this.lastSlotForCheckpointProposalJob = slot;
|
|
379
|
+
await this.p2pClient.prepareForSlot(slot);
|
|
376
380
|
this.log.info(`Preparing checkpoint proposal ${checkpointNumber} at slot ${slot}`, { ...logCtx, proposer });
|
|
377
381
|
|
|
378
382
|
// Create and return the checkpoint proposal job
|
|
@@ -424,8 +428,8 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
424
428
|
this.metrics,
|
|
425
429
|
this,
|
|
426
430
|
this.setState.bind(this),
|
|
427
|
-
this.log,
|
|
428
431
|
this.tracer,
|
|
432
|
+
this.log.getBindings(),
|
|
429
433
|
);
|
|
430
434
|
}
|
|
431
435
|
|
|
@@ -529,7 +533,7 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
529
533
|
};
|
|
530
534
|
}
|
|
531
535
|
|
|
532
|
-
const block = await this.l2BlockSource.
|
|
536
|
+
const block = await this.l2BlockSource.getL2Block(blockNumber);
|
|
533
537
|
if (!block) {
|
|
534
538
|
// this shouldn't really happen because a moment ago we checked that all components were in sync
|
|
535
539
|
this.log.error(`Failed to get L2 block ${blockNumber} from the archiver with all components in sync`);
|
|
@@ -557,7 +561,10 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
557
561
|
proposer = await this.epochCache.getProposerAttesterAddressInSlot(slot);
|
|
558
562
|
} catch (e) {
|
|
559
563
|
if (e instanceof NoCommitteeError) {
|
|
560
|
-
this.
|
|
564
|
+
if (this.lastSlotForNoCommitteeWarning !== slot) {
|
|
565
|
+
this.lastSlotForNoCommitteeWarning = slot;
|
|
566
|
+
this.log.warn(`Cannot propose at next L2 slot ${slot} since the committee does not exist on L1`);
|
|
567
|
+
}
|
|
561
568
|
return [false, undefined];
|
|
562
569
|
}
|
|
563
570
|
this.log.error(`Error getting proposer for slot ${slot}`, e);
|
|
@@ -870,7 +877,7 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
870
877
|
}
|
|
871
878
|
|
|
872
879
|
type SequencerSyncCheckResult = {
|
|
873
|
-
block?:
|
|
880
|
+
block?: L2Block;
|
|
874
881
|
checkpointNumber: CheckpointNumber;
|
|
875
882
|
blockNumber: BlockNumber;
|
|
876
883
|
archive: Fr;
|
|
@@ -1,14 +1,15 @@
|
|
|
1
1
|
import { createLogger } from '@aztec/aztec.js/log';
|
|
2
|
+
import {
|
|
3
|
+
CHECKPOINT_ASSEMBLE_TIME,
|
|
4
|
+
CHECKPOINT_INITIALIZATION_TIME,
|
|
5
|
+
DEFAULT_P2P_PROPAGATION_TIME,
|
|
6
|
+
MIN_EXECUTION_TIME,
|
|
7
|
+
} from '@aztec/stdlib/timetable';
|
|
2
8
|
|
|
3
|
-
import { DEFAULT_ATTESTATION_PROPAGATION_TIME as DEFAULT_P2P_PROPAGATION_TIME } from '../config.js';
|
|
4
9
|
import { SequencerTooSlowError } from './errors.js';
|
|
5
10
|
import type { SequencerMetrics } from './metrics.js';
|
|
6
11
|
import { SequencerState } from './utils.js';
|
|
7
12
|
|
|
8
|
-
export const MIN_EXECUTION_TIME = 2;
|
|
9
|
-
export const CHECKPOINT_INITIALIZATION_TIME = 1;
|
|
10
|
-
export const CHECKPOINT_ASSEMBLE_TIME = 1;
|
|
11
|
-
|
|
12
13
|
export class SequencerTimetable {
|
|
13
14
|
/**
|
|
14
15
|
* How late into the slot can we be to start working. Computed as the total time needed for assembling and publishing a block,
|