@aztec/sequencer-client 0.0.1-commit.0b941701 → 0.0.1-commit.0c875d939
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dest/client/sequencer-client.js +1 -1
- package/dest/config.d.ts +1 -2
- package/dest/config.d.ts.map +1 -1
- package/dest/config.js +4 -8
- package/dest/publisher/sequencer-publisher-metrics.d.ts +1 -1
- package/dest/publisher/sequencer-publisher-metrics.d.ts.map +1 -1
- package/dest/publisher/sequencer-publisher-metrics.js +12 -4
- package/dest/publisher/sequencer-publisher.d.ts +8 -2
- package/dest/publisher/sequencer-publisher.d.ts.map +1 -1
- package/dest/publisher/sequencer-publisher.js +53 -23
- package/dest/sequencer/checkpoint_proposal_job.d.ts +32 -9
- package/dest/sequencer/checkpoint_proposal_job.d.ts.map +1 -1
- package/dest/sequencer/checkpoint_proposal_job.js +79 -51
- package/dest/sequencer/metrics.d.ts +5 -2
- package/dest/sequencer/metrics.d.ts.map +1 -1
- package/dest/sequencer/metrics.js +52 -17
- package/dest/sequencer/sequencer.d.ts +3 -1
- package/dest/sequencer/sequencer.d.ts.map +1 -1
- package/dest/sequencer/sequencer.js +7 -2
- package/dest/sequencer/timetable.d.ts +1 -4
- package/dest/sequencer/timetable.d.ts.map +1 -1
- package/dest/sequencer/timetable.js +1 -4
- package/dest/test/mock_checkpoint_builder.d.ts +7 -5
- package/dest/test/mock_checkpoint_builder.d.ts.map +1 -1
- package/dest/test/mock_checkpoint_builder.js +6 -6
- package/dest/test/utils.d.ts +3 -3
- package/dest/test/utils.d.ts.map +1 -1
- package/dest/test/utils.js +5 -4
- package/package.json +28 -28
- package/src/client/sequencer-client.ts +1 -1
- package/src/config.ts +9 -11
- package/src/publisher/sequencer-publisher-metrics.ts +7 -3
- package/src/publisher/sequencer-publisher.ts +60 -22
- package/src/sequencer/checkpoint_proposal_job.ts +104 -67
- package/src/sequencer/metrics.ts +60 -18
- package/src/sequencer/sequencer.ts +9 -2
- package/src/sequencer/timetable.ts +6 -5
- package/src/test/mock_checkpoint_builder.ts +14 -5
- package/src/test/utils.ts +5 -2
|
@@ -1,16 +1,22 @@
|
|
|
1
1
|
import { NUM_CHECKPOINT_END_MARKER_FIELDS, getNumBlockEndBlobFields } from '@aztec/blob-lib/encoding';
|
|
2
2
|
import { BLOBS_PER_CHECKPOINT, FIELDS_PER_BLOB } from '@aztec/constants';
|
|
3
3
|
import type { EpochCache } from '@aztec/epoch-cache';
|
|
4
|
-
import {
|
|
4
|
+
import {
|
|
5
|
+
BlockNumber,
|
|
6
|
+
CheckpointNumber,
|
|
7
|
+
EpochNumber,
|
|
8
|
+
IndexWithinCheckpoint,
|
|
9
|
+
SlotNumber,
|
|
10
|
+
} from '@aztec/foundation/branded-types';
|
|
5
11
|
import { randomInt } from '@aztec/foundation/crypto/random';
|
|
6
12
|
import { Fr } from '@aztec/foundation/curves/bn254';
|
|
7
13
|
import { EthAddress } from '@aztec/foundation/eth-address';
|
|
8
14
|
import { Signature } from '@aztec/foundation/eth-signature';
|
|
9
15
|
import { filter } from '@aztec/foundation/iterator';
|
|
10
|
-
import type
|
|
16
|
+
import { type Logger, type LoggerBindings, createLogger } from '@aztec/foundation/log';
|
|
11
17
|
import { sleep, sleepUntil } from '@aztec/foundation/sleep';
|
|
12
18
|
import { type DateProvider, Timer } from '@aztec/foundation/timer';
|
|
13
|
-
import { type TypedEventEmitter, unfreeze } from '@aztec/foundation/types';
|
|
19
|
+
import { type TypedEventEmitter, isErrorClass, unfreeze } from '@aztec/foundation/types';
|
|
14
20
|
import type { P2P } from '@aztec/p2p';
|
|
15
21
|
import type { SlasherClientInterface } from '@aztec/slasher';
|
|
16
22
|
import {
|
|
@@ -24,10 +30,11 @@ import {
|
|
|
24
30
|
import type { Checkpoint } from '@aztec/stdlib/checkpoint';
|
|
25
31
|
import { getSlotStartBuildTimestamp } from '@aztec/stdlib/epoch-helpers';
|
|
26
32
|
import { Gas } from '@aztec/stdlib/gas';
|
|
27
|
-
import
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
33
|
+
import {
|
|
34
|
+
NoValidTxsError,
|
|
35
|
+
type PublicProcessorLimits,
|
|
36
|
+
type ResolvedSequencerConfig,
|
|
37
|
+
type WorldStateSynchronizer,
|
|
31
38
|
} from '@aztec/stdlib/interfaces/server';
|
|
32
39
|
import { type L1ToL2MessageSource, computeInHashFromL1ToL2Messages } from '@aztec/stdlib/messaging';
|
|
33
40
|
import type { BlockProposalOptions, CheckpointProposal, CheckpointProposalOptions } from '@aztec/stdlib/p2p';
|
|
@@ -59,6 +66,8 @@ const TXS_POLLING_MS = 500;
|
|
|
59
66
|
* the Sequencer once the check for being the proposer for the slot has succeeded.
|
|
60
67
|
*/
|
|
61
68
|
export class CheckpointProposalJob implements Traceable {
|
|
69
|
+
protected readonly log: Logger;
|
|
70
|
+
|
|
62
71
|
constructor(
|
|
63
72
|
private readonly epoch: EpochNumber,
|
|
64
73
|
private readonly slot: SlotNumber,
|
|
@@ -86,9 +95,11 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
86
95
|
private readonly metrics: SequencerMetrics,
|
|
87
96
|
private readonly eventEmitter: TypedEventEmitter<SequencerEvents>,
|
|
88
97
|
private readonly setStateFn: (state: SequencerState, slot?: SlotNumber) => void,
|
|
89
|
-
protected readonly log: Logger,
|
|
90
98
|
public readonly tracer: Tracer,
|
|
91
|
-
|
|
99
|
+
bindings?: LoggerBindings,
|
|
100
|
+
) {
|
|
101
|
+
this.log = createLogger('sequencer:checkpoint-proposal', { ...bindings, instanceId: `slot-${slot}` });
|
|
102
|
+
}
|
|
92
103
|
|
|
93
104
|
/**
|
|
94
105
|
* Executes the checkpoint proposal job.
|
|
@@ -180,6 +191,9 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
180
191
|
);
|
|
181
192
|
const previousCheckpointOutHashes = previousCheckpoints.map(c => c.getCheckpointOutHash());
|
|
182
193
|
|
|
194
|
+
// Get the fee asset price modifier from the oracle
|
|
195
|
+
const feeAssetPriceModifier = await this.publisher.getFeeAssetPriceModifier();
|
|
196
|
+
|
|
183
197
|
// Create a long-lived forked world state for the checkpoint builder
|
|
184
198
|
using fork = await this.worldState.fork(this.syncedToBlockNumber, { closeDelayMs: 12_000 });
|
|
185
199
|
|
|
@@ -187,9 +201,11 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
187
201
|
const checkpointBuilder = await this.checkpointsBuilder.startCheckpoint(
|
|
188
202
|
this.checkpointNumber,
|
|
189
203
|
checkpointGlobalVariables,
|
|
204
|
+
feeAssetPriceModifier,
|
|
190
205
|
l1ToL2Messages,
|
|
191
206
|
previousCheckpointOutHashes,
|
|
192
207
|
fork,
|
|
208
|
+
this.log.getBindings(),
|
|
193
209
|
);
|
|
194
210
|
|
|
195
211
|
// Options for the validator client when creating block and checkpoint proposals
|
|
@@ -220,19 +236,7 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
220
236
|
// These errors are expected in HA mode, so we yield and let another HA node handle the slot
|
|
221
237
|
// The only distinction between the 2 errors is SlashingProtectionError throws when the payload is different,
|
|
222
238
|
// which is normal for block building (may have picked different txs)
|
|
223
|
-
if (err
|
|
224
|
-
this.log.info(`Checkpoint proposal for slot ${this.slot} already signed by another HA node, yielding`, {
|
|
225
|
-
slot: this.slot,
|
|
226
|
-
signedByNode: err.signedByNode,
|
|
227
|
-
});
|
|
228
|
-
return undefined;
|
|
229
|
-
}
|
|
230
|
-
if (err instanceof SlashingProtectionError) {
|
|
231
|
-
this.log.info(`Checkpoint proposal for slot ${this.slot} blocked by slashing protection, yielding`, {
|
|
232
|
-
slot: this.slot,
|
|
233
|
-
existingMessageHash: err.existingMessageHash,
|
|
234
|
-
attemptedMessageHash: err.attemptedMessageHash,
|
|
235
|
-
});
|
|
239
|
+
if (this.handleHASigningError(err, 'Block proposal')) {
|
|
236
240
|
return undefined;
|
|
237
241
|
}
|
|
238
242
|
throw err;
|
|
@@ -275,6 +279,7 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
275
279
|
const proposal = await this.validatorClient.createCheckpointProposal(
|
|
276
280
|
checkpoint.header,
|
|
277
281
|
checkpoint.archive.root,
|
|
282
|
+
feeAssetPriceModifier,
|
|
278
283
|
lastBlock,
|
|
279
284
|
this.proposer,
|
|
280
285
|
checkpointProposalOptions,
|
|
@@ -301,20 +306,8 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
301
306
|
);
|
|
302
307
|
} catch (err) {
|
|
303
308
|
// We shouldn't really get here since we yield to another HA node
|
|
304
|
-
// as soon as we see these errors when creating block proposals.
|
|
305
|
-
if (err
|
|
306
|
-
this.log.info(`Attestations signature for slot ${this.slot} already signed by another HA node, yielding`, {
|
|
307
|
-
slot: this.slot,
|
|
308
|
-
signedByNode: err.signedByNode,
|
|
309
|
-
});
|
|
310
|
-
return undefined;
|
|
311
|
-
}
|
|
312
|
-
if (err instanceof SlashingProtectionError) {
|
|
313
|
-
this.log.info(`Attestations signature for slot ${this.slot} blocked by slashing protection, yielding`, {
|
|
314
|
-
slot: this.slot,
|
|
315
|
-
existingMessageHash: err.existingMessageHash,
|
|
316
|
-
attemptedMessageHash: err.attemptedMessageHash,
|
|
317
|
-
});
|
|
309
|
+
// as soon as we see these errors when creating block or checkpoint proposals.
|
|
310
|
+
if (this.handleHASigningError(err, 'Attestations signature')) {
|
|
318
311
|
return undefined;
|
|
319
312
|
}
|
|
320
313
|
throw err;
|
|
@@ -367,7 +360,7 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
367
360
|
|
|
368
361
|
while (true) {
|
|
369
362
|
const blocksBuilt = blocksInCheckpoint.length;
|
|
370
|
-
const indexWithinCheckpoint = blocksBuilt;
|
|
363
|
+
const indexWithinCheckpoint = IndexWithinCheckpoint(blocksBuilt);
|
|
371
364
|
const blockNumber = BlockNumber(initialBlockNumber + blocksBuilt);
|
|
372
365
|
|
|
373
366
|
const secondsIntoSlot = this.getSecondsIntoSlot();
|
|
@@ -397,6 +390,7 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
397
390
|
remainingBlobFields,
|
|
398
391
|
});
|
|
399
392
|
|
|
393
|
+
// TODO(palla/mbps): Review these conditions. We may want to keep trying in some scenarios.
|
|
400
394
|
if (!buildResult && timingInfo.isLastBlock) {
|
|
401
395
|
// If no block was produced due to not enough txs and this was the last subslot, exit
|
|
402
396
|
break;
|
|
@@ -483,13 +477,13 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
483
477
|
|
|
484
478
|
/** Builds a single block. Called from the main block building loop. */
|
|
485
479
|
@trackSpan('CheckpointProposalJob.buildSingleBlock')
|
|
486
|
-
|
|
480
|
+
protected async buildSingleBlock(
|
|
487
481
|
checkpointBuilder: CheckpointBuilder,
|
|
488
482
|
opts: {
|
|
489
483
|
forceCreate?: boolean;
|
|
490
484
|
blockTimestamp: bigint;
|
|
491
485
|
blockNumber: BlockNumber;
|
|
492
|
-
indexWithinCheckpoint:
|
|
486
|
+
indexWithinCheckpoint: IndexWithinCheckpoint;
|
|
493
487
|
buildDeadline: Date | undefined;
|
|
494
488
|
txHashesAlreadyIncluded: Set<string>;
|
|
495
489
|
remainingBlobFields: number;
|
|
@@ -527,7 +521,7 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
527
521
|
// Create iterator to pending txs. We filter out txs already included in previous blocks in the checkpoint
|
|
528
522
|
// just in case p2p failed to sync the provisional block and didn't get to remove those txs from the mempool yet.
|
|
529
523
|
const pendingTxs = filter(
|
|
530
|
-
this.p2pClient.
|
|
524
|
+
this.p2pClient.iterateEligiblePendingTxs(),
|
|
531
525
|
tx => !txHashesAlreadyIncluded.has(tx.txHash.toString()),
|
|
532
526
|
);
|
|
533
527
|
|
|
@@ -550,45 +544,38 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
550
544
|
};
|
|
551
545
|
|
|
552
546
|
// Actually build the block by executing txs
|
|
553
|
-
const
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
usedTxs,
|
|
561
|
-
failedTxs,
|
|
562
|
-
usedTxBlobFields,
|
|
563
|
-
} = await checkpointBuilder.buildBlock(pendingTxs, blockNumber, blockTimestamp, blockBuilderOptions);
|
|
564
|
-
const blockBuildDuration = workTimer.ms();
|
|
547
|
+
const buildResult = await this.buildSingleBlockWithCheckpointBuilder(
|
|
548
|
+
checkpointBuilder,
|
|
549
|
+
pendingTxs,
|
|
550
|
+
blockNumber,
|
|
551
|
+
blockTimestamp,
|
|
552
|
+
blockBuilderOptions,
|
|
553
|
+
);
|
|
565
554
|
|
|
566
555
|
// If any txs failed during execution, drop them from the mempool so we don't pick them up again
|
|
567
|
-
await this.dropFailedTxsFromP2P(failedTxs);
|
|
556
|
+
await this.dropFailedTxsFromP2P(buildResult.failedTxs);
|
|
568
557
|
|
|
569
558
|
// Check if we have created a block with enough txs. If there were invalid txs in the pool, or if execution took
|
|
570
559
|
// too long, then we may not get to minTxsPerBlock after executing public functions.
|
|
571
560
|
const minValidTxs = this.config.minValidTxsPerBlock ?? minTxs;
|
|
572
|
-
|
|
561
|
+
const numTxs = buildResult.status === 'no-valid-txs' ? 0 : buildResult.numTxs;
|
|
562
|
+
if (buildResult.status === 'no-valid-txs' || (!forceCreate && numTxs < minValidTxs)) {
|
|
573
563
|
this.log.warn(
|
|
574
|
-
`Block ${blockNumber} at index ${indexWithinCheckpoint} on slot ${this.slot} has too few valid txs to be proposed
|
|
575
|
-
{ slot: this.slot, blockNumber, numTxs, indexWithinCheckpoint },
|
|
564
|
+
`Block ${blockNumber} at index ${indexWithinCheckpoint} on slot ${this.slot} has too few valid txs to be proposed`,
|
|
565
|
+
{ slot: this.slot, blockNumber, numTxs, indexWithinCheckpoint, minValidTxs, buildResult: buildResult.status },
|
|
576
566
|
);
|
|
577
|
-
this.eventEmitter.emit('block-
|
|
578
|
-
minTxs: minValidTxs,
|
|
579
|
-
availableTxs: numTxs,
|
|
580
|
-
slot: this.slot,
|
|
581
|
-
});
|
|
567
|
+
this.eventEmitter.emit('block-build-failed', { reason: `Insufficient valid txs`, slot: this.slot });
|
|
582
568
|
this.metrics.recordBlockProposalFailed('insufficient_valid_txs');
|
|
583
569
|
return undefined;
|
|
584
570
|
}
|
|
585
571
|
|
|
586
572
|
// Block creation succeeded, emit stats and metrics
|
|
573
|
+
const { publicGas, block, publicProcessorDuration, usedTxs, usedTxBlobFields, blockBuildDuration } = buildResult;
|
|
574
|
+
|
|
587
575
|
const blockStats = {
|
|
588
576
|
eventName: 'l2-block-built',
|
|
589
577
|
duration: blockBuildDuration,
|
|
590
578
|
publicProcessDuration: publicProcessorDuration,
|
|
591
|
-
rollupCircuitsDuration: blockBuildingTimer.ms(),
|
|
592
579
|
...block.getStats(),
|
|
593
580
|
} satisfies L2BlockBuiltStats;
|
|
594
581
|
|
|
@@ -614,17 +601,40 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
614
601
|
}
|
|
615
602
|
}
|
|
616
603
|
|
|
604
|
+
/** Uses the checkpoint builder to build a block, catching specific txs */
|
|
605
|
+
private async buildSingleBlockWithCheckpointBuilder(
|
|
606
|
+
checkpointBuilder: CheckpointBuilder,
|
|
607
|
+
pendingTxs: AsyncIterable<Tx>,
|
|
608
|
+
blockNumber: BlockNumber,
|
|
609
|
+
blockTimestamp: bigint,
|
|
610
|
+
blockBuilderOptions: PublicProcessorLimits,
|
|
611
|
+
) {
|
|
612
|
+
try {
|
|
613
|
+
const workTimer = new Timer();
|
|
614
|
+
const result = await checkpointBuilder.buildBlock(pendingTxs, blockNumber, blockTimestamp, blockBuilderOptions);
|
|
615
|
+
const blockBuildDuration = workTimer.ms();
|
|
616
|
+
return { ...result, blockBuildDuration, status: 'success' as const };
|
|
617
|
+
} catch (err: unknown) {
|
|
618
|
+
if (isErrorClass(err, NoValidTxsError)) {
|
|
619
|
+
return { failedTxs: err.failedTxs, status: 'no-valid-txs' as const };
|
|
620
|
+
}
|
|
621
|
+
throw err;
|
|
622
|
+
}
|
|
623
|
+
}
|
|
624
|
+
|
|
617
625
|
/** Waits until minTxs are available on the pool for building a block. */
|
|
618
626
|
@trackSpan('CheckpointProposalJob.waitForMinTxs')
|
|
619
627
|
private async waitForMinTxs(opts: {
|
|
620
628
|
forceCreate?: boolean;
|
|
621
629
|
blockNumber: BlockNumber;
|
|
622
|
-
indexWithinCheckpoint:
|
|
630
|
+
indexWithinCheckpoint: IndexWithinCheckpoint;
|
|
623
631
|
buildDeadline: Date | undefined;
|
|
624
632
|
}): Promise<{ canStartBuilding: boolean; availableTxs: number }> {
|
|
625
|
-
const minTxs = this.config.minTxsPerBlock;
|
|
626
633
|
const { indexWithinCheckpoint, blockNumber, buildDeadline, forceCreate } = opts;
|
|
627
634
|
|
|
635
|
+
// We only allow a block with 0 txs in the first block of the checkpoint
|
|
636
|
+
const minTxs = indexWithinCheckpoint > 0 && this.config.minTxsPerBlock === 0 ? 1 : this.config.minTxsPerBlock;
|
|
637
|
+
|
|
628
638
|
// Deadline is undefined if we are not enforcing the timetable, meaning we'll exit immediately when out of time
|
|
629
639
|
const startBuildingDeadline = buildDeadline
|
|
630
640
|
? new Date(buildDeadline.getTime() - this.timetable.minExecutionTime * 1000)
|
|
@@ -645,7 +655,7 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
645
655
|
`Waiting for enough txs to build block ${blockNumber} at index ${indexWithinCheckpoint} in slot ${this.slot} (have ${availableTxs} but need ${minTxs})`,
|
|
646
656
|
{ blockNumber, slot: this.slot, indexWithinCheckpoint },
|
|
647
657
|
);
|
|
648
|
-
await
|
|
658
|
+
await this.waitForTxsPollingInterval();
|
|
649
659
|
availableTxs = await this.p2pClient.getPendingTxCount();
|
|
650
660
|
}
|
|
651
661
|
|
|
@@ -686,7 +696,7 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
686
696
|
const attestationTimeAllowed = this.config.enforceTimeTable
|
|
687
697
|
? this.timetable.getMaxAllowedTime(SequencerState.PUBLISHING_CHECKPOINT)!
|
|
688
698
|
: this.l1Constants.slotDuration;
|
|
689
|
-
const attestationDeadline = new Date(this.
|
|
699
|
+
const attestationDeadline = new Date((this.getSlotStartBuildTimestamp() + attestationTimeAllowed) * 1000);
|
|
690
700
|
|
|
691
701
|
this.metrics.recordRequiredAttestations(numberOfRequiredAttestations, attestationTimeAllowed);
|
|
692
702
|
|
|
@@ -774,7 +784,7 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
774
784
|
const failedTxData = failedTxs.map(fail => fail.tx);
|
|
775
785
|
const failedTxHashes = failedTxData.map(tx => tx.getTxHash());
|
|
776
786
|
this.log.verbose(`Dropping failed txs ${failedTxHashes.join(', ')}`);
|
|
777
|
-
await this.p2pClient.
|
|
787
|
+
await this.p2pClient.handleFailedExecution(failedTxHashes);
|
|
778
788
|
}
|
|
779
789
|
|
|
780
790
|
/**
|
|
@@ -822,6 +832,28 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
822
832
|
this.publisher.clearPendingRequests();
|
|
823
833
|
}
|
|
824
834
|
|
|
835
|
+
/**
|
|
836
|
+
* Helper to handle HA double-signing errors. Returns true if the error was handled (caller should yield).
|
|
837
|
+
*/
|
|
838
|
+
private handleHASigningError(err: any, errorContext: string): boolean {
|
|
839
|
+
if (err instanceof DutyAlreadySignedError) {
|
|
840
|
+
this.log.info(`${errorContext} for slot ${this.slot} already signed by another HA node, yielding`, {
|
|
841
|
+
slot: this.slot,
|
|
842
|
+
signedByNode: err.signedByNode,
|
|
843
|
+
});
|
|
844
|
+
return true;
|
|
845
|
+
}
|
|
846
|
+
if (err instanceof SlashingProtectionError) {
|
|
847
|
+
this.log.info(`${errorContext} for slot ${this.slot} blocked by slashing protection, yielding`, {
|
|
848
|
+
slot: this.slot,
|
|
849
|
+
existingMessageHash: err.existingMessageHash,
|
|
850
|
+
attemptedMessageHash: err.attemptedMessageHash,
|
|
851
|
+
});
|
|
852
|
+
return true;
|
|
853
|
+
}
|
|
854
|
+
return false;
|
|
855
|
+
}
|
|
856
|
+
|
|
825
857
|
/** Waits until a specific time within the current slot */
|
|
826
858
|
@trackSpan('CheckpointProposalJob.waitUntilTimeInSlot')
|
|
827
859
|
protected async waitUntilTimeInSlot(targetSecondsIntoSlot: number): Promise<void> {
|
|
@@ -830,6 +862,11 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
830
862
|
await sleepUntil(new Date(targetTimestamp * 1000), this.dateProvider.nowAsDate());
|
|
831
863
|
}
|
|
832
864
|
|
|
865
|
+
/** Waits the polling interval for transactions. Extracted for test overriding. */
|
|
866
|
+
protected async waitForTxsPollingInterval(): Promise<void> {
|
|
867
|
+
await sleep(TXS_POLLING_MS);
|
|
868
|
+
}
|
|
869
|
+
|
|
833
870
|
private getSlotStartBuildTimestamp(): number {
|
|
834
871
|
return getSlotStartBuildTimestamp(this.slot, this.l1Constants);
|
|
835
872
|
}
|
package/src/sequencer/metrics.ts
CHANGED
|
@@ -11,6 +11,7 @@ import {
|
|
|
11
11
|
type TelemetryClient,
|
|
12
12
|
type Tracer,
|
|
13
13
|
type UpDownCounter,
|
|
14
|
+
createUpDownCounterWithDefault,
|
|
14
15
|
} from '@aztec/telemetry-client';
|
|
15
16
|
|
|
16
17
|
import { type Hex, formatUnits } from 'viem';
|
|
@@ -50,6 +51,9 @@ export class SequencerMetrics {
|
|
|
50
51
|
private fishermanTimeBeforeBlock: Histogram;
|
|
51
52
|
private fishermanPendingBlobTxCount: Histogram;
|
|
52
53
|
private fishermanIncludedBlobTxCount: Histogram;
|
|
54
|
+
private fishermanPendingBlobCount: Histogram;
|
|
55
|
+
private fishermanIncludedBlobCount: Histogram;
|
|
56
|
+
private fishermanBlockBlobsFull: UpDownCounter;
|
|
53
57
|
private fishermanCalculatedPriorityFee: Histogram;
|
|
54
58
|
private fishermanPriorityFeeDelta: Histogram;
|
|
55
59
|
private fishermanEstimatedCost: Histogram;
|
|
@@ -67,7 +71,9 @@ export class SequencerMetrics {
|
|
|
67
71
|
this.meter = client.getMeter(name);
|
|
68
72
|
this.tracer = client.getTracer(name);
|
|
69
73
|
|
|
70
|
-
this.blockCounter = this.meter
|
|
74
|
+
this.blockCounter = createUpDownCounterWithDefault(this.meter, Metrics.SEQUENCER_BLOCK_COUNT, {
|
|
75
|
+
[Attributes.STATUS]: ['failed', 'built'],
|
|
76
|
+
});
|
|
71
77
|
|
|
72
78
|
this.blockBuildDuration = this.meter.createHistogram(Metrics.SEQUENCER_BLOCK_BUILD_DURATION);
|
|
73
79
|
|
|
@@ -77,23 +83,15 @@ export class SequencerMetrics {
|
|
|
77
83
|
|
|
78
84
|
this.checkpointAttestationDelay = this.meter.createHistogram(Metrics.SEQUENCER_CHECKPOINT_ATTESTATION_DELAY);
|
|
79
85
|
|
|
80
|
-
// Init gauges and counters
|
|
81
|
-
this.blockCounter.add(0, {
|
|
82
|
-
[Attributes.STATUS]: 'failed',
|
|
83
|
-
});
|
|
84
|
-
this.blockCounter.add(0, {
|
|
85
|
-
[Attributes.STATUS]: 'built',
|
|
86
|
-
});
|
|
87
|
-
|
|
88
86
|
this.rewards = this.meter.createGauge(Metrics.SEQUENCER_CURRENT_BLOCK_REWARDS);
|
|
89
87
|
|
|
90
|
-
this.slots = this.meter
|
|
88
|
+
this.slots = createUpDownCounterWithDefault(this.meter, Metrics.SEQUENCER_SLOT_COUNT);
|
|
91
89
|
|
|
92
90
|
/**
|
|
93
91
|
* NOTE: we do not track missed slots as a separate metric. That would be difficult to determine
|
|
94
92
|
* Instead, use a computed metric, `slots - filledSlots` to get the number of slots a sequencer has missed.
|
|
95
93
|
*/
|
|
96
|
-
this.filledSlots = this.meter
|
|
94
|
+
this.filledSlots = createUpDownCounterWithDefault(this.meter, Metrics.SEQUENCER_FILLED_SLOT_COUNT);
|
|
97
95
|
|
|
98
96
|
this.timeToCollectAttestations = this.meter.createGauge(Metrics.SEQUENCER_COLLECT_ATTESTATIONS_DURATION);
|
|
99
97
|
|
|
@@ -103,20 +101,41 @@ export class SequencerMetrics {
|
|
|
103
101
|
|
|
104
102
|
this.collectedAttestions = this.meter.createGauge(Metrics.SEQUENCER_COLLECTED_ATTESTATIONS_COUNT);
|
|
105
103
|
|
|
106
|
-
this.blockProposalFailed =
|
|
104
|
+
this.blockProposalFailed = createUpDownCounterWithDefault(
|
|
105
|
+
this.meter,
|
|
106
|
+
Metrics.SEQUENCER_BLOCK_PROPOSAL_FAILED_COUNT,
|
|
107
|
+
);
|
|
107
108
|
|
|
108
|
-
this.blockProposalSuccess =
|
|
109
|
+
this.blockProposalSuccess = createUpDownCounterWithDefault(
|
|
110
|
+
this.meter,
|
|
111
|
+
Metrics.SEQUENCER_BLOCK_PROPOSAL_SUCCESS_COUNT,
|
|
112
|
+
);
|
|
109
113
|
|
|
110
|
-
this.checkpointSuccess = this.meter
|
|
114
|
+
this.checkpointSuccess = createUpDownCounterWithDefault(this.meter, Metrics.SEQUENCER_CHECKPOINT_SUCCESS_COUNT);
|
|
111
115
|
|
|
112
|
-
this.blockProposalPrecheckFailed =
|
|
116
|
+
this.blockProposalPrecheckFailed = createUpDownCounterWithDefault(
|
|
117
|
+
this.meter,
|
|
113
118
|
Metrics.SEQUENCER_BLOCK_PROPOSAL_PRECHECK_FAILED_COUNT,
|
|
119
|
+
{
|
|
120
|
+
[Attributes.ERROR_TYPE]: [
|
|
121
|
+
'slot_already_taken',
|
|
122
|
+
'rollup_contract_check_failed',
|
|
123
|
+
'slot_mismatch',
|
|
124
|
+
'block_number_mismatch',
|
|
125
|
+
],
|
|
126
|
+
},
|
|
114
127
|
);
|
|
115
128
|
|
|
116
|
-
this.slashingAttempts = this.meter
|
|
129
|
+
this.slashingAttempts = createUpDownCounterWithDefault(this.meter, Metrics.SEQUENCER_SLASHING_ATTEMPTS_COUNT);
|
|
117
130
|
|
|
118
131
|
// Fisherman fee analysis metrics
|
|
119
|
-
this.fishermanWouldBeIncluded =
|
|
132
|
+
this.fishermanWouldBeIncluded = createUpDownCounterWithDefault(
|
|
133
|
+
this.meter,
|
|
134
|
+
Metrics.FISHERMAN_FEE_ANALYSIS_WOULD_BE_INCLUDED,
|
|
135
|
+
{
|
|
136
|
+
[Attributes.OK]: [true, false],
|
|
137
|
+
},
|
|
138
|
+
);
|
|
120
139
|
|
|
121
140
|
this.fishermanTimeBeforeBlock = this.meter.createHistogram(Metrics.FISHERMAN_FEE_ANALYSIS_TIME_BEFORE_BLOCK);
|
|
122
141
|
|
|
@@ -145,6 +164,18 @@ export class SequencerMetrics {
|
|
|
145
164
|
this.fishermanMinedBlobTxTotalCost = this.meter.createHistogram(
|
|
146
165
|
Metrics.FISHERMAN_FEE_ANALYSIS_MINED_BLOB_TX_TOTAL_COST,
|
|
147
166
|
);
|
|
167
|
+
|
|
168
|
+
this.fishermanPendingBlobCount = this.meter.createHistogram(Metrics.FISHERMAN_FEE_ANALYSIS_PENDING_BLOB_COUNT);
|
|
169
|
+
|
|
170
|
+
this.fishermanIncludedBlobCount = this.meter.createHistogram(Metrics.FISHERMAN_FEE_ANALYSIS_INCLUDED_BLOB_COUNT);
|
|
171
|
+
|
|
172
|
+
this.fishermanBlockBlobsFull = createUpDownCounterWithDefault(
|
|
173
|
+
this.meter,
|
|
174
|
+
Metrics.FISHERMAN_FEE_ANALYSIS_BLOCK_BLOBS_FULL,
|
|
175
|
+
{
|
|
176
|
+
[Attributes.OK]: [true, false],
|
|
177
|
+
},
|
|
178
|
+
);
|
|
148
179
|
}
|
|
149
180
|
|
|
150
181
|
public recordRequiredAttestations(requiredAttestationsCount: number, allowanceMs: number) {
|
|
@@ -231,7 +262,9 @@ export class SequencerMetrics {
|
|
|
231
262
|
this.blockProposalSuccess.add(1);
|
|
232
263
|
}
|
|
233
264
|
|
|
234
|
-
recordBlockProposalPrecheckFailed(
|
|
265
|
+
recordBlockProposalPrecheckFailed(
|
|
266
|
+
checkType: 'slot_already_taken' | 'rollup_contract_check_failed' | 'slot_mismatch' | 'block_number_mismatch',
|
|
267
|
+
) {
|
|
235
268
|
this.blockProposalPrecheckFailed.add(1, {
|
|
236
269
|
[Attributes.ERROR_TYPE]: checkType,
|
|
237
270
|
});
|
|
@@ -263,10 +296,12 @@ export class SequencerMetrics {
|
|
|
263
296
|
|
|
264
297
|
// Record pending block snapshot data (once per strategy for comparison)
|
|
265
298
|
this.fishermanPendingBlobTxCount.record(analysis.pendingSnapshot.pendingBlobTxCount, strategyAttributes);
|
|
299
|
+
this.fishermanPendingBlobCount.record(analysis.pendingSnapshot.pendingBlobCount, strategyAttributes);
|
|
266
300
|
|
|
267
301
|
// Record mined block data if available
|
|
268
302
|
if (analysis.minedBlock) {
|
|
269
303
|
this.fishermanIncludedBlobTxCount.record(analysis.minedBlock.includedBlobTxCount, strategyAttributes);
|
|
304
|
+
this.fishermanIncludedBlobCount.record(analysis.minedBlock.includedBlobCount, strategyAttributes);
|
|
270
305
|
|
|
271
306
|
// Record actual fees from blob transactions in the mined block
|
|
272
307
|
for (const blobTx of analysis.minedBlock.includedBlobTxs) {
|
|
@@ -300,6 +335,13 @@ export class SequencerMetrics {
|
|
|
300
335
|
if (analysis.analysis) {
|
|
301
336
|
this.fishermanTimeBeforeBlock.record(Math.ceil(analysis.analysis.timeBeforeBlockMs), strategyAttributes);
|
|
302
337
|
|
|
338
|
+
// Record whether the block reached 100% blob capacity
|
|
339
|
+
if (analysis.analysis.blockBlobsFull) {
|
|
340
|
+
this.fishermanBlockBlobsFull.add(1, { ...strategyAttributes, [Attributes.OK]: true });
|
|
341
|
+
} else {
|
|
342
|
+
this.fishermanBlockBlobsFull.add(1, { ...strategyAttributes, [Attributes.OK]: false });
|
|
343
|
+
}
|
|
344
|
+
|
|
303
345
|
// Record strategy-specific inclusion result
|
|
304
346
|
if (strategyResult.wouldBeIncluded !== undefined) {
|
|
305
347
|
if (strategyResult.wouldBeIncluded) {
|
|
@@ -60,6 +60,9 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
60
60
|
/** The last slot for which we attempted to perform our voting duties with degraded block production */
|
|
61
61
|
private lastSlotForFallbackVote: SlotNumber | undefined;
|
|
62
62
|
|
|
63
|
+
/** The last slot for which we logged "no committee" warning, to avoid spam */
|
|
64
|
+
private lastSlotForNoCommitteeWarning: SlotNumber | undefined;
|
|
65
|
+
|
|
63
66
|
/** The last slot for which we triggered a checkpoint proposal job, to prevent duplicate attempts. */
|
|
64
67
|
private lastSlotForCheckpointProposalJob: SlotNumber | undefined;
|
|
65
68
|
|
|
@@ -373,6 +376,7 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
373
376
|
}
|
|
374
377
|
|
|
375
378
|
this.lastSlotForCheckpointProposalJob = slot;
|
|
379
|
+
await this.p2pClient.prepareForSlot(slot);
|
|
376
380
|
this.log.info(`Preparing checkpoint proposal ${checkpointNumber} at slot ${slot}`, { ...logCtx, proposer });
|
|
377
381
|
|
|
378
382
|
// Create and return the checkpoint proposal job
|
|
@@ -424,8 +428,8 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
424
428
|
this.metrics,
|
|
425
429
|
this,
|
|
426
430
|
this.setState.bind(this),
|
|
427
|
-
this.log,
|
|
428
431
|
this.tracer,
|
|
432
|
+
this.log.getBindings(),
|
|
429
433
|
);
|
|
430
434
|
}
|
|
431
435
|
|
|
@@ -557,7 +561,10 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
557
561
|
proposer = await this.epochCache.getProposerAttesterAddressInSlot(slot);
|
|
558
562
|
} catch (e) {
|
|
559
563
|
if (e instanceof NoCommitteeError) {
|
|
560
|
-
this.
|
|
564
|
+
if (this.lastSlotForNoCommitteeWarning !== slot) {
|
|
565
|
+
this.lastSlotForNoCommitteeWarning = slot;
|
|
566
|
+
this.log.warn(`Cannot propose at next L2 slot ${slot} since the committee does not exist on L1`);
|
|
567
|
+
}
|
|
561
568
|
return [false, undefined];
|
|
562
569
|
}
|
|
563
570
|
this.log.error(`Error getting proposer for slot ${slot}`, e);
|
|
@@ -1,14 +1,15 @@
|
|
|
1
1
|
import { createLogger } from '@aztec/aztec.js/log';
|
|
2
|
+
import {
|
|
3
|
+
CHECKPOINT_ASSEMBLE_TIME,
|
|
4
|
+
CHECKPOINT_INITIALIZATION_TIME,
|
|
5
|
+
DEFAULT_P2P_PROPAGATION_TIME,
|
|
6
|
+
MIN_EXECUTION_TIME,
|
|
7
|
+
} from '@aztec/stdlib/timetable';
|
|
2
8
|
|
|
3
|
-
import { DEFAULT_ATTESTATION_PROPAGATION_TIME as DEFAULT_P2P_PROPAGATION_TIME } from '../config.js';
|
|
4
9
|
import { SequencerTooSlowError } from './errors.js';
|
|
5
10
|
import type { SequencerMetrics } from './metrics.js';
|
|
6
11
|
import { SequencerState } from './utils.js';
|
|
7
12
|
|
|
8
|
-
export const MIN_EXECUTION_TIME = 2;
|
|
9
|
-
export const CHECKPOINT_INITIALIZATION_TIME = 1;
|
|
10
|
-
export const CHECKPOINT_ASSEMBLE_TIME = 1;
|
|
11
|
-
|
|
12
13
|
export class SequencerTimetable {
|
|
13
14
|
/**
|
|
14
15
|
* How late into the slot can we be to start working. Computed as the total time needed for assembling and publishing a block,
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import { type BlockNumber, CheckpointNumber } from '@aztec/foundation/branded-types';
|
|
2
2
|
import { Fr } from '@aztec/foundation/curves/bn254';
|
|
3
|
-
import { Timer } from '@aztec/foundation/timer';
|
|
4
3
|
import { L2Block } from '@aztec/stdlib/block';
|
|
5
4
|
import { Checkpoint } from '@aztec/stdlib/checkpoint';
|
|
6
5
|
import { Gas } from '@aztec/stdlib/gas';
|
|
@@ -14,7 +13,7 @@ import type {
|
|
|
14
13
|
import { CheckpointHeader } from '@aztec/stdlib/rollup';
|
|
15
14
|
import { makeAppendOnlyTreeSnapshot } from '@aztec/stdlib/testing';
|
|
16
15
|
import type { CheckpointGlobalVariables, Tx } from '@aztec/stdlib/tx';
|
|
17
|
-
import type {
|
|
16
|
+
import type { BuildBlockInCheckpointResult } from '@aztec/validator-client';
|
|
18
17
|
|
|
19
18
|
/**
|
|
20
19
|
* A fake CheckpointBuilder for testing that implements the same interface as the real one.
|
|
@@ -76,7 +75,7 @@ export class MockCheckpointBuilder implements ICheckpointBlockBuilder {
|
|
|
76
75
|
blockNumber: BlockNumber,
|
|
77
76
|
timestamp: bigint,
|
|
78
77
|
opts: PublicProcessorLimits,
|
|
79
|
-
): Promise<
|
|
78
|
+
): Promise<BuildBlockInCheckpointResult> {
|
|
80
79
|
this.buildBlockCalls.push({ blockNumber, timestamp, opts });
|
|
81
80
|
|
|
82
81
|
if (this.errorOnBuild) {
|
|
@@ -117,7 +116,6 @@ export class MockCheckpointBuilder implements ICheckpointBlockBuilder {
|
|
|
117
116
|
publicGas: Gas.empty(),
|
|
118
117
|
publicProcessorDuration: 0,
|
|
119
118
|
numTxs: block?.body?.txEffects?.length ?? usedTxs.length,
|
|
120
|
-
blockBuildingTimer: new Timer(),
|
|
121
119
|
usedTxs,
|
|
122
120
|
failedTxs: [],
|
|
123
121
|
usedTxBlobFields: block?.body?.txEffects?.reduce((sum, tx) => sum + tx.getNumBlobFields(), 0) ?? 0,
|
|
@@ -207,6 +205,7 @@ export class MockCheckpointsBuilder implements ICheckpointsBuilder {
|
|
|
207
205
|
constants: CheckpointGlobalVariables;
|
|
208
206
|
l1ToL2Messages: Fr[];
|
|
209
207
|
previousCheckpointOutHashes: Fr[];
|
|
208
|
+
feeAssetPriceModifier: bigint;
|
|
210
209
|
}> = [];
|
|
211
210
|
public openCheckpointCalls: Array<{
|
|
212
211
|
checkpointNumber: CheckpointNumber;
|
|
@@ -214,6 +213,7 @@ export class MockCheckpointsBuilder implements ICheckpointsBuilder {
|
|
|
214
213
|
l1ToL2Messages: Fr[];
|
|
215
214
|
previousCheckpointOutHashes: Fr[];
|
|
216
215
|
existingBlocks: L2Block[];
|
|
216
|
+
feeAssetPriceModifier: bigint;
|
|
217
217
|
}> = [];
|
|
218
218
|
public updateConfigCalls: Array<Partial<FullNodeBlockBuilderConfig>> = [];
|
|
219
219
|
|
|
@@ -259,11 +259,18 @@ export class MockCheckpointsBuilder implements ICheckpointsBuilder {
|
|
|
259
259
|
startCheckpoint(
|
|
260
260
|
checkpointNumber: CheckpointNumber,
|
|
261
261
|
constants: CheckpointGlobalVariables,
|
|
262
|
+
feeAssetPriceModifier: bigint,
|
|
262
263
|
l1ToL2Messages: Fr[],
|
|
263
264
|
previousCheckpointOutHashes: Fr[],
|
|
264
265
|
_fork: MerkleTreeWriteOperations,
|
|
265
266
|
): Promise<ICheckpointBlockBuilder> {
|
|
266
|
-
this.startCheckpointCalls.push({
|
|
267
|
+
this.startCheckpointCalls.push({
|
|
268
|
+
checkpointNumber,
|
|
269
|
+
constants,
|
|
270
|
+
l1ToL2Messages,
|
|
271
|
+
previousCheckpointOutHashes,
|
|
272
|
+
feeAssetPriceModifier,
|
|
273
|
+
});
|
|
267
274
|
|
|
268
275
|
if (!this.checkpointBuilder) {
|
|
269
276
|
// Auto-create a builder if none was set
|
|
@@ -276,6 +283,7 @@ export class MockCheckpointsBuilder implements ICheckpointsBuilder {
|
|
|
276
283
|
openCheckpoint(
|
|
277
284
|
checkpointNumber: CheckpointNumber,
|
|
278
285
|
constants: CheckpointGlobalVariables,
|
|
286
|
+
feeAssetPriceModifier: bigint,
|
|
279
287
|
l1ToL2Messages: Fr[],
|
|
280
288
|
previousCheckpointOutHashes: Fr[],
|
|
281
289
|
_fork: MerkleTreeWriteOperations,
|
|
@@ -287,6 +295,7 @@ export class MockCheckpointsBuilder implements ICheckpointsBuilder {
|
|
|
287
295
|
l1ToL2Messages,
|
|
288
296
|
previousCheckpointOutHashes,
|
|
289
297
|
existingBlocks,
|
|
298
|
+
feeAssetPriceModifier,
|
|
290
299
|
});
|
|
291
300
|
|
|
292
301
|
if (!this.checkpointBuilder) {
|