@aztec/sequencer-client 0.0.1-commit.c80b6263 → 0.0.1-commit.cf93bcc56
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dest/client/sequencer-client.d.ts +12 -7
- package/dest/client/sequencer-client.d.ts.map +1 -1
- package/dest/client/sequencer-client.js +15 -4
- package/dest/config.d.ts +3 -4
- package/dest/config.d.ts.map +1 -1
- package/dest/config.js +17 -12
- package/dest/global_variable_builder/global_builder.d.ts +2 -4
- package/dest/global_variable_builder/global_builder.d.ts.map +1 -1
- package/dest/publisher/config.d.ts +31 -17
- package/dest/publisher/config.d.ts.map +1 -1
- package/dest/publisher/config.js +101 -42
- package/dest/publisher/sequencer-publisher-factory.d.ts +11 -3
- package/dest/publisher/sequencer-publisher-factory.d.ts.map +1 -1
- package/dest/publisher/sequencer-publisher-factory.js +13 -2
- package/dest/publisher/sequencer-publisher.d.ts +16 -8
- package/dest/publisher/sequencer-publisher.d.ts.map +1 -1
- package/dest/publisher/sequencer-publisher.js +80 -39
- package/dest/sequencer/checkpoint_proposal_job.d.ts +29 -6
- package/dest/sequencer/checkpoint_proposal_job.d.ts.map +1 -1
- package/dest/sequencer/checkpoint_proposal_job.js +97 -53
- package/dest/sequencer/metrics.d.ts +17 -5
- package/dest/sequencer/metrics.d.ts.map +1 -1
- package/dest/sequencer/metrics.js +86 -15
- package/dest/sequencer/sequencer.d.ts +12 -7
- package/dest/sequencer/sequencer.d.ts.map +1 -1
- package/dest/sequencer/sequencer.js +24 -26
- package/dest/sequencer/timetable.d.ts +1 -4
- package/dest/sequencer/timetable.d.ts.map +1 -1
- package/dest/sequencer/timetable.js +1 -4
- package/dest/test/index.d.ts +3 -5
- package/dest/test/index.d.ts.map +1 -1
- package/dest/test/mock_checkpoint_builder.d.ts +7 -5
- package/dest/test/mock_checkpoint_builder.d.ts.map +1 -1
- package/dest/test/mock_checkpoint_builder.js +6 -6
- package/dest/test/utils.d.ts +3 -3
- package/dest/test/utils.d.ts.map +1 -1
- package/dest/test/utils.js +5 -4
- package/package.json +28 -28
- package/src/client/sequencer-client.ts +25 -7
- package/src/config.ts +26 -19
- package/src/global_variable_builder/global_builder.ts +1 -1
- package/src/publisher/config.ts +112 -43
- package/src/publisher/sequencer-publisher-factory.ts +23 -6
- package/src/publisher/sequencer-publisher.ts +96 -45
- package/src/sequencer/checkpoint_proposal_job.ts +134 -70
- package/src/sequencer/metrics.ts +92 -18
- package/src/sequencer/sequencer.ts +32 -31
- package/src/sequencer/timetable.ts +6 -5
- package/src/test/index.ts +2 -4
- package/src/test/mock_checkpoint_builder.ts +14 -5
- package/src/test/utils.ts +5 -2
|
@@ -1,7 +1,13 @@
|
|
|
1
1
|
import { NUM_CHECKPOINT_END_MARKER_FIELDS, getNumBlockEndBlobFields } from '@aztec/blob-lib/encoding';
|
|
2
2
|
import { BLOBS_PER_CHECKPOINT, FIELDS_PER_BLOB } from '@aztec/constants';
|
|
3
3
|
import type { EpochCache } from '@aztec/epoch-cache';
|
|
4
|
-
import {
|
|
4
|
+
import {
|
|
5
|
+
BlockNumber,
|
|
6
|
+
CheckpointNumber,
|
|
7
|
+
EpochNumber,
|
|
8
|
+
IndexWithinCheckpoint,
|
|
9
|
+
SlotNumber,
|
|
10
|
+
} from '@aztec/foundation/branded-types';
|
|
5
11
|
import { randomInt } from '@aztec/foundation/crypto/random';
|
|
6
12
|
import { Fr } from '@aztec/foundation/curves/bn254';
|
|
7
13
|
import { EthAddress } from '@aztec/foundation/eth-address';
|
|
@@ -10,7 +16,7 @@ import { filter } from '@aztec/foundation/iterator';
|
|
|
10
16
|
import { type Logger, type LoggerBindings, createLogger } from '@aztec/foundation/log';
|
|
11
17
|
import { sleep, sleepUntil } from '@aztec/foundation/sleep';
|
|
12
18
|
import { type DateProvider, Timer } from '@aztec/foundation/timer';
|
|
13
|
-
import { type TypedEventEmitter, unfreeze } from '@aztec/foundation/types';
|
|
19
|
+
import { type TypedEventEmitter, isErrorClass, unfreeze } from '@aztec/foundation/types';
|
|
14
20
|
import type { P2P } from '@aztec/p2p';
|
|
15
21
|
import type { SlasherClientInterface } from '@aztec/slasher';
|
|
16
22
|
import {
|
|
@@ -24,10 +30,11 @@ import {
|
|
|
24
30
|
import type { Checkpoint } from '@aztec/stdlib/checkpoint';
|
|
25
31
|
import { getSlotStartBuildTimestamp } from '@aztec/stdlib/epoch-helpers';
|
|
26
32
|
import { Gas } from '@aztec/stdlib/gas';
|
|
27
|
-
import
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
33
|
+
import {
|
|
34
|
+
NoValidTxsError,
|
|
35
|
+
type PublicProcessorLimits,
|
|
36
|
+
type ResolvedSequencerConfig,
|
|
37
|
+
type WorldStateSynchronizer,
|
|
31
38
|
} from '@aztec/stdlib/interfaces/server';
|
|
32
39
|
import { type L1ToL2MessageSource, computeInHashFromL1ToL2Messages } from '@aztec/stdlib/messaging';
|
|
33
40
|
import type { BlockProposalOptions, CheckpointProposal, CheckpointProposalOptions } from '@aztec/stdlib/p2p';
|
|
@@ -122,7 +129,7 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
122
129
|
await Promise.all(votesPromises);
|
|
123
130
|
|
|
124
131
|
if (checkpoint) {
|
|
125
|
-
this.metrics.
|
|
132
|
+
this.metrics.recordCheckpointProposalSuccess();
|
|
126
133
|
}
|
|
127
134
|
|
|
128
135
|
// Do not post anything to L1 if we are fishermen, but do perform L1 fee analysis
|
|
@@ -179,18 +186,21 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
179
186
|
const inHash = computeInHashFromL1ToL2Messages(l1ToL2Messages);
|
|
180
187
|
|
|
181
188
|
// Collect the out hashes of all the checkpoints before this one in the same epoch
|
|
182
|
-
const
|
|
183
|
-
c => c.
|
|
184
|
-
|
|
185
|
-
|
|
189
|
+
const previousCheckpointOutHashes = (await this.l2BlockSource.getCheckpointsDataForEpoch(this.epoch))
|
|
190
|
+
.filter(c => c.checkpointNumber < this.checkpointNumber)
|
|
191
|
+
.map(c => c.checkpointOutHash);
|
|
192
|
+
|
|
193
|
+
// Get the fee asset price modifier from the oracle
|
|
194
|
+
const feeAssetPriceModifier = await this.publisher.getFeeAssetPriceModifier();
|
|
186
195
|
|
|
187
196
|
// Create a long-lived forked world state for the checkpoint builder
|
|
188
|
-
using fork = await this.worldState.fork(this.syncedToBlockNumber, { closeDelayMs: 12_000 });
|
|
197
|
+
await using fork = await this.worldState.fork(this.syncedToBlockNumber, { closeDelayMs: 12_000 });
|
|
189
198
|
|
|
190
199
|
// Create checkpoint builder for the entire slot
|
|
191
200
|
const checkpointBuilder = await this.checkpointsBuilder.startCheckpoint(
|
|
192
201
|
this.checkpointNumber,
|
|
193
202
|
checkpointGlobalVariables,
|
|
203
|
+
feeAssetPriceModifier,
|
|
194
204
|
l1ToL2Messages,
|
|
195
205
|
previousCheckpointOutHashes,
|
|
196
206
|
fork,
|
|
@@ -210,6 +220,7 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
210
220
|
|
|
211
221
|
let blocksInCheckpoint: L2Block[] = [];
|
|
212
222
|
let blockPendingBroadcast: { block: L2Block; txs: Tx[] } | undefined = undefined;
|
|
223
|
+
const checkpointBuildTimer = new Timer();
|
|
213
224
|
|
|
214
225
|
try {
|
|
215
226
|
// Main loop: build blocks for the checkpoint
|
|
@@ -225,19 +236,7 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
225
236
|
// These errors are expected in HA mode, so we yield and let another HA node handle the slot
|
|
226
237
|
// The only distinction between the 2 errors is SlashingProtectionError throws when the payload is different,
|
|
227
238
|
// which is normal for block building (may have picked different txs)
|
|
228
|
-
if (err
|
|
229
|
-
this.log.info(`Checkpoint proposal for slot ${this.slot} already signed by another HA node, yielding`, {
|
|
230
|
-
slot: this.slot,
|
|
231
|
-
signedByNode: err.signedByNode,
|
|
232
|
-
});
|
|
233
|
-
return undefined;
|
|
234
|
-
}
|
|
235
|
-
if (err instanceof SlashingProtectionError) {
|
|
236
|
-
this.log.info(`Checkpoint proposal for slot ${this.slot} blocked by slashing protection, yielding`, {
|
|
237
|
-
slot: this.slot,
|
|
238
|
-
existingMessageHash: err.existingMessageHash,
|
|
239
|
-
attemptedMessageHash: err.attemptedMessageHash,
|
|
240
|
-
});
|
|
239
|
+
if (this.handleHASigningError(err, 'Block proposal')) {
|
|
241
240
|
return undefined;
|
|
242
241
|
}
|
|
243
242
|
throw err;
|
|
@@ -249,11 +248,28 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
249
248
|
return undefined;
|
|
250
249
|
}
|
|
251
250
|
|
|
251
|
+
const minBlocksForCheckpoint = this.config.minBlocksForCheckpoint;
|
|
252
|
+
if (minBlocksForCheckpoint !== undefined && blocksInCheckpoint.length < minBlocksForCheckpoint) {
|
|
253
|
+
this.log.warn(
|
|
254
|
+
`Checkpoint has fewer blocks than minimum (${blocksInCheckpoint.length} < ${minBlocksForCheckpoint}), skipping proposal`,
|
|
255
|
+
{ slot: this.slot, blocksBuilt: blocksInCheckpoint.length, minBlocksForCheckpoint },
|
|
256
|
+
);
|
|
257
|
+
return undefined;
|
|
258
|
+
}
|
|
259
|
+
|
|
252
260
|
// Assemble and broadcast the checkpoint proposal, including the last block that was not
|
|
253
261
|
// broadcasted yet, and wait to collect the committee attestations.
|
|
254
262
|
this.setStateFn(SequencerState.ASSEMBLING_CHECKPOINT, this.slot);
|
|
255
263
|
const checkpoint = await checkpointBuilder.completeCheckpoint();
|
|
256
264
|
|
|
265
|
+
// Record checkpoint-level build metrics
|
|
266
|
+
this.metrics.recordCheckpointBuild(
|
|
267
|
+
checkpointBuildTimer.ms(),
|
|
268
|
+
blocksInCheckpoint.length,
|
|
269
|
+
checkpoint.getStats().txCount,
|
|
270
|
+
Number(checkpoint.header.totalManaUsed.toBigInt()),
|
|
271
|
+
);
|
|
272
|
+
|
|
257
273
|
// Do not collect attestations nor publish to L1 in fisherman mode
|
|
258
274
|
if (this.config.fishermanMode) {
|
|
259
275
|
this.log.info(
|
|
@@ -280,6 +296,7 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
280
296
|
const proposal = await this.validatorClient.createCheckpointProposal(
|
|
281
297
|
checkpoint.header,
|
|
282
298
|
checkpoint.archive.root,
|
|
299
|
+
feeAssetPriceModifier,
|
|
283
300
|
lastBlock,
|
|
284
301
|
this.proposer,
|
|
285
302
|
checkpointProposalOptions,
|
|
@@ -306,20 +323,8 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
306
323
|
);
|
|
307
324
|
} catch (err) {
|
|
308
325
|
// We shouldn't really get here since we yield to another HA node
|
|
309
|
-
// as soon as we see these errors when creating block proposals.
|
|
310
|
-
if (err
|
|
311
|
-
this.log.info(`Attestations signature for slot ${this.slot} already signed by another HA node, yielding`, {
|
|
312
|
-
slot: this.slot,
|
|
313
|
-
signedByNode: err.signedByNode,
|
|
314
|
-
});
|
|
315
|
-
return undefined;
|
|
316
|
-
}
|
|
317
|
-
if (err instanceof SlashingProtectionError) {
|
|
318
|
-
this.log.info(`Attestations signature for slot ${this.slot} blocked by slashing protection, yielding`, {
|
|
319
|
-
slot: this.slot,
|
|
320
|
-
existingMessageHash: err.existingMessageHash,
|
|
321
|
-
attemptedMessageHash: err.attemptedMessageHash,
|
|
322
|
-
});
|
|
326
|
+
// as soon as we see these errors when creating block or checkpoint proposals.
|
|
327
|
+
if (this.handleHASigningError(err, 'Attestations signature')) {
|
|
323
328
|
return undefined;
|
|
324
329
|
}
|
|
325
330
|
throw err;
|
|
@@ -330,6 +335,21 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
330
335
|
const aztecSlotDuration = this.l1Constants.slotDuration;
|
|
331
336
|
const slotStartBuildTimestamp = this.getSlotStartBuildTimestamp();
|
|
332
337
|
const txTimeoutAt = new Date((slotStartBuildTimestamp + aztecSlotDuration) * 1000);
|
|
338
|
+
|
|
339
|
+
// If we have been configured to potentially skip publishing checkpoint then roll the dice here
|
|
340
|
+
if (
|
|
341
|
+
this.config.skipPublishingCheckpointsPercent !== undefined &&
|
|
342
|
+
this.config.skipPublishingCheckpointsPercent > 0
|
|
343
|
+
) {
|
|
344
|
+
const result = Math.max(0, randomInt(100));
|
|
345
|
+
if (result < this.config.skipPublishingCheckpointsPercent) {
|
|
346
|
+
this.log.warn(
|
|
347
|
+
`Skipping publishing proposal for checkpoint ${checkpoint.number}. Configured percentage: ${this.config.skipPublishingCheckpointsPercent}, generated value: ${result}`,
|
|
348
|
+
);
|
|
349
|
+
return checkpoint;
|
|
350
|
+
}
|
|
351
|
+
}
|
|
352
|
+
|
|
333
353
|
await this.publisher.enqueueProposeCheckpoint(checkpoint, attestations, attestationsSignature, {
|
|
334
354
|
txTimeoutAt,
|
|
335
355
|
forcePendingCheckpointNumber: this.invalidateCheckpoint?.forcePendingCheckpointNumber,
|
|
@@ -372,7 +392,7 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
372
392
|
|
|
373
393
|
while (true) {
|
|
374
394
|
const blocksBuilt = blocksInCheckpoint.length;
|
|
375
|
-
const indexWithinCheckpoint = blocksBuilt;
|
|
395
|
+
const indexWithinCheckpoint = IndexWithinCheckpoint(blocksBuilt);
|
|
376
396
|
const blockNumber = BlockNumber(initialBlockNumber + blocksBuilt);
|
|
377
397
|
|
|
378
398
|
const secondsIntoSlot = this.getSecondsIntoSlot();
|
|
@@ -402,6 +422,7 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
402
422
|
remainingBlobFields,
|
|
403
423
|
});
|
|
404
424
|
|
|
425
|
+
// TODO(palla/mbps): Review these conditions. We may want to keep trying in some scenarios.
|
|
405
426
|
if (!buildResult && timingInfo.isLastBlock) {
|
|
406
427
|
// If no block was produced due to not enough txs and this was the last subslot, exit
|
|
407
428
|
break;
|
|
@@ -488,13 +509,13 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
488
509
|
|
|
489
510
|
/** Builds a single block. Called from the main block building loop. */
|
|
490
511
|
@trackSpan('CheckpointProposalJob.buildSingleBlock')
|
|
491
|
-
|
|
512
|
+
protected async buildSingleBlock(
|
|
492
513
|
checkpointBuilder: CheckpointBuilder,
|
|
493
514
|
opts: {
|
|
494
515
|
forceCreate?: boolean;
|
|
495
516
|
blockTimestamp: bigint;
|
|
496
517
|
blockNumber: BlockNumber;
|
|
497
|
-
indexWithinCheckpoint:
|
|
518
|
+
indexWithinCheckpoint: IndexWithinCheckpoint;
|
|
498
519
|
buildDeadline: Date | undefined;
|
|
499
520
|
txHashesAlreadyIncluded: Set<string>;
|
|
500
521
|
remainingBlobFields: number;
|
|
@@ -532,7 +553,7 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
532
553
|
// Create iterator to pending txs. We filter out txs already included in previous blocks in the checkpoint
|
|
533
554
|
// just in case p2p failed to sync the provisional block and didn't get to remove those txs from the mempool yet.
|
|
534
555
|
const pendingTxs = filter(
|
|
535
|
-
this.p2pClient.
|
|
556
|
+
this.p2pClient.iterateEligiblePendingTxs(),
|
|
536
557
|
tx => !txHashesAlreadyIncluded.has(tx.txHash.toString()),
|
|
537
558
|
);
|
|
538
559
|
|
|
@@ -555,45 +576,38 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
555
576
|
};
|
|
556
577
|
|
|
557
578
|
// Actually build the block by executing txs
|
|
558
|
-
const
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
usedTxs,
|
|
566
|
-
failedTxs,
|
|
567
|
-
usedTxBlobFields,
|
|
568
|
-
} = await checkpointBuilder.buildBlock(pendingTxs, blockNumber, blockTimestamp, blockBuilderOptions);
|
|
569
|
-
const blockBuildDuration = workTimer.ms();
|
|
579
|
+
const buildResult = await this.buildSingleBlockWithCheckpointBuilder(
|
|
580
|
+
checkpointBuilder,
|
|
581
|
+
pendingTxs,
|
|
582
|
+
blockNumber,
|
|
583
|
+
blockTimestamp,
|
|
584
|
+
blockBuilderOptions,
|
|
585
|
+
);
|
|
570
586
|
|
|
571
587
|
// If any txs failed during execution, drop them from the mempool so we don't pick them up again
|
|
572
|
-
await this.dropFailedTxsFromP2P(failedTxs);
|
|
588
|
+
await this.dropFailedTxsFromP2P(buildResult.failedTxs);
|
|
573
589
|
|
|
574
590
|
// Check if we have created a block with enough txs. If there were invalid txs in the pool, or if execution took
|
|
575
591
|
// too long, then we may not get to minTxsPerBlock after executing public functions.
|
|
576
592
|
const minValidTxs = this.config.minValidTxsPerBlock ?? minTxs;
|
|
577
|
-
|
|
593
|
+
const numTxs = buildResult.status === 'no-valid-txs' ? 0 : buildResult.numTxs;
|
|
594
|
+
if (buildResult.status === 'no-valid-txs' || (!forceCreate && numTxs < minValidTxs)) {
|
|
578
595
|
this.log.warn(
|
|
579
|
-
`Block ${blockNumber} at index ${indexWithinCheckpoint} on slot ${this.slot} has too few valid txs to be proposed
|
|
580
|
-
{ slot: this.slot, blockNumber, numTxs, indexWithinCheckpoint },
|
|
596
|
+
`Block ${blockNumber} at index ${indexWithinCheckpoint} on slot ${this.slot} has too few valid txs to be proposed`,
|
|
597
|
+
{ slot: this.slot, blockNumber, numTxs, indexWithinCheckpoint, minValidTxs, buildResult: buildResult.status },
|
|
581
598
|
);
|
|
582
|
-
this.eventEmitter.emit('block-
|
|
583
|
-
minTxs: minValidTxs,
|
|
584
|
-
availableTxs: numTxs,
|
|
585
|
-
slot: this.slot,
|
|
586
|
-
});
|
|
599
|
+
this.eventEmitter.emit('block-build-failed', { reason: `Insufficient valid txs`, slot: this.slot });
|
|
587
600
|
this.metrics.recordBlockProposalFailed('insufficient_valid_txs');
|
|
588
601
|
return undefined;
|
|
589
602
|
}
|
|
590
603
|
|
|
591
604
|
// Block creation succeeded, emit stats and metrics
|
|
605
|
+
const { publicGas, block, publicProcessorDuration, usedTxs, usedTxBlobFields, blockBuildDuration } = buildResult;
|
|
606
|
+
|
|
592
607
|
const blockStats = {
|
|
593
608
|
eventName: 'l2-block-built',
|
|
594
609
|
duration: blockBuildDuration,
|
|
595
610
|
publicProcessDuration: publicProcessorDuration,
|
|
596
|
-
rollupCircuitsDuration: blockBuildingTimer.ms(),
|
|
597
611
|
...block.getStats(),
|
|
598
612
|
} satisfies L2BlockBuiltStats;
|
|
599
613
|
|
|
@@ -619,17 +633,40 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
619
633
|
}
|
|
620
634
|
}
|
|
621
635
|
|
|
636
|
+
/** Uses the checkpoint builder to build a block, catching specific txs */
|
|
637
|
+
private async buildSingleBlockWithCheckpointBuilder(
|
|
638
|
+
checkpointBuilder: CheckpointBuilder,
|
|
639
|
+
pendingTxs: AsyncIterable<Tx>,
|
|
640
|
+
blockNumber: BlockNumber,
|
|
641
|
+
blockTimestamp: bigint,
|
|
642
|
+
blockBuilderOptions: PublicProcessorLimits,
|
|
643
|
+
) {
|
|
644
|
+
try {
|
|
645
|
+
const workTimer = new Timer();
|
|
646
|
+
const result = await checkpointBuilder.buildBlock(pendingTxs, blockNumber, blockTimestamp, blockBuilderOptions);
|
|
647
|
+
const blockBuildDuration = workTimer.ms();
|
|
648
|
+
return { ...result, blockBuildDuration, status: 'success' as const };
|
|
649
|
+
} catch (err: unknown) {
|
|
650
|
+
if (isErrorClass(err, NoValidTxsError)) {
|
|
651
|
+
return { failedTxs: err.failedTxs, status: 'no-valid-txs' as const };
|
|
652
|
+
}
|
|
653
|
+
throw err;
|
|
654
|
+
}
|
|
655
|
+
}
|
|
656
|
+
|
|
622
657
|
/** Waits until minTxs are available on the pool for building a block. */
|
|
623
658
|
@trackSpan('CheckpointProposalJob.waitForMinTxs')
|
|
624
659
|
private async waitForMinTxs(opts: {
|
|
625
660
|
forceCreate?: boolean;
|
|
626
661
|
blockNumber: BlockNumber;
|
|
627
|
-
indexWithinCheckpoint:
|
|
662
|
+
indexWithinCheckpoint: IndexWithinCheckpoint;
|
|
628
663
|
buildDeadline: Date | undefined;
|
|
629
664
|
}): Promise<{ canStartBuilding: boolean; availableTxs: number }> {
|
|
630
|
-
const minTxs = this.config.minTxsPerBlock;
|
|
631
665
|
const { indexWithinCheckpoint, blockNumber, buildDeadline, forceCreate } = opts;
|
|
632
666
|
|
|
667
|
+
// We only allow a block with 0 txs in the first block of the checkpoint
|
|
668
|
+
const minTxs = indexWithinCheckpoint > 0 && this.config.minTxsPerBlock === 0 ? 1 : this.config.minTxsPerBlock;
|
|
669
|
+
|
|
633
670
|
// Deadline is undefined if we are not enforcing the timetable, meaning we'll exit immediately when out of time
|
|
634
671
|
const startBuildingDeadline = buildDeadline
|
|
635
672
|
? new Date(buildDeadline.getTime() - this.timetable.minExecutionTime * 1000)
|
|
@@ -650,7 +687,7 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
650
687
|
`Waiting for enough txs to build block ${blockNumber} at index ${indexWithinCheckpoint} in slot ${this.slot} (have ${availableTxs} but need ${minTxs})`,
|
|
651
688
|
{ blockNumber, slot: this.slot, indexWithinCheckpoint },
|
|
652
689
|
);
|
|
653
|
-
await
|
|
690
|
+
await this.waitForTxsPollingInterval();
|
|
654
691
|
availableTxs = await this.p2pClient.getPendingTxCount();
|
|
655
692
|
}
|
|
656
693
|
|
|
@@ -779,7 +816,7 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
779
816
|
const failedTxData = failedTxs.map(fail => fail.tx);
|
|
780
817
|
const failedTxHashes = failedTxData.map(tx => tx.getTxHash());
|
|
781
818
|
this.log.verbose(`Dropping failed txs ${failedTxHashes.join(', ')}`);
|
|
782
|
-
await this.p2pClient.
|
|
819
|
+
await this.p2pClient.handleFailedExecution(failedTxHashes);
|
|
783
820
|
}
|
|
784
821
|
|
|
785
822
|
/**
|
|
@@ -821,12 +858,34 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
821
858
|
slot: this.slot,
|
|
822
859
|
feeAnalysisId: feeAnalysis?.id,
|
|
823
860
|
});
|
|
824
|
-
this.metrics.
|
|
861
|
+
this.metrics.recordCheckpointProposalFailed('block_build_failed');
|
|
825
862
|
}
|
|
826
863
|
|
|
827
864
|
this.publisher.clearPendingRequests();
|
|
828
865
|
}
|
|
829
866
|
|
|
867
|
+
/**
|
|
868
|
+
* Helper to handle HA double-signing errors. Returns true if the error was handled (caller should yield).
|
|
869
|
+
*/
|
|
870
|
+
private handleHASigningError(err: any, errorContext: string): boolean {
|
|
871
|
+
if (err instanceof DutyAlreadySignedError) {
|
|
872
|
+
this.log.info(`${errorContext} for slot ${this.slot} already signed by another HA node, yielding`, {
|
|
873
|
+
slot: this.slot,
|
|
874
|
+
signedByNode: err.signedByNode,
|
|
875
|
+
});
|
|
876
|
+
return true;
|
|
877
|
+
}
|
|
878
|
+
if (err instanceof SlashingProtectionError) {
|
|
879
|
+
this.log.info(`${errorContext} for slot ${this.slot} blocked by slashing protection, yielding`, {
|
|
880
|
+
slot: this.slot,
|
|
881
|
+
existingMessageHash: err.existingMessageHash,
|
|
882
|
+
attemptedMessageHash: err.attemptedMessageHash,
|
|
883
|
+
});
|
|
884
|
+
return true;
|
|
885
|
+
}
|
|
886
|
+
return false;
|
|
887
|
+
}
|
|
888
|
+
|
|
830
889
|
/** Waits until a specific time within the current slot */
|
|
831
890
|
@trackSpan('CheckpointProposalJob.waitUntilTimeInSlot')
|
|
832
891
|
protected async waitUntilTimeInSlot(targetSecondsIntoSlot: number): Promise<void> {
|
|
@@ -835,6 +894,11 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
835
894
|
await sleepUntil(new Date(targetTimestamp * 1000), this.dateProvider.nowAsDate());
|
|
836
895
|
}
|
|
837
896
|
|
|
897
|
+
/** Waits the polling interval for transactions. Extracted for test overriding. */
|
|
898
|
+
protected async waitForTxsPollingInterval(): Promise<void> {
|
|
899
|
+
await sleep(TXS_POLLING_MS);
|
|
900
|
+
}
|
|
901
|
+
|
|
838
902
|
private getSlotStartBuildTimestamp(): number {
|
|
839
903
|
return getSlotStartBuildTimestamp(this.slot, this.l1Constants);
|
|
840
904
|
}
|
package/src/sequencer/metrics.ts
CHANGED
|
@@ -18,7 +18,6 @@ import { type Hex, formatUnits } from 'viem';
|
|
|
18
18
|
|
|
19
19
|
import type { SequencerState } from './utils.js';
|
|
20
20
|
|
|
21
|
-
// TODO(palla/mbps): Review all metrics and add any missing ones per checkpoint
|
|
22
21
|
export class SequencerMetrics {
|
|
23
22
|
public readonly tracer: Tracer;
|
|
24
23
|
private meter: Meter;
|
|
@@ -40,17 +39,26 @@ export class SequencerMetrics {
|
|
|
40
39
|
private filledSlots: UpDownCounter;
|
|
41
40
|
|
|
42
41
|
private blockProposalFailed: UpDownCounter;
|
|
43
|
-
private
|
|
44
|
-
private
|
|
42
|
+
private checkpointProposalSuccess: UpDownCounter;
|
|
43
|
+
private checkpointPrecheckFailed: UpDownCounter;
|
|
44
|
+
private checkpointProposalFailed: UpDownCounter;
|
|
45
45
|
private checkpointSuccess: UpDownCounter;
|
|
46
46
|
private slashingAttempts: UpDownCounter;
|
|
47
47
|
private checkpointAttestationDelay: Histogram;
|
|
48
|
+
private checkpointBuildDuration: Histogram;
|
|
49
|
+
private checkpointBlockCount: Gauge;
|
|
50
|
+
private checkpointTxCount: Gauge;
|
|
51
|
+
private checkpointTotalMana: Gauge;
|
|
48
52
|
|
|
49
53
|
// Fisherman fee analysis metrics
|
|
50
54
|
private fishermanWouldBeIncluded: UpDownCounter;
|
|
51
55
|
private fishermanTimeBeforeBlock: Histogram;
|
|
52
56
|
private fishermanPendingBlobTxCount: Histogram;
|
|
53
57
|
private fishermanIncludedBlobTxCount: Histogram;
|
|
58
|
+
private fishermanPendingBlobCount: Histogram;
|
|
59
|
+
private fishermanIncludedBlobCount: Histogram;
|
|
60
|
+
private fishermanBlockBlobsFull: UpDownCounter;
|
|
61
|
+
private fishermanMaxBlobCapacity: Histogram;
|
|
54
62
|
private fishermanCalculatedPriorityFee: Histogram;
|
|
55
63
|
private fishermanPriorityFeeDelta: Histogram;
|
|
56
64
|
private fishermanEstimatedCost: Histogram;
|
|
@@ -80,7 +88,7 @@ export class SequencerMetrics {
|
|
|
80
88
|
|
|
81
89
|
this.checkpointAttestationDelay = this.meter.createHistogram(Metrics.SEQUENCER_CHECKPOINT_ATTESTATION_DELAY);
|
|
82
90
|
|
|
83
|
-
this.rewards = this.meter.createGauge(Metrics.
|
|
91
|
+
this.rewards = this.meter.createGauge(Metrics.SEQUENCER_CURRENT_SLOT_REWARDS);
|
|
84
92
|
|
|
85
93
|
this.slots = createUpDownCounterWithDefault(this.meter, Metrics.SEQUENCER_SLOT_COUNT);
|
|
86
94
|
|
|
@@ -103,16 +111,16 @@ export class SequencerMetrics {
|
|
|
103
111
|
Metrics.SEQUENCER_BLOCK_PROPOSAL_FAILED_COUNT,
|
|
104
112
|
);
|
|
105
113
|
|
|
106
|
-
this.
|
|
114
|
+
this.checkpointProposalSuccess = createUpDownCounterWithDefault(
|
|
107
115
|
this.meter,
|
|
108
|
-
Metrics.
|
|
116
|
+
Metrics.SEQUENCER_CHECKPOINT_PROPOSAL_SUCCESS_COUNT,
|
|
109
117
|
);
|
|
110
118
|
|
|
111
119
|
this.checkpointSuccess = createUpDownCounterWithDefault(this.meter, Metrics.SEQUENCER_CHECKPOINT_SUCCESS_COUNT);
|
|
112
120
|
|
|
113
|
-
this.
|
|
121
|
+
this.checkpointPrecheckFailed = createUpDownCounterWithDefault(
|
|
114
122
|
this.meter,
|
|
115
|
-
Metrics.
|
|
123
|
+
Metrics.SEQUENCER_CHECKPOINT_PRECHECK_FAILED_COUNT,
|
|
116
124
|
{
|
|
117
125
|
[Attributes.ERROR_TYPE]: [
|
|
118
126
|
'slot_already_taken',
|
|
@@ -123,6 +131,16 @@ export class SequencerMetrics {
|
|
|
123
131
|
},
|
|
124
132
|
);
|
|
125
133
|
|
|
134
|
+
this.checkpointProposalFailed = createUpDownCounterWithDefault(
|
|
135
|
+
this.meter,
|
|
136
|
+
Metrics.SEQUENCER_CHECKPOINT_PROPOSAL_FAILED_COUNT,
|
|
137
|
+
);
|
|
138
|
+
|
|
139
|
+
this.checkpointBuildDuration = this.meter.createHistogram(Metrics.SEQUENCER_CHECKPOINT_BUILD_DURATION);
|
|
140
|
+
this.checkpointBlockCount = this.meter.createGauge(Metrics.SEQUENCER_CHECKPOINT_BLOCK_COUNT);
|
|
141
|
+
this.checkpointTxCount = this.meter.createGauge(Metrics.SEQUENCER_CHECKPOINT_TX_COUNT);
|
|
142
|
+
this.checkpointTotalMana = this.meter.createGauge(Metrics.SEQUENCER_CHECKPOINT_TOTAL_MANA);
|
|
143
|
+
|
|
126
144
|
this.slashingAttempts = createUpDownCounterWithDefault(this.meter, Metrics.SEQUENCER_SLASHING_ATTEMPTS_COUNT);
|
|
127
145
|
|
|
128
146
|
// Fisherman fee analysis metrics
|
|
@@ -131,6 +149,7 @@ export class SequencerMetrics {
|
|
|
131
149
|
Metrics.FISHERMAN_FEE_ANALYSIS_WOULD_BE_INCLUDED,
|
|
132
150
|
{
|
|
133
151
|
[Attributes.OK]: [true, false],
|
|
152
|
+
[Attributes.BLOCK_FULL]: ['true', 'false'],
|
|
134
153
|
},
|
|
135
154
|
);
|
|
136
155
|
|
|
@@ -161,6 +180,20 @@ export class SequencerMetrics {
|
|
|
161
180
|
this.fishermanMinedBlobTxTotalCost = this.meter.createHistogram(
|
|
162
181
|
Metrics.FISHERMAN_FEE_ANALYSIS_MINED_BLOB_TX_TOTAL_COST,
|
|
163
182
|
);
|
|
183
|
+
|
|
184
|
+
this.fishermanPendingBlobCount = this.meter.createHistogram(Metrics.FISHERMAN_FEE_ANALYSIS_PENDING_BLOB_COUNT);
|
|
185
|
+
|
|
186
|
+
this.fishermanIncludedBlobCount = this.meter.createHistogram(Metrics.FISHERMAN_FEE_ANALYSIS_INCLUDED_BLOB_COUNT);
|
|
187
|
+
|
|
188
|
+
this.fishermanBlockBlobsFull = createUpDownCounterWithDefault(
|
|
189
|
+
this.meter,
|
|
190
|
+
Metrics.FISHERMAN_FEE_ANALYSIS_BLOCK_BLOBS_FULL,
|
|
191
|
+
{
|
|
192
|
+
[Attributes.OK]: [true, false],
|
|
193
|
+
},
|
|
194
|
+
);
|
|
195
|
+
|
|
196
|
+
this.fishermanMaxBlobCapacity = this.meter.createHistogram(Metrics.FISHERMAN_FEE_ANALYSIS_MAX_BLOB_CAPACITY);
|
|
164
197
|
}
|
|
165
198
|
|
|
166
199
|
public recordRequiredAttestations(requiredAttestationsCount: number, allowanceMs: number) {
|
|
@@ -243,18 +276,30 @@ export class SequencerMetrics {
|
|
|
243
276
|
});
|
|
244
277
|
}
|
|
245
278
|
|
|
246
|
-
|
|
247
|
-
this.
|
|
279
|
+
recordCheckpointProposalSuccess() {
|
|
280
|
+
this.checkpointProposalSuccess.add(1);
|
|
248
281
|
}
|
|
249
282
|
|
|
250
|
-
|
|
283
|
+
recordCheckpointPrecheckFailed(
|
|
251
284
|
checkType: 'slot_already_taken' | 'rollup_contract_check_failed' | 'slot_mismatch' | 'block_number_mismatch',
|
|
252
285
|
) {
|
|
253
|
-
this.
|
|
254
|
-
|
|
286
|
+
this.checkpointPrecheckFailed.add(1, { [Attributes.ERROR_TYPE]: checkType });
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
recordCheckpointProposalFailed(reason?: string) {
|
|
290
|
+
this.checkpointProposalFailed.add(1, {
|
|
291
|
+
...(reason && { [Attributes.ERROR_TYPE]: reason }),
|
|
255
292
|
});
|
|
256
293
|
}
|
|
257
294
|
|
|
295
|
+
/** Records aggregate metrics for a completed checkpoint build. */
|
|
296
|
+
recordCheckpointBuild(durationMs: number, blockCount: number, txCount: number, totalMana: number) {
|
|
297
|
+
this.checkpointBuildDuration.record(Math.ceil(durationMs));
|
|
298
|
+
this.checkpointBlockCount.record(blockCount);
|
|
299
|
+
this.checkpointTxCount.record(txCount);
|
|
300
|
+
this.checkpointTotalMana.record(totalMana);
|
|
301
|
+
}
|
|
302
|
+
|
|
258
303
|
recordSlashingAttempt(actionCount: number) {
|
|
259
304
|
this.slashingAttempts.add(actionCount);
|
|
260
305
|
}
|
|
@@ -281,10 +326,12 @@ export class SequencerMetrics {
|
|
|
281
326
|
|
|
282
327
|
// Record pending block snapshot data (once per strategy for comparison)
|
|
283
328
|
this.fishermanPendingBlobTxCount.record(analysis.pendingSnapshot.pendingBlobTxCount, strategyAttributes);
|
|
329
|
+
this.fishermanPendingBlobCount.record(analysis.pendingSnapshot.pendingBlobCount, strategyAttributes);
|
|
284
330
|
|
|
285
331
|
// Record mined block data if available
|
|
286
332
|
if (analysis.minedBlock) {
|
|
287
333
|
this.fishermanIncludedBlobTxCount.record(analysis.minedBlock.includedBlobTxCount, strategyAttributes);
|
|
334
|
+
this.fishermanIncludedBlobCount.record(analysis.minedBlock.includedBlobCount, strategyAttributes);
|
|
288
335
|
|
|
289
336
|
// Record actual fees from blob transactions in the mined block
|
|
290
337
|
for (const blobTx of analysis.minedBlock.includedBlobTxs) {
|
|
@@ -318,13 +365,28 @@ export class SequencerMetrics {
|
|
|
318
365
|
if (analysis.analysis) {
|
|
319
366
|
this.fishermanTimeBeforeBlock.record(Math.ceil(analysis.analysis.timeBeforeBlockMs), strategyAttributes);
|
|
320
367
|
|
|
368
|
+
// Record whether the block reached 100% blob capacity
|
|
369
|
+
if (analysis.analysis.blockBlobsFull) {
|
|
370
|
+
this.fishermanBlockBlobsFull.add(1, { ...strategyAttributes, [Attributes.OK]: true });
|
|
371
|
+
} else {
|
|
372
|
+
this.fishermanBlockBlobsFull.add(1, { ...strategyAttributes, [Attributes.OK]: false });
|
|
373
|
+
}
|
|
374
|
+
|
|
375
|
+
// Record the max blob capacity for this block
|
|
376
|
+
this.fishermanMaxBlobCapacity.record(analysis.analysis.maxBlobCapacity, strategyAttributes);
|
|
377
|
+
|
|
321
378
|
// Record strategy-specific inclusion result
|
|
322
379
|
if (strategyResult.wouldBeIncluded !== undefined) {
|
|
380
|
+
const inclusionAttributes = {
|
|
381
|
+
...strategyAttributes,
|
|
382
|
+
[Attributes.BLOCK_FULL]: analysis.analysis.blockBlobsFull ? 'true' : 'false',
|
|
383
|
+
};
|
|
384
|
+
|
|
323
385
|
if (strategyResult.wouldBeIncluded) {
|
|
324
|
-
this.fishermanWouldBeIncluded.add(1, { ...
|
|
386
|
+
this.fishermanWouldBeIncluded.add(1, { ...inclusionAttributes, [Attributes.OK]: true });
|
|
325
387
|
} else {
|
|
326
388
|
this.fishermanWouldBeIncluded.add(1, {
|
|
327
|
-
...
|
|
389
|
+
...inclusionAttributes,
|
|
328
390
|
[Attributes.OK]: false,
|
|
329
391
|
...(strategyResult.exclusionReason && { [Attributes.ERROR_TYPE]: strategyResult.exclusionReason }),
|
|
330
392
|
});
|
|
@@ -334,17 +396,29 @@ export class SequencerMetrics {
|
|
|
334
396
|
// Record strategy-specific priority fee delta
|
|
335
397
|
if (strategyResult.priorityFeeDelta !== undefined) {
|
|
336
398
|
const priorityFeeDeltaGwei = Number(strategyResult.priorityFeeDelta) / 1e9;
|
|
337
|
-
|
|
399
|
+
const deltaAttributes = {
|
|
400
|
+
...strategyAttributes,
|
|
401
|
+
[Attributes.BLOCK_FULL]: analysis.analysis.blockBlobsFull ? 'true' : 'false',
|
|
402
|
+
};
|
|
403
|
+
this.fishermanPriorityFeeDelta.record(priorityFeeDeltaGwei, deltaAttributes);
|
|
338
404
|
}
|
|
339
405
|
|
|
340
406
|
// Record estimated cost if available
|
|
341
407
|
if (strategyResult.estimatedCostEth !== undefined) {
|
|
342
|
-
|
|
408
|
+
const costAttributes = {
|
|
409
|
+
...strategyAttributes,
|
|
410
|
+
[Attributes.BLOCK_FULL]: analysis.analysis.blockBlobsFull ? 'true' : 'false',
|
|
411
|
+
};
|
|
412
|
+
this.fishermanEstimatedCost.record(strategyResult.estimatedCostEth, costAttributes);
|
|
343
413
|
}
|
|
344
414
|
|
|
345
415
|
// Record estimated overpayment if available
|
|
346
416
|
if (strategyResult.estimatedOverpaymentEth !== undefined) {
|
|
347
|
-
|
|
417
|
+
const overpaymentAttributes = {
|
|
418
|
+
...strategyAttributes,
|
|
419
|
+
[Attributes.BLOCK_FULL]: analysis.analysis.blockBlobsFull ? 'true' : 'false',
|
|
420
|
+
};
|
|
421
|
+
this.fishermanEstimatedOverpayment.record(strategyResult.estimatedOverpaymentEth, overpaymentAttributes);
|
|
348
422
|
}
|
|
349
423
|
}
|
|
350
424
|
}
|