@aztec/sequencer-client 0.0.1-commit.f295ac2 → 0.0.1-commit.f2ce05ee
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dest/client/sequencer-client.js +1 -1
- package/dest/config.d.ts +1 -2
- package/dest/config.d.ts.map +1 -1
- package/dest/config.js +5 -11
- package/dest/global_variable_builder/global_builder.js +2 -2
- package/dest/publisher/sequencer-publisher-metrics.d.ts +1 -1
- package/dest/publisher/sequencer-publisher-metrics.d.ts.map +1 -1
- package/dest/publisher/sequencer-publisher-metrics.js +12 -4
- package/dest/publisher/sequencer-publisher.d.ts +1 -2
- package/dest/publisher/sequencer-publisher.d.ts.map +1 -1
- package/dest/publisher/sequencer-publisher.js +39 -18
- package/dest/sequencer/checkpoint_proposal_job.d.ts +32 -9
- package/dest/sequencer/checkpoint_proposal_job.d.ts.map +1 -1
- package/dest/sequencer/checkpoint_proposal_job.js +86 -50
- package/dest/sequencer/metrics.d.ts +2 -2
- package/dest/sequencer/metrics.d.ts.map +1 -1
- package/dest/sequencer/metrics.js +27 -17
- package/dest/sequencer/sequencer.d.ts +5 -3
- package/dest/sequencer/sequencer.d.ts.map +1 -1
- package/dest/sequencer/sequencer.js +7 -3
- package/dest/sequencer/timetable.d.ts +1 -4
- package/dest/sequencer/timetable.d.ts.map +1 -1
- package/dest/sequencer/timetable.js +1 -4
- package/dest/test/mock_checkpoint_builder.d.ts +11 -8
- package/dest/test/mock_checkpoint_builder.d.ts.map +1 -1
- package/dest/test/mock_checkpoint_builder.js +18 -6
- package/dest/test/utils.d.ts +8 -8
- package/dest/test/utils.d.ts.map +1 -1
- package/dest/test/utils.js +5 -5
- package/package.json +28 -28
- package/src/client/sequencer-client.ts +1 -1
- package/src/config.ts +10 -14
- package/src/global_variable_builder/global_builder.ts +2 -2
- package/src/publisher/sequencer-publisher-metrics.ts +7 -3
- package/src/publisher/sequencer-publisher.ts +34 -18
- package/src/sequencer/checkpoint_proposal_job.ts +117 -76
- package/src/sequencer/metrics.ts +36 -18
- package/src/sequencer/sequencer.ts +11 -5
- package/src/sequencer/timetable.ts +6 -5
- package/src/test/mock_checkpoint_builder.ts +32 -18
- package/src/test/utils.ts +17 -11
|
@@ -1,22 +1,28 @@
|
|
|
1
1
|
import { NUM_CHECKPOINT_END_MARKER_FIELDS, getNumBlockEndBlobFields } from '@aztec/blob-lib/encoding';
|
|
2
2
|
import { BLOBS_PER_CHECKPOINT, FIELDS_PER_BLOB } from '@aztec/constants';
|
|
3
3
|
import type { EpochCache } from '@aztec/epoch-cache';
|
|
4
|
-
import {
|
|
4
|
+
import {
|
|
5
|
+
BlockNumber,
|
|
6
|
+
CheckpointNumber,
|
|
7
|
+
EpochNumber,
|
|
8
|
+
IndexWithinCheckpoint,
|
|
9
|
+
SlotNumber,
|
|
10
|
+
} from '@aztec/foundation/branded-types';
|
|
5
11
|
import { randomInt } from '@aztec/foundation/crypto/random';
|
|
6
12
|
import { Fr } from '@aztec/foundation/curves/bn254';
|
|
7
13
|
import { EthAddress } from '@aztec/foundation/eth-address';
|
|
8
14
|
import { Signature } from '@aztec/foundation/eth-signature';
|
|
9
15
|
import { filter } from '@aztec/foundation/iterator';
|
|
10
|
-
import type
|
|
16
|
+
import { type Logger, type LoggerBindings, createLogger } from '@aztec/foundation/log';
|
|
11
17
|
import { sleep, sleepUntil } from '@aztec/foundation/sleep';
|
|
12
18
|
import { type DateProvider, Timer } from '@aztec/foundation/timer';
|
|
13
|
-
import { type TypedEventEmitter, unfreeze } from '@aztec/foundation/types';
|
|
19
|
+
import { type TypedEventEmitter, isErrorClass, unfreeze } from '@aztec/foundation/types';
|
|
14
20
|
import type { P2P } from '@aztec/p2p';
|
|
15
21
|
import type { SlasherClientInterface } from '@aztec/slasher';
|
|
16
22
|
import {
|
|
17
23
|
CommitteeAttestation,
|
|
18
24
|
CommitteeAttestationsAndSigners,
|
|
19
|
-
|
|
25
|
+
L2Block,
|
|
20
26
|
type L2BlockSink,
|
|
21
27
|
type L2BlockSource,
|
|
22
28
|
MaliciousCommitteeAttestationsAndSigners,
|
|
@@ -24,10 +30,11 @@ import {
|
|
|
24
30
|
import type { Checkpoint } from '@aztec/stdlib/checkpoint';
|
|
25
31
|
import { getSlotStartBuildTimestamp } from '@aztec/stdlib/epoch-helpers';
|
|
26
32
|
import { Gas } from '@aztec/stdlib/gas';
|
|
27
|
-
import
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
33
|
+
import {
|
|
34
|
+
NoValidTxsError,
|
|
35
|
+
type PublicProcessorLimits,
|
|
36
|
+
type ResolvedSequencerConfig,
|
|
37
|
+
type WorldStateSynchronizer,
|
|
31
38
|
} from '@aztec/stdlib/interfaces/server';
|
|
32
39
|
import { type L1ToL2MessageSource, computeInHashFromL1ToL2Messages } from '@aztec/stdlib/messaging';
|
|
33
40
|
import type { BlockProposalOptions, CheckpointProposal, CheckpointProposalOptions } from '@aztec/stdlib/p2p';
|
|
@@ -59,6 +66,8 @@ const TXS_POLLING_MS = 500;
|
|
|
59
66
|
* the Sequencer once the check for being the proposer for the slot has succeeded.
|
|
60
67
|
*/
|
|
61
68
|
export class CheckpointProposalJob implements Traceable {
|
|
69
|
+
protected readonly log: Logger;
|
|
70
|
+
|
|
62
71
|
constructor(
|
|
63
72
|
private readonly epoch: EpochNumber,
|
|
64
73
|
private readonly slot: SlotNumber,
|
|
@@ -86,9 +95,11 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
86
95
|
private readonly metrics: SequencerMetrics,
|
|
87
96
|
private readonly eventEmitter: TypedEventEmitter<SequencerEvents>,
|
|
88
97
|
private readonly setStateFn: (state: SequencerState, slot?: SlotNumber) => void,
|
|
89
|
-
protected readonly log: Logger,
|
|
90
98
|
public readonly tracer: Tracer,
|
|
91
|
-
|
|
99
|
+
bindings?: LoggerBindings,
|
|
100
|
+
) {
|
|
101
|
+
this.log = createLogger('sequencer:checkpoint-proposal', { ...bindings, instanceId: `slot-${slot}` });
|
|
102
|
+
}
|
|
92
103
|
|
|
93
104
|
/**
|
|
94
105
|
* Executes the checkpoint proposal job.
|
|
@@ -190,6 +201,7 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
190
201
|
l1ToL2Messages,
|
|
191
202
|
previousCheckpointOutHashes,
|
|
192
203
|
fork,
|
|
204
|
+
this.log.getBindings(),
|
|
193
205
|
);
|
|
194
206
|
|
|
195
207
|
// Options for the validator client when creating block and checkpoint proposals
|
|
@@ -203,8 +215,8 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
203
215
|
broadcastInvalidCheckpointProposal: this.config.broadcastInvalidBlockProposal,
|
|
204
216
|
};
|
|
205
217
|
|
|
206
|
-
let blocksInCheckpoint:
|
|
207
|
-
let blockPendingBroadcast: { block:
|
|
218
|
+
let blocksInCheckpoint: L2Block[] = [];
|
|
219
|
+
let blockPendingBroadcast: { block: L2Block; txs: Tx[] } | undefined = undefined;
|
|
208
220
|
|
|
209
221
|
try {
|
|
210
222
|
// Main loop: build blocks for the checkpoint
|
|
@@ -220,19 +232,7 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
220
232
|
// These errors are expected in HA mode, so we yield and let another HA node handle the slot
|
|
221
233
|
// The only distinction between the 2 errors is SlashingProtectionError throws when the payload is different,
|
|
222
234
|
// which is normal for block building (may have picked different txs)
|
|
223
|
-
if (err
|
|
224
|
-
this.log.info(`Checkpoint proposal for slot ${this.slot} already signed by another HA node, yielding`, {
|
|
225
|
-
slot: this.slot,
|
|
226
|
-
signedByNode: err.signedByNode,
|
|
227
|
-
});
|
|
228
|
-
return undefined;
|
|
229
|
-
}
|
|
230
|
-
if (err instanceof SlashingProtectionError) {
|
|
231
|
-
this.log.info(`Checkpoint proposal for slot ${this.slot} blocked by slashing protection, yielding`, {
|
|
232
|
-
slot: this.slot,
|
|
233
|
-
existingMessageHash: err.existingMessageHash,
|
|
234
|
-
attemptedMessageHash: err.attemptedMessageHash,
|
|
235
|
-
});
|
|
235
|
+
if (this.handleHASigningError(err, 'Block proposal')) {
|
|
236
236
|
return undefined;
|
|
237
237
|
}
|
|
238
238
|
throw err;
|
|
@@ -301,20 +301,8 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
301
301
|
);
|
|
302
302
|
} catch (err) {
|
|
303
303
|
// We shouldn't really get here since we yield to another HA node
|
|
304
|
-
// as soon as we see these errors when creating block proposals.
|
|
305
|
-
if (err
|
|
306
|
-
this.log.info(`Attestations signature for slot ${this.slot} already signed by another HA node, yielding`, {
|
|
307
|
-
slot: this.slot,
|
|
308
|
-
signedByNode: err.signedByNode,
|
|
309
|
-
});
|
|
310
|
-
return undefined;
|
|
311
|
-
}
|
|
312
|
-
if (err instanceof SlashingProtectionError) {
|
|
313
|
-
this.log.info(`Attestations signature for slot ${this.slot} blocked by slashing protection, yielding`, {
|
|
314
|
-
slot: this.slot,
|
|
315
|
-
existingMessageHash: err.existingMessageHash,
|
|
316
|
-
attemptedMessageHash: err.attemptedMessageHash,
|
|
317
|
-
});
|
|
304
|
+
// as soon as we see these errors when creating block or checkpoint proposals.
|
|
305
|
+
if (this.handleHASigningError(err, 'Attestations signature')) {
|
|
318
306
|
return undefined;
|
|
319
307
|
}
|
|
320
308
|
throw err;
|
|
@@ -332,6 +320,11 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
332
320
|
|
|
333
321
|
return checkpoint;
|
|
334
322
|
} catch (err) {
|
|
323
|
+
if (err && (err instanceof DutyAlreadySignedError || err instanceof SlashingProtectionError)) {
|
|
324
|
+
// swallow this error. It's already been logged by a function deeper in the stack
|
|
325
|
+
return undefined;
|
|
326
|
+
}
|
|
327
|
+
|
|
335
328
|
this.log.error(`Error building checkpoint at slot ${this.slot}`, err);
|
|
336
329
|
return undefined;
|
|
337
330
|
}
|
|
@@ -347,10 +340,10 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
347
340
|
inHash: Fr,
|
|
348
341
|
blockProposalOptions: BlockProposalOptions,
|
|
349
342
|
): Promise<{
|
|
350
|
-
blocksInCheckpoint:
|
|
351
|
-
blockPendingBroadcast: { block:
|
|
343
|
+
blocksInCheckpoint: L2Block[];
|
|
344
|
+
blockPendingBroadcast: { block: L2Block; txs: Tx[] } | undefined;
|
|
352
345
|
}> {
|
|
353
|
-
const blocksInCheckpoint:
|
|
346
|
+
const blocksInCheckpoint: L2Block[] = [];
|
|
354
347
|
const txHashesAlreadyIncluded = new Set<string>();
|
|
355
348
|
const initialBlockNumber = BlockNumber(this.syncedToBlockNumber + 1);
|
|
356
349
|
|
|
@@ -358,11 +351,11 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
358
351
|
let remainingBlobFields = BLOBS_PER_CHECKPOINT * FIELDS_PER_BLOB - NUM_CHECKPOINT_END_MARKER_FIELDS;
|
|
359
352
|
|
|
360
353
|
// Last block in the checkpoint will usually be flagged as pending broadcast, so we send it along with the checkpoint proposal
|
|
361
|
-
let blockPendingBroadcast: { block:
|
|
354
|
+
let blockPendingBroadcast: { block: L2Block; txs: Tx[] } | undefined = undefined;
|
|
362
355
|
|
|
363
356
|
while (true) {
|
|
364
357
|
const blocksBuilt = blocksInCheckpoint.length;
|
|
365
|
-
const indexWithinCheckpoint = blocksBuilt;
|
|
358
|
+
const indexWithinCheckpoint = IndexWithinCheckpoint(blocksBuilt);
|
|
366
359
|
const blockNumber = BlockNumber(initialBlockNumber + blocksBuilt);
|
|
367
360
|
|
|
368
361
|
const secondsIntoSlot = this.getSecondsIntoSlot();
|
|
@@ -392,6 +385,7 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
392
385
|
remainingBlobFields,
|
|
393
386
|
});
|
|
394
387
|
|
|
388
|
+
// TODO(palla/mbps): Review these conditions. We may want to keep trying in some scenarios.
|
|
395
389
|
if (!buildResult && timingInfo.isLastBlock) {
|
|
396
390
|
// If no block was produced due to not enough txs and this was the last subslot, exit
|
|
397
391
|
break;
|
|
@@ -423,7 +417,12 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
423
417
|
// Sync the proposed block to the archiver to make it available
|
|
424
418
|
// Note that the checkpoint builder uses its own fork so it should not need to wait for this syncing
|
|
425
419
|
// Eventually we should refactor the checkpoint builder to not need a separate long-lived fork
|
|
426
|
-
|
|
420
|
+
// Fire and forget - don't block the critical path, but log errors
|
|
421
|
+
this.syncProposedBlockToArchiver(block).catch(err => {
|
|
422
|
+
this.log.error(`Failed to sync proposed block ${block.number} to archiver`, { blockNumber: block.number, err });
|
|
423
|
+
});
|
|
424
|
+
|
|
425
|
+
usedTxs.forEach(tx => txHashesAlreadyIncluded.add(tx.txHash.toString()));
|
|
427
426
|
|
|
428
427
|
// If this is the last block, exit the loop now so we start collecting attestations
|
|
429
428
|
if (timingInfo.isLastBlock) {
|
|
@@ -473,18 +472,18 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
473
472
|
|
|
474
473
|
/** Builds a single block. Called from the main block building loop. */
|
|
475
474
|
@trackSpan('CheckpointProposalJob.buildSingleBlock')
|
|
476
|
-
|
|
475
|
+
protected async buildSingleBlock(
|
|
477
476
|
checkpointBuilder: CheckpointBuilder,
|
|
478
477
|
opts: {
|
|
479
478
|
forceCreate?: boolean;
|
|
480
479
|
blockTimestamp: bigint;
|
|
481
480
|
blockNumber: BlockNumber;
|
|
482
|
-
indexWithinCheckpoint:
|
|
481
|
+
indexWithinCheckpoint: IndexWithinCheckpoint;
|
|
483
482
|
buildDeadline: Date | undefined;
|
|
484
483
|
txHashesAlreadyIncluded: Set<string>;
|
|
485
484
|
remainingBlobFields: number;
|
|
486
485
|
},
|
|
487
|
-
): Promise<{ block:
|
|
486
|
+
): Promise<{ block: L2Block; usedTxs: Tx[]; remainingBlobFields: number } | { error: Error } | undefined> {
|
|
488
487
|
const {
|
|
489
488
|
blockTimestamp,
|
|
490
489
|
forceCreate,
|
|
@@ -540,45 +539,38 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
540
539
|
};
|
|
541
540
|
|
|
542
541
|
// Actually build the block by executing txs
|
|
543
|
-
const
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
usedTxs,
|
|
551
|
-
failedTxs,
|
|
552
|
-
usedTxBlobFields,
|
|
553
|
-
} = await checkpointBuilder.buildBlock(pendingTxs, blockNumber, blockTimestamp, blockBuilderOptions);
|
|
554
|
-
const blockBuildDuration = workTimer.ms();
|
|
542
|
+
const buildResult = await this.buildSingleBlockWithCheckpointBuilder(
|
|
543
|
+
checkpointBuilder,
|
|
544
|
+
pendingTxs,
|
|
545
|
+
blockNumber,
|
|
546
|
+
blockTimestamp,
|
|
547
|
+
blockBuilderOptions,
|
|
548
|
+
);
|
|
555
549
|
|
|
556
550
|
// If any txs failed during execution, drop them from the mempool so we don't pick them up again
|
|
557
|
-
await this.dropFailedTxsFromP2P(failedTxs);
|
|
551
|
+
await this.dropFailedTxsFromP2P(buildResult.failedTxs);
|
|
558
552
|
|
|
559
553
|
// Check if we have created a block with enough txs. If there were invalid txs in the pool, or if execution took
|
|
560
554
|
// too long, then we may not get to minTxsPerBlock after executing public functions.
|
|
561
555
|
const minValidTxs = this.config.minValidTxsPerBlock ?? minTxs;
|
|
562
|
-
|
|
556
|
+
const numTxs = buildResult.status === 'no-valid-txs' ? 0 : buildResult.numTxs;
|
|
557
|
+
if (buildResult.status === 'no-valid-txs' || (!forceCreate && numTxs < minValidTxs)) {
|
|
563
558
|
this.log.warn(
|
|
564
|
-
`Block ${blockNumber} at index ${indexWithinCheckpoint} on slot ${this.slot} has too few valid txs to be proposed
|
|
565
|
-
{ slot: this.slot, blockNumber, numTxs, indexWithinCheckpoint },
|
|
559
|
+
`Block ${blockNumber} at index ${indexWithinCheckpoint} on slot ${this.slot} has too few valid txs to be proposed`,
|
|
560
|
+
{ slot: this.slot, blockNumber, numTxs, indexWithinCheckpoint, minValidTxs, buildResult: buildResult.status },
|
|
566
561
|
);
|
|
567
|
-
this.eventEmitter.emit('block-
|
|
568
|
-
minTxs: minValidTxs,
|
|
569
|
-
availableTxs: numTxs,
|
|
570
|
-
slot: this.slot,
|
|
571
|
-
});
|
|
562
|
+
this.eventEmitter.emit('block-build-failed', { reason: `Insufficient valid txs`, slot: this.slot });
|
|
572
563
|
this.metrics.recordBlockProposalFailed('insufficient_valid_txs');
|
|
573
564
|
return undefined;
|
|
574
565
|
}
|
|
575
566
|
|
|
576
567
|
// Block creation succeeded, emit stats and metrics
|
|
568
|
+
const { publicGas, block, publicProcessorDuration, usedTxs, usedTxBlobFields, blockBuildDuration } = buildResult;
|
|
569
|
+
|
|
577
570
|
const blockStats = {
|
|
578
571
|
eventName: 'l2-block-built',
|
|
579
572
|
duration: blockBuildDuration,
|
|
580
573
|
publicProcessDuration: publicProcessorDuration,
|
|
581
|
-
rollupCircuitsDuration: blockBuildingTimer.ms(),
|
|
582
574
|
...block.getStats(),
|
|
583
575
|
} satisfies L2BlockBuiltStats;
|
|
584
576
|
|
|
@@ -604,17 +596,40 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
604
596
|
}
|
|
605
597
|
}
|
|
606
598
|
|
|
599
|
+
/** Uses the checkpoint builder to build a block, catching specific txs */
|
|
600
|
+
private async buildSingleBlockWithCheckpointBuilder(
|
|
601
|
+
checkpointBuilder: CheckpointBuilder,
|
|
602
|
+
pendingTxs: AsyncIterable<Tx>,
|
|
603
|
+
blockNumber: BlockNumber,
|
|
604
|
+
blockTimestamp: bigint,
|
|
605
|
+
blockBuilderOptions: PublicProcessorLimits,
|
|
606
|
+
) {
|
|
607
|
+
try {
|
|
608
|
+
const workTimer = new Timer();
|
|
609
|
+
const result = await checkpointBuilder.buildBlock(pendingTxs, blockNumber, blockTimestamp, blockBuilderOptions);
|
|
610
|
+
const blockBuildDuration = workTimer.ms();
|
|
611
|
+
return { ...result, blockBuildDuration, status: 'success' as const };
|
|
612
|
+
} catch (err: unknown) {
|
|
613
|
+
if (isErrorClass(err, NoValidTxsError)) {
|
|
614
|
+
return { failedTxs: err.failedTxs, status: 'no-valid-txs' as const };
|
|
615
|
+
}
|
|
616
|
+
throw err;
|
|
617
|
+
}
|
|
618
|
+
}
|
|
619
|
+
|
|
607
620
|
/** Waits until minTxs are available on the pool for building a block. */
|
|
608
621
|
@trackSpan('CheckpointProposalJob.waitForMinTxs')
|
|
609
622
|
private async waitForMinTxs(opts: {
|
|
610
623
|
forceCreate?: boolean;
|
|
611
624
|
blockNumber: BlockNumber;
|
|
612
|
-
indexWithinCheckpoint:
|
|
625
|
+
indexWithinCheckpoint: IndexWithinCheckpoint;
|
|
613
626
|
buildDeadline: Date | undefined;
|
|
614
627
|
}): Promise<{ canStartBuilding: boolean; availableTxs: number }> {
|
|
615
|
-
const minTxs = this.config.minTxsPerBlock;
|
|
616
628
|
const { indexWithinCheckpoint, blockNumber, buildDeadline, forceCreate } = opts;
|
|
617
629
|
|
|
630
|
+
// We only allow a block with 0 txs in the first block of the checkpoint
|
|
631
|
+
const minTxs = indexWithinCheckpoint > 0 && this.config.minTxsPerBlock === 0 ? 1 : this.config.minTxsPerBlock;
|
|
632
|
+
|
|
618
633
|
// Deadline is undefined if we are not enforcing the timetable, meaning we'll exit immediately when out of time
|
|
619
634
|
const startBuildingDeadline = buildDeadline
|
|
620
635
|
? new Date(buildDeadline.getTime() - this.timetable.minExecutionTime * 1000)
|
|
@@ -635,7 +650,7 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
635
650
|
`Waiting for enough txs to build block ${blockNumber} at index ${indexWithinCheckpoint} in slot ${this.slot} (have ${availableTxs} but need ${minTxs})`,
|
|
636
651
|
{ blockNumber, slot: this.slot, indexWithinCheckpoint },
|
|
637
652
|
);
|
|
638
|
-
await
|
|
653
|
+
await this.waitForTxsPollingInterval();
|
|
639
654
|
availableTxs = await this.p2pClient.getPendingTxCount();
|
|
640
655
|
}
|
|
641
656
|
|
|
@@ -676,7 +691,7 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
676
691
|
const attestationTimeAllowed = this.config.enforceTimeTable
|
|
677
692
|
? this.timetable.getMaxAllowedTime(SequencerState.PUBLISHING_CHECKPOINT)!
|
|
678
693
|
: this.l1Constants.slotDuration;
|
|
679
|
-
const attestationDeadline = new Date(this.
|
|
694
|
+
const attestationDeadline = new Date((this.getSlotStartBuildTimestamp() + attestationTimeAllowed) * 1000);
|
|
680
695
|
|
|
681
696
|
this.metrics.recordRequiredAttestations(numberOfRequiredAttestations, attestationTimeAllowed);
|
|
682
697
|
|
|
@@ -772,8 +787,7 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
772
787
|
* Gossip doesn't echo messages back to the sender, so the proposer's archiver/world-state
|
|
773
788
|
* would never receive its own block without this explicit sync.
|
|
774
789
|
*/
|
|
775
|
-
private async syncProposedBlockToArchiver(block:
|
|
776
|
-
// TODO(palla/mbps): Change default to false once block sync is stable.
|
|
790
|
+
private async syncProposedBlockToArchiver(block: L2Block): Promise<void> {
|
|
777
791
|
if (this.config.skipPushProposedBlocksToArchiver !== false) {
|
|
778
792
|
this.log.warn(`Skipping push of proposed block ${block.number} to archiver`, {
|
|
779
793
|
blockNumber: block.number,
|
|
@@ -813,6 +827,28 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
813
827
|
this.publisher.clearPendingRequests();
|
|
814
828
|
}
|
|
815
829
|
|
|
830
|
+
/**
|
|
831
|
+
* Helper to handle HA double-signing errors. Returns true if the error was handled (caller should yield).
|
|
832
|
+
*/
|
|
833
|
+
private handleHASigningError(err: any, errorContext: string): boolean {
|
|
834
|
+
if (err instanceof DutyAlreadySignedError) {
|
|
835
|
+
this.log.info(`${errorContext} for slot ${this.slot} already signed by another HA node, yielding`, {
|
|
836
|
+
slot: this.slot,
|
|
837
|
+
signedByNode: err.signedByNode,
|
|
838
|
+
});
|
|
839
|
+
return true;
|
|
840
|
+
}
|
|
841
|
+
if (err instanceof SlashingProtectionError) {
|
|
842
|
+
this.log.info(`${errorContext} for slot ${this.slot} blocked by slashing protection, yielding`, {
|
|
843
|
+
slot: this.slot,
|
|
844
|
+
existingMessageHash: err.existingMessageHash,
|
|
845
|
+
attemptedMessageHash: err.attemptedMessageHash,
|
|
846
|
+
});
|
|
847
|
+
return true;
|
|
848
|
+
}
|
|
849
|
+
return false;
|
|
850
|
+
}
|
|
851
|
+
|
|
816
852
|
/** Waits until a specific time within the current slot */
|
|
817
853
|
@trackSpan('CheckpointProposalJob.waitUntilTimeInSlot')
|
|
818
854
|
protected async waitUntilTimeInSlot(targetSecondsIntoSlot: number): Promise<void> {
|
|
@@ -821,6 +857,11 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
821
857
|
await sleepUntil(new Date(targetTimestamp * 1000), this.dateProvider.nowAsDate());
|
|
822
858
|
}
|
|
823
859
|
|
|
860
|
+
/** Waits the polling interval for transactions. Extracted for test overriding. */
|
|
861
|
+
protected async waitForTxsPollingInterval(): Promise<void> {
|
|
862
|
+
await sleep(TXS_POLLING_MS);
|
|
863
|
+
}
|
|
864
|
+
|
|
824
865
|
private getSlotStartBuildTimestamp(): number {
|
|
825
866
|
return getSlotStartBuildTimestamp(this.slot, this.l1Constants);
|
|
826
867
|
}
|
package/src/sequencer/metrics.ts
CHANGED
|
@@ -11,6 +11,7 @@ import {
|
|
|
11
11
|
type TelemetryClient,
|
|
12
12
|
type Tracer,
|
|
13
13
|
type UpDownCounter,
|
|
14
|
+
createUpDownCounterWithDefault,
|
|
14
15
|
} from '@aztec/telemetry-client';
|
|
15
16
|
|
|
16
17
|
import { type Hex, formatUnits } from 'viem';
|
|
@@ -67,7 +68,9 @@ export class SequencerMetrics {
|
|
|
67
68
|
this.meter = client.getMeter(name);
|
|
68
69
|
this.tracer = client.getTracer(name);
|
|
69
70
|
|
|
70
|
-
this.blockCounter = this.meter
|
|
71
|
+
this.blockCounter = createUpDownCounterWithDefault(this.meter, Metrics.SEQUENCER_BLOCK_COUNT, {
|
|
72
|
+
[Attributes.STATUS]: ['failed', 'built'],
|
|
73
|
+
});
|
|
71
74
|
|
|
72
75
|
this.blockBuildDuration = this.meter.createHistogram(Metrics.SEQUENCER_BLOCK_BUILD_DURATION);
|
|
73
76
|
|
|
@@ -77,23 +80,15 @@ export class SequencerMetrics {
|
|
|
77
80
|
|
|
78
81
|
this.checkpointAttestationDelay = this.meter.createHistogram(Metrics.SEQUENCER_CHECKPOINT_ATTESTATION_DELAY);
|
|
79
82
|
|
|
80
|
-
// Init gauges and counters
|
|
81
|
-
this.blockCounter.add(0, {
|
|
82
|
-
[Attributes.STATUS]: 'failed',
|
|
83
|
-
});
|
|
84
|
-
this.blockCounter.add(0, {
|
|
85
|
-
[Attributes.STATUS]: 'built',
|
|
86
|
-
});
|
|
87
|
-
|
|
88
83
|
this.rewards = this.meter.createGauge(Metrics.SEQUENCER_CURRENT_BLOCK_REWARDS);
|
|
89
84
|
|
|
90
|
-
this.slots = this.meter
|
|
85
|
+
this.slots = createUpDownCounterWithDefault(this.meter, Metrics.SEQUENCER_SLOT_COUNT);
|
|
91
86
|
|
|
92
87
|
/**
|
|
93
88
|
* NOTE: we do not track missed slots as a separate metric. That would be difficult to determine
|
|
94
89
|
* Instead, use a computed metric, `slots - filledSlots` to get the number of slots a sequencer has missed.
|
|
95
90
|
*/
|
|
96
|
-
this.filledSlots = this.meter
|
|
91
|
+
this.filledSlots = createUpDownCounterWithDefault(this.meter, Metrics.SEQUENCER_FILLED_SLOT_COUNT);
|
|
97
92
|
|
|
98
93
|
this.timeToCollectAttestations = this.meter.createGauge(Metrics.SEQUENCER_COLLECT_ATTESTATIONS_DURATION);
|
|
99
94
|
|
|
@@ -103,20 +98,41 @@ export class SequencerMetrics {
|
|
|
103
98
|
|
|
104
99
|
this.collectedAttestions = this.meter.createGauge(Metrics.SEQUENCER_COLLECTED_ATTESTATIONS_COUNT);
|
|
105
100
|
|
|
106
|
-
this.blockProposalFailed =
|
|
101
|
+
this.blockProposalFailed = createUpDownCounterWithDefault(
|
|
102
|
+
this.meter,
|
|
103
|
+
Metrics.SEQUENCER_BLOCK_PROPOSAL_FAILED_COUNT,
|
|
104
|
+
);
|
|
107
105
|
|
|
108
|
-
this.blockProposalSuccess =
|
|
106
|
+
this.blockProposalSuccess = createUpDownCounterWithDefault(
|
|
107
|
+
this.meter,
|
|
108
|
+
Metrics.SEQUENCER_BLOCK_PROPOSAL_SUCCESS_COUNT,
|
|
109
|
+
);
|
|
109
110
|
|
|
110
|
-
this.checkpointSuccess = this.meter
|
|
111
|
+
this.checkpointSuccess = createUpDownCounterWithDefault(this.meter, Metrics.SEQUENCER_CHECKPOINT_SUCCESS_COUNT);
|
|
111
112
|
|
|
112
|
-
this.blockProposalPrecheckFailed =
|
|
113
|
+
this.blockProposalPrecheckFailed = createUpDownCounterWithDefault(
|
|
114
|
+
this.meter,
|
|
113
115
|
Metrics.SEQUENCER_BLOCK_PROPOSAL_PRECHECK_FAILED_COUNT,
|
|
116
|
+
{
|
|
117
|
+
[Attributes.ERROR_TYPE]: [
|
|
118
|
+
'slot_already_taken',
|
|
119
|
+
'rollup_contract_check_failed',
|
|
120
|
+
'slot_mismatch',
|
|
121
|
+
'block_number_mismatch',
|
|
122
|
+
],
|
|
123
|
+
},
|
|
114
124
|
);
|
|
115
125
|
|
|
116
|
-
this.slashingAttempts = this.meter
|
|
126
|
+
this.slashingAttempts = createUpDownCounterWithDefault(this.meter, Metrics.SEQUENCER_SLASHING_ATTEMPTS_COUNT);
|
|
117
127
|
|
|
118
128
|
// Fisherman fee analysis metrics
|
|
119
|
-
this.fishermanWouldBeIncluded =
|
|
129
|
+
this.fishermanWouldBeIncluded = createUpDownCounterWithDefault(
|
|
130
|
+
this.meter,
|
|
131
|
+
Metrics.FISHERMAN_FEE_ANALYSIS_WOULD_BE_INCLUDED,
|
|
132
|
+
{
|
|
133
|
+
[Attributes.OK]: [true, false],
|
|
134
|
+
},
|
|
135
|
+
);
|
|
120
136
|
|
|
121
137
|
this.fishermanTimeBeforeBlock = this.meter.createHistogram(Metrics.FISHERMAN_FEE_ANALYSIS_TIME_BEFORE_BLOCK);
|
|
122
138
|
|
|
@@ -231,7 +247,9 @@ export class SequencerMetrics {
|
|
|
231
247
|
this.blockProposalSuccess.add(1);
|
|
232
248
|
}
|
|
233
249
|
|
|
234
|
-
recordBlockProposalPrecheckFailed(
|
|
250
|
+
recordBlockProposalPrecheckFailed(
|
|
251
|
+
checkType: 'slot_already_taken' | 'rollup_contract_check_failed' | 'slot_mismatch' | 'block_number_mismatch',
|
|
252
|
+
) {
|
|
235
253
|
this.blockProposalPrecheckFailed.add(1, {
|
|
236
254
|
[Attributes.ERROR_TYPE]: checkType,
|
|
237
255
|
});
|
|
@@ -12,7 +12,7 @@ import type { DateProvider } from '@aztec/foundation/timer';
|
|
|
12
12
|
import type { TypedEventEmitter } from '@aztec/foundation/types';
|
|
13
13
|
import type { P2P } from '@aztec/p2p';
|
|
14
14
|
import type { SlasherClientInterface } from '@aztec/slasher';
|
|
15
|
-
import type {
|
|
15
|
+
import type { L2Block, L2BlockSink, L2BlockSource, ValidateCheckpointResult } from '@aztec/stdlib/block';
|
|
16
16
|
import type { Checkpoint } from '@aztec/stdlib/checkpoint';
|
|
17
17
|
import { getSlotAtTimestamp, getSlotStartBuildTimestamp } from '@aztec/stdlib/epoch-helpers';
|
|
18
18
|
import {
|
|
@@ -60,6 +60,9 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
60
60
|
/** The last slot for which we attempted to perform our voting duties with degraded block production */
|
|
61
61
|
private lastSlotForFallbackVote: SlotNumber | undefined;
|
|
62
62
|
|
|
63
|
+
/** The last slot for which we logged "no committee" warning, to avoid spam */
|
|
64
|
+
private lastSlotForNoCommitteeWarning: SlotNumber | undefined;
|
|
65
|
+
|
|
63
66
|
/** The last slot for which we triggered a checkpoint proposal job, to prevent duplicate attempts. */
|
|
64
67
|
private lastSlotForCheckpointProposalJob: SlotNumber | undefined;
|
|
65
68
|
|
|
@@ -424,8 +427,8 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
424
427
|
this.metrics,
|
|
425
428
|
this,
|
|
426
429
|
this.setState.bind(this),
|
|
427
|
-
this.log,
|
|
428
430
|
this.tracer,
|
|
431
|
+
this.log.getBindings(),
|
|
429
432
|
);
|
|
430
433
|
}
|
|
431
434
|
|
|
@@ -529,7 +532,7 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
529
532
|
};
|
|
530
533
|
}
|
|
531
534
|
|
|
532
|
-
const block = await this.l2BlockSource.
|
|
535
|
+
const block = await this.l2BlockSource.getL2Block(blockNumber);
|
|
533
536
|
if (!block) {
|
|
534
537
|
// this shouldn't really happen because a moment ago we checked that all components were in sync
|
|
535
538
|
this.log.error(`Failed to get L2 block ${blockNumber} from the archiver with all components in sync`);
|
|
@@ -557,7 +560,10 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
557
560
|
proposer = await this.epochCache.getProposerAttesterAddressInSlot(slot);
|
|
558
561
|
} catch (e) {
|
|
559
562
|
if (e instanceof NoCommitteeError) {
|
|
560
|
-
this.
|
|
563
|
+
if (this.lastSlotForNoCommitteeWarning !== slot) {
|
|
564
|
+
this.lastSlotForNoCommitteeWarning = slot;
|
|
565
|
+
this.log.warn(`Cannot propose at next L2 slot ${slot} since the committee does not exist on L1`);
|
|
566
|
+
}
|
|
561
567
|
return [false, undefined];
|
|
562
568
|
}
|
|
563
569
|
this.log.error(`Error getting proposer for slot ${slot}`, e);
|
|
@@ -870,7 +876,7 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
870
876
|
}
|
|
871
877
|
|
|
872
878
|
type SequencerSyncCheckResult = {
|
|
873
|
-
block?:
|
|
879
|
+
block?: L2Block;
|
|
874
880
|
checkpointNumber: CheckpointNumber;
|
|
875
881
|
blockNumber: BlockNumber;
|
|
876
882
|
archive: Fr;
|
|
@@ -1,14 +1,15 @@
|
|
|
1
1
|
import { createLogger } from '@aztec/aztec.js/log';
|
|
2
|
+
import {
|
|
3
|
+
CHECKPOINT_ASSEMBLE_TIME,
|
|
4
|
+
CHECKPOINT_INITIALIZATION_TIME,
|
|
5
|
+
DEFAULT_P2P_PROPAGATION_TIME,
|
|
6
|
+
MIN_EXECUTION_TIME,
|
|
7
|
+
} from '@aztec/stdlib/timetable';
|
|
2
8
|
|
|
3
|
-
import { DEFAULT_ATTESTATION_PROPAGATION_TIME as DEFAULT_P2P_PROPAGATION_TIME } from '../config.js';
|
|
4
9
|
import { SequencerTooSlowError } from './errors.js';
|
|
5
10
|
import type { SequencerMetrics } from './metrics.js';
|
|
6
11
|
import { SequencerState } from './utils.js';
|
|
7
12
|
|
|
8
|
-
export const MIN_EXECUTION_TIME = 2;
|
|
9
|
-
export const CHECKPOINT_INITIALIZATION_TIME = 1;
|
|
10
|
-
export const CHECKPOINT_ASSEMBLE_TIME = 1;
|
|
11
|
-
|
|
12
13
|
export class SequencerTimetable {
|
|
13
14
|
/**
|
|
14
15
|
* How late into the slot can we be to start working. Computed as the total time needed for assembling and publishing a block,
|