@aztec/sequencer-client 0.0.1-commit.e61ad554 → 0.0.1-commit.ee80a48
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dest/client/sequencer-client.js +1 -1
- package/dest/publisher/sequencer-publisher-metrics.d.ts +1 -1
- package/dest/publisher/sequencer-publisher-metrics.d.ts.map +1 -1
- package/dest/publisher/sequencer-publisher-metrics.js +12 -4
- package/dest/publisher/sequencer-publisher.d.ts +1 -2
- package/dest/publisher/sequencer-publisher.d.ts.map +1 -1
- package/dest/publisher/sequencer-publisher.js +39 -18
- package/dest/sequencer/checkpoint_proposal_job.d.ts +30 -9
- package/dest/sequencer/checkpoint_proposal_job.d.ts.map +1 -1
- package/dest/sequencer/checkpoint_proposal_job.js +70 -47
- package/dest/sequencer/metrics.d.ts +2 -2
- package/dest/sequencer/metrics.d.ts.map +1 -1
- package/dest/sequencer/metrics.js +27 -17
- package/dest/sequencer/sequencer.d.ts +3 -1
- package/dest/sequencer/sequencer.d.ts.map +1 -1
- package/dest/sequencer/sequencer.js +6 -2
- package/dest/test/mock_checkpoint_builder.d.ts +3 -3
- package/dest/test/mock_checkpoint_builder.d.ts.map +1 -1
- package/dest/test/mock_checkpoint_builder.js +0 -2
- package/package.json +28 -28
- package/src/client/sequencer-client.ts +1 -1
- package/src/publisher/sequencer-publisher-metrics.ts +7 -3
- package/src/publisher/sequencer-publisher.ts +34 -18
- package/src/sequencer/checkpoint_proposal_job.ts +91 -64
- package/src/sequencer/metrics.ts +36 -18
- package/src/sequencer/sequencer.ts +8 -2
- package/src/test/mock_checkpoint_builder.ts +2 -4
|
@@ -7,6 +7,7 @@ import {
|
|
|
7
7
|
Metrics,
|
|
8
8
|
type TelemetryClient,
|
|
9
9
|
type UpDownCounter,
|
|
10
|
+
createUpDownCounterWithDefault,
|
|
10
11
|
} from '@aztec/telemetry-client';
|
|
11
12
|
|
|
12
13
|
import { formatEther } from 'viem/utils';
|
|
@@ -41,7 +42,10 @@ export class SequencerPublisherMetrics {
|
|
|
41
42
|
|
|
42
43
|
this.gasPrice = meter.createHistogram(Metrics.L1_PUBLISHER_GAS_PRICE);
|
|
43
44
|
|
|
44
|
-
this.txCount = meter
|
|
45
|
+
this.txCount = createUpDownCounterWithDefault(meter, Metrics.L1_PUBLISHER_TX_COUNT, {
|
|
46
|
+
[Attributes.L1_TX_TYPE]: ['process'],
|
|
47
|
+
[Attributes.OK]: [true, false],
|
|
48
|
+
});
|
|
45
49
|
|
|
46
50
|
this.txDuration = meter.createHistogram(Metrics.L1_PUBLISHER_TX_DURATION);
|
|
47
51
|
|
|
@@ -59,9 +63,9 @@ export class SequencerPublisherMetrics {
|
|
|
59
63
|
|
|
60
64
|
this.blobInclusionBlocksHistogram = meter.createHistogram(Metrics.L1_PUBLISHER_BLOB_INCLUSION_BLOCKS);
|
|
61
65
|
|
|
62
|
-
this.blobTxSuccessCounter = meter
|
|
66
|
+
this.blobTxSuccessCounter = createUpDownCounterWithDefault(meter, Metrics.L1_PUBLISHER_BLOB_TX_SUCCESS);
|
|
63
67
|
|
|
64
|
-
this.blobTxFailureCounter = meter
|
|
68
|
+
this.blobTxFailureCounter = createUpDownCounterWithDefault(meter, Metrics.L1_PUBLISHER_BLOB_TX_FAILURE);
|
|
65
69
|
|
|
66
70
|
this.txTotalFee = meter.createHistogram(Metrics.L1_PUBLISHER_TX_TOTAL_FEE);
|
|
67
71
|
|
|
@@ -18,11 +18,12 @@ import {
|
|
|
18
18
|
type L1BlobInputs,
|
|
19
19
|
type L1TxConfig,
|
|
20
20
|
type L1TxRequest,
|
|
21
|
+
MAX_L1_TX_LIMIT,
|
|
21
22
|
type TransactionStats,
|
|
22
23
|
WEI_CONST,
|
|
23
24
|
} from '@aztec/ethereum/l1-tx-utils';
|
|
24
25
|
import type { L1TxUtilsWithBlobs } from '@aztec/ethereum/l1-tx-utils-with-blobs';
|
|
25
|
-
import { FormattedViemError, formatViemError, tryExtractEvent } from '@aztec/ethereum/utils';
|
|
26
|
+
import { FormattedViemError, formatViemError, mergeAbis, tryExtractEvent } from '@aztec/ethereum/utils';
|
|
26
27
|
import { sumBigint } from '@aztec/foundation/bigint';
|
|
27
28
|
import { toHex as toPaddedHex } from '@aztec/foundation/bigint-buffer';
|
|
28
29
|
import { CheckpointNumber, SlotNumber } from '@aztec/foundation/branded-types';
|
|
@@ -122,11 +123,6 @@ export class SequencerPublisher {
|
|
|
122
123
|
|
|
123
124
|
/** L1 fee analyzer for fisherman mode */
|
|
124
125
|
private l1FeeAnalyzer?: L1FeeAnalyzer;
|
|
125
|
-
// @note - with blobs, the below estimate seems too large.
|
|
126
|
-
// Total used for full block from int_l1_pub e2e test: 1m (of which 86k is 1x blob)
|
|
127
|
-
// Total used for emptier block from above test: 429k (of which 84k is 1x blob)
|
|
128
|
-
public static PROPOSE_GAS_GUESS: bigint = 12_000_000n;
|
|
129
|
-
|
|
130
126
|
// A CALL to a cold address is 2700 gas
|
|
131
127
|
public static MULTICALL_OVERHEAD_GAS_GUESS = 5000n;
|
|
132
128
|
|
|
@@ -273,7 +269,7 @@ export class SequencerPublisher {
|
|
|
273
269
|
// Start the analysis
|
|
274
270
|
const analysisId = await this.l1FeeAnalyzer.startAnalysis(
|
|
275
271
|
l2SlotNumber,
|
|
276
|
-
gasLimit > 0n ? gasLimit :
|
|
272
|
+
gasLimit > 0n ? gasLimit : MAX_L1_TX_LIMIT,
|
|
277
273
|
l1Requests,
|
|
278
274
|
blobConfig,
|
|
279
275
|
onComplete,
|
|
@@ -346,7 +342,16 @@ export class SequencerPublisher {
|
|
|
346
342
|
|
|
347
343
|
// Merge gasConfigs. Yields the sum of gasLimits, and the earliest txTimeoutAt, or undefined if no gasConfig sets them.
|
|
348
344
|
const gasLimits = gasConfigs.map(g => g?.gasLimit).filter((g): g is bigint => g !== undefined);
|
|
349
|
-
|
|
345
|
+
let gasLimit = gasLimits.length > 0 ? sumBigint(gasLimits) : undefined; // sum
|
|
346
|
+
// Cap at L1 block gas limit so the node accepts the tx ("gas limit too high" otherwise).
|
|
347
|
+
const maxGas = MAX_L1_TX_LIMIT;
|
|
348
|
+
if (gasLimit !== undefined && gasLimit > maxGas) {
|
|
349
|
+
this.log.debug('Capping bundled tx gas limit to L1 max', {
|
|
350
|
+
requested: gasLimit,
|
|
351
|
+
capped: maxGas,
|
|
352
|
+
});
|
|
353
|
+
gasLimit = maxGas;
|
|
354
|
+
}
|
|
350
355
|
const txTimeoutAts = gasConfigs.map(g => g?.txTimeoutAt).filter((g): g is Date => g !== undefined);
|
|
351
356
|
const txTimeoutAt = txTimeoutAts.length > 0 ? new Date(Math.min(...txTimeoutAts.map(g => g.getTime()))) : undefined; // earliest
|
|
352
357
|
const txConfig: RequestWithExpiry['gasConfig'] = { gasLimit, txTimeoutAt };
|
|
@@ -517,7 +522,12 @@ export class SequencerPublisher {
|
|
|
517
522
|
this.log.debug(`Simulating invalidate checkpoint ${checkpointNumber}`, { ...logData, request });
|
|
518
523
|
|
|
519
524
|
try {
|
|
520
|
-
const { gasUsed } = await this.l1TxUtils.simulate(
|
|
525
|
+
const { gasUsed } = await this.l1TxUtils.simulate(
|
|
526
|
+
request,
|
|
527
|
+
undefined,
|
|
528
|
+
undefined,
|
|
529
|
+
mergeAbis([request.abi ?? [], ErrorsAbi]),
|
|
530
|
+
);
|
|
521
531
|
this.log.verbose(`Simulation for invalidate checkpoint ${checkpointNumber} succeeded`, {
|
|
522
532
|
...logData,
|
|
523
533
|
request,
|
|
@@ -536,7 +546,7 @@ export class SequencerPublisher {
|
|
|
536
546
|
|
|
537
547
|
// If the error is due to the checkpoint not being in the pending chain, and it was indeed removed by someone else,
|
|
538
548
|
// we can safely ignore it and return undefined so we go ahead with checkpoint building.
|
|
539
|
-
if (viemError.message?.includes('
|
|
549
|
+
if (viemError.message?.includes('Rollup__CheckpointNotInPendingChain')) {
|
|
540
550
|
this.log.verbose(
|
|
541
551
|
`Simulation for invalidate checkpoint ${checkpointNumber} failed due to checkpoint not being in pending chain`,
|
|
542
552
|
{ ...logData, request, error: viemError.message },
|
|
@@ -700,7 +710,7 @@ export class SequencerPublisher {
|
|
|
700
710
|
});
|
|
701
711
|
|
|
702
712
|
try {
|
|
703
|
-
await this.l1TxUtils.simulate(request, { time: timestamp }, [], ErrorsAbi);
|
|
713
|
+
await this.l1TxUtils.simulate(request, { time: timestamp }, [], mergeAbis([request.abi ?? [], ErrorsAbi]));
|
|
704
714
|
this.log.debug(`Simulation for ${action} at slot ${slotNumber} succeeded`, { request });
|
|
705
715
|
} catch (err) {
|
|
706
716
|
this.log.error(`Failed simulation for ${action} at slot ${slotNumber} (enqueuing the action anyway)`, err);
|
|
@@ -999,12 +1009,14 @@ export class SequencerPublisher {
|
|
|
999
1009
|
this.log.debug(`Simulating ${action} for slot ${slotNumber}`, logData);
|
|
1000
1010
|
|
|
1001
1011
|
let gasUsed: bigint;
|
|
1012
|
+
const simulateAbi = mergeAbis([request.abi ?? [], ErrorsAbi]);
|
|
1002
1013
|
try {
|
|
1003
|
-
({ gasUsed } = await this.l1TxUtils.simulate(request, { time: timestamp }, [],
|
|
1014
|
+
({ gasUsed } = await this.l1TxUtils.simulate(request, { time: timestamp }, [], simulateAbi)); // TODO(palla/slash): Check the timestamp logic
|
|
1004
1015
|
this.log.verbose(`Simulation for ${action} succeeded`, { ...logData, request, gasUsed });
|
|
1005
1016
|
} catch (err) {
|
|
1006
|
-
const viemError = formatViemError(err);
|
|
1017
|
+
const viemError = formatViemError(err, simulateAbi);
|
|
1007
1018
|
this.log.error(`Simulation for ${action} at ${slotNumber} failed`, viemError, logData);
|
|
1019
|
+
|
|
1008
1020
|
return false;
|
|
1009
1021
|
}
|
|
1010
1022
|
|
|
@@ -1012,10 +1024,14 @@ export class SequencerPublisher {
|
|
|
1012
1024
|
const gasLimit = this.l1TxUtils.bumpGasLimit(BigInt(Math.ceil((Number(gasUsed) * 64) / 63)));
|
|
1013
1025
|
logData.gasLimit = gasLimit;
|
|
1014
1026
|
|
|
1027
|
+
// Store the ABI used for simulation on the request so Multicall3.forward can decode errors
|
|
1028
|
+
// when the tx is sent and a revert is diagnosed via simulation.
|
|
1029
|
+
const requestWithAbi = { ...request, abi: simulateAbi };
|
|
1030
|
+
|
|
1015
1031
|
this.log.debug(`Enqueuing ${action}`, logData);
|
|
1016
1032
|
this.addRequest({
|
|
1017
1033
|
action,
|
|
1018
|
-
request,
|
|
1034
|
+
request: requestWithAbi,
|
|
1019
1035
|
gasConfig: { gasLimit },
|
|
1020
1036
|
lastValidL2Slot: slotNumber,
|
|
1021
1037
|
checkSuccess: (_req, result) => {
|
|
@@ -1171,20 +1187,20 @@ export class SequencerPublisher {
|
|
|
1171
1187
|
{
|
|
1172
1188
|
to: this.rollupContract.address,
|
|
1173
1189
|
data: rollupData,
|
|
1174
|
-
gas:
|
|
1190
|
+
gas: MAX_L1_TX_LIMIT,
|
|
1175
1191
|
...(this.proposerAddressForSimulation && { from: this.proposerAddressForSimulation.toString() }),
|
|
1176
1192
|
},
|
|
1177
1193
|
{
|
|
1178
1194
|
// @note we add 1n to the timestamp because geth implementation doesn't like simulation timestamp to be equal to the current block timestamp
|
|
1179
1195
|
time: timestamp + 1n,
|
|
1180
1196
|
// @note reth should have a 30m gas limit per block but throws errors that this tx is beyond limit so we increase here
|
|
1181
|
-
gasLimit:
|
|
1197
|
+
gasLimit: MAX_L1_TX_LIMIT * 2n,
|
|
1182
1198
|
},
|
|
1183
1199
|
stateOverrides,
|
|
1184
1200
|
RollupAbi,
|
|
1185
1201
|
{
|
|
1186
1202
|
// @note fallback gas estimate to use if the node doesn't support simulation API
|
|
1187
|
-
fallbackGasEstimate:
|
|
1203
|
+
fallbackGasEstimate: MAX_L1_TX_LIMIT,
|
|
1188
1204
|
},
|
|
1189
1205
|
)
|
|
1190
1206
|
.catch(err => {
|
|
@@ -1194,7 +1210,7 @@ export class SequencerPublisher {
|
|
|
1194
1210
|
this.log.debug(`Ignoring expected ValidatorSelection__MissingProposerSignature error in fisherman mode`);
|
|
1195
1211
|
// Return a minimal simulation result with the fallback gas estimate
|
|
1196
1212
|
return {
|
|
1197
|
-
gasUsed:
|
|
1213
|
+
gasUsed: MAX_L1_TX_LIMIT,
|
|
1198
1214
|
logs: [],
|
|
1199
1215
|
};
|
|
1200
1216
|
}
|
|
@@ -1,16 +1,22 @@
|
|
|
1
1
|
import { NUM_CHECKPOINT_END_MARKER_FIELDS, getNumBlockEndBlobFields } from '@aztec/blob-lib/encoding';
|
|
2
2
|
import { BLOBS_PER_CHECKPOINT, FIELDS_PER_BLOB } from '@aztec/constants';
|
|
3
3
|
import type { EpochCache } from '@aztec/epoch-cache';
|
|
4
|
-
import {
|
|
4
|
+
import {
|
|
5
|
+
BlockNumber,
|
|
6
|
+
CheckpointNumber,
|
|
7
|
+
EpochNumber,
|
|
8
|
+
IndexWithinCheckpoint,
|
|
9
|
+
SlotNumber,
|
|
10
|
+
} from '@aztec/foundation/branded-types';
|
|
5
11
|
import { randomInt } from '@aztec/foundation/crypto/random';
|
|
6
12
|
import { Fr } from '@aztec/foundation/curves/bn254';
|
|
7
13
|
import { EthAddress } from '@aztec/foundation/eth-address';
|
|
8
14
|
import { Signature } from '@aztec/foundation/eth-signature';
|
|
9
15
|
import { filter } from '@aztec/foundation/iterator';
|
|
10
|
-
import type
|
|
16
|
+
import { type Logger, type LoggerBindings, createLogger } from '@aztec/foundation/log';
|
|
11
17
|
import { sleep, sleepUntil } from '@aztec/foundation/sleep';
|
|
12
18
|
import { type DateProvider, Timer } from '@aztec/foundation/timer';
|
|
13
|
-
import { type TypedEventEmitter, unfreeze } from '@aztec/foundation/types';
|
|
19
|
+
import { type TypedEventEmitter, isErrorClass, unfreeze } from '@aztec/foundation/types';
|
|
14
20
|
import type { P2P } from '@aztec/p2p';
|
|
15
21
|
import type { SlasherClientInterface } from '@aztec/slasher';
|
|
16
22
|
import {
|
|
@@ -24,10 +30,11 @@ import {
|
|
|
24
30
|
import type { Checkpoint } from '@aztec/stdlib/checkpoint';
|
|
25
31
|
import { getSlotStartBuildTimestamp } from '@aztec/stdlib/epoch-helpers';
|
|
26
32
|
import { Gas } from '@aztec/stdlib/gas';
|
|
27
|
-
import
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
33
|
+
import {
|
|
34
|
+
NoValidTxsError,
|
|
35
|
+
type PublicProcessorLimits,
|
|
36
|
+
type ResolvedSequencerConfig,
|
|
37
|
+
type WorldStateSynchronizer,
|
|
31
38
|
} from '@aztec/stdlib/interfaces/server';
|
|
32
39
|
import { type L1ToL2MessageSource, computeInHashFromL1ToL2Messages } from '@aztec/stdlib/messaging';
|
|
33
40
|
import type { BlockProposalOptions, CheckpointProposal, CheckpointProposalOptions } from '@aztec/stdlib/p2p';
|
|
@@ -59,6 +66,8 @@ const TXS_POLLING_MS = 500;
|
|
|
59
66
|
* the Sequencer once the check for being the proposer for the slot has succeeded.
|
|
60
67
|
*/
|
|
61
68
|
export class CheckpointProposalJob implements Traceable {
|
|
69
|
+
protected readonly log: Logger;
|
|
70
|
+
|
|
62
71
|
constructor(
|
|
63
72
|
private readonly epoch: EpochNumber,
|
|
64
73
|
private readonly slot: SlotNumber,
|
|
@@ -86,9 +95,11 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
86
95
|
private readonly metrics: SequencerMetrics,
|
|
87
96
|
private readonly eventEmitter: TypedEventEmitter<SequencerEvents>,
|
|
88
97
|
private readonly setStateFn: (state: SequencerState, slot?: SlotNumber) => void,
|
|
89
|
-
protected readonly log: Logger,
|
|
90
98
|
public readonly tracer: Tracer,
|
|
91
|
-
|
|
99
|
+
bindings?: LoggerBindings,
|
|
100
|
+
) {
|
|
101
|
+
this.log = createLogger('sequencer:checkpoint-proposal', { ...bindings, instanceId: `slot-${slot}` });
|
|
102
|
+
}
|
|
92
103
|
|
|
93
104
|
/**
|
|
94
105
|
* Executes the checkpoint proposal job.
|
|
@@ -190,6 +201,7 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
190
201
|
l1ToL2Messages,
|
|
191
202
|
previousCheckpointOutHashes,
|
|
192
203
|
fork,
|
|
204
|
+
this.log.getBindings(),
|
|
193
205
|
);
|
|
194
206
|
|
|
195
207
|
// Options for the validator client when creating block and checkpoint proposals
|
|
@@ -220,19 +232,7 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
220
232
|
// These errors are expected in HA mode, so we yield and let another HA node handle the slot
|
|
221
233
|
// The only distinction between the 2 errors is SlashingProtectionError throws when the payload is different,
|
|
222
234
|
// which is normal for block building (may have picked different txs)
|
|
223
|
-
if (err
|
|
224
|
-
this.log.info(`Checkpoint proposal for slot ${this.slot} already signed by another HA node, yielding`, {
|
|
225
|
-
slot: this.slot,
|
|
226
|
-
signedByNode: err.signedByNode,
|
|
227
|
-
});
|
|
228
|
-
return undefined;
|
|
229
|
-
}
|
|
230
|
-
if (err instanceof SlashingProtectionError) {
|
|
231
|
-
this.log.info(`Checkpoint proposal for slot ${this.slot} blocked by slashing protection, yielding`, {
|
|
232
|
-
slot: this.slot,
|
|
233
|
-
existingMessageHash: err.existingMessageHash,
|
|
234
|
-
attemptedMessageHash: err.attemptedMessageHash,
|
|
235
|
-
});
|
|
235
|
+
if (this.handleHASigningError(err, 'Block proposal')) {
|
|
236
236
|
return undefined;
|
|
237
237
|
}
|
|
238
238
|
throw err;
|
|
@@ -301,20 +301,8 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
301
301
|
);
|
|
302
302
|
} catch (err) {
|
|
303
303
|
// We shouldn't really get here since we yield to another HA node
|
|
304
|
-
// as soon as we see these errors when creating block proposals.
|
|
305
|
-
if (err
|
|
306
|
-
this.log.info(`Attestations signature for slot ${this.slot} already signed by another HA node, yielding`, {
|
|
307
|
-
slot: this.slot,
|
|
308
|
-
signedByNode: err.signedByNode,
|
|
309
|
-
});
|
|
310
|
-
return undefined;
|
|
311
|
-
}
|
|
312
|
-
if (err instanceof SlashingProtectionError) {
|
|
313
|
-
this.log.info(`Attestations signature for slot ${this.slot} blocked by slashing protection, yielding`, {
|
|
314
|
-
slot: this.slot,
|
|
315
|
-
existingMessageHash: err.existingMessageHash,
|
|
316
|
-
attemptedMessageHash: err.attemptedMessageHash,
|
|
317
|
-
});
|
|
304
|
+
// as soon as we see these errors when creating block or checkpoint proposals.
|
|
305
|
+
if (this.handleHASigningError(err, 'Attestations signature')) {
|
|
318
306
|
return undefined;
|
|
319
307
|
}
|
|
320
308
|
throw err;
|
|
@@ -367,7 +355,7 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
367
355
|
|
|
368
356
|
while (true) {
|
|
369
357
|
const blocksBuilt = blocksInCheckpoint.length;
|
|
370
|
-
const indexWithinCheckpoint = blocksBuilt;
|
|
358
|
+
const indexWithinCheckpoint = IndexWithinCheckpoint(blocksBuilt);
|
|
371
359
|
const blockNumber = BlockNumber(initialBlockNumber + blocksBuilt);
|
|
372
360
|
|
|
373
361
|
const secondsIntoSlot = this.getSecondsIntoSlot();
|
|
@@ -397,6 +385,7 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
397
385
|
remainingBlobFields,
|
|
398
386
|
});
|
|
399
387
|
|
|
388
|
+
// TODO(palla/mbps): Review these conditions. We may want to keep trying in some scenarios.
|
|
400
389
|
if (!buildResult && timingInfo.isLastBlock) {
|
|
401
390
|
// If no block was produced due to not enough txs and this was the last subslot, exit
|
|
402
391
|
break;
|
|
@@ -483,13 +472,13 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
483
472
|
|
|
484
473
|
/** Builds a single block. Called from the main block building loop. */
|
|
485
474
|
@trackSpan('CheckpointProposalJob.buildSingleBlock')
|
|
486
|
-
|
|
475
|
+
protected async buildSingleBlock(
|
|
487
476
|
checkpointBuilder: CheckpointBuilder,
|
|
488
477
|
opts: {
|
|
489
478
|
forceCreate?: boolean;
|
|
490
479
|
blockTimestamp: bigint;
|
|
491
480
|
blockNumber: BlockNumber;
|
|
492
|
-
indexWithinCheckpoint:
|
|
481
|
+
indexWithinCheckpoint: IndexWithinCheckpoint;
|
|
493
482
|
buildDeadline: Date | undefined;
|
|
494
483
|
txHashesAlreadyIncluded: Set<string>;
|
|
495
484
|
remainingBlobFields: number;
|
|
@@ -550,45 +539,38 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
550
539
|
};
|
|
551
540
|
|
|
552
541
|
// Actually build the block by executing txs
|
|
553
|
-
const
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
usedTxs,
|
|
561
|
-
failedTxs,
|
|
562
|
-
usedTxBlobFields,
|
|
563
|
-
} = await checkpointBuilder.buildBlock(pendingTxs, blockNumber, blockTimestamp, blockBuilderOptions);
|
|
564
|
-
const blockBuildDuration = workTimer.ms();
|
|
542
|
+
const buildResult = await this.buildSingleBlockWithCheckpointBuilder(
|
|
543
|
+
checkpointBuilder,
|
|
544
|
+
pendingTxs,
|
|
545
|
+
blockNumber,
|
|
546
|
+
blockTimestamp,
|
|
547
|
+
blockBuilderOptions,
|
|
548
|
+
);
|
|
565
549
|
|
|
566
550
|
// If any txs failed during execution, drop them from the mempool so we don't pick them up again
|
|
567
|
-
await this.dropFailedTxsFromP2P(failedTxs);
|
|
551
|
+
await this.dropFailedTxsFromP2P(buildResult.failedTxs);
|
|
568
552
|
|
|
569
553
|
// Check if we have created a block with enough txs. If there were invalid txs in the pool, or if execution took
|
|
570
554
|
// too long, then we may not get to minTxsPerBlock after executing public functions.
|
|
571
555
|
const minValidTxs = this.config.minValidTxsPerBlock ?? minTxs;
|
|
572
|
-
|
|
556
|
+
const numTxs = buildResult.status === 'no-valid-txs' ? 0 : buildResult.numTxs;
|
|
557
|
+
if (buildResult.status === 'no-valid-txs' || (!forceCreate && numTxs < minValidTxs)) {
|
|
573
558
|
this.log.warn(
|
|
574
|
-
`Block ${blockNumber} at index ${indexWithinCheckpoint} on slot ${this.slot} has too few valid txs to be proposed
|
|
575
|
-
{ slot: this.slot, blockNumber, numTxs, indexWithinCheckpoint },
|
|
559
|
+
`Block ${blockNumber} at index ${indexWithinCheckpoint} on slot ${this.slot} has too few valid txs to be proposed`,
|
|
560
|
+
{ slot: this.slot, blockNumber, numTxs, indexWithinCheckpoint, minValidTxs, buildResult: buildResult.status },
|
|
576
561
|
);
|
|
577
|
-
this.eventEmitter.emit('block-
|
|
578
|
-
minTxs: minValidTxs,
|
|
579
|
-
availableTxs: numTxs,
|
|
580
|
-
slot: this.slot,
|
|
581
|
-
});
|
|
562
|
+
this.eventEmitter.emit('block-build-failed', { reason: `Insufficient valid txs`, slot: this.slot });
|
|
582
563
|
this.metrics.recordBlockProposalFailed('insufficient_valid_txs');
|
|
583
564
|
return undefined;
|
|
584
565
|
}
|
|
585
566
|
|
|
586
567
|
// Block creation succeeded, emit stats and metrics
|
|
568
|
+
const { publicGas, block, publicProcessorDuration, usedTxs, usedTxBlobFields, blockBuildDuration } = buildResult;
|
|
569
|
+
|
|
587
570
|
const blockStats = {
|
|
588
571
|
eventName: 'l2-block-built',
|
|
589
572
|
duration: blockBuildDuration,
|
|
590
573
|
publicProcessDuration: publicProcessorDuration,
|
|
591
|
-
rollupCircuitsDuration: blockBuildingTimer.ms(),
|
|
592
574
|
...block.getStats(),
|
|
593
575
|
} satisfies L2BlockBuiltStats;
|
|
594
576
|
|
|
@@ -614,17 +596,40 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
614
596
|
}
|
|
615
597
|
}
|
|
616
598
|
|
|
599
|
+
/** Uses the checkpoint builder to build a block, catching specific txs */
|
|
600
|
+
private async buildSingleBlockWithCheckpointBuilder(
|
|
601
|
+
checkpointBuilder: CheckpointBuilder,
|
|
602
|
+
pendingTxs: AsyncIterable<Tx>,
|
|
603
|
+
blockNumber: BlockNumber,
|
|
604
|
+
blockTimestamp: bigint,
|
|
605
|
+
blockBuilderOptions: PublicProcessorLimits,
|
|
606
|
+
) {
|
|
607
|
+
try {
|
|
608
|
+
const workTimer = new Timer();
|
|
609
|
+
const result = await checkpointBuilder.buildBlock(pendingTxs, blockNumber, blockTimestamp, blockBuilderOptions);
|
|
610
|
+
const blockBuildDuration = workTimer.ms();
|
|
611
|
+
return { ...result, blockBuildDuration, status: 'success' as const };
|
|
612
|
+
} catch (err: unknown) {
|
|
613
|
+
if (isErrorClass(err, NoValidTxsError)) {
|
|
614
|
+
return { failedTxs: err.failedTxs, status: 'no-valid-txs' as const };
|
|
615
|
+
}
|
|
616
|
+
throw err;
|
|
617
|
+
}
|
|
618
|
+
}
|
|
619
|
+
|
|
617
620
|
/** Waits until minTxs are available on the pool for building a block. */
|
|
618
621
|
@trackSpan('CheckpointProposalJob.waitForMinTxs')
|
|
619
622
|
private async waitForMinTxs(opts: {
|
|
620
623
|
forceCreate?: boolean;
|
|
621
624
|
blockNumber: BlockNumber;
|
|
622
|
-
indexWithinCheckpoint:
|
|
625
|
+
indexWithinCheckpoint: IndexWithinCheckpoint;
|
|
623
626
|
buildDeadline: Date | undefined;
|
|
624
627
|
}): Promise<{ canStartBuilding: boolean; availableTxs: number }> {
|
|
625
|
-
const minTxs = this.config.minTxsPerBlock;
|
|
626
628
|
const { indexWithinCheckpoint, blockNumber, buildDeadline, forceCreate } = opts;
|
|
627
629
|
|
|
630
|
+
// We only allow a block with 0 txs in the first block of the checkpoint
|
|
631
|
+
const minTxs = indexWithinCheckpoint > 0 && this.config.minTxsPerBlock === 0 ? 1 : this.config.minTxsPerBlock;
|
|
632
|
+
|
|
628
633
|
// Deadline is undefined if we are not enforcing the timetable, meaning we'll exit immediately when out of time
|
|
629
634
|
const startBuildingDeadline = buildDeadline
|
|
630
635
|
? new Date(buildDeadline.getTime() - this.timetable.minExecutionTime * 1000)
|
|
@@ -686,7 +691,7 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
686
691
|
const attestationTimeAllowed = this.config.enforceTimeTable
|
|
687
692
|
? this.timetable.getMaxAllowedTime(SequencerState.PUBLISHING_CHECKPOINT)!
|
|
688
693
|
: this.l1Constants.slotDuration;
|
|
689
|
-
const attestationDeadline = new Date(this.
|
|
694
|
+
const attestationDeadline = new Date((this.getSlotStartBuildTimestamp() + attestationTimeAllowed) * 1000);
|
|
690
695
|
|
|
691
696
|
this.metrics.recordRequiredAttestations(numberOfRequiredAttestations, attestationTimeAllowed);
|
|
692
697
|
|
|
@@ -822,6 +827,28 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
822
827
|
this.publisher.clearPendingRequests();
|
|
823
828
|
}
|
|
824
829
|
|
|
830
|
+
/**
|
|
831
|
+
* Helper to handle HA double-signing errors. Returns true if the error was handled (caller should yield).
|
|
832
|
+
*/
|
|
833
|
+
private handleHASigningError(err: any, errorContext: string): boolean {
|
|
834
|
+
if (err instanceof DutyAlreadySignedError) {
|
|
835
|
+
this.log.info(`${errorContext} for slot ${this.slot} already signed by another HA node, yielding`, {
|
|
836
|
+
slot: this.slot,
|
|
837
|
+
signedByNode: err.signedByNode,
|
|
838
|
+
});
|
|
839
|
+
return true;
|
|
840
|
+
}
|
|
841
|
+
if (err instanceof SlashingProtectionError) {
|
|
842
|
+
this.log.info(`${errorContext} for slot ${this.slot} blocked by slashing protection, yielding`, {
|
|
843
|
+
slot: this.slot,
|
|
844
|
+
existingMessageHash: err.existingMessageHash,
|
|
845
|
+
attemptedMessageHash: err.attemptedMessageHash,
|
|
846
|
+
});
|
|
847
|
+
return true;
|
|
848
|
+
}
|
|
849
|
+
return false;
|
|
850
|
+
}
|
|
851
|
+
|
|
825
852
|
/** Waits until a specific time within the current slot */
|
|
826
853
|
@trackSpan('CheckpointProposalJob.waitUntilTimeInSlot')
|
|
827
854
|
protected async waitUntilTimeInSlot(targetSecondsIntoSlot: number): Promise<void> {
|
package/src/sequencer/metrics.ts
CHANGED
|
@@ -11,6 +11,7 @@ import {
|
|
|
11
11
|
type TelemetryClient,
|
|
12
12
|
type Tracer,
|
|
13
13
|
type UpDownCounter,
|
|
14
|
+
createUpDownCounterWithDefault,
|
|
14
15
|
} from '@aztec/telemetry-client';
|
|
15
16
|
|
|
16
17
|
import { type Hex, formatUnits } from 'viem';
|
|
@@ -67,7 +68,9 @@ export class SequencerMetrics {
|
|
|
67
68
|
this.meter = client.getMeter(name);
|
|
68
69
|
this.tracer = client.getTracer(name);
|
|
69
70
|
|
|
70
|
-
this.blockCounter = this.meter
|
|
71
|
+
this.blockCounter = createUpDownCounterWithDefault(this.meter, Metrics.SEQUENCER_BLOCK_COUNT, {
|
|
72
|
+
[Attributes.STATUS]: ['failed', 'built'],
|
|
73
|
+
});
|
|
71
74
|
|
|
72
75
|
this.blockBuildDuration = this.meter.createHistogram(Metrics.SEQUENCER_BLOCK_BUILD_DURATION);
|
|
73
76
|
|
|
@@ -77,23 +80,15 @@ export class SequencerMetrics {
|
|
|
77
80
|
|
|
78
81
|
this.checkpointAttestationDelay = this.meter.createHistogram(Metrics.SEQUENCER_CHECKPOINT_ATTESTATION_DELAY);
|
|
79
82
|
|
|
80
|
-
// Init gauges and counters
|
|
81
|
-
this.blockCounter.add(0, {
|
|
82
|
-
[Attributes.STATUS]: 'failed',
|
|
83
|
-
});
|
|
84
|
-
this.blockCounter.add(0, {
|
|
85
|
-
[Attributes.STATUS]: 'built',
|
|
86
|
-
});
|
|
87
|
-
|
|
88
83
|
this.rewards = this.meter.createGauge(Metrics.SEQUENCER_CURRENT_BLOCK_REWARDS);
|
|
89
84
|
|
|
90
|
-
this.slots = this.meter
|
|
85
|
+
this.slots = createUpDownCounterWithDefault(this.meter, Metrics.SEQUENCER_SLOT_COUNT);
|
|
91
86
|
|
|
92
87
|
/**
|
|
93
88
|
* NOTE: we do not track missed slots as a separate metric. That would be difficult to determine
|
|
94
89
|
* Instead, use a computed metric, `slots - filledSlots` to get the number of slots a sequencer has missed.
|
|
95
90
|
*/
|
|
96
|
-
this.filledSlots = this.meter
|
|
91
|
+
this.filledSlots = createUpDownCounterWithDefault(this.meter, Metrics.SEQUENCER_FILLED_SLOT_COUNT);
|
|
97
92
|
|
|
98
93
|
this.timeToCollectAttestations = this.meter.createGauge(Metrics.SEQUENCER_COLLECT_ATTESTATIONS_DURATION);
|
|
99
94
|
|
|
@@ -103,20 +98,41 @@ export class SequencerMetrics {
|
|
|
103
98
|
|
|
104
99
|
this.collectedAttestions = this.meter.createGauge(Metrics.SEQUENCER_COLLECTED_ATTESTATIONS_COUNT);
|
|
105
100
|
|
|
106
|
-
this.blockProposalFailed =
|
|
101
|
+
this.blockProposalFailed = createUpDownCounterWithDefault(
|
|
102
|
+
this.meter,
|
|
103
|
+
Metrics.SEQUENCER_BLOCK_PROPOSAL_FAILED_COUNT,
|
|
104
|
+
);
|
|
107
105
|
|
|
108
|
-
this.blockProposalSuccess =
|
|
106
|
+
this.blockProposalSuccess = createUpDownCounterWithDefault(
|
|
107
|
+
this.meter,
|
|
108
|
+
Metrics.SEQUENCER_BLOCK_PROPOSAL_SUCCESS_COUNT,
|
|
109
|
+
);
|
|
109
110
|
|
|
110
|
-
this.checkpointSuccess = this.meter
|
|
111
|
+
this.checkpointSuccess = createUpDownCounterWithDefault(this.meter, Metrics.SEQUENCER_CHECKPOINT_SUCCESS_COUNT);
|
|
111
112
|
|
|
112
|
-
this.blockProposalPrecheckFailed =
|
|
113
|
+
this.blockProposalPrecheckFailed = createUpDownCounterWithDefault(
|
|
114
|
+
this.meter,
|
|
113
115
|
Metrics.SEQUENCER_BLOCK_PROPOSAL_PRECHECK_FAILED_COUNT,
|
|
116
|
+
{
|
|
117
|
+
[Attributes.ERROR_TYPE]: [
|
|
118
|
+
'slot_already_taken',
|
|
119
|
+
'rollup_contract_check_failed',
|
|
120
|
+
'slot_mismatch',
|
|
121
|
+
'block_number_mismatch',
|
|
122
|
+
],
|
|
123
|
+
},
|
|
114
124
|
);
|
|
115
125
|
|
|
116
|
-
this.slashingAttempts = this.meter
|
|
126
|
+
this.slashingAttempts = createUpDownCounterWithDefault(this.meter, Metrics.SEQUENCER_SLASHING_ATTEMPTS_COUNT);
|
|
117
127
|
|
|
118
128
|
// Fisherman fee analysis metrics
|
|
119
|
-
this.fishermanWouldBeIncluded =
|
|
129
|
+
this.fishermanWouldBeIncluded = createUpDownCounterWithDefault(
|
|
130
|
+
this.meter,
|
|
131
|
+
Metrics.FISHERMAN_FEE_ANALYSIS_WOULD_BE_INCLUDED,
|
|
132
|
+
{
|
|
133
|
+
[Attributes.OK]: [true, false],
|
|
134
|
+
},
|
|
135
|
+
);
|
|
120
136
|
|
|
121
137
|
this.fishermanTimeBeforeBlock = this.meter.createHistogram(Metrics.FISHERMAN_FEE_ANALYSIS_TIME_BEFORE_BLOCK);
|
|
122
138
|
|
|
@@ -231,7 +247,9 @@ export class SequencerMetrics {
|
|
|
231
247
|
this.blockProposalSuccess.add(1);
|
|
232
248
|
}
|
|
233
249
|
|
|
234
|
-
recordBlockProposalPrecheckFailed(
|
|
250
|
+
recordBlockProposalPrecheckFailed(
|
|
251
|
+
checkType: 'slot_already_taken' | 'rollup_contract_check_failed' | 'slot_mismatch' | 'block_number_mismatch',
|
|
252
|
+
) {
|
|
235
253
|
this.blockProposalPrecheckFailed.add(1, {
|
|
236
254
|
[Attributes.ERROR_TYPE]: checkType,
|
|
237
255
|
});
|
|
@@ -60,6 +60,9 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
60
60
|
/** The last slot for which we attempted to perform our voting duties with degraded block production */
|
|
61
61
|
private lastSlotForFallbackVote: SlotNumber | undefined;
|
|
62
62
|
|
|
63
|
+
/** The last slot for which we logged "no committee" warning, to avoid spam */
|
|
64
|
+
private lastSlotForNoCommitteeWarning: SlotNumber | undefined;
|
|
65
|
+
|
|
63
66
|
/** The last slot for which we triggered a checkpoint proposal job, to prevent duplicate attempts. */
|
|
64
67
|
private lastSlotForCheckpointProposalJob: SlotNumber | undefined;
|
|
65
68
|
|
|
@@ -424,8 +427,8 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
424
427
|
this.metrics,
|
|
425
428
|
this,
|
|
426
429
|
this.setState.bind(this),
|
|
427
|
-
this.log,
|
|
428
430
|
this.tracer,
|
|
431
|
+
this.log.getBindings(),
|
|
429
432
|
);
|
|
430
433
|
}
|
|
431
434
|
|
|
@@ -557,7 +560,10 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
557
560
|
proposer = await this.epochCache.getProposerAttesterAddressInSlot(slot);
|
|
558
561
|
} catch (e) {
|
|
559
562
|
if (e instanceof NoCommitteeError) {
|
|
560
|
-
this.
|
|
563
|
+
if (this.lastSlotForNoCommitteeWarning !== slot) {
|
|
564
|
+
this.lastSlotForNoCommitteeWarning = slot;
|
|
565
|
+
this.log.warn(`Cannot propose at next L2 slot ${slot} since the committee does not exist on L1`);
|
|
566
|
+
}
|
|
561
567
|
return [false, undefined];
|
|
562
568
|
}
|
|
563
569
|
this.log.error(`Error getting proposer for slot ${slot}`, e);
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import { type BlockNumber, CheckpointNumber } from '@aztec/foundation/branded-types';
|
|
2
2
|
import { Fr } from '@aztec/foundation/curves/bn254';
|
|
3
|
-
import { Timer } from '@aztec/foundation/timer';
|
|
4
3
|
import { L2Block } from '@aztec/stdlib/block';
|
|
5
4
|
import { Checkpoint } from '@aztec/stdlib/checkpoint';
|
|
6
5
|
import { Gas } from '@aztec/stdlib/gas';
|
|
@@ -14,7 +13,7 @@ import type {
|
|
|
14
13
|
import { CheckpointHeader } from '@aztec/stdlib/rollup';
|
|
15
14
|
import { makeAppendOnlyTreeSnapshot } from '@aztec/stdlib/testing';
|
|
16
15
|
import type { CheckpointGlobalVariables, Tx } from '@aztec/stdlib/tx';
|
|
17
|
-
import type {
|
|
16
|
+
import type { BuildBlockInCheckpointResult } from '@aztec/validator-client';
|
|
18
17
|
|
|
19
18
|
/**
|
|
20
19
|
* A fake CheckpointBuilder for testing that implements the same interface as the real one.
|
|
@@ -76,7 +75,7 @@ export class MockCheckpointBuilder implements ICheckpointBlockBuilder {
|
|
|
76
75
|
blockNumber: BlockNumber,
|
|
77
76
|
timestamp: bigint,
|
|
78
77
|
opts: PublicProcessorLimits,
|
|
79
|
-
): Promise<
|
|
78
|
+
): Promise<BuildBlockInCheckpointResult> {
|
|
80
79
|
this.buildBlockCalls.push({ blockNumber, timestamp, opts });
|
|
81
80
|
|
|
82
81
|
if (this.errorOnBuild) {
|
|
@@ -117,7 +116,6 @@ export class MockCheckpointBuilder implements ICheckpointBlockBuilder {
|
|
|
117
116
|
publicGas: Gas.empty(),
|
|
118
117
|
publicProcessorDuration: 0,
|
|
119
118
|
numTxs: block?.body?.txEffects?.length ?? usedTxs.length,
|
|
120
|
-
blockBuildingTimer: new Timer(),
|
|
121
119
|
usedTxs,
|
|
122
120
|
failedTxs: [],
|
|
123
121
|
usedTxBlobFields: block?.body?.txEffects?.reduce((sum, tx) => sum + tx.getNumBlobFields(), 0) ?? 0,
|