@aztec/sequencer-client 0.0.1-commit.e3c1de76 → 0.0.1-commit.e57c76e
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +281 -21
- package/dest/client/sequencer-client.d.ts +19 -9
- package/dest/client/sequencer-client.d.ts.map +1 -1
- package/dest/client/sequencer-client.js +65 -32
- package/dest/config.d.ts +28 -6
- package/dest/config.d.ts.map +1 -1
- package/dest/config.js +76 -37
- package/dest/global_variable_builder/fee_predictor.d.ts +37 -0
- package/dest/global_variable_builder/fee_predictor.d.ts.map +1 -0
- package/dest/global_variable_builder/fee_predictor.js +128 -0
- package/dest/global_variable_builder/fee_provider.d.ts +21 -0
- package/dest/global_variable_builder/fee_provider.d.ts.map +1 -0
- package/dest/global_variable_builder/fee_provider.js +58 -0
- package/dest/global_variable_builder/global_builder.d.ts +14 -16
- package/dest/global_variable_builder/global_builder.d.ts.map +1 -1
- package/dest/global_variable_builder/global_builder.js +16 -50
- package/dest/global_variable_builder/index.d.ts +4 -2
- package/dest/global_variable_builder/index.d.ts.map +1 -1
- package/dest/global_variable_builder/index.js +2 -0
- package/dest/index.d.ts +2 -2
- package/dest/index.d.ts.map +1 -1
- package/dest/index.js +1 -1
- package/dest/publisher/config.d.ts +47 -17
- package/dest/publisher/config.d.ts.map +1 -1
- package/dest/publisher/config.js +121 -42
- package/dest/publisher/index.d.ts +2 -1
- package/dest/publisher/index.d.ts.map +1 -1
- package/dest/publisher/l1_tx_failed_store/factory.d.ts +11 -0
- package/dest/publisher/l1_tx_failed_store/factory.d.ts.map +1 -0
- package/dest/publisher/l1_tx_failed_store/factory.js +22 -0
- package/dest/publisher/l1_tx_failed_store/failed_tx_store.d.ts +58 -0
- package/dest/publisher/l1_tx_failed_store/failed_tx_store.d.ts.map +1 -0
- package/dest/publisher/l1_tx_failed_store/failed_tx_store.js +1 -0
- package/dest/publisher/l1_tx_failed_store/file_store_failed_tx_store.d.ts +15 -0
- package/dest/publisher/l1_tx_failed_store/file_store_failed_tx_store.d.ts.map +1 -0
- package/dest/publisher/l1_tx_failed_store/file_store_failed_tx_store.js +34 -0
- package/dest/publisher/l1_tx_failed_store/index.d.ts +4 -0
- package/dest/publisher/l1_tx_failed_store/index.d.ts.map +1 -0
- package/dest/publisher/l1_tx_failed_store/index.js +2 -0
- package/dest/publisher/sequencer-bundle-simulator.d.ts +96 -0
- package/dest/publisher/sequencer-bundle-simulator.d.ts.map +1 -0
- package/dest/publisher/sequencer-bundle-simulator.js +198 -0
- package/dest/publisher/sequencer-publisher-factory.d.ts +11 -5
- package/dest/publisher/sequencer-publisher-factory.d.ts.map +1 -1
- package/dest/publisher/sequencer-publisher-factory.js +27 -3
- package/dest/publisher/sequencer-publisher.d.ts +94 -67
- package/dest/publisher/sequencer-publisher.d.ts.map +1 -1
- package/dest/publisher/sequencer-publisher.js +439 -378
- package/dest/sequencer/automine/automine_factory.d.ts +54 -0
- package/dest/sequencer/automine/automine_factory.d.ts.map +1 -0
- package/dest/sequencer/automine/automine_factory.js +84 -0
- package/dest/sequencer/automine/automine_sequencer.d.ts +151 -0
- package/dest/sequencer/automine/automine_sequencer.d.ts.map +1 -0
- package/dest/sequencer/automine/automine_sequencer.js +509 -0
- package/dest/sequencer/chain_state_overrides.d.ts +61 -0
- package/dest/sequencer/chain_state_overrides.d.ts.map +1 -0
- package/dest/sequencer/chain_state_overrides.js +98 -0
- package/dest/sequencer/checkpoint_proposal_job.d.ts +65 -11
- package/dest/sequencer/checkpoint_proposal_job.d.ts.map +1 -1
- package/dest/sequencer/checkpoint_proposal_job.js +849 -249
- package/dest/sequencer/checkpoint_proposal_job_metrics.d.ts +34 -0
- package/dest/sequencer/checkpoint_proposal_job_metrics.d.ts.map +1 -0
- package/dest/sequencer/checkpoint_proposal_job_metrics.js +72 -0
- package/dest/sequencer/checkpoint_voter.d.ts +1 -2
- package/dest/sequencer/checkpoint_voter.d.ts.map +1 -1
- package/dest/sequencer/checkpoint_voter.js +2 -5
- package/dest/sequencer/events.d.ts +48 -2
- package/dest/sequencer/events.d.ts.map +1 -1
- package/dest/sequencer/index.d.ts +3 -1
- package/dest/sequencer/index.d.ts.map +1 -1
- package/dest/sequencer/index.js +2 -0
- package/dest/sequencer/metrics.d.ts +23 -8
- package/dest/sequencer/metrics.d.ts.map +1 -1
- package/dest/sequencer/metrics.js +117 -21
- package/dest/sequencer/sequencer.d.ts +85 -22
- package/dest/sequencer/sequencer.d.ts.map +1 -1
- package/dest/sequencer/sequencer.js +380 -137
- package/dest/sequencer/timetable.d.ts +14 -6
- package/dest/sequencer/timetable.d.ts.map +1 -1
- package/dest/sequencer/timetable.js +47 -46
- package/dest/sequencer/types.d.ts +2 -2
- package/dest/sequencer/types.d.ts.map +1 -1
- package/dest/test/index.d.ts +3 -5
- package/dest/test/index.d.ts.map +1 -1
- package/dest/test/mock_checkpoint_builder.d.ts +11 -11
- package/dest/test/mock_checkpoint_builder.d.ts.map +1 -1
- package/dest/test/mock_checkpoint_builder.js +45 -34
- package/dest/test/utils.d.ts +3 -3
- package/dest/test/utils.d.ts.map +1 -1
- package/dest/test/utils.js +10 -8
- package/package.json +27 -28
- package/src/client/sequencer-client.ts +91 -35
- package/src/config.ts +98 -46
- package/src/global_variable_builder/README.md +44 -0
- package/src/global_variable_builder/fee_predictor.ts +172 -0
- package/src/global_variable_builder/fee_provider.ts +75 -0
- package/src/global_variable_builder/global_builder.ts +26 -63
- package/src/global_variable_builder/index.ts +3 -1
- package/src/index.ts +10 -1
- package/src/publisher/config.ts +157 -45
- package/src/publisher/index.ts +3 -0
- package/src/publisher/l1_tx_failed_store/factory.ts +32 -0
- package/src/publisher/l1_tx_failed_store/failed_tx_store.ts +57 -0
- package/src/publisher/l1_tx_failed_store/file_store_failed_tx_store.ts +46 -0
- package/src/publisher/l1_tx_failed_store/index.ts +3 -0
- package/src/publisher/sequencer-bundle-simulator.ts +254 -0
- package/src/publisher/sequencer-publisher-factory.ts +38 -9
- package/src/publisher/sequencer-publisher.ts +532 -469
- package/src/sequencer/README.md +162 -450
- package/src/sequencer/automine/README.md +46 -0
- package/src/sequencer/automine/automine_factory.ts +145 -0
- package/src/sequencer/automine/automine_sequencer.ts +595 -0
- package/src/sequencer/chain_state_overrides.ts +162 -0
- package/src/sequencer/checkpoint_proposal_job.ts +1026 -268
- package/src/sequencer/checkpoint_proposal_job_metrics.ts +128 -0
- package/src/sequencer/checkpoint_voter.ts +1 -12
- package/src/sequencer/events.ts +51 -2
- package/src/sequencer/index.ts +2 -0
- package/src/sequencer/metrics.ts +132 -25
- package/src/sequencer/sequencer.ts +470 -156
- package/src/sequencer/timetable.ts +62 -56
- package/src/sequencer/types.ts +1 -1
- package/src/test/index.ts +2 -4
- package/src/test/mock_checkpoint_builder.ts +63 -49
- package/src/test/utils.ts +31 -10
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
import {
|
|
2
|
+
Attributes,
|
|
3
|
+
type Gauge,
|
|
4
|
+
type Histogram,
|
|
5
|
+
type Meter,
|
|
6
|
+
Metrics,
|
|
7
|
+
type TelemetryClient,
|
|
8
|
+
} from '@aztec/telemetry-client';
|
|
9
|
+
|
|
10
|
+
type CheckpointProposalJobInstruments = {
|
|
11
|
+
checkpointAttestationDelay: Histogram;
|
|
12
|
+
checkpointBuildDuration: Histogram;
|
|
13
|
+
checkpointStartToFirstBlockDuration: Histogram;
|
|
14
|
+
checkpointLastBlockToBroadcastDuration: Histogram;
|
|
15
|
+
pipelinedCheckpointBuildStartOffsetFromSlotBoundary: Histogram;
|
|
16
|
+
checkpointBlockCount: Gauge;
|
|
17
|
+
checkpointTxCount: Gauge;
|
|
18
|
+
checkpointTotalMana: Gauge;
|
|
19
|
+
};
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* Per-job recording surface used by {@link CheckpointProposalJob}.
|
|
23
|
+
*
|
|
24
|
+
* Ownership split:
|
|
25
|
+
* - {@link CheckpointProposalJobMetrics} owns the OpenTelemetry meter and instruments and should live for the
|
|
26
|
+
* lifetime of the sequencer process.
|
|
27
|
+
* - A recorder owns only mutable timing state for one checkpoint proposal job so overlapping jobs cannot
|
|
28
|
+
* overwrite each other's timing markers.
|
|
29
|
+
*/
|
|
30
|
+
export interface CheckpointProposalJobMetricsRecorder {
|
|
31
|
+
recordCheckpointAttestationDelay(durationMs: number): void;
|
|
32
|
+
recordCheckpointBuild(durationMs: number, blockCount: number, txCount: number, totalMana: number): void;
|
|
33
|
+
recordPipelinedCheckpointBuildStartOffsetFromSlotBoundary(offsetMs: number): void;
|
|
34
|
+
startCheckpointTiming(nowMs: number): void;
|
|
35
|
+
noteCheckpointBlockBuilt(nowMs: number, opts: { isFirstBlock: boolean; isLastBlock: boolean }): void;
|
|
36
|
+
noteCheckpointBroadcast(nowMs: number): void;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* Concrete per-job recorder.
|
|
41
|
+
*
|
|
42
|
+
* This class should be short-lived: create one recorder per checkpoint proposal job and discard it when the job
|
|
43
|
+
* completes. It intentionally does not create instruments; it only holds the job-local timestamps needed to derive
|
|
44
|
+
* timing metrics safely.
|
|
45
|
+
*/
|
|
46
|
+
class CheckpointProposalJobMetricsRecorderImpl implements CheckpointProposalJobMetricsRecorder {
|
|
47
|
+
private checkpointStartedAt?: number;
|
|
48
|
+
private checkpointLastBlockBuiltAt?: number;
|
|
49
|
+
|
|
50
|
+
constructor(private readonly instruments: CheckpointProposalJobInstruments) {}
|
|
51
|
+
|
|
52
|
+
public recordCheckpointAttestationDelay(durationMs: number) {
|
|
53
|
+
this.instruments.checkpointAttestationDelay.record(Math.ceil(durationMs));
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
public recordCheckpointBuild(durationMs: number, blockCount: number, txCount: number, totalMana: number) {
|
|
57
|
+
this.instruments.checkpointBuildDuration.record(Math.ceil(durationMs));
|
|
58
|
+
this.instruments.checkpointBlockCount.record(blockCount);
|
|
59
|
+
this.instruments.checkpointTxCount.record(txCount);
|
|
60
|
+
this.instruments.checkpointTotalMana.record(totalMana);
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
public recordPipelinedCheckpointBuildStartOffsetFromSlotBoundary(offsetMs: number) {
|
|
64
|
+
this.instruments.pipelinedCheckpointBuildStartOffsetFromSlotBoundary.record(Math.ceil(Math.abs(offsetMs)), {
|
|
65
|
+
[Attributes.SLOT_BOUNDARY_SIDE]: offsetMs < 0 ? 'before' : 'after',
|
|
66
|
+
});
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
public startCheckpointTiming(nowMs: number) {
|
|
70
|
+
this.checkpointStartedAt = nowMs;
|
|
71
|
+
this.checkpointLastBlockBuiltAt = undefined;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
public noteCheckpointBlockBuilt(nowMs: number, opts: { isFirstBlock: boolean; isLastBlock: boolean }) {
|
|
75
|
+
if (opts.isFirstBlock && this.checkpointStartedAt !== undefined) {
|
|
76
|
+
this.instruments.checkpointStartToFirstBlockDuration.record(Math.ceil(nowMs - this.checkpointStartedAt));
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
if (opts.isLastBlock) {
|
|
80
|
+
this.checkpointLastBlockBuiltAt = nowMs;
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
public noteCheckpointBroadcast(nowMs: number) {
|
|
85
|
+
if (this.checkpointLastBlockBuiltAt !== undefined) {
|
|
86
|
+
this.instruments.checkpointLastBlockToBroadcastDuration.record(
|
|
87
|
+
Math.ceil(nowMs - this.checkpointLastBlockBuiltAt),
|
|
88
|
+
);
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
this.checkpointLastBlockBuiltAt = undefined;
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* Long-lived owner of checkpoint proposal job telemetry instruments.
|
|
97
|
+
*
|
|
98
|
+
* The sequencer should construct this once and reuse it across jobs. Each call to {@link createRecorder} returns a
|
|
99
|
+
* lightweight recorder with isolated mutable state for a single checkpoint proposal job.
|
|
100
|
+
*/
|
|
101
|
+
export class CheckpointProposalJobMetrics {
|
|
102
|
+
private readonly meter: Meter;
|
|
103
|
+
private readonly instruments: CheckpointProposalJobInstruments;
|
|
104
|
+
|
|
105
|
+
constructor(client: TelemetryClient, name = 'CheckpointProposalJob') {
|
|
106
|
+
this.meter = client.getMeter(name);
|
|
107
|
+
this.instruments = {
|
|
108
|
+
checkpointAttestationDelay: this.meter.createHistogram(Metrics.SEQUENCER_CHECKPOINT_ATTESTATION_DELAY),
|
|
109
|
+
checkpointBuildDuration: this.meter.createHistogram(Metrics.SEQUENCER_CHECKPOINT_BUILD_DURATION),
|
|
110
|
+
checkpointStartToFirstBlockDuration: this.meter.createHistogram(
|
|
111
|
+
Metrics.SEQUENCER_CHECKPOINT_START_TO_FIRST_BLOCK_DURATION,
|
|
112
|
+
),
|
|
113
|
+
checkpointLastBlockToBroadcastDuration: this.meter.createHistogram(
|
|
114
|
+
Metrics.SEQUENCER_CHECKPOINT_LAST_BLOCK_TO_BROADCAST_DURATION,
|
|
115
|
+
),
|
|
116
|
+
pipelinedCheckpointBuildStartOffsetFromSlotBoundary: this.meter.createHistogram(
|
|
117
|
+
Metrics.SEQUENCER_PIPELINED_CHECKPOINT_BUILD_START_OFFSET_FROM_SLOT_BOUNDARY,
|
|
118
|
+
),
|
|
119
|
+
checkpointBlockCount: this.meter.createGauge(Metrics.SEQUENCER_CHECKPOINT_BLOCK_COUNT),
|
|
120
|
+
checkpointTxCount: this.meter.createGauge(Metrics.SEQUENCER_CHECKPOINT_TX_COUNT),
|
|
121
|
+
checkpointTotalMana: this.meter.createGauge(Metrics.SEQUENCER_CHECKPOINT_TOTAL_MANA),
|
|
122
|
+
};
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
public createRecorder(): CheckpointProposalJobMetricsRecorder {
|
|
126
|
+
return new CheckpointProposalJobMetricsRecorderImpl(this.instruments);
|
|
127
|
+
}
|
|
128
|
+
}
|
|
@@ -2,7 +2,6 @@ import type { SlotNumber } from '@aztec/foundation/branded-types';
|
|
|
2
2
|
import type { EthAddress } from '@aztec/foundation/eth-address';
|
|
3
3
|
import type { Logger } from '@aztec/foundation/log';
|
|
4
4
|
import type { SlasherClientInterface } from '@aztec/slasher';
|
|
5
|
-
import { getTimestampForSlot } from '@aztec/stdlib/epoch-helpers';
|
|
6
5
|
import type { ResolvedSequencerConfig } from '@aztec/stdlib/interfaces/server';
|
|
7
6
|
import type { ValidatorClient } from '@aztec/validator-client';
|
|
8
7
|
import { DutyAlreadySignedError } from '@aztec/validator-ha-signer/errors';
|
|
@@ -18,7 +17,6 @@ import type { SequencerRollupConstants } from './types.js';
|
|
|
18
17
|
* Handles governance and slashing voting for a given slot.
|
|
19
18
|
*/
|
|
20
19
|
export class CheckpointVoter {
|
|
21
|
-
private slotTimestamp: bigint;
|
|
22
20
|
private governanceSigner: (msg: TypedDataDefinition) => Promise<`0x${string}`>;
|
|
23
21
|
private slashingSigner: (msg: TypedDataDefinition) => Promise<`0x${string}`>;
|
|
24
22
|
|
|
@@ -33,8 +31,6 @@ export class CheckpointVoter {
|
|
|
33
31
|
private readonly metrics: SequencerMetrics,
|
|
34
32
|
private readonly log: Logger,
|
|
35
33
|
) {
|
|
36
|
-
this.slotTimestamp = getTimestampForSlot(this.slot, this.l1Constants);
|
|
37
|
-
|
|
38
34
|
// Create separate signers with appropriate duty contexts for governance and slashing votes
|
|
39
35
|
// These use HA protection to ensure only one node signs per slot/duty
|
|
40
36
|
const governanceContext: SigningContext = { slot: this.slot, dutyType: DutyType.GOVERNANCE_VOTE };
|
|
@@ -77,7 +73,6 @@ export class CheckpointVoter {
|
|
|
77
73
|
return await this.publisher.enqueueGovernanceCastSignal(
|
|
78
74
|
governanceProposerPayload,
|
|
79
75
|
this.slot,
|
|
80
|
-
this.slotTimestamp,
|
|
81
76
|
this.attestorAddress,
|
|
82
77
|
this.governanceSigner,
|
|
83
78
|
);
|
|
@@ -108,13 +103,7 @@ export class CheckpointVoter {
|
|
|
108
103
|
|
|
109
104
|
this.metrics.recordSlashingAttempt(actions.length);
|
|
110
105
|
|
|
111
|
-
return await this.publisher.enqueueSlashingActions(
|
|
112
|
-
actions,
|
|
113
|
-
this.slot,
|
|
114
|
-
this.slotTimestamp,
|
|
115
|
-
this.attestorAddress,
|
|
116
|
-
this.slashingSigner,
|
|
117
|
-
);
|
|
106
|
+
return await this.publisher.enqueueSlashingActions(actions, this.slot, this.attestorAddress, this.slashingSigner);
|
|
118
107
|
} catch (err) {
|
|
119
108
|
if (err instanceof DutyAlreadySignedError) {
|
|
120
109
|
this.log.info(`Slashing vote already signed by another node`, {
|
package/src/sequencer/events.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import type { BlockNumber, CheckpointNumber, SlotNumber } from '@aztec/foundation/branded-types';
|
|
1
|
+
import type { BlockNumber, CheckpointNumber, IndexWithinCheckpoint, SlotNumber } from '@aztec/foundation/branded-types';
|
|
2
|
+
import type { BlockHash } from '@aztec/stdlib/block';
|
|
2
3
|
|
|
3
4
|
import type { Action } from '../publisher/sequencer-publisher.js';
|
|
4
5
|
import type { SequencerState } from './utils.js';
|
|
@@ -9,12 +10,55 @@ export type SequencerEvents = {
|
|
|
9
10
|
newState: SequencerState;
|
|
10
11
|
secondsIntoSlot?: number;
|
|
11
12
|
slot?: SlotNumber;
|
|
13
|
+
timeReferenceSlot?: SlotNumber;
|
|
14
|
+
}) => void;
|
|
15
|
+
/**
|
|
16
|
+
* Emitted by the sequencer once it has decided it is going to attempt to build a
|
|
17
|
+
* checkpoint for `targetSlot`, after computing the L1 simulation overrides used by
|
|
18
|
+
* `canProposeAt`. Fired BEFORE the L1 simulation is run, so consumers can observe the
|
|
19
|
+
* decision regardless of whether the propose ultimately lands.
|
|
20
|
+
*
|
|
21
|
+
* - `hadProposedParent` indicates whether the build saw a proposed (pipelined) parent
|
|
22
|
+
* checkpoint that hasn't landed on L1 yet.
|
|
23
|
+
* - `provenOverride` is the assumed proven checkpoint number pinned for the L1
|
|
24
|
+
* simulation. The plan always pins both chain tips to short-circuit `canPruneAtTime`,
|
|
25
|
+
* so this is populated whenever a simulation plan was built — the value either
|
|
26
|
+
* matches the on-chain proven snapshot (defensive pin) or the assumed-proven
|
|
27
|
+
* checkpoint when building optimistically across a pruning boundary.
|
|
28
|
+
* - `simulatedPending` is the pending checkpoint passed to L1 simulation. The plan
|
|
29
|
+
* always pins both chain tips to short-circuit `canPruneAtTime`, so this reflects
|
|
30
|
+
* either the pipelined/invalidated tip or the on-chain pending snapshot.
|
|
31
|
+
*/
|
|
32
|
+
['preparing-checkpoint']: (args: {
|
|
33
|
+
targetSlot: SlotNumber;
|
|
34
|
+
checkpointNumber: CheckpointNumber;
|
|
35
|
+
hadProposedParent: boolean;
|
|
36
|
+
provenOverride: CheckpointNumber | undefined;
|
|
37
|
+
simulatedPending: CheckpointNumber | undefined;
|
|
12
38
|
}) => void;
|
|
13
39
|
['proposer-rollup-check-failed']: (args: { reason: string; slot: SlotNumber }) => void;
|
|
14
40
|
['block-tx-count-check-failed']: (args: { minTxs: number; availableTxs: number; slot: SlotNumber }) => void;
|
|
15
41
|
['block-build-failed']: (args: { reason: string; slot: SlotNumber }) => void;
|
|
16
|
-
['block-proposed']: (args: {
|
|
42
|
+
['block-proposed']: (args: {
|
|
43
|
+
blockNumber: BlockNumber;
|
|
44
|
+
blockHash: BlockHash;
|
|
45
|
+
checkpointNumber: CheckpointNumber;
|
|
46
|
+
indexWithinCheckpoint: IndexWithinCheckpoint;
|
|
47
|
+
slot: SlotNumber;
|
|
48
|
+
buildSlot: SlotNumber;
|
|
49
|
+
}) => void;
|
|
17
50
|
['checkpoint-empty']: (args: { slot: SlotNumber }) => void;
|
|
51
|
+
/**
|
|
52
|
+
* Emitted when the proposer's pre-broadcast `validateBlockHeader` simulation fails. This is a
|
|
53
|
+
* last-chance check before we gossip a checkpoint proposal: a failure here means the header
|
|
54
|
+
* would not be accepted by L1 (e.g. archive mismatch, stale chain tip, or some other state
|
|
55
|
+
* drift between when we built the checkpoint and when we are about to broadcast it).
|
|
56
|
+
*/
|
|
57
|
+
['header-validation-failed']: (args: {
|
|
58
|
+
slot: SlotNumber;
|
|
59
|
+
checkpointNumber: CheckpointNumber;
|
|
60
|
+
reason: string;
|
|
61
|
+
}) => void;
|
|
18
62
|
['checkpoint-publish-failed']: (args: {
|
|
19
63
|
slot: SlotNumber;
|
|
20
64
|
successfulActions?: Action[];
|
|
@@ -24,4 +68,9 @@ export type SequencerEvents = {
|
|
|
24
68
|
}) => void;
|
|
25
69
|
['checkpoint-published']: (args: { checkpoint: CheckpointNumber; slot: SlotNumber }) => void;
|
|
26
70
|
['checkpoint-error']: (args: { error: Error }) => void;
|
|
71
|
+
['pipelined-checkpoint-discarded']: (args: {
|
|
72
|
+
slot: SlotNumber;
|
|
73
|
+
checkpointNumber: CheckpointNumber;
|
|
74
|
+
reason: string;
|
|
75
|
+
}) => void;
|
|
27
76
|
};
|
package/src/sequencer/index.ts
CHANGED
package/src/sequencer/metrics.ts
CHANGED
|
@@ -18,7 +18,6 @@ import { type Hex, formatUnits } from 'viem';
|
|
|
18
18
|
|
|
19
19
|
import type { SequencerState } from './utils.js';
|
|
20
20
|
|
|
21
|
-
// TODO(palla/mbps): Review all metrics and add any missing ones per checkpoint
|
|
22
21
|
export class SequencerMetrics {
|
|
23
22
|
public readonly tracer: Tracer;
|
|
24
23
|
private meter: Meter;
|
|
@@ -27,6 +26,7 @@ export class SequencerMetrics {
|
|
|
27
26
|
private blockBuildDuration: Histogram;
|
|
28
27
|
private blockBuildManaPerSecond: Gauge;
|
|
29
28
|
private stateTransitionBufferDuration: Histogram;
|
|
29
|
+
private stateDuration: Histogram;
|
|
30
30
|
|
|
31
31
|
// these are gauges because for individual sequencers building a block is not something that happens often enough to warrant a histogram
|
|
32
32
|
private timeToCollectAttestations: Gauge;
|
|
@@ -40,17 +40,24 @@ export class SequencerMetrics {
|
|
|
40
40
|
private filledSlots: UpDownCounter;
|
|
41
41
|
|
|
42
42
|
private blockProposalFailed: UpDownCounter;
|
|
43
|
-
private
|
|
44
|
-
private
|
|
43
|
+
private checkpointProposalSuccess: UpDownCounter;
|
|
44
|
+
private checkpointPrecheckFailed: UpDownCounter;
|
|
45
|
+
private checkpointProposalFailed: UpDownCounter;
|
|
45
46
|
private checkpointSuccess: UpDownCounter;
|
|
46
47
|
private slashingAttempts: UpDownCounter;
|
|
47
|
-
private
|
|
48
|
+
private pipelineDepth: Gauge;
|
|
49
|
+
private pipelineDiscards: UpDownCounter;
|
|
50
|
+
private pipelineParentCheckpointMismatches: UpDownCounter;
|
|
48
51
|
|
|
49
52
|
// Fisherman fee analysis metrics
|
|
50
53
|
private fishermanWouldBeIncluded: UpDownCounter;
|
|
51
54
|
private fishermanTimeBeforeBlock: Histogram;
|
|
52
55
|
private fishermanPendingBlobTxCount: Histogram;
|
|
53
56
|
private fishermanIncludedBlobTxCount: Histogram;
|
|
57
|
+
private fishermanPendingBlobCount: Histogram;
|
|
58
|
+
private fishermanIncludedBlobCount: Histogram;
|
|
59
|
+
private fishermanBlockBlobsFull: UpDownCounter;
|
|
60
|
+
private fishermanMaxBlobCapacity: Histogram;
|
|
54
61
|
private fishermanCalculatedPriorityFee: Histogram;
|
|
55
62
|
private fishermanPriorityFeeDelta: Histogram;
|
|
56
63
|
private fishermanEstimatedCost: Histogram;
|
|
@@ -58,6 +65,10 @@ export class SequencerMetrics {
|
|
|
58
65
|
private fishermanMinedBlobTxPriorityFee: Histogram;
|
|
59
66
|
private fishermanMinedBlobTxTotalCost: Histogram;
|
|
60
67
|
|
|
68
|
+
private blockInterBlockTime: Histogram;
|
|
69
|
+
private lastBlockBuiltTimestamp?: number;
|
|
70
|
+
private lastBlockBuiltSlot?: SlotNumber;
|
|
71
|
+
|
|
61
72
|
private lastSeenSlot?: SlotNumber;
|
|
62
73
|
|
|
63
74
|
constructor(
|
|
@@ -76,11 +87,13 @@ export class SequencerMetrics {
|
|
|
76
87
|
|
|
77
88
|
this.blockBuildManaPerSecond = this.meter.createGauge(Metrics.SEQUENCER_BLOCK_BUILD_MANA_PER_SECOND);
|
|
78
89
|
|
|
90
|
+
this.blockInterBlockTime = this.meter.createHistogram(Metrics.SEQUENCER_BLOCK_INTER_BLOCK_TIME);
|
|
91
|
+
|
|
79
92
|
this.stateTransitionBufferDuration = this.meter.createHistogram(Metrics.SEQUENCER_STATE_TRANSITION_BUFFER_DURATION);
|
|
80
93
|
|
|
81
|
-
this.
|
|
94
|
+
this.stateDuration = this.meter.createHistogram(Metrics.SEQUENCER_STATE_DURATION);
|
|
82
95
|
|
|
83
|
-
this.rewards = this.meter.createGauge(Metrics.
|
|
96
|
+
this.rewards = this.meter.createGauge(Metrics.SEQUENCER_CURRENT_SLOT_REWARDS);
|
|
84
97
|
|
|
85
98
|
this.slots = createUpDownCounterWithDefault(this.meter, Metrics.SEQUENCER_SLOT_COUNT);
|
|
86
99
|
|
|
@@ -103,16 +116,16 @@ export class SequencerMetrics {
|
|
|
103
116
|
Metrics.SEQUENCER_BLOCK_PROPOSAL_FAILED_COUNT,
|
|
104
117
|
);
|
|
105
118
|
|
|
106
|
-
this.
|
|
119
|
+
this.checkpointProposalSuccess = createUpDownCounterWithDefault(
|
|
107
120
|
this.meter,
|
|
108
|
-
Metrics.
|
|
121
|
+
Metrics.SEQUENCER_CHECKPOINT_PROPOSAL_SUCCESS_COUNT,
|
|
109
122
|
);
|
|
110
123
|
|
|
111
124
|
this.checkpointSuccess = createUpDownCounterWithDefault(this.meter, Metrics.SEQUENCER_CHECKPOINT_SUCCESS_COUNT);
|
|
112
125
|
|
|
113
|
-
this.
|
|
126
|
+
this.checkpointPrecheckFailed = createUpDownCounterWithDefault(
|
|
114
127
|
this.meter,
|
|
115
|
-
Metrics.
|
|
128
|
+
Metrics.SEQUENCER_CHECKPOINT_PRECHECK_FAILED_COUNT,
|
|
116
129
|
{
|
|
117
130
|
[Attributes.ERROR_TYPE]: [
|
|
118
131
|
'slot_already_taken',
|
|
@@ -123,14 +136,37 @@ export class SequencerMetrics {
|
|
|
123
136
|
},
|
|
124
137
|
);
|
|
125
138
|
|
|
139
|
+
this.checkpointProposalFailed = createUpDownCounterWithDefault(
|
|
140
|
+
this.meter,
|
|
141
|
+
Metrics.SEQUENCER_CHECKPOINT_PROPOSAL_FAILED_COUNT,
|
|
142
|
+
);
|
|
143
|
+
|
|
126
144
|
this.slashingAttempts = createUpDownCounterWithDefault(this.meter, Metrics.SEQUENCER_SLASHING_ATTEMPTS_COUNT);
|
|
127
145
|
|
|
146
|
+
this.pipelineDepth = this.meter.createGauge(Metrics.SEQUENCER_PIPELINE_DEPTH);
|
|
147
|
+
this.pipelineDiscards = createUpDownCounterWithDefault(this.meter, Metrics.SEQUENCER_PIPELINE_DISCARDS_COUNT);
|
|
148
|
+
this.pipelineParentCheckpointMismatches = createUpDownCounterWithDefault(
|
|
149
|
+
this.meter,
|
|
150
|
+
Metrics.SEQUENCER_PIPELINE_PARENT_CHECKPOINT_MISMATCH_COUNT,
|
|
151
|
+
{
|
|
152
|
+
[Attributes.ERROR_TYPE]: [
|
|
153
|
+
'archiver-sync-timeout',
|
|
154
|
+
'parent-not-on-l1',
|
|
155
|
+
'parent-hash-mismatch',
|
|
156
|
+
'parent-invalid-attestations',
|
|
157
|
+
'unexpected-parent-appeared',
|
|
158
|
+
],
|
|
159
|
+
},
|
|
160
|
+
);
|
|
161
|
+
this.pipelineDepth.record(0);
|
|
162
|
+
|
|
128
163
|
// Fisherman fee analysis metrics
|
|
129
164
|
this.fishermanWouldBeIncluded = createUpDownCounterWithDefault(
|
|
130
165
|
this.meter,
|
|
131
166
|
Metrics.FISHERMAN_FEE_ANALYSIS_WOULD_BE_INCLUDED,
|
|
132
167
|
{
|
|
133
168
|
[Attributes.OK]: [true, false],
|
|
169
|
+
[Attributes.BLOCK_FULL]: ['true', 'false'],
|
|
134
170
|
},
|
|
135
171
|
);
|
|
136
172
|
|
|
@@ -161,6 +197,20 @@ export class SequencerMetrics {
|
|
|
161
197
|
this.fishermanMinedBlobTxTotalCost = this.meter.createHistogram(
|
|
162
198
|
Metrics.FISHERMAN_FEE_ANALYSIS_MINED_BLOB_TX_TOTAL_COST,
|
|
163
199
|
);
|
|
200
|
+
|
|
201
|
+
this.fishermanPendingBlobCount = this.meter.createHistogram(Metrics.FISHERMAN_FEE_ANALYSIS_PENDING_BLOB_COUNT);
|
|
202
|
+
|
|
203
|
+
this.fishermanIncludedBlobCount = this.meter.createHistogram(Metrics.FISHERMAN_FEE_ANALYSIS_INCLUDED_BLOB_COUNT);
|
|
204
|
+
|
|
205
|
+
this.fishermanBlockBlobsFull = createUpDownCounterWithDefault(
|
|
206
|
+
this.meter,
|
|
207
|
+
Metrics.FISHERMAN_FEE_ANALYSIS_BLOCK_BLOBS_FULL,
|
|
208
|
+
{
|
|
209
|
+
[Attributes.OK]: [true, false],
|
|
210
|
+
},
|
|
211
|
+
);
|
|
212
|
+
|
|
213
|
+
this.fishermanMaxBlobCapacity = this.meter.createHistogram(Metrics.FISHERMAN_FEE_ANALYSIS_MAX_BLOB_CAPACITY);
|
|
164
214
|
}
|
|
165
215
|
|
|
166
216
|
public recordRequiredAttestations(requiredAttestationsCount: number, allowanceMs: number) {
|
|
@@ -172,21 +222,25 @@ export class SequencerMetrics {
|
|
|
172
222
|
this.timeToCollectAttestations.record(0);
|
|
173
223
|
}
|
|
174
224
|
|
|
175
|
-
public recordCheckpointAttestationDelay(duration: number) {
|
|
176
|
-
this.checkpointAttestationDelay.record(duration);
|
|
177
|
-
}
|
|
178
|
-
|
|
179
225
|
public recordCollectedAttestations(count: number, durationMs: number) {
|
|
180
226
|
this.collectedAttestions.record(count);
|
|
181
227
|
this.timeToCollectAttestations.record(Math.ceil(durationMs));
|
|
182
228
|
}
|
|
183
229
|
|
|
184
|
-
recordBuiltBlock(buildDurationMs: number, totalMana: number) {
|
|
230
|
+
recordBuiltBlock(buildDurationMs: number, totalMana: number, slot: SlotNumber) {
|
|
185
231
|
this.blockCounter.add(1, {
|
|
186
232
|
[Attributes.STATUS]: 'built',
|
|
187
233
|
});
|
|
188
234
|
this.blockBuildDuration.record(Math.ceil(buildDurationMs));
|
|
189
235
|
this.blockBuildManaPerSecond.record(Math.ceil((totalMana * 1000) / buildDurationMs));
|
|
236
|
+
|
|
237
|
+
// Only record inter-block time between blocks built within the same slot.
|
|
238
|
+
const now = Date.now();
|
|
239
|
+
if (this.lastBlockBuiltTimestamp !== undefined && this.lastBlockBuiltSlot === slot) {
|
|
240
|
+
this.blockInterBlockTime.record(now - this.lastBlockBuiltTimestamp);
|
|
241
|
+
}
|
|
242
|
+
this.lastBlockBuiltTimestamp = now;
|
|
243
|
+
this.lastBlockBuiltSlot = slot;
|
|
190
244
|
}
|
|
191
245
|
|
|
192
246
|
recordFailedBlock() {
|
|
@@ -201,6 +255,26 @@ export class SequencerMetrics {
|
|
|
201
255
|
});
|
|
202
256
|
}
|
|
203
257
|
|
|
258
|
+
recordStateDuration(durationMs: number, state: SequencerState) {
|
|
259
|
+
this.stateDuration.record(Math.ceil(durationMs), {
|
|
260
|
+
[Attributes.SEQUENCER_STATE]: state,
|
|
261
|
+
});
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
recordPipelineDepth(depth: number) {
|
|
265
|
+
this.pipelineDepth.record(depth);
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
recordPipelineDiscard(count = 1) {
|
|
269
|
+
this.pipelineDiscards.add(count);
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
recordPipelineParentCheckpointMismatch(reason: string) {
|
|
273
|
+
this.pipelineParentCheckpointMismatches.add(1, {
|
|
274
|
+
[Attributes.ERROR_TYPE]: reason,
|
|
275
|
+
});
|
|
276
|
+
}
|
|
277
|
+
|
|
204
278
|
incOpenSlot(slot: SlotNumber, proposer: string) {
|
|
205
279
|
// sequencer went through the loop a second time. Noop
|
|
206
280
|
if (slot === this.lastSeenSlot) {
|
|
@@ -243,15 +317,19 @@ export class SequencerMetrics {
|
|
|
243
317
|
});
|
|
244
318
|
}
|
|
245
319
|
|
|
246
|
-
|
|
247
|
-
this.
|
|
320
|
+
recordCheckpointProposalSuccess() {
|
|
321
|
+
this.checkpointProposalSuccess.add(1);
|
|
248
322
|
}
|
|
249
323
|
|
|
250
|
-
|
|
324
|
+
recordCheckpointPrecheckFailed(
|
|
251
325
|
checkType: 'slot_already_taken' | 'rollup_contract_check_failed' | 'slot_mismatch' | 'block_number_mismatch',
|
|
252
326
|
) {
|
|
253
|
-
this.
|
|
254
|
-
|
|
327
|
+
this.checkpointPrecheckFailed.add(1, { [Attributes.ERROR_TYPE]: checkType });
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
recordCheckpointProposalFailed(reason?: string) {
|
|
331
|
+
this.checkpointProposalFailed.add(1, {
|
|
332
|
+
...(reason && { [Attributes.ERROR_TYPE]: reason }),
|
|
255
333
|
});
|
|
256
334
|
}
|
|
257
335
|
|
|
@@ -281,10 +359,12 @@ export class SequencerMetrics {
|
|
|
281
359
|
|
|
282
360
|
// Record pending block snapshot data (once per strategy for comparison)
|
|
283
361
|
this.fishermanPendingBlobTxCount.record(analysis.pendingSnapshot.pendingBlobTxCount, strategyAttributes);
|
|
362
|
+
this.fishermanPendingBlobCount.record(analysis.pendingSnapshot.pendingBlobCount, strategyAttributes);
|
|
284
363
|
|
|
285
364
|
// Record mined block data if available
|
|
286
365
|
if (analysis.minedBlock) {
|
|
287
366
|
this.fishermanIncludedBlobTxCount.record(analysis.minedBlock.includedBlobTxCount, strategyAttributes);
|
|
367
|
+
this.fishermanIncludedBlobCount.record(analysis.minedBlock.includedBlobCount, strategyAttributes);
|
|
288
368
|
|
|
289
369
|
// Record actual fees from blob transactions in the mined block
|
|
290
370
|
for (const blobTx of analysis.minedBlock.includedBlobTxs) {
|
|
@@ -318,13 +398,28 @@ export class SequencerMetrics {
|
|
|
318
398
|
if (analysis.analysis) {
|
|
319
399
|
this.fishermanTimeBeforeBlock.record(Math.ceil(analysis.analysis.timeBeforeBlockMs), strategyAttributes);
|
|
320
400
|
|
|
401
|
+
// Record whether the block reached 100% blob capacity
|
|
402
|
+
if (analysis.analysis.blockBlobsFull) {
|
|
403
|
+
this.fishermanBlockBlobsFull.add(1, { ...strategyAttributes, [Attributes.OK]: true });
|
|
404
|
+
} else {
|
|
405
|
+
this.fishermanBlockBlobsFull.add(1, { ...strategyAttributes, [Attributes.OK]: false });
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
// Record the max blob capacity for this block
|
|
409
|
+
this.fishermanMaxBlobCapacity.record(analysis.analysis.maxBlobCapacity, strategyAttributes);
|
|
410
|
+
|
|
321
411
|
// Record strategy-specific inclusion result
|
|
322
412
|
if (strategyResult.wouldBeIncluded !== undefined) {
|
|
413
|
+
const inclusionAttributes = {
|
|
414
|
+
...strategyAttributes,
|
|
415
|
+
[Attributes.BLOCK_FULL]: analysis.analysis.blockBlobsFull ? 'true' : 'false',
|
|
416
|
+
};
|
|
417
|
+
|
|
323
418
|
if (strategyResult.wouldBeIncluded) {
|
|
324
|
-
this.fishermanWouldBeIncluded.add(1, { ...
|
|
419
|
+
this.fishermanWouldBeIncluded.add(1, { ...inclusionAttributes, [Attributes.OK]: true });
|
|
325
420
|
} else {
|
|
326
421
|
this.fishermanWouldBeIncluded.add(1, {
|
|
327
|
-
...
|
|
422
|
+
...inclusionAttributes,
|
|
328
423
|
[Attributes.OK]: false,
|
|
329
424
|
...(strategyResult.exclusionReason && { [Attributes.ERROR_TYPE]: strategyResult.exclusionReason }),
|
|
330
425
|
});
|
|
@@ -334,17 +429,29 @@ export class SequencerMetrics {
|
|
|
334
429
|
// Record strategy-specific priority fee delta
|
|
335
430
|
if (strategyResult.priorityFeeDelta !== undefined) {
|
|
336
431
|
const priorityFeeDeltaGwei = Number(strategyResult.priorityFeeDelta) / 1e9;
|
|
337
|
-
|
|
432
|
+
const deltaAttributes = {
|
|
433
|
+
...strategyAttributes,
|
|
434
|
+
[Attributes.BLOCK_FULL]: analysis.analysis.blockBlobsFull ? 'true' : 'false',
|
|
435
|
+
};
|
|
436
|
+
this.fishermanPriorityFeeDelta.record(priorityFeeDeltaGwei, deltaAttributes);
|
|
338
437
|
}
|
|
339
438
|
|
|
340
439
|
// Record estimated cost if available
|
|
341
440
|
if (strategyResult.estimatedCostEth !== undefined) {
|
|
342
|
-
|
|
441
|
+
const costAttributes = {
|
|
442
|
+
...strategyAttributes,
|
|
443
|
+
[Attributes.BLOCK_FULL]: analysis.analysis.blockBlobsFull ? 'true' : 'false',
|
|
444
|
+
};
|
|
445
|
+
this.fishermanEstimatedCost.record(strategyResult.estimatedCostEth, costAttributes);
|
|
343
446
|
}
|
|
344
447
|
|
|
345
448
|
// Record estimated overpayment if available
|
|
346
449
|
if (strategyResult.estimatedOverpaymentEth !== undefined) {
|
|
347
|
-
|
|
450
|
+
const overpaymentAttributes = {
|
|
451
|
+
...strategyAttributes,
|
|
452
|
+
[Attributes.BLOCK_FULL]: analysis.analysis.blockBlobsFull ? 'true' : 'false',
|
|
453
|
+
};
|
|
454
|
+
this.fishermanEstimatedOverpayment.record(strategyResult.estimatedOverpaymentEth, overpaymentAttributes);
|
|
348
455
|
}
|
|
349
456
|
}
|
|
350
457
|
}
|