@aztec/aztec-node 0.0.1-commit.03f7ef2 → 0.0.1-commit.04d373f
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dest/aztec-node/block_response_helpers.d.ts +25 -0
- package/dest/aztec-node/block_response_helpers.d.ts.map +1 -0
- package/dest/aztec-node/block_response_helpers.js +112 -0
- package/dest/aztec-node/config.d.ts +14 -8
- package/dest/aztec-node/config.d.ts.map +1 -1
- package/dest/aztec-node/config.js +16 -8
- package/dest/aztec-node/node_metrics.d.ts +1 -1
- package/dest/aztec-node/node_metrics.d.ts.map +1 -1
- package/dest/aztec-node/node_metrics.js +9 -16
- package/dest/aztec-node/public_data_overrides.d.ts +13 -0
- package/dest/aztec-node/public_data_overrides.d.ts.map +1 -0
- package/dest/aztec-node/public_data_overrides.js +21 -0
- package/dest/aztec-node/server.d.ts +101 -187
- package/dest/aztec-node/server.d.ts.map +1 -1
- package/dest/aztec-node/server.js +1528 -513
- package/dest/sentinel/config.d.ts +3 -2
- package/dest/sentinel/config.d.ts.map +1 -1
- package/dest/sentinel/config.js +15 -5
- package/dest/sentinel/factory.d.ts +5 -3
- package/dest/sentinel/factory.d.ts.map +1 -1
- package/dest/sentinel/factory.js +5 -5
- package/dest/sentinel/sentinel.d.ts +137 -12
- package/dest/sentinel/sentinel.d.ts.map +1 -1
- package/dest/sentinel/sentinel.js +273 -100
- package/dest/sentinel/store.d.ts +8 -8
- package/dest/sentinel/store.d.ts.map +1 -1
- package/dest/sentinel/store.js +33 -21
- package/dest/test/index.d.ts +3 -3
- package/dest/test/index.d.ts.map +1 -1
- package/package.json +28 -26
- package/src/aztec-node/block_response_helpers.ts +161 -0
- package/src/aztec-node/config.ts +36 -16
- package/src/aztec-node/node_metrics.ts +6 -17
- package/src/aztec-node/public_data_overrides.ts +35 -0
- package/src/aztec-node/server.ts +1282 -663
- package/src/sentinel/README.md +103 -0
- package/src/sentinel/config.ts +18 -6
- package/src/sentinel/factory.ts +9 -11
- package/src/sentinel/sentinel.ts +336 -109
- package/src/sentinel/store.ts +35 -27
- package/src/test/index.ts +2 -2
package/src/sentinel/sentinel.ts
CHANGED
|
@@ -1,5 +1,11 @@
|
|
|
1
1
|
import type { EpochCache } from '@aztec/epoch-cache';
|
|
2
|
-
import {
|
|
2
|
+
import {
|
|
3
|
+
BlockNumber,
|
|
4
|
+
CheckpointNumber,
|
|
5
|
+
CheckpointProposalHash,
|
|
6
|
+
EpochNumber,
|
|
7
|
+
SlotNumber,
|
|
8
|
+
} from '@aztec/foundation/branded-types';
|
|
3
9
|
import { countWhile, filterAsync, fromEntries, getEntries, mapValues } from '@aztec/foundation/collection';
|
|
4
10
|
import { EthAddress } from '@aztec/foundation/eth-address';
|
|
5
11
|
import { createLogger } from '@aztec/foundation/log';
|
|
@@ -12,6 +18,7 @@ import {
|
|
|
12
18
|
type WantToSlashArgs,
|
|
13
19
|
type Watcher,
|
|
14
20
|
type WatcherEmitter,
|
|
21
|
+
getOffenseTypeName,
|
|
15
22
|
} from '@aztec/slasher';
|
|
16
23
|
import type { SlasherConfig } from '@aztec/slasher/config';
|
|
17
24
|
import {
|
|
@@ -19,9 +26,12 @@ import {
|
|
|
19
26
|
L2BlockStream,
|
|
20
27
|
type L2BlockStreamEvent,
|
|
21
28
|
type L2BlockStreamEventHandler,
|
|
22
|
-
|
|
29
|
+
getAttestationInfoFromPublishedCheckpoint,
|
|
23
30
|
} from '@aztec/stdlib/block';
|
|
31
|
+
import type { CheckpointReexecutionTracker } from '@aztec/stdlib/checkpoint';
|
|
32
|
+
import type { ChainConfig } from '@aztec/stdlib/config';
|
|
24
33
|
import { getEpochAtSlot, getSlotRangeForEpoch, getTimestampForSlot } from '@aztec/stdlib/epoch-helpers';
|
|
34
|
+
import { ConsensusPayload, type CoordinationSignatureContext } from '@aztec/stdlib/p2p';
|
|
25
35
|
import type {
|
|
26
36
|
SingleValidatorStats,
|
|
27
37
|
ValidatorStats,
|
|
@@ -34,8 +44,97 @@ import type {
|
|
|
34
44
|
|
|
35
45
|
import EventEmitter from 'node:events';
|
|
36
46
|
|
|
47
|
+
import type { SentinelConfig } from './config.js';
|
|
37
48
|
import { SentinelStore } from './store.js';
|
|
38
49
|
|
|
50
|
+
export type SentinelRuntimeConfig = Pick<
|
|
51
|
+
SlasherConfig,
|
|
52
|
+
'slashInactivityTargetPercentage' | 'slashInactivityPenalty' | 'slashInactivityConsecutiveEpochThreshold'
|
|
53
|
+
> &
|
|
54
|
+
Pick<SentinelConfig, 'sentinelEpochEndBufferSlots'> &
|
|
55
|
+
Pick<ChainConfig, 'l1ChainId' | 'rollupAddress'>;
|
|
56
|
+
|
|
57
|
+
/** Maps a validator status to its category: proposer or attestation. */
|
|
58
|
+
function statusToCategory(status: ValidatorStatusInSlot): ValidatorStatusType {
|
|
59
|
+
switch (status) {
|
|
60
|
+
case 'attestation-sent':
|
|
61
|
+
case 'attestation-missed':
|
|
62
|
+
return 'attestation';
|
|
63
|
+
default:
|
|
64
|
+
return 'proposer';
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* The Sentinel observes validator behaviour every L2 slot, classifies it into a per-slot status,
|
|
70
|
+
* aggregates those statuses into per-epoch performance once each epoch is fully observed, and
|
|
71
|
+
* emits inactivity slash payloads when a validator has been inactive for the configured number
|
|
72
|
+
* of consecutive epochs.
|
|
73
|
+
*
|
|
74
|
+
* ## Two cadences
|
|
75
|
+
*
|
|
76
|
+
* The sentinel runs `work()` every quarter L2 slot and drives two independent pipelines:
|
|
77
|
+
*
|
|
78
|
+
* 1. **Per-slot activity recording.** `processSlot(currentSlot - 2)` runs once per slot, with a
|
|
79
|
+
* two-slot lag to let P2P attestations settle and the archiver catch up. It classifies each
|
|
80
|
+
* committee member's behaviour for that slot via `getSlotActivity` and persists the result to
|
|
81
|
+
* `SentinelStore.historyMap` (sliding window of `sentinelHistoryLengthInEpochs * epochDuration`
|
|
82
|
+
* slots, default 24 epochs).
|
|
83
|
+
*
|
|
84
|
+
* 2. **Per-epoch evaluation.** `processEpochEnds(currentSlot)` runs every tick too. Once
|
|
85
|
+
* `sentinelEpochEndBufferSlots` (default 2) has elapsed past an epoch's last slot AND the
|
|
86
|
+
* per-slot recorder has covered that last slot, the sentinel calls `handleEpochEnd(epoch)`.
|
|
87
|
+
* That aggregates the slot-level statuses for the epoch into per-validator `{missed, total}`,
|
|
88
|
+
* persists it to `SentinelStore.epochMap` (default 2000-epoch window), and runs the slashing
|
|
89
|
+
* decision.
|
|
90
|
+
*
|
|
91
|
+
* Triggering per-epoch evaluation off local L2 state — rather than waiting for L1 proof
|
|
92
|
+
* publication — decouples slashing from prover availability.
|
|
93
|
+
*
|
|
94
|
+
* ## Six-case taxonomy in `getSlotActivity`
|
|
95
|
+
*
|
|
96
|
+
* For each slot, the sentinel assigns the proposer one of six statuses, ranked highest-confidence
|
|
97
|
+
* first:
|
|
98
|
+
*
|
|
99
|
+
* - `checkpoint-mined` — a checkpoint covering this slot has landed on L1
|
|
100
|
+
* (`slotNumberToCheckpoint` populated from `chain-checkpointed`).
|
|
101
|
+
* - `checkpoint-valid` — the local node re-executed a checkpoint proposal for this slot
|
|
102
|
+
* successfully (consulted via `CheckpointReexecutionTracker`).
|
|
103
|
+
* - `checkpoint-invalid` — the local node re-executed a checkpoint proposal for this slot
|
|
104
|
+
* and rejected it (e.g. header/archive/out-hash mismatch, limit
|
|
105
|
+
* breach). Proposer-fault.
|
|
106
|
+
* - `checkpoint-unvalidated` — the local node observed a checkpoint proposal but could not
|
|
107
|
+
* validate it (missing blocks/txs, timeouts). Treated as
|
|
108
|
+
* proposer-fault for slashing.
|
|
109
|
+
* - `checkpoint-missed` — block proposals seen on P2P but no checkpoint proposal at all.
|
|
110
|
+
* - `blocks-missed` — no block proposals seen for this slot.
|
|
111
|
+
*
|
|
112
|
+
* Missing-attestor faults are recorded only in `checkpoint-mined` and `checkpoint-valid`, where
|
|
113
|
+
* the local node has positive evidence the checkpoint was canonical or valid. In the other four
|
|
114
|
+
* cases the proposer is at fault and no attestor penalty applies.
|
|
115
|
+
*
|
|
116
|
+
* ## Re-execution tracker
|
|
117
|
+
*
|
|
118
|
+
* `CheckpointReexecutionTracker` is populated by the validator client's checkpoint proposal
|
|
119
|
+
* handler. Every early return in `validateCheckpointProposal` records an outcome
|
|
120
|
+
* (`valid` / `invalid` / `unvalidated`) keyed by slot.
|
|
121
|
+
*
|
|
122
|
+
* ## Inactivity slashing
|
|
123
|
+
*
|
|
124
|
+
* `handleEpochPerformance` filters the epoch's per-validator stats by
|
|
125
|
+
* `slashInactivityTargetPercentage` and then calls `checkPastInactivity` to require
|
|
126
|
+
* `slashInactivityConsecutiveEpochThreshold` consecutive past epochs over the same threshold
|
|
127
|
+
* (read from `SentinelStore.epochMap`). Only validators meeting both conditions are emitted as
|
|
128
|
+
* `WANT_TO_SLASH_EVENT` with `OffenseType.INACTIVITY`. The slot-level counters that feed this —
|
|
129
|
+
* `missedProposals` and `missedAttestations` — include the four proposer-fault statuses plus
|
|
130
|
+
* `attestation-missed`.
|
|
131
|
+
*
|
|
132
|
+
* ## Escape hatch
|
|
133
|
+
*
|
|
134
|
+
* If `epochCache.getCommittee(slot)` reports `isEscapeHatchOpen`, per-slot recording is skipped
|
|
135
|
+
* (no history entries for that slot) and per-epoch evaluation writes an empty performance map
|
|
136
|
+
* (no slashing).
|
|
137
|
+
*/
|
|
39
138
|
export class Sentinel extends (EventEmitter as new () => WatcherEmitter) implements L2BlockStreamEventHandler, Watcher {
|
|
40
139
|
protected runningPromise: RunningPromise;
|
|
41
140
|
protected blockStream!: L2BlockStream;
|
|
@@ -43,27 +142,41 @@ export class Sentinel extends (EventEmitter as new () => WatcherEmitter) impleme
|
|
|
43
142
|
|
|
44
143
|
protected initialSlot: SlotNumber | undefined;
|
|
45
144
|
protected lastProcessedSlot: SlotNumber | undefined;
|
|
46
|
-
|
|
47
|
-
protected
|
|
48
|
-
|
|
145
|
+
/** Largest epoch number for which the end-of-epoch aggregator has run. */
|
|
146
|
+
protected lastEvaluatedEpoch: EpochNumber | undefined;
|
|
147
|
+
protected slotNumberToCheckpoint: Map<
|
|
148
|
+
SlotNumber,
|
|
149
|
+
{
|
|
150
|
+
checkpointNumber: CheckpointNumber;
|
|
151
|
+
archive: string;
|
|
152
|
+
/** Hex keccak256 of the consensus payload bytes; used to fetch matching p2p attestations. */
|
|
153
|
+
proposalPayloadHash: CheckpointProposalHash;
|
|
154
|
+
attestors: EthAddress[];
|
|
155
|
+
}
|
|
156
|
+
> = new Map();
|
|
49
157
|
|
|
50
158
|
constructor(
|
|
51
159
|
protected epochCache: EpochCache,
|
|
52
160
|
protected archiver: L2BlockSource,
|
|
53
161
|
protected p2p: P2PClient,
|
|
54
162
|
protected store: SentinelStore,
|
|
55
|
-
protected
|
|
56
|
-
|
|
57
|
-
'slashInactivityTargetPercentage' | 'slashInactivityPenalty' | 'slashInactivityConsecutiveEpochThreshold'
|
|
58
|
-
>,
|
|
163
|
+
protected reexecutionTracker: CheckpointReexecutionTracker,
|
|
164
|
+
protected config: SentinelRuntimeConfig,
|
|
59
165
|
protected logger = createLogger('node:sentinel'),
|
|
60
166
|
) {
|
|
61
167
|
super();
|
|
62
|
-
this.l2TipsStore = new L2TipsMemoryStore();
|
|
168
|
+
this.l2TipsStore = new L2TipsMemoryStore(archiver.getGenesisBlockHash());
|
|
63
169
|
const interval = (epochCache.getL1Constants().ethereumSlotDuration * 1000) / 4;
|
|
64
170
|
this.runningPromise = new RunningPromise(this.work.bind(this), logger, interval);
|
|
65
171
|
}
|
|
66
172
|
|
|
173
|
+
private getSignatureContext(): CoordinationSignatureContext {
|
|
174
|
+
return {
|
|
175
|
+
chainId: this.config.l1ChainId,
|
|
176
|
+
rollupAddress: this.config.rollupAddress,
|
|
177
|
+
};
|
|
178
|
+
}
|
|
179
|
+
|
|
67
180
|
public updateConfig(config: Partial<SlasherConfig>) {
|
|
68
181
|
this.config = { ...this.config, ...config };
|
|
69
182
|
}
|
|
@@ -73,72 +186,96 @@ export class Sentinel extends (EventEmitter as new () => WatcherEmitter) impleme
|
|
|
73
186
|
this.runningPromise.start();
|
|
74
187
|
}
|
|
75
188
|
|
|
76
|
-
/**
|
|
189
|
+
/**
|
|
190
|
+
* Loads initial slot and initializes blockstream. We will not process anything at or before
|
|
191
|
+
* the initial slot. Floors at the archiver's synced L2 slot so the sentinel keeps making
|
|
192
|
+
* forward progress when L1 is advancing but L2 has no activity (the synced slot is driven by
|
|
193
|
+
* L1 sync, not by L2 blocks). Falls back to the wallclock if the archiver isn't ready yet
|
|
194
|
+
* (cold start).
|
|
195
|
+
*/
|
|
77
196
|
protected async init() {
|
|
78
|
-
this.initialSlot = this.
|
|
197
|
+
this.initialSlot = await this.getCurrentSlot();
|
|
79
198
|
const startingBlock = BlockNumber(await this.archiver.getBlockNumber());
|
|
80
199
|
this.logger.info(`Starting validator sentinel with initial slot ${this.initialSlot} and block ${startingBlock}`);
|
|
81
200
|
this.blockStream = new L2BlockStream(this.archiver, this.l2TipsStore, this, this.logger, { startingBlock });
|
|
82
201
|
}
|
|
83
202
|
|
|
203
|
+
/**
|
|
204
|
+
* Returns the L2 slot the sentinel should treat as "current": the archiver's last fully
|
|
205
|
+
* synced L2 slot, falling back to the wallclock slot when the archiver isn't ready yet
|
|
206
|
+
* (cold start). Anchoring to the synced slot keeps timing arithmetic (initial floor,
|
|
207
|
+
* per-slot lag, end-of-epoch buffer, stats-range fallback) from speculating ahead of where
|
|
208
|
+
* L1 actually is.
|
|
209
|
+
*/
|
|
210
|
+
protected async getCurrentSlot(): Promise<SlotNumber> {
|
|
211
|
+
return (await this.archiver.getSyncedL2SlotNumber()) ?? this.epochCache.getSlotNow();
|
|
212
|
+
}
|
|
213
|
+
|
|
84
214
|
public stop() {
|
|
85
215
|
return this.runningPromise.stop();
|
|
86
216
|
}
|
|
87
217
|
|
|
88
218
|
public async handleBlockStreamEvent(event: L2BlockStreamEvent): Promise<void> {
|
|
89
219
|
await this.l2TipsStore.handleBlockStreamEvent(event);
|
|
90
|
-
if (event.type === '
|
|
91
|
-
|
|
92
|
-
for (const block of event.blocks) {
|
|
93
|
-
this.slotNumberToBlock.set(block.block.header.getSlot(), {
|
|
94
|
-
blockNumber: BlockNumber(block.block.number),
|
|
95
|
-
archive: block.block.archive.root.toString(),
|
|
96
|
-
attestors: getAttestationInfoFromPublishedL2Block(block)
|
|
97
|
-
.filter(a => a.status === 'recovered-from-signature')
|
|
98
|
-
.map(a => a.address!),
|
|
99
|
-
});
|
|
100
|
-
}
|
|
101
|
-
|
|
102
|
-
// Prune the archive map to only keep at most N entries
|
|
103
|
-
const historyLength = this.store.getHistoryLength();
|
|
104
|
-
if (this.slotNumberToBlock.size > historyLength) {
|
|
105
|
-
const toDelete = Array.from(this.slotNumberToBlock.keys())
|
|
106
|
-
.sort((a, b) => Number(a - b))
|
|
107
|
-
.slice(0, this.slotNumberToBlock.size - historyLength);
|
|
108
|
-
for (const key of toDelete) {
|
|
109
|
-
this.slotNumberToBlock.delete(key);
|
|
110
|
-
}
|
|
111
|
-
}
|
|
112
|
-
} else if (event.type === 'chain-proven') {
|
|
113
|
-
await this.handleChainProven(event);
|
|
220
|
+
if (event.type === 'chain-checkpointed') {
|
|
221
|
+
this.handleCheckpoint(event);
|
|
114
222
|
}
|
|
115
223
|
}
|
|
116
224
|
|
|
117
|
-
protected
|
|
118
|
-
if (event.type !== 'chain-
|
|
225
|
+
protected handleCheckpoint(event: L2BlockStreamEvent) {
|
|
226
|
+
if (event.type !== 'chain-checkpointed') {
|
|
119
227
|
return;
|
|
120
228
|
}
|
|
121
|
-
const
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
229
|
+
const checkpoint = event.checkpoint;
|
|
230
|
+
|
|
231
|
+
// Store mapping from slot to archive, checkpoint number, attestors, and the consensus payload
|
|
232
|
+
// hash (used to query matching p2p attestations regardless of feeAssetPriceModifier variants).
|
|
233
|
+
const signatureContext = this.getSignatureContext();
|
|
234
|
+
const proposalPayloadHash = CheckpointProposalHash.fromBuffer(
|
|
235
|
+
ConsensusPayload.fromCheckpoint(checkpoint.checkpoint, signatureContext).getPayloadHash(),
|
|
236
|
+
);
|
|
237
|
+
this.slotNumberToCheckpoint.set(checkpoint.checkpoint.header.slotNumber, {
|
|
238
|
+
checkpointNumber: checkpoint.checkpoint.number,
|
|
239
|
+
archive: checkpoint.checkpoint.archive.root.toString(),
|
|
240
|
+
proposalPayloadHash,
|
|
241
|
+
attestors: getAttestationInfoFromPublishedCheckpoint(checkpoint, signatureContext)
|
|
242
|
+
.filter(a => a.status === 'recovered-from-signature')
|
|
243
|
+
.map(a => a.address!),
|
|
244
|
+
});
|
|
245
|
+
|
|
246
|
+
// Prune the archive map to only keep at most N entries
|
|
247
|
+
const historyLength = this.store.getHistoryLength();
|
|
248
|
+
if (this.slotNumberToCheckpoint.size > historyLength) {
|
|
249
|
+
const toDelete = Array.from(this.slotNumberToCheckpoint.keys())
|
|
250
|
+
.sort((a, b) => Number(a - b))
|
|
251
|
+
.slice(0, this.slotNumberToCheckpoint.size - historyLength);
|
|
252
|
+
for (const key of toDelete) {
|
|
253
|
+
this.slotNumberToCheckpoint.delete(key);
|
|
254
|
+
}
|
|
126
255
|
}
|
|
256
|
+
}
|
|
127
257
|
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
258
|
+
/**
|
|
259
|
+
* Called once per epoch, after the configured end-of-epoch buffer has elapsed beyond the
|
|
260
|
+
* epoch's last slot. Computes per-epoch performance from the slot-level history collected
|
|
261
|
+
* by `processSlot` and emits any inactivity slash payloads.
|
|
262
|
+
*/
|
|
263
|
+
protected async handleEpochEnd(epoch: EpochNumber) {
|
|
264
|
+
this.logger.debug(`Computing epoch performance for epoch ${epoch}`);
|
|
265
|
+
const performance = await this.computeEpochPerformance(epoch);
|
|
266
|
+
this.logger.info(`Computed epoch performance for epoch ${epoch}`, performance);
|
|
134
267
|
|
|
135
|
-
await this.store.
|
|
136
|
-
await this.
|
|
268
|
+
await this.store.updateEpochPerformance(epoch, performance);
|
|
269
|
+
await this.handleEpochPerformance(epoch, performance);
|
|
137
270
|
}
|
|
138
271
|
|
|
139
|
-
protected async
|
|
272
|
+
protected async computeEpochPerformance(epoch: EpochNumber): Promise<ValidatorsEpochPerformance> {
|
|
140
273
|
const [fromSlot, toSlot] = getSlotRangeForEpoch(epoch, this.epochCache.getL1Constants());
|
|
141
|
-
const { committee } = await this.epochCache.getCommittee(fromSlot);
|
|
274
|
+
const { committee, isEscapeHatchOpen } = await this.epochCache.getCommittee(fromSlot);
|
|
275
|
+
if (isEscapeHatchOpen) {
|
|
276
|
+
this.logger.info(`Skipping epoch performance for epoch ${epoch} - escape hatch is open`);
|
|
277
|
+
return {};
|
|
278
|
+
}
|
|
142
279
|
if (!committee) {
|
|
143
280
|
this.logger.trace(`No committee found for slot ${fromSlot}`);
|
|
144
281
|
return {};
|
|
@@ -178,8 +315,8 @@ export class Sentinel extends (EventEmitter as new () => WatcherEmitter) impleme
|
|
|
178
315
|
return true;
|
|
179
316
|
}
|
|
180
317
|
|
|
181
|
-
// Get all historical performance for this validator
|
|
182
|
-
const allPerformance = await this.store.
|
|
318
|
+
// Get all historical per-epoch performance for this validator
|
|
319
|
+
const allPerformance = await this.store.getEpochPerformance(validator);
|
|
183
320
|
|
|
184
321
|
// Sort by epoch descending to get most recent first, keep only epochs strictly before the current one, and get the first N
|
|
185
322
|
const pastEpochs = allPerformance.sort((a, b) => Number(b.epoch - a.epoch)).filter(p => p.epoch < currentEpoch);
|
|
@@ -195,16 +332,12 @@ export class Sentinel extends (EventEmitter as new () => WatcherEmitter) impleme
|
|
|
195
332
|
// Check that we have at least requiredConsecutiveEpochs and that all of them are above the inactivity threshold
|
|
196
333
|
return pastEpochs
|
|
197
334
|
.slice(0, requiredConsecutiveEpochs)
|
|
198
|
-
.every(p => p.missed / p.total >= this.config.slashInactivityTargetPercentage);
|
|
335
|
+
.every(p => (p.total === 0 ? false : p.missed / p.total >= this.config.slashInactivityTargetPercentage));
|
|
199
336
|
}
|
|
200
337
|
|
|
201
|
-
protected async
|
|
202
|
-
if (this.config.slashInactivityPenalty === 0n) {
|
|
203
|
-
return;
|
|
204
|
-
}
|
|
205
|
-
|
|
338
|
+
protected async handleEpochPerformance(epoch: EpochNumber, performance: ValidatorsEpochPerformance) {
|
|
206
339
|
const inactiveValidators = getEntries(performance)
|
|
207
|
-
.filter(([_, { missed, total }]) => missed / total >= this.config.slashInactivityTargetPercentage)
|
|
340
|
+
.filter(([_, { missed, total }]) => total > 0 && missed / total >= this.config.slashInactivityTargetPercentage)
|
|
208
341
|
.map(([address]) => address);
|
|
209
342
|
|
|
210
343
|
this.logger.debug(`Found ${inactiveValidators.length} inactive validators in epoch ${epoch}`, {
|
|
@@ -226,9 +359,17 @@ export class Sentinel extends (EventEmitter as new () => WatcherEmitter) impleme
|
|
|
226
359
|
}));
|
|
227
360
|
|
|
228
361
|
if (criminals.length > 0) {
|
|
229
|
-
this.logger.
|
|
230
|
-
`Identified ${criminals.length}
|
|
231
|
-
{
|
|
362
|
+
this.logger.info(
|
|
363
|
+
`Identified ${criminals.length} inactivity offenses in at least ${epochThreshold} consecutive epochs`,
|
|
364
|
+
{
|
|
365
|
+
offenses: args.map(arg => ({
|
|
366
|
+
validator: arg.validator.toString(),
|
|
367
|
+
amount: arg.amount,
|
|
368
|
+
offenseType: getOffenseTypeName(arg.offenseType),
|
|
369
|
+
epochOrSlot: arg.epochOrSlot,
|
|
370
|
+
})),
|
|
371
|
+
epochThreshold,
|
|
372
|
+
},
|
|
232
373
|
);
|
|
233
374
|
this.emit(WANT_TO_SLASH_EVENT, args);
|
|
234
375
|
}
|
|
@@ -238,26 +379,72 @@ export class Sentinel extends (EventEmitter as new () => WatcherEmitter) impleme
|
|
|
238
379
|
* Process data for two L2 slots ago.
|
|
239
380
|
* Note that we do not process historical data, since we rely on p2p data for processing,
|
|
240
381
|
* and we don't have that data if we were offline during the period.
|
|
382
|
+
*
|
|
383
|
+
* `currentSlot` is anchored to the archiver's last synced L2 slot rather than the wallclock,
|
|
384
|
+
* so the per-slot lag (`isReadyToProcess`) and the end-of-epoch buffer (`processEpochEnds`)
|
|
385
|
+
* advance with archiver.
|
|
241
386
|
*/
|
|
242
387
|
public async work() {
|
|
243
|
-
const
|
|
388
|
+
const currentSlot = await this.getCurrentSlot();
|
|
244
389
|
try {
|
|
245
390
|
// Manually sync the block stream to ensure we have the latest data.
|
|
246
391
|
// Note we never `start` the blockstream, so it loops at the same pace as we do.
|
|
247
392
|
await this.blockStream.sync();
|
|
248
393
|
|
|
249
|
-
//
|
|
394
|
+
// Per-slot activity recording (lag = 2 slots for P2P attestation settlement).
|
|
250
395
|
const targetSlot = await this.isReadyToProcess(currentSlot);
|
|
251
|
-
|
|
252
|
-
// And process it if we are.
|
|
253
396
|
if (targetSlot !== false) {
|
|
254
397
|
await this.processSlot(targetSlot);
|
|
255
398
|
}
|
|
399
|
+
|
|
400
|
+
// End-of-epoch evaluation (lag = sentinelEpochEndBufferSlots beyond the epoch's last slot).
|
|
401
|
+
await this.processEpochEnds(currentSlot);
|
|
256
402
|
} catch (err) {
|
|
257
403
|
this.logger.error(`Failed to process slot ${currentSlot}`, err);
|
|
258
404
|
}
|
|
259
405
|
}
|
|
260
406
|
|
|
407
|
+
/**
|
|
408
|
+
* After the configured buffer has elapsed past an epoch's last slot, runs the end-of-epoch
|
|
409
|
+
* aggregator for that epoch. Catches up if multiple epochs become eligible at once.
|
|
410
|
+
*/
|
|
411
|
+
protected async processEpochEnds(currentSlot: SlotNumber) {
|
|
412
|
+
const constants = this.epochCache.getL1Constants();
|
|
413
|
+
const buffer = this.config.sentinelEpochEndBufferSlots;
|
|
414
|
+
if (currentSlot < buffer) {
|
|
415
|
+
return;
|
|
416
|
+
}
|
|
417
|
+
if (this.initialSlot === undefined) {
|
|
418
|
+
return;
|
|
419
|
+
}
|
|
420
|
+
|
|
421
|
+
// We can close epoch E iff:
|
|
422
|
+
// - the per-slot recorder has covered the epoch's last slot (lastProcessedSlot ≥ toSlot(E))
|
|
423
|
+
// - the buffer has elapsed past the epoch's last slot (currentSlot − buffer ≥ toSlot(E))
|
|
424
|
+
// - the epoch is not in the past relative to when the sentinel started (toSlot(E) > initialSlot)
|
|
425
|
+
if (this.lastProcessedSlot === undefined) {
|
|
426
|
+
return;
|
|
427
|
+
}
|
|
428
|
+
const slotForBuffer = SlotNumber(currentSlot - buffer);
|
|
429
|
+
|
|
430
|
+
// First eligible epoch to close is the one after lastEvaluatedEpoch, or the epoch containing
|
|
431
|
+
// the initial slot if we haven't evaluated any yet (the initialSlot epoch may be partial — we
|
|
432
|
+
// don't try to evaluate it, we start from initialSlot's epoch + 1).
|
|
433
|
+
const startEpoch =
|
|
434
|
+
this.lastEvaluatedEpoch !== undefined
|
|
435
|
+
? EpochNumber(this.lastEvaluatedEpoch + 1)
|
|
436
|
+
: EpochNumber(getEpochAtSlot(this.initialSlot, constants) + 1);
|
|
437
|
+
|
|
438
|
+
for (let epoch = startEpoch; ; epoch = EpochNumber(epoch + 1)) {
|
|
439
|
+
const [, toSlot] = getSlotRangeForEpoch(epoch, constants);
|
|
440
|
+
if (toSlot > this.lastProcessedSlot || toSlot > slotForBuffer) {
|
|
441
|
+
break;
|
|
442
|
+
}
|
|
443
|
+
await this.handleEpochEnd(epoch);
|
|
444
|
+
this.lastEvaluatedEpoch = epoch;
|
|
445
|
+
}
|
|
446
|
+
}
|
|
447
|
+
|
|
261
448
|
/**
|
|
262
449
|
* Check if we are ready to process data for two L2 slots ago, so we allow plenty of time for p2p to process all in-flight attestations.
|
|
263
450
|
* We also don't move past the archiver last synced L2 slot, as we don't want to process data that is not yet available.
|
|
@@ -285,14 +472,14 @@ export class Sentinel extends (EventEmitter as new () => WatcherEmitter) impleme
|
|
|
285
472
|
return false;
|
|
286
473
|
}
|
|
287
474
|
|
|
288
|
-
const
|
|
289
|
-
if (
|
|
290
|
-
this.logger.debug(`Waiting for archiver to sync with L2 slot ${targetSlot}`, {
|
|
475
|
+
const syncedSlot = await this.archiver.getSyncedL2SlotNumber();
|
|
476
|
+
if (syncedSlot === undefined || syncedSlot < targetSlot) {
|
|
477
|
+
this.logger.debug(`Waiting for archiver to sync with L2 slot ${targetSlot}`, { syncedSlot, targetSlot });
|
|
291
478
|
return false;
|
|
292
479
|
}
|
|
293
480
|
|
|
294
|
-
const archiverLastBlockHash = await this.l2TipsStore.getL2Tips().then(tip => tip.
|
|
295
|
-
const p2pLastBlockHash = await this.p2p.getL2Tips().then(tips => tips.
|
|
481
|
+
const archiverLastBlockHash = await this.l2TipsStore.getL2Tips().then(tip => tip.proposed.hash);
|
|
482
|
+
const p2pLastBlockHash = await this.p2p.getL2Tips().then(tips => tips.proposed.hash);
|
|
296
483
|
const isP2pSynced = archiverLastBlockHash === p2pLastBlockHash;
|
|
297
484
|
if (!isP2pSynced) {
|
|
298
485
|
this.logger.debug(`Waiting for P2P client to sync with archiver`, { archiverLastBlockHash, p2pLastBlockHash });
|
|
@@ -307,7 +494,12 @@ export class Sentinel extends (EventEmitter as new () => WatcherEmitter) impleme
|
|
|
307
494
|
* and updates overall stats.
|
|
308
495
|
*/
|
|
309
496
|
protected async processSlot(slot: SlotNumber) {
|
|
310
|
-
const { epoch, seed, committee } = await this.epochCache.getCommittee(slot);
|
|
497
|
+
const { epoch, seed, committee, isEscapeHatchOpen } = await this.epochCache.getCommittee(slot);
|
|
498
|
+
if (isEscapeHatchOpen) {
|
|
499
|
+
this.logger.info(`Skipping slot ${slot} at epoch ${epoch} - escape hatch is open`);
|
|
500
|
+
this.lastProcessedSlot = slot;
|
|
501
|
+
return;
|
|
502
|
+
}
|
|
311
503
|
if (!committee || committee.length === 0) {
|
|
312
504
|
this.logger.trace(`No committee found for slot ${slot} at epoch ${epoch}`);
|
|
313
505
|
this.lastProcessedSlot = slot;
|
|
@@ -321,46 +513,74 @@ export class Sentinel extends (EventEmitter as new () => WatcherEmitter) impleme
|
|
|
321
513
|
this.lastProcessedSlot = slot;
|
|
322
514
|
}
|
|
323
515
|
|
|
324
|
-
/**
|
|
516
|
+
/**
|
|
517
|
+
* Computes activity for a given slot using the six-case taxonomy.
|
|
518
|
+
*
|
|
519
|
+
* Proposer status:
|
|
520
|
+
* - case 6 `checkpoint-mined` — a checkpoint covering this slot has landed on L1.
|
|
521
|
+
* - case 5 `checkpoint-valid` — the local node re-executed a checkpoint proposal for this
|
|
522
|
+
* slot successfully.
|
|
523
|
+
* - case 4 `checkpoint-invalid` — the local node re-executed a checkpoint proposal for this
|
|
524
|
+
* slot and rejected it.
|
|
525
|
+
* - case 3 `checkpoint-unvalidated` — the local node observed a checkpoint proposal for this
|
|
526
|
+
* slot but could not validate it (missing data, timeouts).
|
|
527
|
+
* - case 2 `checkpoint-missed` — block proposals seen on P2P but no checkpoint proposal.
|
|
528
|
+
* - case 1 `blocks-missed` — no block proposals seen for this slot.
|
|
529
|
+
*
|
|
530
|
+
* Missing-attestor penalties apply only in cases 5 and 6, where the local node has positive
|
|
531
|
+
* evidence the checkpoint was valid or has been canonicalised on L1.
|
|
532
|
+
*/
|
|
325
533
|
protected async getSlotActivity(slot: SlotNumber, epoch: EpochNumber, proposer: EthAddress, committee: EthAddress[]) {
|
|
326
534
|
this.logger.debug(`Computing stats for slot ${slot} at epoch ${epoch}`, { slot, epoch, proposer, committee });
|
|
327
535
|
|
|
328
|
-
//
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
// We gather from both p2p (contains the ones seen on the p2p layer) and archiver
|
|
333
|
-
// (contains the ones synced from mined blocks, which we may have missed from p2p).
|
|
334
|
-
const block = this.slotNumberToBlock.get(slot);
|
|
335
|
-
const p2pAttested = await this.p2p.getAttestationsForSlot(slot, block?.archive);
|
|
336
|
-
// Filter out attestations with invalid signatures
|
|
536
|
+
// Gather attestors from both p2p (live attestations) and the archiver (signers on the
|
|
537
|
+
// checkpoint if one has landed on L1). Used regardless of which case applies.
|
|
538
|
+
const checkpoint = this.slotNumberToCheckpoint.get(slot);
|
|
539
|
+
const p2pAttested = await this.p2p.getCheckpointAttestationsForSlot(slot, checkpoint?.proposalPayloadHash);
|
|
337
540
|
const p2pAttestors = p2pAttested.map(a => a.getSender()).filter((s): s is EthAddress => s !== undefined);
|
|
338
541
|
const attestors = new Set(
|
|
339
|
-
[...p2pAttestors.map(a => a.toString()), ...(
|
|
340
|
-
addr => proposer.toString() !== addr,
|
|
542
|
+
[...p2pAttestors.map(a => a.toString()), ...(checkpoint?.attestors.map(a => a.toString()) ?? [])].filter(
|
|
543
|
+
addr => proposer.toString() !== addr,
|
|
341
544
|
),
|
|
342
545
|
);
|
|
343
546
|
|
|
344
|
-
//
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
547
|
+
// Determine the proposer status from the six-case taxonomy.
|
|
548
|
+
const reexecutionOutcome = this.reexecutionTracker.getOutcomeForSlot(slot);
|
|
549
|
+
let status:
|
|
550
|
+
| 'checkpoint-mined'
|
|
551
|
+
| 'checkpoint-valid'
|
|
552
|
+
| 'checkpoint-invalid'
|
|
553
|
+
| 'checkpoint-unvalidated'
|
|
554
|
+
| 'checkpoint-missed'
|
|
555
|
+
| 'blocks-missed';
|
|
556
|
+
if (checkpoint) {
|
|
557
|
+
status = 'checkpoint-mined';
|
|
558
|
+
} else if (reexecutionOutcome === 'valid') {
|
|
559
|
+
status = 'checkpoint-valid';
|
|
560
|
+
} else if (reexecutionOutcome === 'invalid') {
|
|
561
|
+
status = 'checkpoint-invalid';
|
|
562
|
+
} else if (reexecutionOutcome === 'unvalidated') {
|
|
563
|
+
status = 'checkpoint-unvalidated';
|
|
564
|
+
} else {
|
|
565
|
+
// No L1 checkpoint, no local re-execution outcome for this slot. Distinguish "proposer
|
|
566
|
+
// sent block proposals but never made a checkpoint" from "proposer sent nothing".
|
|
567
|
+
const hasBlockProposals = await this.p2p.hasBlockProposalsForSlot(slot);
|
|
568
|
+
status = hasBlockProposals ? 'checkpoint-missed' : 'blocks-missed';
|
|
569
|
+
}
|
|
570
|
+
this.logger.debug(`Checkpoint status for slot ${slot}: ${status}`, { ...checkpoint, slot });
|
|
352
571
|
|
|
353
|
-
//
|
|
572
|
+
// Missing-attestor faults only apply when we have positive evidence the proposal was valid.
|
|
573
|
+
const attestorsExpected = status === 'checkpoint-mined' || status === 'checkpoint-valid';
|
|
354
574
|
const missedAttestors = new Set(
|
|
355
|
-
|
|
356
|
-
?
|
|
357
|
-
:
|
|
575
|
+
attestorsExpected
|
|
576
|
+
? committee.filter(v => !attestors.has(v.toString()) && !proposer.equals(v)).map(v => v.toString())
|
|
577
|
+
: [],
|
|
358
578
|
);
|
|
359
579
|
|
|
360
580
|
this.logger.debug(`Retrieved ${attestors.size} attestors out of ${committee.length} for slot ${slot}`, {
|
|
361
|
-
|
|
581
|
+
status,
|
|
362
582
|
proposer: proposer.toString(),
|
|
363
|
-
...
|
|
583
|
+
...checkpoint,
|
|
364
584
|
slot,
|
|
365
585
|
attestors: [...attestors],
|
|
366
586
|
missedAttestors: [...missedAttestors],
|
|
@@ -370,7 +590,7 @@ export class Sentinel extends (EventEmitter as new () => WatcherEmitter) impleme
|
|
|
370
590
|
// Compute the status for each validator in the committee
|
|
371
591
|
const statusFor = (who: `0x${string}`): ValidatorStatusInSlot | undefined => {
|
|
372
592
|
if (who === proposer.toString()) {
|
|
373
|
-
return
|
|
593
|
+
return status;
|
|
374
594
|
} else if (attestors.has(who)) {
|
|
375
595
|
return 'attestation-sent';
|
|
376
596
|
} else if (missedAttestors.has(who)) {
|
|
@@ -398,7 +618,7 @@ export class Sentinel extends (EventEmitter as new () => WatcherEmitter) impleme
|
|
|
398
618
|
? fromEntries(await Promise.all(validators.map(async v => [v.toString(), await this.store.getHistory(v)])))
|
|
399
619
|
: await this.store.getHistories();
|
|
400
620
|
|
|
401
|
-
const slotNow = this.
|
|
621
|
+
const slotNow = await this.getCurrentSlot();
|
|
402
622
|
fromSlot ??= SlotNumber(Math.max((this.lastProcessedSlot ?? slotNow) - this.store.getHistoryLength(), 0));
|
|
403
623
|
toSlot ??= this.lastProcessedSlot ?? slotNow;
|
|
404
624
|
|
|
@@ -426,7 +646,7 @@ export class Sentinel extends (EventEmitter as new () => WatcherEmitter) impleme
|
|
|
426
646
|
return undefined;
|
|
427
647
|
}
|
|
428
648
|
|
|
429
|
-
const slotNow = this.
|
|
649
|
+
const slotNow = await this.getCurrentSlot();
|
|
430
650
|
const effectiveFromSlot =
|
|
431
651
|
fromSlot ?? SlotNumber(Math.max((this.lastProcessedSlot ?? slotNow) - this.store.getHistoryLength(), 0));
|
|
432
652
|
const effectiveToSlot = toSlot ?? this.lastProcessedSlot ?? slotNow;
|
|
@@ -448,7 +668,7 @@ export class Sentinel extends (EventEmitter as new () => WatcherEmitter) impleme
|
|
|
448
668
|
|
|
449
669
|
return {
|
|
450
670
|
validator,
|
|
451
|
-
|
|
671
|
+
allTimeEpochPerformance: await this.store.getEpochPerformance(validatorAddress),
|
|
452
672
|
lastProcessedSlot: this.lastProcessedSlot,
|
|
453
673
|
initialSlot: this.initialSlot,
|
|
454
674
|
slotWindow: this.store.getHistoryLength(),
|
|
@@ -463,14 +683,19 @@ export class Sentinel extends (EventEmitter as new () => WatcherEmitter) impleme
|
|
|
463
683
|
): ValidatorStats {
|
|
464
684
|
let history = fromSlot ? allHistory.filter(h => BigInt(h.slot) >= fromSlot) : allHistory;
|
|
465
685
|
history = toSlot ? history.filter(h => BigInt(h.slot) <= toSlot) : history;
|
|
466
|
-
const lastProposal = history.filter(h => h.status === '
|
|
686
|
+
const lastProposal = history.filter(h => h.status === 'checkpoint-valid' || h.status === 'checkpoint-mined').at(-1);
|
|
467
687
|
const lastAttestation = history.filter(h => h.status === 'attestation-sent').at(-1);
|
|
468
688
|
return {
|
|
469
689
|
address: EthAddress.fromString(address),
|
|
470
690
|
lastProposal: this.computeFromSlot(lastProposal?.slot),
|
|
471
691
|
lastAttestation: this.computeFromSlot(lastAttestation?.slot),
|
|
472
692
|
totalSlots: history.length,
|
|
473
|
-
missedProposals: this.computeMissed(history, '
|
|
693
|
+
missedProposals: this.computeMissed(history, 'proposer', [
|
|
694
|
+
'checkpoint-missed',
|
|
695
|
+
'blocks-missed',
|
|
696
|
+
'checkpoint-invalid',
|
|
697
|
+
'checkpoint-unvalidated',
|
|
698
|
+
]),
|
|
474
699
|
missedAttestations: this.computeMissed(history, 'attestation', ['attestation-missed']),
|
|
475
700
|
history,
|
|
476
701
|
};
|
|
@@ -478,10 +703,12 @@ export class Sentinel extends (EventEmitter as new () => WatcherEmitter) impleme
|
|
|
478
703
|
|
|
479
704
|
protected computeMissed(
|
|
480
705
|
history: ValidatorStatusHistory,
|
|
481
|
-
|
|
706
|
+
computeOverCategory: ValidatorStatusType | undefined,
|
|
482
707
|
filter: ValidatorStatusInSlot[],
|
|
483
708
|
) {
|
|
484
|
-
const relevantHistory = history.filter(
|
|
709
|
+
const relevantHistory = history.filter(
|
|
710
|
+
h => !computeOverCategory || statusToCategory(h.status) === computeOverCategory,
|
|
711
|
+
);
|
|
485
712
|
const filteredHistory = relevantHistory.filter(h => filter.includes(h.status));
|
|
486
713
|
return {
|
|
487
714
|
currentStreak: countWhile([...relevantHistory].reverse(), h => filter.includes(h.status)),
|