@aztec/aztec-node 0.0.1-commit.24de95ac → 0.0.1-commit.2606882

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. package/dest/aztec-node/block_response_helpers.d.ts +25 -0
  2. package/dest/aztec-node/block_response_helpers.d.ts.map +1 -0
  3. package/dest/aztec-node/block_response_helpers.js +112 -0
  4. package/dest/aztec-node/config.d.ts +17 -8
  5. package/dest/aztec-node/config.d.ts.map +1 -1
  6. package/dest/aztec-node/config.js +22 -8
  7. package/dest/aztec-node/node_metrics.d.ts +1 -1
  8. package/dest/aztec-node/node_metrics.d.ts.map +1 -1
  9. package/dest/aztec-node/node_metrics.js +9 -16
  10. package/dest/aztec-node/public_data_overrides.d.ts +13 -0
  11. package/dest/aztec-node/public_data_overrides.d.ts.map +1 -0
  12. package/dest/aztec-node/public_data_overrides.js +21 -0
  13. package/dest/aztec-node/server.d.ts +109 -199
  14. package/dest/aztec-node/server.d.ts.map +1 -1
  15. package/dest/aztec-node/server.js +1602 -506
  16. package/dest/bin/index.d.ts +1 -1
  17. package/dest/index.d.ts +1 -1
  18. package/dest/sentinel/config.d.ts +3 -2
  19. package/dest/sentinel/config.d.ts.map +1 -1
  20. package/dest/sentinel/config.js +15 -5
  21. package/dest/sentinel/factory.d.ts +5 -3
  22. package/dest/sentinel/factory.d.ts.map +1 -1
  23. package/dest/sentinel/factory.js +5 -5
  24. package/dest/sentinel/index.d.ts +1 -1
  25. package/dest/sentinel/sentinel.d.ts +151 -25
  26. package/dest/sentinel/sentinel.d.ts.map +1 -1
  27. package/dest/sentinel/sentinel.js +290 -110
  28. package/dest/sentinel/store.d.ts +11 -10
  29. package/dest/sentinel/store.d.ts.map +1 -1
  30. package/dest/sentinel/store.js +36 -23
  31. package/dest/test/index.d.ts +3 -3
  32. package/dest/test/index.d.ts.map +1 -1
  33. package/package.json +31 -28
  34. package/src/aztec-node/block_response_helpers.ts +161 -0
  35. package/src/aztec-node/config.ts +45 -21
  36. package/src/aztec-node/node_metrics.ts +6 -17
  37. package/src/aztec-node/public_data_overrides.ts +35 -0
  38. package/src/aztec-node/server.ts +1385 -640
  39. package/src/sentinel/README.md +103 -0
  40. package/src/sentinel/config.ts +18 -6
  41. package/src/sentinel/factory.ts +9 -11
  42. package/src/sentinel/sentinel.ts +375 -136
  43. package/src/sentinel/store.ts +43 -34
  44. package/src/test/index.ts +2 -2
@@ -1,4 +1,11 @@
1
1
  import type { EpochCache } from '@aztec/epoch-cache';
2
+ import {
3
+ BlockNumber,
4
+ CheckpointNumber,
5
+ CheckpointProposalHash,
6
+ EpochNumber,
7
+ SlotNumber,
8
+ } from '@aztec/foundation/branded-types';
2
9
  import { countWhile, filterAsync, fromEntries, getEntries, mapValues } from '@aztec/foundation/collection';
3
10
  import { EthAddress } from '@aztec/foundation/eth-address';
4
11
  import { createLogger } from '@aztec/foundation/log';
@@ -11,6 +18,7 @@ import {
11
18
  type WantToSlashArgs,
12
19
  type Watcher,
13
20
  type WatcherEmitter,
21
+ getOffenseTypeName,
14
22
  } from '@aztec/slasher';
15
23
  import type { SlasherConfig } from '@aztec/slasher/config';
16
24
  import {
@@ -18,9 +26,12 @@ import {
18
26
  L2BlockStream,
19
27
  type L2BlockStreamEvent,
20
28
  type L2BlockStreamEventHandler,
21
- getAttestationInfoFromPublishedL2Block,
29
+ getAttestationInfoFromPublishedCheckpoint,
22
30
  } from '@aztec/stdlib/block';
31
+ import type { CheckpointReexecutionTracker } from '@aztec/stdlib/checkpoint';
32
+ import type { ChainConfig } from '@aztec/stdlib/config';
23
33
  import { getEpochAtSlot, getSlotRangeForEpoch, getTimestampForSlot } from '@aztec/stdlib/epoch-helpers';
34
+ import { ConsensusPayload, type CoordinationSignatureContext } from '@aztec/stdlib/p2p';
24
35
  import type {
25
36
  SingleValidatorStats,
26
37
  ValidatorStats,
@@ -33,35 +44,139 @@ import type {
33
44
 
34
45
  import EventEmitter from 'node:events';
35
46
 
47
+ import type { SentinelConfig } from './config.js';
36
48
  import { SentinelStore } from './store.js';
37
49
 
50
+ export type SentinelRuntimeConfig = Pick<
51
+ SlasherConfig,
52
+ 'slashInactivityTargetPercentage' | 'slashInactivityPenalty' | 'slashInactivityConsecutiveEpochThreshold'
53
+ > &
54
+ Pick<SentinelConfig, 'sentinelEpochEndBufferSlots'> &
55
+ Pick<ChainConfig, 'l1ChainId' | 'rollupAddress'>;
56
+
57
+ /** Maps a validator status to its category: proposer or attestation. */
58
+ function statusToCategory(status: ValidatorStatusInSlot): ValidatorStatusType {
59
+ switch (status) {
60
+ case 'attestation-sent':
61
+ case 'attestation-missed':
62
+ return 'attestation';
63
+ default:
64
+ return 'proposer';
65
+ }
66
+ }
67
+
68
+ /**
69
+ * The Sentinel observes validator behaviour every L2 slot, classifies it into a per-slot status,
70
+ * aggregates those statuses into per-epoch performance once each epoch is fully observed, and
71
+ * emits inactivity slash payloads when a validator has been inactive for the configured number
72
+ * of consecutive epochs.
73
+ *
74
+ * ## Two cadences
75
+ *
76
+ * The sentinel runs `work()` every quarter L2 slot and drives two independent pipelines:
77
+ *
78
+ * 1. **Per-slot activity recording.** `processSlot(currentSlot - 2)` runs once per slot, with a
79
+ * two-slot lag to let P2P attestations settle and the archiver catch up. It classifies each
80
+ * committee member's behaviour for that slot via `getSlotActivity` and persists the result to
81
+ * `SentinelStore.historyMap` (sliding window of `sentinelHistoryLengthInEpochs * epochDuration`
82
+ * slots, default 24 epochs).
83
+ *
84
+ * 2. **Per-epoch evaluation.** `processEpochEnds(currentSlot)` runs every tick too. Once
85
+ * `sentinelEpochEndBufferSlots` (default 2) has elapsed past an epoch's last slot AND the
86
+ * per-slot recorder has covered that last slot, the sentinel calls `handleEpochEnd(epoch)`.
87
+ * That aggregates the slot-level statuses for the epoch into per-validator `{missed, total}`,
88
+ * persists it to `SentinelStore.epochMap` (default 2000-epoch window), and runs the slashing
89
+ * decision.
90
+ *
91
+ * Triggering per-epoch evaluation off local L2 state — rather than waiting for L1 proof
92
+ * publication — decouples slashing from prover availability.
93
+ *
94
+ * ## Six-case taxonomy in `getSlotActivity`
95
+ *
96
+ * For each slot, the sentinel assigns the proposer one of six statuses, ranked highest-confidence
97
+ * first:
98
+ *
99
+ * - `checkpoint-mined` — a checkpoint covering this slot has landed on L1
100
+ * (`slotNumberToCheckpoint` populated from `chain-checkpointed`).
101
+ * - `checkpoint-valid` — the local node re-executed a checkpoint proposal for this slot
102
+ * successfully (consulted via `CheckpointReexecutionTracker`).
103
+ * - `checkpoint-invalid` — the local node re-executed a checkpoint proposal for this slot
104
+ * and rejected it (e.g. header/archive/out-hash mismatch, limit
105
+ * breach). Proposer-fault.
106
+ * - `checkpoint-unvalidated` — the local node observed a checkpoint proposal but could not
107
+ * validate it (missing blocks/txs, timeouts). Treated as
108
+ * proposer-fault for slashing.
109
+ * - `checkpoint-missed` — block proposals seen on P2P but no checkpoint proposal at all.
110
+ * - `blocks-missed` — no block proposals seen for this slot.
111
+ *
112
+ * Missing-attestor faults are recorded only in `checkpoint-mined` and `checkpoint-valid`, where
113
+ * the local node has positive evidence the checkpoint was canonical or valid. In the other four
114
+ * cases the proposer is at fault and no attestor penalty applies.
115
+ *
116
+ * ## Re-execution tracker
117
+ *
118
+ * `CheckpointReexecutionTracker` is populated by the validator client's checkpoint proposal
119
+ * handler. Every early return in `validateCheckpointProposal` records an outcome
120
+ * (`valid` / `invalid` / `unvalidated`) keyed by slot.
121
+ *
122
+ * ## Inactivity slashing
123
+ *
124
+ * `handleEpochPerformance` filters the epoch's per-validator stats by
125
+ * `slashInactivityTargetPercentage` and then calls `checkPastInactivity` to require
126
+ * `slashInactivityConsecutiveEpochThreshold` consecutive past epochs over the same threshold
127
+ * (read from `SentinelStore.epochMap`). Only validators meeting both conditions are emitted as
128
+ * `WANT_TO_SLASH_EVENT` with `OffenseType.INACTIVITY`. The slot-level counters that feed this —
129
+ * `missedProposals` and `missedAttestations` — include the four proposer-fault statuses plus
130
+ * `attestation-missed`.
131
+ *
132
+ * ## Escape hatch
133
+ *
134
+ * If `epochCache.getCommittee(slot)` reports `isEscapeHatchOpen`, per-slot recording is skipped
135
+ * (no history entries for that slot) and per-epoch evaluation writes an empty performance map
136
+ * (no slashing).
137
+ */
38
138
  export class Sentinel extends (EventEmitter as new () => WatcherEmitter) implements L2BlockStreamEventHandler, Watcher {
39
139
  protected runningPromise: RunningPromise;
40
140
  protected blockStream!: L2BlockStream;
41
141
  protected l2TipsStore: L2TipsStore;
42
142
 
43
- protected initialSlot: bigint | undefined;
44
- protected lastProcessedSlot: bigint | undefined;
45
- protected slotNumberToBlock: Map<bigint, { blockNumber: number; archive: string; attestors: EthAddress[] }> =
46
- new Map();
143
+ protected initialSlot: SlotNumber | undefined;
144
+ protected lastProcessedSlot: SlotNumber | undefined;
145
+ /** Largest epoch number for which the end-of-epoch aggregator has run. */
146
+ protected lastEvaluatedEpoch: EpochNumber | undefined;
147
+ protected slotNumberToCheckpoint: Map<
148
+ SlotNumber,
149
+ {
150
+ checkpointNumber: CheckpointNumber;
151
+ archive: string;
152
+ /** Hex keccak256 of the consensus payload bytes; used to fetch matching p2p attestations. */
153
+ proposalPayloadHash: CheckpointProposalHash;
154
+ attestors: EthAddress[];
155
+ }
156
+ > = new Map();
47
157
 
48
158
  constructor(
49
159
  protected epochCache: EpochCache,
50
160
  protected archiver: L2BlockSource,
51
161
  protected p2p: P2PClient,
52
162
  protected store: SentinelStore,
53
- protected config: Pick<
54
- SlasherConfig,
55
- 'slashInactivityTargetPercentage' | 'slashInactivityPenalty' | 'slashInactivityConsecutiveEpochThreshold'
56
- >,
163
+ protected reexecutionTracker: CheckpointReexecutionTracker,
164
+ protected config: SentinelRuntimeConfig,
57
165
  protected logger = createLogger('node:sentinel'),
58
166
  ) {
59
167
  super();
60
- this.l2TipsStore = new L2TipsMemoryStore();
168
+ this.l2TipsStore = new L2TipsMemoryStore(archiver.getGenesisBlockHash());
61
169
  const interval = (epochCache.getL1Constants().ethereumSlotDuration * 1000) / 4;
62
170
  this.runningPromise = new RunningPromise(this.work.bind(this), logger, interval);
63
171
  }
64
172
 
173
+ private getSignatureContext(): CoordinationSignatureContext {
174
+ return {
175
+ chainId: this.config.l1ChainId,
176
+ rollupAddress: this.config.rollupAddress,
177
+ };
178
+ }
179
+
65
180
  public updateConfig(config: Partial<SlasherConfig>) {
66
181
  this.config = { ...this.config, ...config };
67
182
  }
@@ -71,78 +186,106 @@ export class Sentinel extends (EventEmitter as new () => WatcherEmitter) impleme
71
186
  this.runningPromise.start();
72
187
  }
73
188
 
74
- /** Loads initial slot and initializes blockstream. We will not process anything at or before the initial slot. */
189
+ /**
190
+ * Loads initial slot and initializes blockstream. We will not process anything at or before
191
+ * the initial slot. Floors at the archiver's synced L2 slot so the sentinel keeps making
192
+ * forward progress when L1 is advancing but L2 has no activity (the synced slot is driven by
193
+ * L1 sync, not by L2 blocks). Falls back to the wallclock if the archiver isn't ready yet
194
+ * (cold start).
195
+ */
75
196
  protected async init() {
76
- this.initialSlot = this.epochCache.getEpochAndSlotNow().slot;
77
- const startingBlock = await this.archiver.getBlockNumber();
197
+ this.initialSlot = await this.getCurrentSlot();
198
+ const startingBlock = BlockNumber(await this.archiver.getBlockNumber());
78
199
  this.logger.info(`Starting validator sentinel with initial slot ${this.initialSlot} and block ${startingBlock}`);
79
200
  this.blockStream = new L2BlockStream(this.archiver, this.l2TipsStore, this, this.logger, { startingBlock });
80
201
  }
81
202
 
203
+ /**
204
+ * Returns the L2 slot the sentinel should treat as "current": the archiver's last fully
205
+ * synced L2 slot, falling back to the wallclock slot when the archiver isn't ready yet
206
+ * (cold start). Anchoring to the synced slot keeps timing arithmetic (initial floor,
207
+ * per-slot lag, end-of-epoch buffer, stats-range fallback) from speculating ahead of where
208
+ * L1 actually is.
209
+ */
210
+ protected async getCurrentSlot(): Promise<SlotNumber> {
211
+ return (await this.archiver.getSyncedL2SlotNumber()) ?? this.epochCache.getSlotNow();
212
+ }
213
+
82
214
  public stop() {
83
215
  return this.runningPromise.stop();
84
216
  }
85
217
 
86
218
  public async handleBlockStreamEvent(event: L2BlockStreamEvent): Promise<void> {
87
219
  await this.l2TipsStore.handleBlockStreamEvent(event);
88
- if (event.type === 'blocks-added') {
89
- // Store mapping from slot to archive, block number, and attestors
90
- for (const block of event.blocks) {
91
- this.slotNumberToBlock.set(block.block.header.getSlot(), {
92
- blockNumber: block.block.number,
93
- archive: block.block.archive.root.toString(),
94
- attestors: getAttestationInfoFromPublishedL2Block(block)
95
- .filter(a => a.status === 'recovered-from-signature')
96
- .map(a => a.address!),
97
- });
98
- }
99
-
100
- // Prune the archive map to only keep at most N entries
101
- const historyLength = this.store.getHistoryLength();
102
- if (this.slotNumberToBlock.size > historyLength) {
103
- const toDelete = Array.from(this.slotNumberToBlock.keys())
104
- .sort((a, b) => Number(a - b))
105
- .slice(0, this.slotNumberToBlock.size - historyLength);
106
- for (const key of toDelete) {
107
- this.slotNumberToBlock.delete(key);
108
- }
109
- }
110
- } else if (event.type === 'chain-proven') {
111
- await this.handleChainProven(event);
220
+ if (event.type === 'chain-checkpointed') {
221
+ this.handleCheckpoint(event);
112
222
  }
113
223
  }
114
224
 
115
- protected async handleChainProven(event: L2BlockStreamEvent) {
116
- if (event.type !== 'chain-proven') {
225
+ protected handleCheckpoint(event: L2BlockStreamEvent) {
226
+ if (event.type !== 'chain-checkpointed') {
117
227
  return;
118
228
  }
119
- const blockNumber = event.block.number;
120
- const block = await this.archiver.getBlock(blockNumber);
121
- if (!block) {
122
- this.logger.error(`Failed to get block ${blockNumber}`, { block });
123
- return;
229
+ const checkpoint = event.checkpoint;
230
+
231
+ // Store mapping from slot to archive, checkpoint number, attestors, and the consensus payload
232
+ // hash (used to query matching p2p attestations regardless of feeAssetPriceModifier variants).
233
+ const signatureContext = this.getSignatureContext();
234
+ const proposalPayloadHash = CheckpointProposalHash.fromBuffer(
235
+ ConsensusPayload.fromCheckpoint(checkpoint.checkpoint, signatureContext).getPayloadHash(),
236
+ );
237
+ this.slotNumberToCheckpoint.set(checkpoint.checkpoint.header.slotNumber, {
238
+ checkpointNumber: checkpoint.checkpoint.number,
239
+ archive: checkpoint.checkpoint.archive.root.toString(),
240
+ proposalPayloadHash,
241
+ attestors: getAttestationInfoFromPublishedCheckpoint(checkpoint, signatureContext)
242
+ .filter(a => a.status === 'recovered-from-signature')
243
+ .map(a => a.address!),
244
+ });
245
+
246
+ // Prune the archive map to only keep at most N entries
247
+ const historyLength = this.store.getHistoryLength();
248
+ if (this.slotNumberToCheckpoint.size > historyLength) {
249
+ const toDelete = Array.from(this.slotNumberToCheckpoint.keys())
250
+ .sort((a, b) => Number(a - b))
251
+ .slice(0, this.slotNumberToCheckpoint.size - historyLength);
252
+ for (const key of toDelete) {
253
+ this.slotNumberToCheckpoint.delete(key);
254
+ }
124
255
  }
256
+ }
125
257
 
126
- // TODO(palla/slash): We should only be computing proven performance if this is
127
- // a full proof epoch and not a partial one, otherwise we'll end up with skewed stats.
128
- const epoch = getEpochAtSlot(block.header.getSlot(), this.epochCache.getL1Constants());
129
- this.logger.debug(`Computing proven performance for epoch ${epoch}`);
130
- const performance = await this.computeProvenPerformance(epoch);
131
- this.logger.info(`Computed proven performance for epoch ${epoch}`, performance);
258
+ /**
259
+ * Called once per epoch, after the configured end-of-epoch buffer has elapsed beyond the
260
+ * epoch's last slot. Computes per-epoch performance from the slot-level history collected
261
+ * by `processSlot` and emits any inactivity slash payloads.
262
+ */
263
+ protected async handleEpochEnd(epoch: EpochNumber) {
264
+ this.logger.debug(`Computing epoch performance for epoch ${epoch}`);
265
+ const performance = await this.computeEpochPerformance(epoch);
266
+ this.logger.info(`Computed epoch performance for epoch ${epoch}`, performance);
132
267
 
133
- await this.store.updateProvenPerformance(epoch, performance);
134
- await this.handleProvenPerformance(epoch, performance);
268
+ await this.store.updateEpochPerformance(epoch, performance);
269
+ await this.handleEpochPerformance(epoch, performance);
135
270
  }
136
271
 
137
- protected async computeProvenPerformance(epoch: bigint): Promise<ValidatorsEpochPerformance> {
272
+ protected async computeEpochPerformance(epoch: EpochNumber): Promise<ValidatorsEpochPerformance> {
138
273
  const [fromSlot, toSlot] = getSlotRangeForEpoch(epoch, this.epochCache.getL1Constants());
139
- const { committee } = await this.epochCache.getCommittee(fromSlot);
274
+ const { committee, isEscapeHatchOpen } = await this.epochCache.getCommittee(fromSlot);
275
+ if (isEscapeHatchOpen) {
276
+ this.logger.info(`Skipping epoch performance for epoch ${epoch} - escape hatch is open`);
277
+ return {};
278
+ }
140
279
  if (!committee) {
141
280
  this.logger.trace(`No committee found for slot ${fromSlot}`);
142
281
  return {};
143
282
  }
144
283
 
145
- const stats = await this.computeStats({ fromSlot, toSlot, validators: committee });
284
+ const stats = await this.computeStats({
285
+ fromSlot,
286
+ toSlot,
287
+ validators: committee,
288
+ });
146
289
  this.logger.debug(`Stats for epoch ${epoch}`, { ...stats, fromSlot, toSlot, epoch });
147
290
 
148
291
  // Note that we are NOT using the total slots in the epoch as `total` here, since we only
@@ -165,39 +308,36 @@ export class Sentinel extends (EventEmitter as new () => WatcherEmitter) impleme
165
308
  */
166
309
  protected async checkPastInactivity(
167
310
  validator: EthAddress,
168
- currentEpoch: bigint,
311
+ currentEpoch: EpochNumber,
169
312
  requiredConsecutiveEpochs: number,
170
313
  ): Promise<boolean> {
171
314
  if (requiredConsecutiveEpochs === 0) {
172
315
  return true;
173
316
  }
174
317
 
175
- // Get all historical performance for this validator
176
- const allPerformance = await this.store.getProvenPerformance(validator);
318
+ // Get all historical per-epoch performance for this validator
319
+ const allPerformance = await this.store.getEpochPerformance(validator);
320
+
321
+ // Sort by epoch descending to get most recent first, keep only epochs strictly before the current one, and get the first N
322
+ const pastEpochs = allPerformance.sort((a, b) => Number(b.epoch - a.epoch)).filter(p => p.epoch < currentEpoch);
177
323
 
178
324
  // If we don't have enough historical data, don't slash
179
- if (allPerformance.length < requiredConsecutiveEpochs) {
325
+ if (pastEpochs.length < requiredConsecutiveEpochs) {
180
326
  this.logger.debug(
181
327
  `Not enough historical data for slashing ${validator} for inactivity (${allPerformance.length} epochs < ${requiredConsecutiveEpochs} required)`,
182
328
  );
183
329
  return false;
184
330
  }
185
331
 
186
- // Sort by epoch descending to get most recent first, keep only epochs strictly before the current one, and get the first N
187
- return allPerformance
188
- .sort((a, b) => Number(b.epoch - a.epoch))
189
- .filter(p => p.epoch < currentEpoch)
332
+ // Check that we have at least requiredConsecutiveEpochs and that all of them are above the inactivity threshold
333
+ return pastEpochs
190
334
  .slice(0, requiredConsecutiveEpochs)
191
- .every(p => p.missed / p.total >= this.config.slashInactivityTargetPercentage);
335
+ .every(p => (p.total === 0 ? false : p.missed / p.total >= this.config.slashInactivityTargetPercentage));
192
336
  }
193
337
 
194
- protected async handleProvenPerformance(epoch: bigint, performance: ValidatorsEpochPerformance) {
195
- if (this.config.slashInactivityPenalty === 0n) {
196
- return;
197
- }
198
-
338
+ protected async handleEpochPerformance(epoch: EpochNumber, performance: ValidatorsEpochPerformance) {
199
339
  const inactiveValidators = getEntries(performance)
200
- .filter(([_, { missed, total }]) => missed / total >= this.config.slashInactivityTargetPercentage)
340
+ .filter(([_, { missed, total }]) => total > 0 && missed / total >= this.config.slashInactivityTargetPercentage)
201
341
  .map(([address]) => address);
202
342
 
203
343
  this.logger.debug(`Found ${inactiveValidators.length} inactive validators in epoch ${epoch}`, {
@@ -215,13 +355,21 @@ export class Sentinel extends (EventEmitter as new () => WatcherEmitter) impleme
215
355
  validator: EthAddress.fromString(address),
216
356
  amount: this.config.slashInactivityPenalty,
217
357
  offenseType: OffenseType.INACTIVITY,
218
- epochOrSlot: epoch,
358
+ epochOrSlot: BigInt(epoch),
219
359
  }));
220
360
 
221
361
  if (criminals.length > 0) {
222
- this.logger.verbose(
223
- `Identified ${criminals.length} validators to slash due to inactivity in at least ${epochThreshold} consecutive epochs`,
224
- { ...args, epochThreshold },
362
+ this.logger.info(
363
+ `Identified ${criminals.length} inactivity offenses in at least ${epochThreshold} consecutive epochs`,
364
+ {
365
+ offenses: args.map(arg => ({
366
+ validator: arg.validator.toString(),
367
+ amount: arg.amount,
368
+ offenseType: getOffenseTypeName(arg.offenseType),
369
+ epochOrSlot: arg.epochOrSlot,
370
+ })),
371
+ epochThreshold,
372
+ },
225
373
  );
226
374
  this.emit(WANT_TO_SLASH_EVENT, args);
227
375
  }
@@ -231,33 +379,84 @@ export class Sentinel extends (EventEmitter as new () => WatcherEmitter) impleme
231
379
  * Process data for two L2 slots ago.
232
380
  * Note that we do not process historical data, since we rely on p2p data for processing,
233
381
  * and we don't have that data if we were offline during the period.
382
+ *
383
+ * `currentSlot` is anchored to the archiver's last synced L2 slot rather than the wallclock,
384
+ * so the per-slot lag (`isReadyToProcess`) and the end-of-epoch buffer (`processEpochEnds`)
385
+ * advance with archiver.
234
386
  */
235
387
  public async work() {
236
- const { slot: currentSlot } = this.epochCache.getEpochAndSlotNow();
388
+ const currentSlot = await this.getCurrentSlot();
237
389
  try {
238
390
  // Manually sync the block stream to ensure we have the latest data.
239
391
  // Note we never `start` the blockstream, so it loops at the same pace as we do.
240
392
  await this.blockStream.sync();
241
393
 
242
- // Check if we are ready to process data for two L2 slots ago.
394
+ // Per-slot activity recording (lag = 2 slots for P2P attestation settlement).
243
395
  const targetSlot = await this.isReadyToProcess(currentSlot);
244
-
245
- // And process it if we are.
246
396
  if (targetSlot !== false) {
247
397
  await this.processSlot(targetSlot);
248
398
  }
399
+
400
+ // End-of-epoch evaluation (lag = sentinelEpochEndBufferSlots beyond the epoch's last slot).
401
+ await this.processEpochEnds(currentSlot);
249
402
  } catch (err) {
250
403
  this.logger.error(`Failed to process slot ${currentSlot}`, err);
251
404
  }
252
405
  }
253
406
 
407
+ /**
408
+ * After the configured buffer has elapsed past an epoch's last slot, runs the end-of-epoch
409
+ * aggregator for that epoch. Catches up if multiple epochs become eligible at once.
410
+ */
411
+ protected async processEpochEnds(currentSlot: SlotNumber) {
412
+ const constants = this.epochCache.getL1Constants();
413
+ const buffer = this.config.sentinelEpochEndBufferSlots;
414
+ if (currentSlot < buffer) {
415
+ return;
416
+ }
417
+ if (this.initialSlot === undefined) {
418
+ return;
419
+ }
420
+
421
+ // We can close epoch E iff:
422
+ // - the per-slot recorder has covered the epoch's last slot (lastProcessedSlot ≥ toSlot(E))
423
+ // - the buffer has elapsed past the epoch's last slot (currentSlot − buffer ≥ toSlot(E))
424
+ // - the epoch is not in the past relative to when the sentinel started (toSlot(E) > initialSlot)
425
+ if (this.lastProcessedSlot === undefined) {
426
+ return;
427
+ }
428
+ const slotForBuffer = SlotNumber(currentSlot - buffer);
429
+
430
+ // First eligible epoch to close is the one after lastEvaluatedEpoch, or the epoch containing
431
+ // the initial slot if we haven't evaluated any yet (the initialSlot epoch may be partial — we
432
+ // don't try to evaluate it, we start from initialSlot's epoch + 1).
433
+ const startEpoch =
434
+ this.lastEvaluatedEpoch !== undefined
435
+ ? EpochNumber(this.lastEvaluatedEpoch + 1)
436
+ : EpochNumber(getEpochAtSlot(this.initialSlot, constants) + 1);
437
+
438
+ for (let epoch = startEpoch; ; epoch = EpochNumber(epoch + 1)) {
439
+ const [, toSlot] = getSlotRangeForEpoch(epoch, constants);
440
+ if (toSlot > this.lastProcessedSlot || toSlot > slotForBuffer) {
441
+ break;
442
+ }
443
+ await this.handleEpochEnd(epoch);
444
+ this.lastEvaluatedEpoch = epoch;
445
+ }
446
+ }
447
+
254
448
  /**
255
449
  * Check if we are ready to process data for two L2 slots ago, so we allow plenty of time for p2p to process all in-flight attestations.
256
450
  * We also don't move past the archiver last synced L2 slot, as we don't want to process data that is not yet available.
257
451
  * Last, we check the p2p is synced with the archiver, so it has pulled all attestations from it.
258
452
  */
259
- protected async isReadyToProcess(currentSlot: bigint) {
260
- const targetSlot = currentSlot - 2n;
453
+ protected async isReadyToProcess(currentSlot: SlotNumber): Promise<SlotNumber | false> {
454
+ if (currentSlot < 2) {
455
+ this.logger.trace(`Current slot ${currentSlot} too early.`);
456
+ return false;
457
+ }
458
+
459
+ const targetSlot = SlotNumber(currentSlot - 2);
261
460
  if (this.lastProcessedSlot && this.lastProcessedSlot >= targetSlot) {
262
461
  this.logger.trace(`Already processed slot ${targetSlot}`, { lastProcessedSlot: this.lastProcessedSlot });
263
462
  return false;
@@ -273,14 +472,14 @@ export class Sentinel extends (EventEmitter as new () => WatcherEmitter) impleme
273
472
  return false;
274
473
  }
275
474
 
276
- const archiverSlot = await this.archiver.getL2SlotNumber();
277
- if (archiverSlot === undefined || archiverSlot < targetSlot) {
278
- this.logger.debug(`Waiting for archiver to sync with L2 slot ${targetSlot}`, { archiverSlot, targetSlot });
475
+ const syncedSlot = await this.archiver.getSyncedL2SlotNumber();
476
+ if (syncedSlot === undefined || syncedSlot < targetSlot) {
477
+ this.logger.debug(`Waiting for archiver to sync with L2 slot ${targetSlot}`, { syncedSlot, targetSlot });
279
478
  return false;
280
479
  }
281
480
 
282
- const archiverLastBlockHash = await this.l2TipsStore.getL2Tips().then(tip => tip.latest.hash);
283
- const p2pLastBlockHash = await this.p2p.getL2Tips().then(tips => tips.latest.hash);
481
+ const archiverLastBlockHash = await this.l2TipsStore.getL2Tips().then(tip => tip.proposed.hash);
482
+ const p2pLastBlockHash = await this.p2p.getL2Tips().then(tips => tips.proposed.hash);
284
483
  const isP2pSynced = archiverLastBlockHash === p2pLastBlockHash;
285
484
  if (!isP2pSynced) {
286
485
  this.logger.debug(`Waiting for P2P client to sync with archiver`, { archiverLastBlockHash, p2pLastBlockHash });
@@ -294,8 +493,13 @@ export class Sentinel extends (EventEmitter as new () => WatcherEmitter) impleme
294
493
  * Gathers committee and proposer data for a given slot, computes slot stats,
295
494
  * and updates overall stats.
296
495
  */
297
- protected async processSlot(slot: bigint) {
298
- const { epoch, seed, committee } = await this.epochCache.getCommittee(slot);
496
+ protected async processSlot(slot: SlotNumber) {
497
+ const { epoch, seed, committee, isEscapeHatchOpen } = await this.epochCache.getCommittee(slot);
498
+ if (isEscapeHatchOpen) {
499
+ this.logger.info(`Skipping slot ${slot} at epoch ${epoch} - escape hatch is open`);
500
+ this.lastProcessedSlot = slot;
501
+ return;
502
+ }
299
503
  if (!committee || committee.length === 0) {
300
504
  this.logger.trace(`No committee found for slot ${slot} at epoch ${epoch}`);
301
505
  this.lastProcessedSlot = slot;
@@ -309,46 +513,74 @@ export class Sentinel extends (EventEmitter as new () => WatcherEmitter) impleme
309
513
  this.lastProcessedSlot = slot;
310
514
  }
311
515
 
312
- /** Computes activity for a given slot. */
313
- protected async getSlotActivity(slot: bigint, epoch: bigint, proposer: EthAddress, committee: EthAddress[]) {
516
+ /**
517
+ * Computes activity for a given slot using the six-case taxonomy.
518
+ *
519
+ * Proposer status:
520
+ * - case 6 `checkpoint-mined` — a checkpoint covering this slot has landed on L1.
521
+ * - case 5 `checkpoint-valid` — the local node re-executed a checkpoint proposal for this
522
+ * slot successfully.
523
+ * - case 4 `checkpoint-invalid` — the local node re-executed a checkpoint proposal for this
524
+ * slot and rejected it.
525
+ * - case 3 `checkpoint-unvalidated` — the local node observed a checkpoint proposal for this
526
+ * slot but could not validate it (missing data, timeouts).
527
+ * - case 2 `checkpoint-missed` — block proposals seen on P2P but no checkpoint proposal.
528
+ * - case 1 `blocks-missed` — no block proposals seen for this slot.
529
+ *
530
+ * Missing-attestor penalties apply only in cases 5 and 6, where the local node has positive
531
+ * evidence the checkpoint was valid or has been canonicalised on L1.
532
+ */
533
+ protected async getSlotActivity(slot: SlotNumber, epoch: EpochNumber, proposer: EthAddress, committee: EthAddress[]) {
314
534
  this.logger.debug(`Computing stats for slot ${slot} at epoch ${epoch}`, { slot, epoch, proposer, committee });
315
535
 
316
- // Check if there is an L2 block in L1 for this L2 slot
317
-
318
- // Here we get all attestations for the block mined at the given slot,
319
- // or all attestations for all proposals in the slot if no block was mined.
320
- // We gather from both p2p (contains the ones seen on the p2p layer) and archiver
321
- // (contains the ones synced from mined blocks, which we may have missed from p2p).
322
- const block = this.slotNumberToBlock.get(slot);
323
- const p2pAttested = await this.p2p.getAttestationsForSlot(slot, block?.archive);
324
- // Filter out attestations with invalid signatures
536
+ // Gather attestors from both p2p (live attestations) and the archiver (signers on the
537
+ // checkpoint if one has landed on L1). Used regardless of which case applies.
538
+ const checkpoint = this.slotNumberToCheckpoint.get(slot);
539
+ const p2pAttested = await this.p2p.getCheckpointAttestationsForSlot(slot, checkpoint?.proposalPayloadHash);
325
540
  const p2pAttestors = p2pAttested.map(a => a.getSender()).filter((s): s is EthAddress => s !== undefined);
326
541
  const attestors = new Set(
327
- [...p2pAttestors.map(a => a.toString()), ...(block?.attestors.map(a => a.toString()) ?? [])].filter(
328
- addr => proposer.toString() !== addr, // Exclude the proposer from the attestors
542
+ [...p2pAttestors.map(a => a.toString()), ...(checkpoint?.attestors.map(a => a.toString()) ?? [])].filter(
543
+ addr => proposer.toString() !== addr,
329
544
  ),
330
545
  );
331
546
 
332
- // We assume that there was a block proposal if at least one of the validators (other than the proposer) attested to it.
333
- // It could be the case that every single validator failed, and we could differentiate it by having
334
- // this node re-execute every block proposal it sees and storing it in the attestation pool.
335
- // But we'll leave that corner case out to reduce pressure on the node.
336
- // TODO(palla/slash): This breaks if a given node has more than one validator in the current committee,
337
- // since they will attest to their own proposal it even if it's not re-executable.
338
- const blockStatus = block ? 'mined' : attestors.size > 0 ? 'proposed' : 'missed';
339
- this.logger.debug(`Block for slot ${slot} was ${blockStatus}`, { ...block, slot });
547
+ // Determine the proposer status from the six-case taxonomy.
548
+ const reexecutionOutcome = this.reexecutionTracker.getOutcomeForSlot(slot);
549
+ let status:
550
+ | 'checkpoint-mined'
551
+ | 'checkpoint-valid'
552
+ | 'checkpoint-invalid'
553
+ | 'checkpoint-unvalidated'
554
+ | 'checkpoint-missed'
555
+ | 'blocks-missed';
556
+ if (checkpoint) {
557
+ status = 'checkpoint-mined';
558
+ } else if (reexecutionOutcome === 'valid') {
559
+ status = 'checkpoint-valid';
560
+ } else if (reexecutionOutcome === 'invalid') {
561
+ status = 'checkpoint-invalid';
562
+ } else if (reexecutionOutcome === 'unvalidated') {
563
+ status = 'checkpoint-unvalidated';
564
+ } else {
565
+ // No L1 checkpoint, no local re-execution outcome for this slot. Distinguish "proposer
566
+ // sent block proposals but never made a checkpoint" from "proposer sent nothing".
567
+ const hasBlockProposals = await this.p2p.hasBlockProposalsForSlot(slot);
568
+ status = hasBlockProposals ? 'checkpoint-missed' : 'blocks-missed';
569
+ }
570
+ this.logger.debug(`Checkpoint status for slot ${slot}: ${status}`, { ...checkpoint, slot });
340
571
 
341
- // Get attestors that failed their duties for this block, but only if there was a block proposed
572
+ // Missing-attestor faults only apply when we have positive evidence the proposal was valid.
573
+ const attestorsExpected = status === 'checkpoint-mined' || status === 'checkpoint-valid';
342
574
  const missedAttestors = new Set(
343
- blockStatus === 'missed'
344
- ? []
345
- : committee.filter(v => !attestors.has(v.toString()) && !proposer.equals(v)).map(v => v.toString()),
575
+ attestorsExpected
576
+ ? committee.filter(v => !attestors.has(v.toString()) && !proposer.equals(v)).map(v => v.toString())
577
+ : [],
346
578
  );
347
579
 
348
580
  this.logger.debug(`Retrieved ${attestors.size} attestors out of ${committee.length} for slot ${slot}`, {
349
- blockStatus,
581
+ status,
350
582
  proposer: proposer.toString(),
351
- ...block,
583
+ ...checkpoint,
352
584
  slot,
353
585
  attestors: [...attestors],
354
586
  missedAttestors: [...missedAttestors],
@@ -358,7 +590,7 @@ export class Sentinel extends (EventEmitter as new () => WatcherEmitter) impleme
358
590
  // Compute the status for each validator in the committee
359
591
  const statusFor = (who: `0x${string}`): ValidatorStatusInSlot | undefined => {
360
592
  if (who === proposer.toString()) {
361
- return `block-${blockStatus}`;
593
+ return status;
362
594
  } else if (attestors.has(who)) {
363
595
  return 'attestation-sent';
364
596
  } else if (missedAttestors.has(who)) {
@@ -372,7 +604,7 @@ export class Sentinel extends (EventEmitter as new () => WatcherEmitter) impleme
372
604
  }
373
605
 
374
606
  /** Push the status for each slot for each validator. */
375
- protected updateValidators(slot: bigint, stats: Record<`0x${string}`, ValidatorStatusInSlot | undefined>) {
607
+ protected updateValidators(slot: SlotNumber, stats: Record<`0x${string}`, ValidatorStatusInSlot | undefined>) {
376
608
  return this.store.updateValidators(slot, stats);
377
609
  }
378
610
 
@@ -381,13 +613,13 @@ export class Sentinel extends (EventEmitter as new () => WatcherEmitter) impleme
381
613
  fromSlot,
382
614
  toSlot,
383
615
  validators,
384
- }: { fromSlot?: bigint; toSlot?: bigint; validators?: EthAddress[] } = {}): Promise<ValidatorsStats> {
616
+ }: { fromSlot?: SlotNumber; toSlot?: SlotNumber; validators?: EthAddress[] } = {}): Promise<ValidatorsStats> {
385
617
  const histories = validators
386
618
  ? fromEntries(await Promise.all(validators.map(async v => [v.toString(), await this.store.getHistory(v)])))
387
619
  : await this.store.getHistories();
388
620
 
389
- const slotNow = this.epochCache.getEpochAndSlotNow().slot;
390
- fromSlot ??= (this.lastProcessedSlot ?? slotNow) - BigInt(this.store.getHistoryLength());
621
+ const slotNow = await this.getCurrentSlot();
622
+ fromSlot ??= SlotNumber(Math.max((this.lastProcessedSlot ?? slotNow) - this.store.getHistoryLength(), 0));
391
623
  toSlot ??= this.lastProcessedSlot ?? slotNow;
392
624
 
393
625
  const stats = mapValues(histories, (history, address) =>
@@ -405,8 +637,8 @@ export class Sentinel extends (EventEmitter as new () => WatcherEmitter) impleme
405
637
  /** Computes stats for a single validator. */
406
638
  public async getValidatorStats(
407
639
  validatorAddress: EthAddress,
408
- fromSlot?: bigint,
409
- toSlot?: bigint,
640
+ fromSlot?: SlotNumber,
641
+ toSlot?: SlotNumber,
410
642
  ): Promise<SingleValidatorStats | undefined> {
411
643
  const history = await this.store.getHistory(validatorAddress);
412
644
 
@@ -414,14 +646,15 @@ export class Sentinel extends (EventEmitter as new () => WatcherEmitter) impleme
414
646
  return undefined;
415
647
  }
416
648
 
417
- const slotNow = this.epochCache.getEpochAndSlotNow().slot;
418
- const effectiveFromSlot = fromSlot ?? (this.lastProcessedSlot ?? slotNow) - BigInt(this.store.getHistoryLength());
649
+ const slotNow = await this.getCurrentSlot();
650
+ const effectiveFromSlot =
651
+ fromSlot ?? SlotNumber(Math.max((this.lastProcessedSlot ?? slotNow) - this.store.getHistoryLength(), 0));
419
652
  const effectiveToSlot = toSlot ?? this.lastProcessedSlot ?? slotNow;
420
653
 
421
654
  const historyLength = BigInt(this.store.getHistoryLength());
422
- if (effectiveToSlot - effectiveFromSlot > historyLength) {
655
+ if (BigInt(effectiveToSlot) - BigInt(effectiveFromSlot) > historyLength) {
423
656
  throw new Error(
424
- `Slot range (${effectiveToSlot - effectiveFromSlot}) exceeds history length (${historyLength}). ` +
657
+ `Slot range (${BigInt(effectiveToSlot) - BigInt(effectiveFromSlot)}) exceeds history length (${historyLength}). ` +
425
658
  `Requested range: ${effectiveFromSlot} to ${effectiveToSlot}.`,
426
659
  );
427
660
  }
@@ -432,11 +665,10 @@ export class Sentinel extends (EventEmitter as new () => WatcherEmitter) impleme
432
665
  effectiveFromSlot,
433
666
  effectiveToSlot,
434
667
  );
435
- const allTimeProvenPerformance = await this.store.getProvenPerformance(validatorAddress);
436
668
 
437
669
  return {
438
670
  validator,
439
- allTimeProvenPerformance,
671
+ allTimeEpochPerformance: await this.store.getEpochPerformance(validatorAddress),
440
672
  lastProcessedSlot: this.lastProcessedSlot,
441
673
  initialSlot: this.initialSlot,
442
674
  slotWindow: this.store.getHistoryLength(),
@@ -446,19 +678,24 @@ export class Sentinel extends (EventEmitter as new () => WatcherEmitter) impleme
446
678
  protected computeStatsForValidator(
447
679
  address: `0x${string}`,
448
680
  allHistory: ValidatorStatusHistory,
449
- fromSlot?: bigint,
450
- toSlot?: bigint,
681
+ fromSlot?: SlotNumber,
682
+ toSlot?: SlotNumber,
451
683
  ): ValidatorStats {
452
- let history = fromSlot ? allHistory.filter(h => h.slot >= fromSlot) : allHistory;
453
- history = toSlot ? history.filter(h => h.slot <= toSlot) : history;
454
- const lastProposal = history.filter(h => h.status === 'block-proposed' || h.status === 'block-mined').at(-1);
684
+ let history = fromSlot ? allHistory.filter(h => BigInt(h.slot) >= fromSlot) : allHistory;
685
+ history = toSlot ? history.filter(h => BigInt(h.slot) <= toSlot) : history;
686
+ const lastProposal = history.filter(h => h.status === 'checkpoint-valid' || h.status === 'checkpoint-mined').at(-1);
455
687
  const lastAttestation = history.filter(h => h.status === 'attestation-sent').at(-1);
456
688
  return {
457
689
  address: EthAddress.fromString(address),
458
690
  lastProposal: this.computeFromSlot(lastProposal?.slot),
459
691
  lastAttestation: this.computeFromSlot(lastAttestation?.slot),
460
692
  totalSlots: history.length,
461
- missedProposals: this.computeMissed(history, 'block', ['block-missed']),
693
+ missedProposals: this.computeMissed(history, 'proposer', [
694
+ 'checkpoint-missed',
695
+ 'blocks-missed',
696
+ 'checkpoint-invalid',
697
+ 'checkpoint-unvalidated',
698
+ ]),
462
699
  missedAttestations: this.computeMissed(history, 'attestation', ['attestation-missed']),
463
700
  history,
464
701
  };
@@ -466,10 +703,12 @@ export class Sentinel extends (EventEmitter as new () => WatcherEmitter) impleme
466
703
 
467
704
  protected computeMissed(
468
705
  history: ValidatorStatusHistory,
469
- computeOverPrefix: ValidatorStatusType | undefined,
706
+ computeOverCategory: ValidatorStatusType | undefined,
470
707
  filter: ValidatorStatusInSlot[],
471
708
  ) {
472
- const relevantHistory = history.filter(h => !computeOverPrefix || h.status.startsWith(computeOverPrefix));
709
+ const relevantHistory = history.filter(
710
+ h => !computeOverCategory || statusToCategory(h.status) === computeOverCategory,
711
+ );
473
712
  const filteredHistory = relevantHistory.filter(h => filter.includes(h.status));
474
713
  return {
475
714
  currentStreak: countWhile([...relevantHistory].reverse(), h => filter.includes(h.status)),
@@ -479,7 +718,7 @@ export class Sentinel extends (EventEmitter as new () => WatcherEmitter) impleme
479
718
  };
480
719
  }
481
720
 
482
- protected computeFromSlot(slot: bigint | undefined) {
721
+ protected computeFromSlot(slot: SlotNumber | undefined) {
483
722
  if (slot === undefined) {
484
723
  return undefined;
485
724
  }