@aztec/aztec-node 0.0.1-commit.03f7ef2 → 0.0.1-commit.04d373f

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. package/dest/aztec-node/block_response_helpers.d.ts +25 -0
  2. package/dest/aztec-node/block_response_helpers.d.ts.map +1 -0
  3. package/dest/aztec-node/block_response_helpers.js +112 -0
  4. package/dest/aztec-node/config.d.ts +14 -8
  5. package/dest/aztec-node/config.d.ts.map +1 -1
  6. package/dest/aztec-node/config.js +16 -8
  7. package/dest/aztec-node/node_metrics.d.ts +1 -1
  8. package/dest/aztec-node/node_metrics.d.ts.map +1 -1
  9. package/dest/aztec-node/node_metrics.js +9 -16
  10. package/dest/aztec-node/public_data_overrides.d.ts +13 -0
  11. package/dest/aztec-node/public_data_overrides.d.ts.map +1 -0
  12. package/dest/aztec-node/public_data_overrides.js +21 -0
  13. package/dest/aztec-node/server.d.ts +101 -187
  14. package/dest/aztec-node/server.d.ts.map +1 -1
  15. package/dest/aztec-node/server.js +1528 -513
  16. package/dest/sentinel/config.d.ts +3 -2
  17. package/dest/sentinel/config.d.ts.map +1 -1
  18. package/dest/sentinel/config.js +15 -5
  19. package/dest/sentinel/factory.d.ts +5 -3
  20. package/dest/sentinel/factory.d.ts.map +1 -1
  21. package/dest/sentinel/factory.js +5 -5
  22. package/dest/sentinel/sentinel.d.ts +137 -12
  23. package/dest/sentinel/sentinel.d.ts.map +1 -1
  24. package/dest/sentinel/sentinel.js +273 -100
  25. package/dest/sentinel/store.d.ts +8 -8
  26. package/dest/sentinel/store.d.ts.map +1 -1
  27. package/dest/sentinel/store.js +33 -21
  28. package/dest/test/index.d.ts +3 -3
  29. package/dest/test/index.d.ts.map +1 -1
  30. package/package.json +28 -26
  31. package/src/aztec-node/block_response_helpers.ts +161 -0
  32. package/src/aztec-node/config.ts +36 -16
  33. package/src/aztec-node/node_metrics.ts +6 -17
  34. package/src/aztec-node/public_data_overrides.ts +35 -0
  35. package/src/aztec-node/server.ts +1282 -663
  36. package/src/sentinel/README.md +103 -0
  37. package/src/sentinel/config.ts +18 -6
  38. package/src/sentinel/factory.ts +9 -11
  39. package/src/sentinel/sentinel.ts +336 -109
  40. package/src/sentinel/store.ts +35 -27
  41. package/src/test/index.ts +2 -2
@@ -1,5 +1,11 @@
1
1
  import type { EpochCache } from '@aztec/epoch-cache';
2
- import { BlockNumber, EpochNumber, SlotNumber } from '@aztec/foundation/branded-types';
2
+ import {
3
+ BlockNumber,
4
+ CheckpointNumber,
5
+ CheckpointProposalHash,
6
+ EpochNumber,
7
+ SlotNumber,
8
+ } from '@aztec/foundation/branded-types';
3
9
  import { countWhile, filterAsync, fromEntries, getEntries, mapValues } from '@aztec/foundation/collection';
4
10
  import { EthAddress } from '@aztec/foundation/eth-address';
5
11
  import { createLogger } from '@aztec/foundation/log';
@@ -12,6 +18,7 @@ import {
12
18
  type WantToSlashArgs,
13
19
  type Watcher,
14
20
  type WatcherEmitter,
21
+ getOffenseTypeName,
15
22
  } from '@aztec/slasher';
16
23
  import type { SlasherConfig } from '@aztec/slasher/config';
17
24
  import {
@@ -19,9 +26,12 @@ import {
19
26
  L2BlockStream,
20
27
  type L2BlockStreamEvent,
21
28
  type L2BlockStreamEventHandler,
22
- getAttestationInfoFromPublishedL2Block,
29
+ getAttestationInfoFromPublishedCheckpoint,
23
30
  } from '@aztec/stdlib/block';
31
+ import type { CheckpointReexecutionTracker } from '@aztec/stdlib/checkpoint';
32
+ import type { ChainConfig } from '@aztec/stdlib/config';
24
33
  import { getEpochAtSlot, getSlotRangeForEpoch, getTimestampForSlot } from '@aztec/stdlib/epoch-helpers';
34
+ import { ConsensusPayload, type CoordinationSignatureContext } from '@aztec/stdlib/p2p';
25
35
  import type {
26
36
  SingleValidatorStats,
27
37
  ValidatorStats,
@@ -34,8 +44,97 @@ import type {
34
44
 
35
45
  import EventEmitter from 'node:events';
36
46
 
47
+ import type { SentinelConfig } from './config.js';
37
48
  import { SentinelStore } from './store.js';
38
49
 
50
+ export type SentinelRuntimeConfig = Pick<
51
+ SlasherConfig,
52
+ 'slashInactivityTargetPercentage' | 'slashInactivityPenalty' | 'slashInactivityConsecutiveEpochThreshold'
53
+ > &
54
+ Pick<SentinelConfig, 'sentinelEpochEndBufferSlots'> &
55
+ Pick<ChainConfig, 'l1ChainId' | 'rollupAddress'>;
56
+
57
+ /** Maps a validator status to its category: proposer or attestation. */
58
+ function statusToCategory(status: ValidatorStatusInSlot): ValidatorStatusType {
59
+ switch (status) {
60
+ case 'attestation-sent':
61
+ case 'attestation-missed':
62
+ return 'attestation';
63
+ default:
64
+ return 'proposer';
65
+ }
66
+ }
67
+
68
+ /**
69
+ * The Sentinel observes validator behaviour every L2 slot, classifies it into a per-slot status,
70
+ * aggregates those statuses into per-epoch performance once each epoch is fully observed, and
71
+ * emits inactivity slash payloads when a validator has been inactive for the configured number
72
+ * of consecutive epochs.
73
+ *
74
+ * ## Two cadences
75
+ *
76
+ * The sentinel runs `work()` every quarter L2 slot and drives two independent pipelines:
77
+ *
78
+ * 1. **Per-slot activity recording.** `processSlot(currentSlot - 2)` runs once per slot, with a
79
+ * two-slot lag to let P2P attestations settle and the archiver catch up. It classifies each
80
+ * committee member's behaviour for that slot via `getSlotActivity` and persists the result to
81
+ * `SentinelStore.historyMap` (sliding window of `sentinelHistoryLengthInEpochs * epochDuration`
82
+ * slots, default 24 epochs).
83
+ *
84
+ * 2. **Per-epoch evaluation.** `processEpochEnds(currentSlot)` runs every tick too. Once
85
+ * `sentinelEpochEndBufferSlots` (default 2) has elapsed past an epoch's last slot AND the
86
+ * per-slot recorder has covered that last slot, the sentinel calls `handleEpochEnd(epoch)`.
87
+ * That aggregates the slot-level statuses for the epoch into per-validator `{missed, total}`,
88
+ * persists it to `SentinelStore.epochMap` (default 2000-epoch window), and runs the slashing
89
+ * decision.
90
+ *
91
+ * Triggering per-epoch evaluation off local L2 state — rather than waiting for L1 proof
92
+ * publication — decouples slashing from prover availability.
93
+ *
94
+ * ## Six-case taxonomy in `getSlotActivity`
95
+ *
96
+ * For each slot, the sentinel assigns the proposer one of six statuses, ranked highest-confidence
97
+ * first:
98
+ *
99
+ * - `checkpoint-mined` — a checkpoint covering this slot has landed on L1
100
+ * (`slotNumberToCheckpoint` populated from `chain-checkpointed`).
101
+ * - `checkpoint-valid` — the local node re-executed a checkpoint proposal for this slot
102
+ * successfully (consulted via `CheckpointReexecutionTracker`).
103
+ * - `checkpoint-invalid` — the local node re-executed a checkpoint proposal for this slot
104
+ * and rejected it (e.g. header/archive/out-hash mismatch, limit
105
+ * breach). Proposer-fault.
106
+ * - `checkpoint-unvalidated` — the local node observed a checkpoint proposal but could not
107
+ * validate it (missing blocks/txs, timeouts). Treated as
108
+ * proposer-fault for slashing.
109
+ * - `checkpoint-missed` — block proposals seen on P2P but no checkpoint proposal at all.
110
+ * - `blocks-missed` — no block proposals seen for this slot.
111
+ *
112
+ * Missing-attestor faults are recorded only in `checkpoint-mined` and `checkpoint-valid`, where
113
+ * the local node has positive evidence the checkpoint was canonical or valid. In the other four
114
+ * cases the proposer is at fault and no attestor penalty applies.
115
+ *
116
+ * ## Re-execution tracker
117
+ *
118
+ * `CheckpointReexecutionTracker` is populated by the validator client's checkpoint proposal
119
+ * handler. Every early return in `validateCheckpointProposal` records an outcome
120
+ * (`valid` / `invalid` / `unvalidated`) keyed by slot.
121
+ *
122
+ * ## Inactivity slashing
123
+ *
124
+ * `handleEpochPerformance` filters the epoch's per-validator stats by
125
+ * `slashInactivityTargetPercentage` and then calls `checkPastInactivity` to require
126
+ * `slashInactivityConsecutiveEpochThreshold` consecutive past epochs over the same threshold
127
+ * (read from `SentinelStore.epochMap`). Only validators meeting both conditions are emitted as
128
+ * `WANT_TO_SLASH_EVENT` with `OffenseType.INACTIVITY`. The slot-level counters that feed this —
129
+ * `missedProposals` and `missedAttestations` — include the four proposer-fault statuses plus
130
+ * `attestation-missed`.
131
+ *
132
+ * ## Escape hatch
133
+ *
134
+ * If `epochCache.getCommittee(slot)` reports `isEscapeHatchOpen`, per-slot recording is skipped
135
+ * (no history entries for that slot) and per-epoch evaluation writes an empty performance map
136
+ * (no slashing).
137
+ */
39
138
  export class Sentinel extends (EventEmitter as new () => WatcherEmitter) implements L2BlockStreamEventHandler, Watcher {
40
139
  protected runningPromise: RunningPromise;
41
140
  protected blockStream!: L2BlockStream;
@@ -43,27 +142,41 @@ export class Sentinel extends (EventEmitter as new () => WatcherEmitter) impleme
43
142
 
44
143
  protected initialSlot: SlotNumber | undefined;
45
144
  protected lastProcessedSlot: SlotNumber | undefined;
46
- // eslint-disable-next-line aztec-custom/no-non-primitive-in-collections
47
- protected slotNumberToBlock: Map<SlotNumber, { blockNumber: BlockNumber; archive: string; attestors: EthAddress[] }> =
48
- new Map();
145
+ /** Largest epoch number for which the end-of-epoch aggregator has run. */
146
+ protected lastEvaluatedEpoch: EpochNumber | undefined;
147
+ protected slotNumberToCheckpoint: Map<
148
+ SlotNumber,
149
+ {
150
+ checkpointNumber: CheckpointNumber;
151
+ archive: string;
152
+ /** Hex keccak256 of the consensus payload bytes; used to fetch matching p2p attestations. */
153
+ proposalPayloadHash: CheckpointProposalHash;
154
+ attestors: EthAddress[];
155
+ }
156
+ > = new Map();
49
157
 
50
158
  constructor(
51
159
  protected epochCache: EpochCache,
52
160
  protected archiver: L2BlockSource,
53
161
  protected p2p: P2PClient,
54
162
  protected store: SentinelStore,
55
- protected config: Pick<
56
- SlasherConfig,
57
- 'slashInactivityTargetPercentage' | 'slashInactivityPenalty' | 'slashInactivityConsecutiveEpochThreshold'
58
- >,
163
+ protected reexecutionTracker: CheckpointReexecutionTracker,
164
+ protected config: SentinelRuntimeConfig,
59
165
  protected logger = createLogger('node:sentinel'),
60
166
  ) {
61
167
  super();
62
- this.l2TipsStore = new L2TipsMemoryStore();
168
+ this.l2TipsStore = new L2TipsMemoryStore(archiver.getGenesisBlockHash());
63
169
  const interval = (epochCache.getL1Constants().ethereumSlotDuration * 1000) / 4;
64
170
  this.runningPromise = new RunningPromise(this.work.bind(this), logger, interval);
65
171
  }
66
172
 
173
+ private getSignatureContext(): CoordinationSignatureContext {
174
+ return {
175
+ chainId: this.config.l1ChainId,
176
+ rollupAddress: this.config.rollupAddress,
177
+ };
178
+ }
179
+
67
180
  public updateConfig(config: Partial<SlasherConfig>) {
68
181
  this.config = { ...this.config, ...config };
69
182
  }
@@ -73,72 +186,96 @@ export class Sentinel extends (EventEmitter as new () => WatcherEmitter) impleme
73
186
  this.runningPromise.start();
74
187
  }
75
188
 
76
- /** Loads initial slot and initializes blockstream. We will not process anything at or before the initial slot. */
189
+ /**
190
+ * Loads initial slot and initializes blockstream. We will not process anything at or before
191
+ * the initial slot. Floors at the archiver's synced L2 slot so the sentinel keeps making
192
+ * forward progress when L1 is advancing but L2 has no activity (the synced slot is driven by
193
+ * L1 sync, not by L2 blocks). Falls back to the wallclock if the archiver isn't ready yet
194
+ * (cold start).
195
+ */
77
196
  protected async init() {
78
- this.initialSlot = this.epochCache.getEpochAndSlotNow().slot;
197
+ this.initialSlot = await this.getCurrentSlot();
79
198
  const startingBlock = BlockNumber(await this.archiver.getBlockNumber());
80
199
  this.logger.info(`Starting validator sentinel with initial slot ${this.initialSlot} and block ${startingBlock}`);
81
200
  this.blockStream = new L2BlockStream(this.archiver, this.l2TipsStore, this, this.logger, { startingBlock });
82
201
  }
83
202
 
203
+ /**
204
+ * Returns the L2 slot the sentinel should treat as "current": the archiver's last fully
205
+ * synced L2 slot, falling back to the wallclock slot when the archiver isn't ready yet
206
+ * (cold start). Anchoring to the synced slot keeps timing arithmetic (initial floor,
207
+ * per-slot lag, end-of-epoch buffer, stats-range fallback) from speculating ahead of where
208
+ * L1 actually is.
209
+ */
210
+ protected async getCurrentSlot(): Promise<SlotNumber> {
211
+ return (await this.archiver.getSyncedL2SlotNumber()) ?? this.epochCache.getSlotNow();
212
+ }
213
+
84
214
  public stop() {
85
215
  return this.runningPromise.stop();
86
216
  }
87
217
 
88
218
  public async handleBlockStreamEvent(event: L2BlockStreamEvent): Promise<void> {
89
219
  await this.l2TipsStore.handleBlockStreamEvent(event);
90
- if (event.type === 'blocks-added') {
91
- // Store mapping from slot to archive, block number, and attestors
92
- for (const block of event.blocks) {
93
- this.slotNumberToBlock.set(block.block.header.getSlot(), {
94
- blockNumber: BlockNumber(block.block.number),
95
- archive: block.block.archive.root.toString(),
96
- attestors: getAttestationInfoFromPublishedL2Block(block)
97
- .filter(a => a.status === 'recovered-from-signature')
98
- .map(a => a.address!),
99
- });
100
- }
101
-
102
- // Prune the archive map to only keep at most N entries
103
- const historyLength = this.store.getHistoryLength();
104
- if (this.slotNumberToBlock.size > historyLength) {
105
- const toDelete = Array.from(this.slotNumberToBlock.keys())
106
- .sort((a, b) => Number(a - b))
107
- .slice(0, this.slotNumberToBlock.size - historyLength);
108
- for (const key of toDelete) {
109
- this.slotNumberToBlock.delete(key);
110
- }
111
- }
112
- } else if (event.type === 'chain-proven') {
113
- await this.handleChainProven(event);
220
+ if (event.type === 'chain-checkpointed') {
221
+ this.handleCheckpoint(event);
114
222
  }
115
223
  }
116
224
 
117
- protected async handleChainProven(event: L2BlockStreamEvent) {
118
- if (event.type !== 'chain-proven') {
225
+ protected handleCheckpoint(event: L2BlockStreamEvent) {
226
+ if (event.type !== 'chain-checkpointed') {
119
227
  return;
120
228
  }
121
- const blockNumber = BlockNumber(event.block.number);
122
- const block = await this.archiver.getBlock(blockNumber);
123
- if (!block) {
124
- this.logger.error(`Failed to get block ${blockNumber}`, { block });
125
- return;
229
+ const checkpoint = event.checkpoint;
230
+
231
+ // Store mapping from slot to archive, checkpoint number, attestors, and the consensus payload
232
+ // hash (used to query matching p2p attestations regardless of feeAssetPriceModifier variants).
233
+ const signatureContext = this.getSignatureContext();
234
+ const proposalPayloadHash = CheckpointProposalHash.fromBuffer(
235
+ ConsensusPayload.fromCheckpoint(checkpoint.checkpoint, signatureContext).getPayloadHash(),
236
+ );
237
+ this.slotNumberToCheckpoint.set(checkpoint.checkpoint.header.slotNumber, {
238
+ checkpointNumber: checkpoint.checkpoint.number,
239
+ archive: checkpoint.checkpoint.archive.root.toString(),
240
+ proposalPayloadHash,
241
+ attestors: getAttestationInfoFromPublishedCheckpoint(checkpoint, signatureContext)
242
+ .filter(a => a.status === 'recovered-from-signature')
243
+ .map(a => a.address!),
244
+ });
245
+
246
+ // Prune the archive map to only keep at most N entries
247
+ const historyLength = this.store.getHistoryLength();
248
+ if (this.slotNumberToCheckpoint.size > historyLength) {
249
+ const toDelete = Array.from(this.slotNumberToCheckpoint.keys())
250
+ .sort((a, b) => Number(a - b))
251
+ .slice(0, this.slotNumberToCheckpoint.size - historyLength);
252
+ for (const key of toDelete) {
253
+ this.slotNumberToCheckpoint.delete(key);
254
+ }
126
255
  }
256
+ }
127
257
 
128
- // TODO(palla/slash): We should only be computing proven performance if this is
129
- // a full proof epoch and not a partial one, otherwise we'll end up with skewed stats.
130
- const epoch = getEpochAtSlot(block.header.getSlot(), this.epochCache.getL1Constants());
131
- this.logger.debug(`Computing proven performance for epoch ${epoch}`);
132
- const performance = await this.computeProvenPerformance(epoch);
133
- this.logger.info(`Computed proven performance for epoch ${epoch}`, performance);
258
+ /**
259
+ * Called once per epoch, after the configured end-of-epoch buffer has elapsed beyond the
260
+ * epoch's last slot. Computes per-epoch performance from the slot-level history collected
261
+ * by `processSlot` and emits any inactivity slash payloads.
262
+ */
263
+ protected async handleEpochEnd(epoch: EpochNumber) {
264
+ this.logger.debug(`Computing epoch performance for epoch ${epoch}`);
265
+ const performance = await this.computeEpochPerformance(epoch);
266
+ this.logger.info(`Computed epoch performance for epoch ${epoch}`, performance);
134
267
 
135
- await this.store.updateProvenPerformance(epoch, performance);
136
- await this.handleProvenPerformance(epoch, performance);
268
+ await this.store.updateEpochPerformance(epoch, performance);
269
+ await this.handleEpochPerformance(epoch, performance);
137
270
  }
138
271
 
139
- protected async computeProvenPerformance(epoch: EpochNumber): Promise<ValidatorsEpochPerformance> {
272
+ protected async computeEpochPerformance(epoch: EpochNumber): Promise<ValidatorsEpochPerformance> {
140
273
  const [fromSlot, toSlot] = getSlotRangeForEpoch(epoch, this.epochCache.getL1Constants());
141
- const { committee } = await this.epochCache.getCommittee(fromSlot);
274
+ const { committee, isEscapeHatchOpen } = await this.epochCache.getCommittee(fromSlot);
275
+ if (isEscapeHatchOpen) {
276
+ this.logger.info(`Skipping epoch performance for epoch ${epoch} - escape hatch is open`);
277
+ return {};
278
+ }
142
279
  if (!committee) {
143
280
  this.logger.trace(`No committee found for slot ${fromSlot}`);
144
281
  return {};
@@ -178,8 +315,8 @@ export class Sentinel extends (EventEmitter as new () => WatcherEmitter) impleme
178
315
  return true;
179
316
  }
180
317
 
181
- // Get all historical performance for this validator
182
- const allPerformance = await this.store.getProvenPerformance(validator);
318
+ // Get all historical per-epoch performance for this validator
319
+ const allPerformance = await this.store.getEpochPerformance(validator);
183
320
 
184
321
  // Sort by epoch descending to get most recent first, keep only epochs strictly before the current one, and get the first N
185
322
  const pastEpochs = allPerformance.sort((a, b) => Number(b.epoch - a.epoch)).filter(p => p.epoch < currentEpoch);
@@ -195,16 +332,12 @@ export class Sentinel extends (EventEmitter as new () => WatcherEmitter) impleme
195
332
  // Check that we have at least requiredConsecutiveEpochs and that all of them are above the inactivity threshold
196
333
  return pastEpochs
197
334
  .slice(0, requiredConsecutiveEpochs)
198
- .every(p => p.missed / p.total >= this.config.slashInactivityTargetPercentage);
335
+ .every(p => (p.total === 0 ? false : p.missed / p.total >= this.config.slashInactivityTargetPercentage));
199
336
  }
200
337
 
201
- protected async handleProvenPerformance(epoch: EpochNumber, performance: ValidatorsEpochPerformance) {
202
- if (this.config.slashInactivityPenalty === 0n) {
203
- return;
204
- }
205
-
338
+ protected async handleEpochPerformance(epoch: EpochNumber, performance: ValidatorsEpochPerformance) {
206
339
  const inactiveValidators = getEntries(performance)
207
- .filter(([_, { missed, total }]) => missed / total >= this.config.slashInactivityTargetPercentage)
340
+ .filter(([_, { missed, total }]) => total > 0 && missed / total >= this.config.slashInactivityTargetPercentage)
208
341
  .map(([address]) => address);
209
342
 
210
343
  this.logger.debug(`Found ${inactiveValidators.length} inactive validators in epoch ${epoch}`, {
@@ -226,9 +359,17 @@ export class Sentinel extends (EventEmitter as new () => WatcherEmitter) impleme
226
359
  }));
227
360
 
228
361
  if (criminals.length > 0) {
229
- this.logger.verbose(
230
- `Identified ${criminals.length} validators to slash due to inactivity in at least ${epochThreshold} consecutive epochs`,
231
- { ...args, epochThreshold },
362
+ this.logger.info(
363
+ `Identified ${criminals.length} inactivity offenses in at least ${epochThreshold} consecutive epochs`,
364
+ {
365
+ offenses: args.map(arg => ({
366
+ validator: arg.validator.toString(),
367
+ amount: arg.amount,
368
+ offenseType: getOffenseTypeName(arg.offenseType),
369
+ epochOrSlot: arg.epochOrSlot,
370
+ })),
371
+ epochThreshold,
372
+ },
232
373
  );
233
374
  this.emit(WANT_TO_SLASH_EVENT, args);
234
375
  }
@@ -238,26 +379,72 @@ export class Sentinel extends (EventEmitter as new () => WatcherEmitter) impleme
238
379
  * Process data for two L2 slots ago.
239
380
  * Note that we do not process historical data, since we rely on p2p data for processing,
240
381
  * and we don't have that data if we were offline during the period.
382
+ *
383
+ * `currentSlot` is anchored to the archiver's last synced L2 slot rather than the wallclock,
384
+ * so the per-slot lag (`isReadyToProcess`) and the end-of-epoch buffer (`processEpochEnds`)
385
+ * advance with archiver.
241
386
  */
242
387
  public async work() {
243
- const { slot: currentSlot } = this.epochCache.getEpochAndSlotNow();
388
+ const currentSlot = await this.getCurrentSlot();
244
389
  try {
245
390
  // Manually sync the block stream to ensure we have the latest data.
246
391
  // Note we never `start` the blockstream, so it loops at the same pace as we do.
247
392
  await this.blockStream.sync();
248
393
 
249
- // Check if we are ready to process data for two L2 slots ago.
394
+ // Per-slot activity recording (lag = 2 slots for P2P attestation settlement).
250
395
  const targetSlot = await this.isReadyToProcess(currentSlot);
251
-
252
- // And process it if we are.
253
396
  if (targetSlot !== false) {
254
397
  await this.processSlot(targetSlot);
255
398
  }
399
+
400
+ // End-of-epoch evaluation (lag = sentinelEpochEndBufferSlots beyond the epoch's last slot).
401
+ await this.processEpochEnds(currentSlot);
256
402
  } catch (err) {
257
403
  this.logger.error(`Failed to process slot ${currentSlot}`, err);
258
404
  }
259
405
  }
260
406
 
407
+ /**
408
+ * After the configured buffer has elapsed past an epoch's last slot, runs the end-of-epoch
409
+ * aggregator for that epoch. Catches up if multiple epochs become eligible at once.
410
+ */
411
+ protected async processEpochEnds(currentSlot: SlotNumber) {
412
+ const constants = this.epochCache.getL1Constants();
413
+ const buffer = this.config.sentinelEpochEndBufferSlots;
414
+ if (currentSlot < buffer) {
415
+ return;
416
+ }
417
+ if (this.initialSlot === undefined) {
418
+ return;
419
+ }
420
+
421
+ // We can close epoch E iff:
422
+ // - the per-slot recorder has covered the epoch's last slot (lastProcessedSlot ≥ toSlot(E))
423
+ // - the buffer has elapsed past the epoch's last slot (currentSlot − buffer ≥ toSlot(E))
424
+ // - the epoch is not in the past relative to when the sentinel started (toSlot(E) > initialSlot)
425
+ if (this.lastProcessedSlot === undefined) {
426
+ return;
427
+ }
428
+ const slotForBuffer = SlotNumber(currentSlot - buffer);
429
+
430
+ // First eligible epoch to close is the one after lastEvaluatedEpoch, or the epoch containing
431
+ // the initial slot if we haven't evaluated any yet (the initialSlot epoch may be partial — we
432
+ // don't try to evaluate it, we start from initialSlot's epoch + 1).
433
+ const startEpoch =
434
+ this.lastEvaluatedEpoch !== undefined
435
+ ? EpochNumber(this.lastEvaluatedEpoch + 1)
436
+ : EpochNumber(getEpochAtSlot(this.initialSlot, constants) + 1);
437
+
438
+ for (let epoch = startEpoch; ; epoch = EpochNumber(epoch + 1)) {
439
+ const [, toSlot] = getSlotRangeForEpoch(epoch, constants);
440
+ if (toSlot > this.lastProcessedSlot || toSlot > slotForBuffer) {
441
+ break;
442
+ }
443
+ await this.handleEpochEnd(epoch);
444
+ this.lastEvaluatedEpoch = epoch;
445
+ }
446
+ }
447
+
261
448
  /**
262
449
  * Check if we are ready to process data for two L2 slots ago, so we allow plenty of time for p2p to process all in-flight attestations.
263
450
  * We also don't move past the archiver last synced L2 slot, as we don't want to process data that is not yet available.
@@ -285,14 +472,14 @@ export class Sentinel extends (EventEmitter as new () => WatcherEmitter) impleme
285
472
  return false;
286
473
  }
287
474
 
288
- const archiverSlot = await this.archiver.getL2SlotNumber();
289
- if (archiverSlot === undefined || archiverSlot < targetSlot) {
290
- this.logger.debug(`Waiting for archiver to sync with L2 slot ${targetSlot}`, { archiverSlot, targetSlot });
475
+ const syncedSlot = await this.archiver.getSyncedL2SlotNumber();
476
+ if (syncedSlot === undefined || syncedSlot < targetSlot) {
477
+ this.logger.debug(`Waiting for archiver to sync with L2 slot ${targetSlot}`, { syncedSlot, targetSlot });
291
478
  return false;
292
479
  }
293
480
 
294
- const archiverLastBlockHash = await this.l2TipsStore.getL2Tips().then(tip => tip.latest.hash);
295
- const p2pLastBlockHash = await this.p2p.getL2Tips().then(tips => tips.latest.hash);
481
+ const archiverLastBlockHash = await this.l2TipsStore.getL2Tips().then(tip => tip.proposed.hash);
482
+ const p2pLastBlockHash = await this.p2p.getL2Tips().then(tips => tips.proposed.hash);
296
483
  const isP2pSynced = archiverLastBlockHash === p2pLastBlockHash;
297
484
  if (!isP2pSynced) {
298
485
  this.logger.debug(`Waiting for P2P client to sync with archiver`, { archiverLastBlockHash, p2pLastBlockHash });
@@ -307,7 +494,12 @@ export class Sentinel extends (EventEmitter as new () => WatcherEmitter) impleme
307
494
  * and updates overall stats.
308
495
  */
309
496
  protected async processSlot(slot: SlotNumber) {
310
- const { epoch, seed, committee } = await this.epochCache.getCommittee(slot);
497
+ const { epoch, seed, committee, isEscapeHatchOpen } = await this.epochCache.getCommittee(slot);
498
+ if (isEscapeHatchOpen) {
499
+ this.logger.info(`Skipping slot ${slot} at epoch ${epoch} - escape hatch is open`);
500
+ this.lastProcessedSlot = slot;
501
+ return;
502
+ }
311
503
  if (!committee || committee.length === 0) {
312
504
  this.logger.trace(`No committee found for slot ${slot} at epoch ${epoch}`);
313
505
  this.lastProcessedSlot = slot;
@@ -321,46 +513,74 @@ export class Sentinel extends (EventEmitter as new () => WatcherEmitter) impleme
321
513
  this.lastProcessedSlot = slot;
322
514
  }
323
515
 
324
- /** Computes activity for a given slot. */
516
+ /**
517
+ * Computes activity for a given slot using the six-case taxonomy.
518
+ *
519
+ * Proposer status:
520
+ * - case 6 `checkpoint-mined` — a checkpoint covering this slot has landed on L1.
521
+ * - case 5 `checkpoint-valid` — the local node re-executed a checkpoint proposal for this
522
+ * slot successfully.
523
+ * - case 4 `checkpoint-invalid` — the local node re-executed a checkpoint proposal for this
524
+ * slot and rejected it.
525
+ * - case 3 `checkpoint-unvalidated` — the local node observed a checkpoint proposal for this
526
+ * slot but could not validate it (missing data, timeouts).
527
+ * - case 2 `checkpoint-missed` — block proposals seen on P2P but no checkpoint proposal.
528
+ * - case 1 `blocks-missed` — no block proposals seen for this slot.
529
+ *
530
+ * Missing-attestor penalties apply only in cases 5 and 6, where the local node has positive
531
+ * evidence the checkpoint was valid or has been canonicalised on L1.
532
+ */
325
533
  protected async getSlotActivity(slot: SlotNumber, epoch: EpochNumber, proposer: EthAddress, committee: EthAddress[]) {
326
534
  this.logger.debug(`Computing stats for slot ${slot} at epoch ${epoch}`, { slot, epoch, proposer, committee });
327
535
 
328
- // Check if there is an L2 block in L1 for this L2 slot
329
-
330
- // Here we get all attestations for the block mined at the given slot,
331
- // or all attestations for all proposals in the slot if no block was mined.
332
- // We gather from both p2p (contains the ones seen on the p2p layer) and archiver
333
- // (contains the ones synced from mined blocks, which we may have missed from p2p).
334
- const block = this.slotNumberToBlock.get(slot);
335
- const p2pAttested = await this.p2p.getAttestationsForSlot(slot, block?.archive);
336
- // Filter out attestations with invalid signatures
536
+ // Gather attestors from both p2p (live attestations) and the archiver (signers on the
537
+ // checkpoint if one has landed on L1). Used regardless of which case applies.
538
+ const checkpoint = this.slotNumberToCheckpoint.get(slot);
539
+ const p2pAttested = await this.p2p.getCheckpointAttestationsForSlot(slot, checkpoint?.proposalPayloadHash);
337
540
  const p2pAttestors = p2pAttested.map(a => a.getSender()).filter((s): s is EthAddress => s !== undefined);
338
541
  const attestors = new Set(
339
- [...p2pAttestors.map(a => a.toString()), ...(block?.attestors.map(a => a.toString()) ?? [])].filter(
340
- addr => proposer.toString() !== addr, // Exclude the proposer from the attestors
542
+ [...p2pAttestors.map(a => a.toString()), ...(checkpoint?.attestors.map(a => a.toString()) ?? [])].filter(
543
+ addr => proposer.toString() !== addr,
341
544
  ),
342
545
  );
343
546
 
344
- // We assume that there was a block proposal if at least one of the validators (other than the proposer) attested to it.
345
- // It could be the case that every single validator failed, and we could differentiate it by having
346
- // this node re-execute every block proposal it sees and storing it in the attestation pool.
347
- // But we'll leave that corner case out to reduce pressure on the node.
348
- // TODO(palla/slash): This breaks if a given node has more than one validator in the current committee,
349
- // since they will attest to their own proposal it even if it's not re-executable.
350
- const blockStatus = block ? 'mined' : attestors.size > 0 ? 'proposed' : 'missed';
351
- this.logger.debug(`Block for slot ${slot} was ${blockStatus}`, { ...block, slot });
547
+ // Determine the proposer status from the six-case taxonomy.
548
+ const reexecutionOutcome = this.reexecutionTracker.getOutcomeForSlot(slot);
549
+ let status:
550
+ | 'checkpoint-mined'
551
+ | 'checkpoint-valid'
552
+ | 'checkpoint-invalid'
553
+ | 'checkpoint-unvalidated'
554
+ | 'checkpoint-missed'
555
+ | 'blocks-missed';
556
+ if (checkpoint) {
557
+ status = 'checkpoint-mined';
558
+ } else if (reexecutionOutcome === 'valid') {
559
+ status = 'checkpoint-valid';
560
+ } else if (reexecutionOutcome === 'invalid') {
561
+ status = 'checkpoint-invalid';
562
+ } else if (reexecutionOutcome === 'unvalidated') {
563
+ status = 'checkpoint-unvalidated';
564
+ } else {
565
+ // No L1 checkpoint, no local re-execution outcome for this slot. Distinguish "proposer
566
+ // sent block proposals but never made a checkpoint" from "proposer sent nothing".
567
+ const hasBlockProposals = await this.p2p.hasBlockProposalsForSlot(slot);
568
+ status = hasBlockProposals ? 'checkpoint-missed' : 'blocks-missed';
569
+ }
570
+ this.logger.debug(`Checkpoint status for slot ${slot}: ${status}`, { ...checkpoint, slot });
352
571
 
353
- // Get attestors that failed their duties for this block, but only if there was a block proposed
572
+ // Missing-attestor faults only apply when we have positive evidence the proposal was valid.
573
+ const attestorsExpected = status === 'checkpoint-mined' || status === 'checkpoint-valid';
354
574
  const missedAttestors = new Set(
355
- blockStatus === 'missed'
356
- ? []
357
- : committee.filter(v => !attestors.has(v.toString()) && !proposer.equals(v)).map(v => v.toString()),
575
+ attestorsExpected
576
+ ? committee.filter(v => !attestors.has(v.toString()) && !proposer.equals(v)).map(v => v.toString())
577
+ : [],
358
578
  );
359
579
 
360
580
  this.logger.debug(`Retrieved ${attestors.size} attestors out of ${committee.length} for slot ${slot}`, {
361
- blockStatus,
581
+ status,
362
582
  proposer: proposer.toString(),
363
- ...block,
583
+ ...checkpoint,
364
584
  slot,
365
585
  attestors: [...attestors],
366
586
  missedAttestors: [...missedAttestors],
@@ -370,7 +590,7 @@ export class Sentinel extends (EventEmitter as new () => WatcherEmitter) impleme
370
590
  // Compute the status for each validator in the committee
371
591
  const statusFor = (who: `0x${string}`): ValidatorStatusInSlot | undefined => {
372
592
  if (who === proposer.toString()) {
373
- return `block-${blockStatus}`;
593
+ return status;
374
594
  } else if (attestors.has(who)) {
375
595
  return 'attestation-sent';
376
596
  } else if (missedAttestors.has(who)) {
@@ -398,7 +618,7 @@ export class Sentinel extends (EventEmitter as new () => WatcherEmitter) impleme
398
618
  ? fromEntries(await Promise.all(validators.map(async v => [v.toString(), await this.store.getHistory(v)])))
399
619
  : await this.store.getHistories();
400
620
 
401
- const slotNow = this.epochCache.getEpochAndSlotNow().slot;
621
+ const slotNow = await this.getCurrentSlot();
402
622
  fromSlot ??= SlotNumber(Math.max((this.lastProcessedSlot ?? slotNow) - this.store.getHistoryLength(), 0));
403
623
  toSlot ??= this.lastProcessedSlot ?? slotNow;
404
624
 
@@ -426,7 +646,7 @@ export class Sentinel extends (EventEmitter as new () => WatcherEmitter) impleme
426
646
  return undefined;
427
647
  }
428
648
 
429
- const slotNow = this.epochCache.getEpochAndSlotNow().slot;
649
+ const slotNow = await this.getCurrentSlot();
430
650
  const effectiveFromSlot =
431
651
  fromSlot ?? SlotNumber(Math.max((this.lastProcessedSlot ?? slotNow) - this.store.getHistoryLength(), 0));
432
652
  const effectiveToSlot = toSlot ?? this.lastProcessedSlot ?? slotNow;
@@ -448,7 +668,7 @@ export class Sentinel extends (EventEmitter as new () => WatcherEmitter) impleme
448
668
 
449
669
  return {
450
670
  validator,
451
- allTimeProvenPerformance: await this.store.getProvenPerformance(validatorAddress),
671
+ allTimeEpochPerformance: await this.store.getEpochPerformance(validatorAddress),
452
672
  lastProcessedSlot: this.lastProcessedSlot,
453
673
  initialSlot: this.initialSlot,
454
674
  slotWindow: this.store.getHistoryLength(),
@@ -463,14 +683,19 @@ export class Sentinel extends (EventEmitter as new () => WatcherEmitter) impleme
463
683
  ): ValidatorStats {
464
684
  let history = fromSlot ? allHistory.filter(h => BigInt(h.slot) >= fromSlot) : allHistory;
465
685
  history = toSlot ? history.filter(h => BigInt(h.slot) <= toSlot) : history;
466
- const lastProposal = history.filter(h => h.status === 'block-proposed' || h.status === 'block-mined').at(-1);
686
+ const lastProposal = history.filter(h => h.status === 'checkpoint-valid' || h.status === 'checkpoint-mined').at(-1);
467
687
  const lastAttestation = history.filter(h => h.status === 'attestation-sent').at(-1);
468
688
  return {
469
689
  address: EthAddress.fromString(address),
470
690
  lastProposal: this.computeFromSlot(lastProposal?.slot),
471
691
  lastAttestation: this.computeFromSlot(lastAttestation?.slot),
472
692
  totalSlots: history.length,
473
- missedProposals: this.computeMissed(history, 'block', ['block-missed']),
693
+ missedProposals: this.computeMissed(history, 'proposer', [
694
+ 'checkpoint-missed',
695
+ 'blocks-missed',
696
+ 'checkpoint-invalid',
697
+ 'checkpoint-unvalidated',
698
+ ]),
474
699
  missedAttestations: this.computeMissed(history, 'attestation', ['attestation-missed']),
475
700
  history,
476
701
  };
@@ -478,10 +703,12 @@ export class Sentinel extends (EventEmitter as new () => WatcherEmitter) impleme
478
703
 
479
704
  protected computeMissed(
480
705
  history: ValidatorStatusHistory,
481
- computeOverPrefix: ValidatorStatusType | undefined,
706
+ computeOverCategory: ValidatorStatusType | undefined,
482
707
  filter: ValidatorStatusInSlot[],
483
708
  ) {
484
- const relevantHistory = history.filter(h => !computeOverPrefix || h.status.startsWith(computeOverPrefix));
709
+ const relevantHistory = history.filter(
710
+ h => !computeOverCategory || statusToCategory(h.status) === computeOverCategory,
711
+ );
485
712
  const filteredHistory = relevantHistory.filter(h => filter.includes(h.status));
486
713
  return {
487
714
  currentStreak: countWhile([...relevantHistory].reverse(), h => filter.includes(h.status)),