@aztec/aztec-node 0.0.1-commit.fce3e4f → 0.0.1-commit.ff7989d6c

package/src/sentinel/factory.ts

@@ -20,12 +20,7 @@ export async function createSentinel(
   if (!config.sentinelEnabled) {
     return undefined;
   }
-  const kvStore = await createStore(
-    'sentinel',
-    SentinelStore.SCHEMA_VERSION,
-    config,
-    createLogger('node:sentinel:lmdb'),
-  );
+  const kvStore = await createStore('sentinel', SentinelStore.SCHEMA_VERSION, config, logger.getBindings());
   const storeHistoryLength = config.sentinelHistoryLengthInEpochs * epochCache.getL1Constants().epochDuration;
   const storeHistoricProvenPerformanceLength = config.sentinelHistoricProvenPerformanceLengthInEpochs;
   const sentinelStore = new SentinelStore(kvStore, {

package/src/sentinel/sentinel.ts

@@ -1,5 +1,5 @@
 import type { EpochCache } from '@aztec/epoch-cache';
-import { EpochNumber, SlotNumber } from '@aztec/foundation/branded-types';
+import { BlockNumber, CheckpointNumber, EpochNumber, SlotNumber } from '@aztec/foundation/branded-types';
 import { countWhile, filterAsync, fromEntries, getEntries, mapValues } from '@aztec/foundation/collection';
 import { EthAddress } from '@aztec/foundation/eth-address';
 import { createLogger } from '@aztec/foundation/log';
@@ -19,7 +19,7 @@ import {
   L2BlockStream,
   type L2BlockStreamEvent,
   type L2BlockStreamEventHandler,
-  getAttestationInfoFromPublishedL2Block,
+  getAttestationInfoFromPublishedCheckpoint,
 } from '@aztec/stdlib/block';
 import { getEpochAtSlot, getSlotRangeForEpoch, getTimestampForSlot } from '@aztec/stdlib/epoch-helpers';
 import type {
@@ -36,6 +36,17 @@ import EventEmitter from 'node:events';
 
 import { SentinelStore } from './store.js';
 
+/** Maps a validator status to its category: proposer or attestation. */
+function statusToCategory(status: ValidatorStatusInSlot): ValidatorStatusType {
+  switch (status) {
+    case 'attestation-sent':
+    case 'attestation-missed':
+      return 'attestation';
+    default:
+      return 'proposer';
+  }
+}
+
 export class Sentinel extends (EventEmitter as new () => WatcherEmitter) implements L2BlockStreamEventHandler, Watcher {
   protected runningPromise: RunningPromise;
   protected blockStream!: L2BlockStream;
@@ -44,8 +55,10 @@ export class Sentinel extends (EventEmitter as new () => WatcherEmitter) impleme
   protected initialSlot: SlotNumber | undefined;
   protected lastProcessedSlot: SlotNumber | undefined;
   // eslint-disable-next-line aztec-custom/no-non-primitive-in-collections
-  protected slotNumberToBlock: Map<SlotNumber, { blockNumber: number; archive: string; attestors: EthAddress[] }> =
-    new Map();
+  protected slotNumberToCheckpoint: Map<
+    SlotNumber,
+    { checkpointNumber: CheckpointNumber; archive: string; attestors: EthAddress[] }
+  > = new Map();
 
   constructor(
     protected epochCache: EpochCache,
@@ -76,7 +89,7 @@ export class Sentinel extends (EventEmitter as new () => WatcherEmitter) impleme
   /** Loads initial slot and initializes blockstream. We will not process anything at or before the initial slot. */
   protected async init() {
     this.initialSlot = this.epochCache.getEpochAndSlotNow().slot;
-    const startingBlock = await this.archiver.getBlockNumber();
+    const startingBlock = BlockNumber(await this.archiver.getBlockNumber());
     this.logger.info(`Starting validator sentinel with initial slot ${this.initialSlot} and block ${startingBlock}`);
     this.blockStream = new L2BlockStream(this.archiver, this.l2TipsStore, this, this.logger, { startingBlock });
   }
@@ -87,47 +100,54 @@ export class Sentinel extends (EventEmitter as new () => WatcherEmitter) impleme
 
   public async handleBlockStreamEvent(event: L2BlockStreamEvent): Promise<void> {
     await this.l2TipsStore.handleBlockStreamEvent(event);
-    if (event.type === 'blocks-added') {
-      // Store mapping from slot to archive, block number, and attestors
-      for (const block of event.blocks) {
-        this.slotNumberToBlock.set(block.block.header.getSlot(), {
-          blockNumber: block.block.number,
-          archive: block.block.archive.root.toString(),
-          attestors: getAttestationInfoFromPublishedL2Block(block)
-            .filter(a => a.status === 'recovered-from-signature')
-            .map(a => a.address!),
-        });
-      }
-
-      // Prune the archive map to only keep at most N entries
-      const historyLength = this.store.getHistoryLength();
-      if (this.slotNumberToBlock.size > historyLength) {
-        const toDelete = Array.from(this.slotNumberToBlock.keys())
-          .sort((a, b) => Number(a - b))
-          .slice(0, this.slotNumberToBlock.size - historyLength);
-        for (const key of toDelete) {
-          this.slotNumberToBlock.delete(key);
-        }
-      }
+    if (event.type === 'chain-checkpointed') {
+      this.handleCheckpoint(event);
     } else if (event.type === 'chain-proven') {
       await this.handleChainProven(event);
     }
   }
 
+  protected handleCheckpoint(event: L2BlockStreamEvent) {
+    if (event.type !== 'chain-checkpointed') {
+      return;
+    }
+    const checkpoint = event.checkpoint;
+
+    // Store mapping from slot to archive, checkpoint number, and attestors
+    this.slotNumberToCheckpoint.set(checkpoint.checkpoint.header.slotNumber, {
+      checkpointNumber: checkpoint.checkpoint.number,
+      archive: checkpoint.checkpoint.archive.root.toString(),
+      attestors: getAttestationInfoFromPublishedCheckpoint(checkpoint)
+        .filter(a => a.status === 'recovered-from-signature')
+        .map(a => a.address!),
+    });
+
+    // Prune the archive map to only keep at most N entries
+    const historyLength = this.store.getHistoryLength();
+    if (this.slotNumberToCheckpoint.size > historyLength) {
+      const toDelete = Array.from(this.slotNumberToCheckpoint.keys())
+        .sort((a, b) => Number(a - b))
+        .slice(0, this.slotNumberToCheckpoint.size - historyLength);
+      for (const key of toDelete) {
+        this.slotNumberToCheckpoint.delete(key);
+      }
+    }
+  }
+
   protected async handleChainProven(event: L2BlockStreamEvent) {
     if (event.type !== 'chain-proven') {
       return;
     }
     const blockNumber = event.block.number;
-    const block = await this.archiver.getBlock(blockNumber);
-    if (!block) {
-      this.logger.error(`Failed to get block ${blockNumber}`, { block });
+    const header = await this.archiver.getBlockHeader(blockNumber);
+    if (!header) {
+      this.logger.error(`Failed to get block header ${blockNumber}`);
       return;
     }
 
     // TODO(palla/slash): We should only be computing proven performance if this is
     // a full proof epoch and not a partial one, otherwise we'll end up with skewed stats.
-    const epoch = getEpochAtSlot(block.header.getSlot(), this.epochCache.getL1Constants());
+    const epoch = getEpochAtSlot(header.getSlot(), this.epochCache.getL1Constants());
     this.logger.debug(`Computing proven performance for epoch ${epoch}`);
     const performance = await this.computeProvenPerformance(epoch);
     this.logger.info(`Computed proven performance for epoch ${epoch}`, performance);
@@ -138,7 +158,11 @@ export class Sentinel extends (EventEmitter as new () => WatcherEmitter) impleme
 
   protected async computeProvenPerformance(epoch: EpochNumber): Promise<ValidatorsEpochPerformance> {
     const [fromSlot, toSlot] = getSlotRangeForEpoch(epoch, this.epochCache.getL1Constants());
-    const { committee } = await this.epochCache.getCommittee(fromSlot);
+    const { committee, isEscapeHatchOpen } = await this.epochCache.getCommittee(fromSlot);
+    if (isEscapeHatchOpen) {
+      this.logger.info(`Skipping proven performance for epoch ${epoch} - escape hatch is open`);
+      return {};
+    }
     if (!committee) {
       this.logger.trace(`No committee found for slot ${fromSlot}`);
       return {};
@@ -291,8 +315,8 @@ export class Sentinel extends (EventEmitter as new () => WatcherEmitter) impleme
       return false;
     }
 
-    const archiverLastBlockHash = await this.l2TipsStore.getL2Tips().then(tip => tip.latest.hash);
-    const p2pLastBlockHash = await this.p2p.getL2Tips().then(tips => tips.latest.hash);
+    const archiverLastBlockHash = await this.l2TipsStore.getL2Tips().then(tip => tip.proposed.hash);
+    const p2pLastBlockHash = await this.p2p.getL2Tips().then(tips => tips.proposed.hash);
     const isP2pSynced = archiverLastBlockHash === p2pLastBlockHash;
     if (!isP2pSynced) {
       this.logger.debug(`Waiting for P2P client to sync with archiver`, { archiverLastBlockHash, p2pLastBlockHash });
@@ -307,7 +331,12 @@ export class Sentinel extends (EventEmitter as new () => WatcherEmitter) impleme
    * and updates overall stats.
    */
   protected async processSlot(slot: SlotNumber) {
-    const { epoch, seed, committee } = await this.epochCache.getCommittee(slot);
+    const { epoch, seed, committee, isEscapeHatchOpen } = await this.epochCache.getCommittee(slot);
+    if (isEscapeHatchOpen) {
+      this.logger.info(`Skipping slot ${slot} at epoch ${epoch} - escape hatch is open`);
+      this.lastProcessedSlot = slot;
+      return;
+    }
     if (!committee || committee.length === 0) {
       this.logger.trace(`No committee found for slot ${slot} at epoch ${epoch}`);
       this.lastProcessedSlot = slot;
@@ -327,16 +356,16 @@ export class Sentinel extends (EventEmitter as new () => WatcherEmitter) impleme
 
     // Check if there is an L2 block in L1 for this L2 slot
 
-    // Here we get all attestations for the block mined at the given slot,
-    // or all attestations for all proposals in the slot if no block was mined.
+    // Here we get all checkpoint attestations for the checkpoint at the given slot,
+    // or all checkpoint attestations for all proposals in the slot if no checkpoint was mined.
     // We gather from both p2p (contains the ones seen on the p2p layer) and archiver
-    // (contains the ones synced from mined blocks, which we may have missed from p2p).
-    const block = this.slotNumberToBlock.get(slot);
-    const p2pAttested = await this.p2p.getAttestationsForSlot(slot, block?.archive);
+    // (contains the ones synced from mined checkpoints, which we may have missed from p2p).
+    const checkpoint = this.slotNumberToCheckpoint.get(slot);
+    const p2pAttested = await this.p2p.getCheckpointAttestationsForSlot(slot, checkpoint?.archive);
     // Filter out attestations with invalid signatures
     const p2pAttestors = p2pAttested.map(a => a.getSender()).filter((s): s is EthAddress => s !== undefined);
     const attestors = new Set(
-      [...p2pAttestors.map(a => a.toString()), ...(block?.attestors.map(a => a.toString()) ?? [])].filter(
+      [...p2pAttestors.map(a => a.toString()), ...(checkpoint?.attestors.map(a => a.toString()) ?? [])].filter(
         addr => proposer.toString() !== addr, // Exclude the proposer from the attestors
       ),
     );
@@ -347,20 +376,29 @@ export class Sentinel extends (EventEmitter as new () => WatcherEmitter) impleme
     // But we'll leave that corner case out to reduce pressure on the node.
     // TODO(palla/slash): This breaks if a given node has more than one validator in the current committee,
     // since they will attest to their own proposal it even if it's not re-executable.
-    const blockStatus = block ? 'mined' : attestors.size > 0 ? 'proposed' : 'missed';
-    this.logger.debug(`Block for slot ${slot} was ${blockStatus}`, { ...block, slot });
+    let status: 'checkpoint-mined' | 'checkpoint-proposed' | 'checkpoint-missed' | 'blocks-missed';
+    if (checkpoint) {
+      status = 'checkpoint-mined';
+    } else if (attestors.size > 0) {
+      status = 'checkpoint-proposed';
+    } else {
+      // No checkpoint on L1 and no checkpoint attestations seen. Check if block proposals were sent for this slot.
+      const hasBlockProposals = await this.p2p.hasBlockProposalsForSlot(slot);
+      status = hasBlockProposals ? 'checkpoint-missed' : 'blocks-missed';
+    }
+    this.logger.debug(`Checkpoint status for slot ${slot}: ${status}`, { ...checkpoint, slot });
 
-    // Get attestors that failed their duties for this block, but only if there was a block proposed
+    // Get attestors that failed their checkpoint attestation duties, but only if there was a checkpoint proposed or mined
     const missedAttestors = new Set(
-      blockStatus === 'missed'
+      status === 'blocks-missed' || status === 'checkpoint-missed'
         ? []
         : committee.filter(v => !attestors.has(v.toString()) && !proposer.equals(v)).map(v => v.toString()),
     );
 
     this.logger.debug(`Retrieved ${attestors.size} attestors out of ${committee.length} for slot ${slot}`, {
-      blockStatus,
+      status,
       proposer: proposer.toString(),
-      ...block,
+      ...checkpoint,
       slot,
       attestors: [...attestors],
       missedAttestors: [...missedAttestors],
@@ -370,7 +408,7 @@ export class Sentinel extends (EventEmitter as new () => WatcherEmitter) impleme
     // Compute the status for each validator in the committee
     const statusFor = (who: `0x${string}`): ValidatorStatusInSlot | undefined => {
       if (who === proposer.toString()) {
-        return `block-${blockStatus}`;
+        return status;
       } else if (attestors.has(who)) {
         return 'attestation-sent';
       } else if (missedAttestors.has(who)) {
@@ -463,14 +501,16 @@ export class Sentinel extends (EventEmitter as new () => WatcherEmitter) impleme
   ): ValidatorStats {
     let history = fromSlot ? allHistory.filter(h => BigInt(h.slot) >= fromSlot) : allHistory;
     history = toSlot ? history.filter(h => BigInt(h.slot) <= toSlot) : history;
-    const lastProposal = history.filter(h => h.status === 'block-proposed' || h.status === 'block-mined').at(-1);
+    const lastProposal = history
+      .filter(h => h.status === 'checkpoint-proposed' || h.status === 'checkpoint-mined')
+      .at(-1);
     const lastAttestation = history.filter(h => h.status === 'attestation-sent').at(-1);
     return {
       address: EthAddress.fromString(address),
       lastProposal: this.computeFromSlot(lastProposal?.slot),
       lastAttestation: this.computeFromSlot(lastAttestation?.slot),
       totalSlots: history.length,
-      missedProposals: this.computeMissed(history, 'block', ['block-missed']),
+      missedProposals: this.computeMissed(history, 'proposer', ['checkpoint-missed', 'blocks-missed']),
       missedAttestations: this.computeMissed(history, 'attestation', ['attestation-missed']),
       history,
     };
@@ -478,10 +518,12 @@ export class Sentinel extends (EventEmitter as new () => WatcherEmitter) impleme
 
   protected computeMissed(
     history: ValidatorStatusHistory,
-    computeOverPrefix: ValidatorStatusType | undefined,
+    computeOverCategory: ValidatorStatusType | undefined,
     filter: ValidatorStatusInSlot[],
   ) {
-    const relevantHistory = history.filter(h => !computeOverPrefix || h.status.startsWith(computeOverPrefix));
+    const relevantHistory = history.filter(
+      h => !computeOverCategory || statusToCategory(h.status) === computeOverCategory,
+    );
     const filteredHistory = relevantHistory.filter(h => filter.includes(h.status));
     return {
       currentStreak: countWhile([...relevantHistory].reverse(), h => filter.includes(h.status)),

package/src/sentinel/store.ts

@@ -9,7 +9,7 @@ import type {
 } from '@aztec/stdlib/validators';
 
 export class SentinelStore {
-  public static readonly SCHEMA_VERSION = 2;
+  public static readonly SCHEMA_VERSION = 3;
 
   // a map from validator address to their ValidatorStatusHistory
   private readonly historyMap: AztecAsyncMap<`0x${string}`, Buffer>;
@@ -86,11 +86,7 @@ export class SentinelStore {
     });
   }
 
-  private async pushValidatorStatusForSlot(
-    who: EthAddress,
-    slot: SlotNumber,
-    status: 'block-mined' | 'block-proposed' | 'block-missed' | 'attestation-sent' | 'attestation-missed',
-  ) {
+  private async pushValidatorStatusForSlot(who: EthAddress, slot: SlotNumber, status: ValidatorStatusInSlot) {
     await this.store.transactionAsync(async () => {
       const currentHistory = (await this.getHistory(who)) ?? [];
       const newHistory = [...currentHistory, { slot, status }].slice(-this.config.historyLength);
@@ -149,16 +145,18 @@ export class SentinelStore {
 
   private statusToNumber(status: ValidatorStatusInSlot): number {
     switch (status) {
-      case 'block-mined':
+      case 'checkpoint-mined':
         return 1;
-      case 'block-proposed':
+      case 'checkpoint-proposed':
         return 2;
-      case 'block-missed':
+      case 'checkpoint-missed':
         return 3;
       case 'attestation-sent':
         return 4;
       case 'attestation-missed':
         return 5;
+      case 'blocks-missed':
+        return 6;
       default: {
         const _exhaustive: never = status;
         throw new Error(`Unknown status: ${status}`);
@@ -169,15 +167,17 @@ export class SentinelStore {
   private statusFromNumber(status: number): ValidatorStatusInSlot {
     switch (status) {
       case 1:
-        return 'block-mined';
+        return 'checkpoint-mined';
       case 2:
-        return 'block-proposed';
+        return 'checkpoint-proposed';
       case 3:
-        return 'block-missed';
+        return 'checkpoint-missed';
       case 4:
         return 'attestation-sent';
       case 5:
         return 'attestation-missed';
+      case 6:
+        return 'blocks-missed';
       default:
         throw new Error(`Unknown status: ${status}`);
     }