@aztec/slasher 0.0.1-commit.c2eed6949 → 0.0.1-commit.c52d6e7

package/src/metrics.ts

@@ -0,0 +1,59 @@
+import {
+  type Gauge,
+  Metrics,
+  type TelemetryClient,
+  type UpDownCounter,
+  createUpDownCounterWithDefault,
+} from '@aztec/telemetry-client';
+
+import { formatEther } from 'viem/utils';
+
+export class SlasherMetrics {
+  private readonly roundExecuted: UpDownCounter;
+  private readonly ownValidatorTargeted: UpDownCounter;
+  private readonly ownValidatorSlashedCount: UpDownCounter;
+  private readonly ownValidatorSlashedAmount: UpDownCounter;
+  private readonly ownValidatorCurrentRoundVotesMax: Gauge;
+  private readonly quorumSize: Gauge;
+
+  constructor(client: TelemetryClient, name = 'Slasher') {
+    const meter = client.getMeter(name);
+    this.roundExecuted = createUpDownCounterWithDefault(meter, Metrics.SLASHER_ROUND_EXECUTED_COUNT);
+    this.ownValidatorTargeted = createUpDownCounterWithDefault(meter, Metrics.SLASHER_OWN_VALIDATOR_TARGETED_COUNT);
+    this.ownValidatorSlashedCount = createUpDownCounterWithDefault(meter, Metrics.SLASHER_OWN_VALIDATOR_SLASHED_COUNT);
+    this.ownValidatorSlashedAmount = createUpDownCounterWithDefault(
+      meter,
+      Metrics.SLASHER_OWN_VALIDATOR_SLASHED_AMOUNT,
+    );
+    this.ownValidatorCurrentRoundVotesMax = meter.createGauge(Metrics.SLASHER_OWN_VALIDATOR_CURRENT_ROUND_VOTES_MAX);
+    this.quorumSize = meter.createGauge(Metrics.SLASHER_QUORUM_SIZE);
+  }
+
+  public recordRoundExecuted(): void {
+    this.roundExecuted.add(1);
+  }
+
+  /** Records the quorum a validator must reach in a round to be slashed, so dashboards can plot the threshold. */
+  public recordQuorumSize(quorum: number): void {
+    this.quorumSize.record(quorum);
+  }
+
+  /** Records that an onchain slashing vote named one of the node's own validators as a target. */
+  public recordOwnValidatorTargeted(): void {
+    this.ownValidatorTargeted.add(1);
+  }
+
+  /**
+   * Records how close the most-voted committee position held by the node's own validators is to quorum this round.
+   * Recorded as an absolute value rather than a delta so a vote seen across a round rollover cannot make it drift.
+   */
+  public recordCurrentRoundVotesMax(votes: number): void {
+    this.ownValidatorCurrentRoundVotesMax.record(votes);
+  }
+
+  /** Records an executed slash against one of the node's own validators. */
+  public recordOwnValidatorSlashed(amount: bigint): void {
+    this.ownValidatorSlashedCount.add(1);
+    this.ownValidatorSlashedAmount.add(parseFloat(formatEther(amount)));
+  }
+}

package/src/null_slasher_client.ts

@@ -1,5 +1,5 @@
 import type { SlotNumber } from '@aztec/foundation/branded-types';
-import type { Offense, ProposerSlashAction, SlashPayloadRound } from '@aztec/stdlib/slashing';
+import type { Offense, ProposerSlashAction } from '@aztec/stdlib/slashing';
 
 import type { SlasherConfig } from './config.js';
 import type { SlasherClientInterface } from './slasher_client_interface.js';
@@ -15,15 +15,11 @@ export class NullSlasherClient implements SlasherClientInterface {
     return Promise.resolve();
   }
 
-  public getSlashPayloads(): Promise<SlashPayloadRound[]> {
-    return Promise.resolve([]);
-  }
-
   public gatherOffensesForRound(_round?: bigint): Promise<Offense[]> {
     return Promise.resolve([]);
   }
 
-  public getPendingOffenses(): Promise<Offense[]> {
+  public getOffenses(): Promise<Offense[]> {
     return Promise.resolve([]);
   }
 

package/src/own_validator_slash_monitor.ts

@@ -0,0 +1,184 @@
+import type { SlashVoteTarget, SlashingProposerContract } from '@aztec/ethereum/contracts';
+import { uniqueBy } from '@aztec/foundation/collection';
+import type { EthAddress } from '@aztec/foundation/eth-address';
+import { createLogger } from '@aztec/foundation/log';
+import { SerialQueue } from '@aztec/foundation/queue';
+
+import type { Hex } from 'viem';
+
+import type { SlasherMetrics } from './metrics.js';
+
+/**
+ * Watches slashing activity that targets the node's own validators: warns and counts on every vote naming one of
+ * them, tracks how close the current round's tally is to quorum, and reports executed slashes. Does nothing when
+ * the node runs no validators. Lifecycle calls must be sequential: start() before events are wired, and stop()
+ * awaited before any restart.
+ */
+export class OwnValidatorSlashMonitor {
+  /**
+   * Tally for the single round being tracked (votes are only ever cast for the current round). `nextVoteIndex` is
+   * the cursor: the index of the first vote in the round not yet processed, advanced from the indices the VoteCast
+   * events carry. It is undefined until the first event of a round that started mid-round, whose own vote is then
+   * the first one counted, so votes cast before the node subscribed are never replayed into the cumulative metrics.
+   * The tally is per flattened committee position because that is the unit the contract tallies quorum by.
+   */
+  private state: { round: bigint; nextVoteIndex: bigint | undefined; countByPosition: Map<number, number> } = {
+    round: -1n,
+    nextVoteIndex: undefined,
+    countByPosition: new Map(),
+  };
+
+  /**
+   * All vote processing runs through this queue, one job at a time in arrival order. Serialization is what makes
+   * the cursor sound: without it, concurrent event handlers could process the same index twice or emit warnings
+   * with out-of-order running tallies. Replaced on every start(), since a queue that has been ended cannot run
+   * jobs again.
+   */
+  private queue = new SerialQueue();
+
+  private stopped = false;
+
+  constructor(
+    private readonly slashingProposer: SlashingProposerContract,
+    private readonly settings: { slashingQuorumSize: number },
+    private readonly ownValidators: EthAddress[],
+    private readonly metrics: SlasherMetrics,
+    private readonly log = createLogger('slasher:own-validators'),
+  ) {
+    this.queue.start();
+  }
+
+  private get enabled(): boolean {
+    return this.ownValidators.length > 0;
+  }
+
+  /**
+   * Starts tracking at the given round, leaving the cursor unset so the round's votes already cast are skipped:
+   * replaying them would warn about votes the operator can no longer react to sooner, and would double-count
+   * cumulative counters across restarts.
+   */
+  public start(currentRound: bigint): void {
+    if (!this.enabled) {
+      return;
+    }
+    this.stopped = false;
+    this.queue = new SerialQueue();
+    this.queue.start();
+    this.state = { round: currentRound, nextVoteIndex: undefined, countByPosition: new Map() };
+    this.metrics.recordQuorumSize(this.settings.slashingQuorumSize);
+    this.metrics.recordCurrentRoundVotesMax(0);
+  }
+
+  /** Stops processing and waits for any in-flight drain, so no warning or metric is emitted after shutdown. */
+  public async stop(): Promise<void> {
+    this.stopped = true;
+    // Jobs already queued are no-ops now that the flag is set, so end() runs through them at no cost while still
+    // settling the promises their callers hold, where cancel() would discard them and leave those promises pending.
+    await this.queue.end();
+  }
+
+  /**
+   * Called by the round monitor clock. Strictly greater so the clock catching up to a round an event already rolled
+   * to keeps the tally.
+   */
+  public handleNewRound(round: bigint): void {
+    if (this.enabled && round > this.state.round) {
+      this.rollTo(round);
+    }
+  }
+
+  /**
+   * Called on each VoteCast event with the index the event carries. Reads every vote from the cursor up to that
+   * index, so an event delivery that never arrived is healed by the next one.
+   */
+  public handleVoteCast(round: bigint, voteIndex: bigint): Promise<void> {
+    if (!this.enabled || this.stopped) {
+      return Promise.resolve();
+    }
+    // Errors are logged rather than propagated: callers fire and forget, and a failed read is retried by the drain
+    // the next event triggers.
+    return this.queue
+      .put(() => this.drainVotesTo(round, voteIndex))
+      .catch(err => this.log.error('Error processing slashing votes', err));
+  }
+
+  /** Reports executed slashes against own validators. Called with the Slashed events already fetched by the client. */
+  public handleSlashes(round: bigint, slashes: { attester: EthAddress; amount: bigint }[], l1BlockHash: Hex): void {
+    if (!this.enabled || this.stopped) {
+      return;
+    }
+    for (const { attester, amount } of slashes.filter(slash => this.isOwnValidator(slash.attester))) {
+      this.log.warn(`Own validator ${attester} was slashed for ${amount}`, {
+        round,
+        validator: attester.toString(),
+        amount,
+        l1BlockHash,
+      });
+      this.metrics.recordOwnValidatorSlashed(amount);
+    }
+  }
+
+  /** Processes votes [cursor, voteIndex] of a round, in order, advancing the cursor after each success. */
+  private async drainVotesTo(round: bigint, voteIndex: bigint): Promise<void> {
+    if (this.stopped) {
+      return; // enqueued before stop() but not yet started: emit nothing and read nothing after shutdown
+    }
+    if (round < this.state.round) {
+      return; // a vote for a round that already closed can no longer reach quorum
+    }
+    if (round > this.state.round) {
+      this.rollTo(round); // the event beat the round-monitor clock to the boundary
+    }
+
+    // The first event of a round tracked from mid-round counts its own vote only: earlier ones predate the subscription
+    const from = this.state.nextVoteIndex ?? voteIndex;
+    if (voteIndex < from) {
+      return; // a duplicate or out-of-order delivery of a vote already processed
+    }
+
+    for (let index = from; index <= voteIndex; index++) {
+      const vote = await this.slashingProposer.getVoteAt(round, index);
+      if (this.stopped || round !== this.state.round) {
+        return; // stopped, or rolled mid-drain: either way this vote must no longer be counted
+      }
+      this.processVote(round, vote);
+      // Advanced only after successful processing, so a failed read is retried on the next event
+      this.state.nextVoteIndex = index + 1n;
+    }
+  }
+
+  private processVote(round: bigint, vote: SlashVoteTarget[]): void {
+    const own = vote.filter(target => this.isOwnValidator(target.validator));
+    for (const { position } of own) {
+      this.state.countByPosition.set(position, (this.state.countByPosition.get(position) ?? 0) + 1);
+    }
+
+    // Warn once per (vote, validator) at the validator's highest position tally: the operator cares about the
+    // validator, not which of its committee seats is being voted on. No slash amount (the amount voted is not
+    // necessarily the amount slashed) and no proposer (a catch-up drain cannot attribute votes to proposers).
+    const quorum = this.settings.slashingQuorumSize;
+    for (const { validator } of uniqueBy(own, target => target.validator.toString())) {
+      const votes = Math.max(
+        ...own.filter(t => t.validator.equals(validator)).map(t => this.state.countByPosition.get(t.position)!),
+      );
+      this.metrics.recordOwnValidatorTargeted();
+      this.log.warn(
+        `Own validator ${validator} targeted by slashing vote (${votes} of ${quorum} votes needed to slash)`,
+        { round, validator: validator.toString(), votes, quorum },
+      );
+    }
+
+    this.metrics.recordCurrentRoundVotesMax(Math.max(0, ...this.state.countByPosition.values()));
+  }
+
+  /** Starts a fresh tally, zeroing the gauge so a quiet round does not keep the previous round's value. */
+  private rollTo(round: bigint): void {
+    this.state = { round, nextVoteIndex: 0n, countByPosition: new Map() };
+    this.metrics.recordCurrentRoundVotesMax(0);
+    this.metrics.recordQuorumSize(this.settings.slashingQuorumSize); // keeps the gauge fresh across export cycles
+  }
+
+  private isOwnValidator(address: EthAddress): boolean {
+    return this.ownValidators.some(validator => validator.equals(address));
+  }
+}

package/src/slash_offenses_collector.ts

@@ -1,12 +1,19 @@
 import type { SlotNumber } from '@aztec/foundation/branded-types';
 import { createLogger } from '@aztec/foundation/log';
+import { SerialQueue } from '@aztec/foundation/queue';
 import type { Prettify } from '@aztec/foundation/types';
 import type { L1RollupConstants } from '@aztec/stdlib/epoch-helpers';
 import type { SlasherConfig } from '@aztec/stdlib/interfaces/server';
-import { type Offense, type OffenseIdentifier, getSlotForOffense } from '@aztec/stdlib/slashing';
+import { type Offense, getOffenseTypeName, getSlotForOffense } from '@aztec/stdlib/slashing';
 
 import type { SlasherOffensesStore } from './stores/offenses_store.js';
-import { WANT_TO_SLASH_EVENT, type WantToSlashArgs, type Watcher } from './watcher.js';
+import {
+  WANT_TO_CLEAR_SLASH_EVENT,
+  WANT_TO_SLASH_EVENT,
+  type WantToClearSlashArgs,
+  type WantToSlashArgs,
+  type Watcher,
+} from './watcher.js';
 
 export type SlashOffensesCollectorConfig = Prettify<Pick<SlasherConfig, 'slashGracePeriodL2Slots'>>;
 export type SlashOffensesCollectorSettings = Prettify<
@@ -24,6 +31,7 @@ export type SlashOffensesCollectorSettings = Prettify<
  */
 export class SlashOffensesCollector {
   private readonly unwatchCallbacks: (() => void)[] = [];
+  private readonly storeMutationQueue = new SerialQueue();
 
   constructor(
     private readonly config: SlashOffensesCollectorConfig,
@@ -35,28 +43,35 @@ export class SlashOffensesCollector {
 
   public start() {
     this.log.debug('Starting SlashOffensesCollector...');
+    this.storeMutationQueue.start();
 
-    // Subscribe to watchers WANT_TO_SLASH_EVENT
+    // Subscribe to watcher slashing events.
     for (const watcher of this.watchers) {
       const wantToSlashCallback = (args: WantToSlashArgs[]) =>
-        void this.handleWantToSlash(args).catch(err => this.log.error('Error handling wantToSlash', err));
+        this.enqueueStoreMutation('wantToSlash', () => this.handleWantToSlash(args));
       watcher.on(WANT_TO_SLASH_EVENT, wantToSlashCallback);
       this.unwatchCallbacks.push(() => watcher.removeListener(WANT_TO_SLASH_EVENT, wantToSlashCallback));
+
+      const wantToClearSlashCallback = (args: WantToClearSlashArgs[]) =>
+        this.enqueueStoreMutation('wantToClearSlash', () => this.handleWantToClearSlash(args));
+      watcher.on(WANT_TO_CLEAR_SLASH_EVENT, wantToClearSlashCallback);
+      this.unwatchCallbacks.push(() => watcher.removeListener(WANT_TO_CLEAR_SLASH_EVENT, wantToClearSlashCallback));
     }
 
     this.log.info('Started SlashOffensesCollector');
     return Promise.resolve();
   }
 
-  public stop() {
+  public async stop() {
     this.log.debug('Stopping SlashOffensesCollector...');
 
     for (const unwatchCallback of this.unwatchCallbacks) {
       unwatchCallback();
     }
 
+    await this.storeMutationQueue.end();
+
     this.log.info('SlashOffensesCollector stopped');
-    return Promise.resolve();
   }
 
   /**
@@ -66,32 +81,47 @@ export class SlashOffensesCollector {
    */
   public async handleWantToSlash(args: WantToSlashArgs[]) {
     for (const arg of args) {
-      const pendingOffense: Offense = {
+      const offense: Offense = {
         validator: arg.validator,
         amount: arg.amount,
         offenseType: arg.offenseType,
         epochOrSlot: arg.epochOrSlot,
       };
 
-      if (this.shouldSkipOffense(pendingOffense)) {
-        this.log.verbose('Skipping offense during grace period', pendingOffense);
-        continue;
-      }
-
-      if (await this.offensesStore.hasOffense(pendingOffense)) {
-        this.log.debug('Skipping repeated offense', pendingOffense);
+      if (this.shouldSkipOffense(offense)) {
+        this.log.verbose('Skipping offense during grace period', this.getOffenseLogData(offense));
         continue;
       }
 
-      if (this.settings.slashingAmounts) {
-        const minSlash = this.settings.slashingAmounts[0];
-        if (arg.amount < minSlash) {
-          this.log.warn(`Offense amount ${arg.amount} is below minimum slashing amount ${minSlash}`);
+      const added = await this.offensesStore.addOffense(offense);
+      if (added) {
+        if (this.settings.slashingAmounts) {
+          const minSlash = this.settings.slashingAmounts[0];
+          if (arg.amount < minSlash) {
+            this.log.warn(
+              `Offense amount ${arg.amount} is below minimum slashing amount ${minSlash}`,
+              this.getOffenseLogData(offense),
+            );
+          }
         }
+
+        this.log.info(`Adding pending offense for validator ${arg.validator}`, this.getOffenseLogData(offense));
+      } else {
+        this.log.debug('Skipping repeated offense', this.getOffenseLogData(offense));
       }
+    }
+  }
 
-      this.log.info(`Adding pending offense for validator ${arg.validator}`, pendingOffense);
-      await this.offensesStore.addPendingOffense(pendingOffense);
+  public async handleWantToClearSlash(args: WantToClearSlashArgs[]) {
+    for (const arg of args) {
+      const cleared = await this.offensesStore.clearOffenses(arg);
+      if (cleared > 0) {
+        this.log.info(`Cleared ${cleared} pending offenses`, {
+          offenseType: getOffenseTypeName(arg.offenseType),
+          epochOrSlot: arg.epochOrSlot,
+          validators: arg.validators?.map(validator => validator.toString()),
+        });
+      }
     }
   }
 
@@ -106,18 +136,21 @@ export class SlashOffensesCollector {
     }
   }
 
-  /**
-   * Marks offenses as slashed (no longer pending)
-   * @param offenses - The offenses to mark as slashed
-   */
-  public markAsSlashed(offenses: OffenseIdentifier[]) {
-    this.log.verbose(`Marking offenses as slashed`, { offenses });
-    return this.offensesStore.markAsSlashed(offenses);
-  }
-
   /** Returns whether to skip an offense if it happened during the grace period after the network upgrade */
   private shouldSkipOffense(offense: Offense): boolean {
     const offenseSlot = getSlotForOffense(offense, this.settings);
     return offenseSlot < this.settings.rollupRegisteredAtL2Slot + this.config.slashGracePeriodL2Slots;
   }
+
+  private getOffenseLogData(offense: Offense) {
+    return {
+      ...offense,
+      validator: offense.validator.toString(),
+      offenseType: getOffenseTypeName(offense.offenseType),
+    };
+  }
+
+  private enqueueStoreMutation(label: string, callback: () => Promise<void>) {
+    void this.storeMutationQueue.put(callback).catch(err => this.log.error(`Error handling ${label}`, err));
+  }
 }