@aztec/validator-client 0.0.1-commit.b2a5d0dd1 → 0.0.1-commit.b3d3157a

package/src/validator.ts

@@ -5,6 +5,7 @@ import { CheckpointNumber, EpochNumber, IndexWithinCheckpoint, SlotNumber } from
 import { Fr } from '@aztec/foundation/curves/bn254';
 import type { EthAddress } from '@aztec/foundation/eth-address';
 import type { Signature } from '@aztec/foundation/eth-signature';
+import { FifoSet } from '@aztec/foundation/fifo-set';
 import { type LogData, type Logger, createLogger } from '@aztec/foundation/log';
 import { RunningPromise } from '@aztec/foundation/running-promise';
 import { sleep } from '@aztec/foundation/sleep';
@@ -12,9 +13,17 @@ import { DateProvider } from '@aztec/foundation/timer';
 import type { KeystoreManager } from '@aztec/node-keystore';
 import type { DuplicateAttestationInfo, DuplicateProposalInfo, P2P, PeerId } from '@aztec/p2p';
 import { AuthRequest, AuthResponse, BlockProposalValidator, ReqRespSubProtocol } from '@aztec/p2p';
-import { OffenseType, WANT_TO_SLASH_EVENT, type Watcher, type WatcherEmitter } from '@aztec/slasher';
+import {
+  OffenseType,
+  WANT_TO_CLEAR_SLASH_EVENT,
+  WANT_TO_SLASH_EVENT,
+  type Watcher,
+  type WatcherEmitter,
+  getOffenseTypeName,
+} from '@aztec/slasher';
 import type { AztecAddress } from '@aztec/stdlib/aztec-address';
 import type { CommitteeAttestationsAndSigners, L2BlockSink, L2BlockSource } from '@aztec/stdlib/block';
+import type { CheckpointReexecutionTracker } from '@aztec/stdlib/checkpoint';
 import { getEpochAtSlot } from '@aztec/stdlib/epoch-helpers';
 import type {
   ITxProvider,
@@ -30,6 +39,7 @@ import {
   CheckpointProposal,
   type CheckpointProposalCore,
   type CheckpointProposalOptions,
+  type CoordinationSignatureContext,
 } from '@aztec/stdlib/p2p';
 import type { CheckpointHeader } from '@aztec/stdlib/rollup';
 import type { BlockHeader, Tx } from '@aztec/stdlib/tx';
@@ -52,18 +62,50 @@ import { HAKeyStore } from './key_store/ha_key_store.js';
 import type { ExtendedValidatorKeyStore } from './key_store/interface.js';
 import { NodeKeystoreAdapter } from './key_store/node_keystore_adapter.js';
 import { ValidatorMetrics } from './metrics.js';
-import { type BlockProposalValidationFailureReason, ProposalHandler } from './proposal_handler.js';
+import {
+  type BlockProposalValidationFailureReason,
+  type CheckpointProposalValidationFailureReason,
+  type CheckpointProposalValidationFailureResult,
+  ProposalHandler,
+} from './proposal_handler.js';
 
 // We maintain a set of proposers who have proposed invalid blocks.
 // Just cap the set to avoid unbounded growth.
 const MAX_PROPOSERS_OF_INVALID_BLOCKS = 1000;
+const MAX_TRACKED_INVALID_PROPOSAL_SLOTS = 1000;
+const MAX_TRACKED_INVALID_CHECKPOINT_PROPOSALS = 1000;
+const MAX_TRACKED_BAD_ATTESTATIONS = 10_000;
 
 // What errors from the block proposal handler result in slashing
 const SLASHABLE_BLOCK_PROPOSAL_VALIDATION_RESULT: BlockProposalValidationFailureReason[] = [
   'state_mismatch',
   'failed_txs',
+  'global_variables_mismatch',
+  'invalid_proposal',
+  'parent_block_wrong_slot',
+  'in_hash_mismatch',
 ];
 
+const SLASHABLE_CHECKPOINT_PROPOSAL_VALIDATION_RESULT: Record<CheckpointProposalValidationFailureReason, boolean> = {
+  // enabled
+  ['invalid_fee_asset_price_modifier']: true,
+  ['checkpoint_header_mismatch']: true,
+  // These late mismatches should normally be caught by earlier checks, but if reached after validating the local
+  // checkpoint inputs, the proposer-signed payload disagrees with deterministic recomputation.
+  ['archive_mismatch']: true,
+  ['out_hash_mismatch']: true,
+  ['no_blocks_for_slot']: true,
+  ['too_many_blocks_in_checkpoint']: true,
+  ['checkpoint_validation_failed']: true,
+  ['last_block_archive_mismatch']: true,
+
+  // disabled
+  ['invalid_signature']: false,
+  ['last_block_not_found']: false,
+  ['block_fetch_error']: false,
+  ['checkpoint_already_published']: false,
+};
+
 /**
  * Validator Client
  */
@@ -86,7 +128,11 @@ export class ValidatorClient extends (EventEmitter as new () => WatcherEmitter)
   /** Tracks the last epoch in which each attester successfully submitted at least one attestation. */
   private lastAttestedEpochByAttester: Map<string, EpochNumber> = new Map();
 
-  private proposersOfInvalidBlocks: Set<string> = new Set();
+  private proposersOfInvalidBlocks = FifoSet.withLimit<string>(MAX_PROPOSERS_OF_INVALID_BLOCKS);
+  private slotsWithInvalidProposals = FifoSet.withLimit<SlotNumber>(MAX_TRACKED_INVALID_PROPOSAL_SLOTS);
+  private invalidCheckpointProposalOffenseKeys = FifoSet.withLimit<string>(MAX_TRACKED_INVALID_CHECKPOINT_PROPOSALS);
+  private badAttestationOffenseKeys = FifoSet.withLimit<string>(MAX_TRACKED_BAD_ATTESTATIONS);
+  private slotsWithProposalEquivocation = FifoSet.withLimit<SlotNumber>(MAX_TRACKED_INVALID_PROPOSAL_SLOTS);
 
   /** Tracks the last checkpoint proposal we attested to, to prevent equivocation. */
   private lastAttestedProposal?: CheckpointProposalCore;
@@ -115,11 +161,17 @@ export class ValidatorClient extends (EventEmitter as new () => WatcherEmitter)
     this.tracer = telemetry.getTracer('Validator');
     this.metrics = new ValidatorMetrics(telemetry);
 
-    this.validationService = new ValidationService(keyStore, this.log.createChild('validation-service'));
+    this.validationService = new ValidationService(
+      keyStore,
+      this.getSignatureContext(),
+      this.log.createChild('validation-service'),
+    );
+    this.proposalHandler.setCheckpointProposalValidationFailureCallback((proposal, result, proposalInfo) =>
+      this.handleInvalidCheckpointProposal(proposal, result, proposalInfo),
+    );
 
     // Refresh epoch cache every second to trigger alert if participation in committee changes
     this.epochCacheUpdateLoop = new RunningPromise(this.handleEpochCommitteeUpdate.bind(this), this.log, 1000);
-
     const myAddresses = this.getValidatorAddresses();
     this.log.verbose(`Initialized validator with addresses: ${myAddresses.map(a => a.toString()).join(', ')}`);
   }
@@ -188,6 +240,7 @@ export class ValidatorClient extends (EventEmitter as new () => WatcherEmitter)
     txProvider: ITxProvider,
     keyStoreManager: KeystoreManager,
     blobClient: BlobClientInterface,
+    reexecutionTracker: CheckpointReexecutionTracker,
     dateProvider: DateProvider = new DateProvider(),
     telemetry: TelemetryClient = getTelemetryClient(),
     slashingProtectionDb?: SlashingProtectionDatabase,
@@ -196,6 +249,12 @@ export class ValidatorClient extends (EventEmitter as new () => WatcherEmitter)
     const blockProposalValidator = new BlockProposalValidator(epochCache, {
       txsPermitted: !config.disableTransactions,
       maxTxsPerBlock: config.validateMaxTxsPerBlock,
+      maxBlocksPerCheckpoint: config.maxBlocksPerCheckpoint,
+      skipSlotValidation: config.skipProposalSlotValidation,
+      signatureContext: {
+        chainId: config.l1ChainId,
+        rollupAddress: config.rollupAddress,
+      },
     });
     const proposalHandler = new ProposalHandler(
       checkpointsBuilder,
@@ -207,6 +266,7 @@ export class ValidatorClient extends (EventEmitter as new () => WatcherEmitter)
       epochCache,
       config,
       blobClient,
+      reexecutionTracker,
       metrics,
       dateProvider,
       telemetry,
@@ -275,6 +335,13 @@ export class ValidatorClient extends (EventEmitter as new () => WatcherEmitter)
     return this.keyStore.signTypedDataWithAddress(addr, msg, context);
   }
 
+  private getSignatureContext(): CoordinationSignatureContext {
+    return {
+      chainId: this.config.l1ChainId,
+      rollupAddress: this.config.rollupAddress,
+    };
+  }
+
   public getCoinbaseForAttestor(attestor: EthAddress): EthAddress {
     return this.keyStore.getCoinbaseAddress(attestor);
   }
@@ -287,14 +354,27 @@ export class ValidatorClient extends (EventEmitter as new () => WatcherEmitter)
     return this.config;
   }
 
+  public hasProposalEquivocation(slotNumber: SlotNumber): boolean {
+    return this.slotsWithProposalEquivocation.has(slotNumber);
+  }
+
+  public hasInvalidProposals(slotNumber: SlotNumber): boolean {
+    return this.slotsWithInvalidProposals.has(slotNumber);
+  }
+
   public updateConfig(config: Partial<ValidatorClientFullConfig>) {
     this.config = { ...this.config, ...config };
+    this.proposalHandler.updateConfig(config);
   }
 
   public reloadKeystore(newManager: KeystoreManager): void {
     const newAdapter = NodeKeystoreAdapter.fromKeyStoreManager(newManager);
     this.keyStore = new HAKeyStore(newAdapter, this.slashingProtectionSigner);
-    this.validationService = new ValidationService(this.keyStore, this.log.createChild('validation-service'));
+    this.validationService = new ValidationService(
+      this.keyStore,
+      this.getSignatureContext(),
+      this.log.createChild('validation-service'),
+    );
   }
 
   public async start() {
@@ -353,6 +433,10 @@ export class ValidatorClient extends (EventEmitter as new () => WatcherEmitter)
         this.handleDuplicateAttestation(info);
       });
 
+      this.p2pClient.registerCheckpointAttestationCallback((attestation: CheckpointAttestation) => {
+        this.handleCheckpointAttestation(attestation);
+      });
+
       const myAddresses = this.getValidatorAddresses();
       this.p2pClient.registerThisValidatorAddresses(myAddresses);
 
@@ -399,21 +483,8 @@ export class ValidatorClient extends (EventEmitter as new () => WatcherEmitter)
       fishermanMode: this.config.fishermanMode || false,
     });
 
-    // Reexecute txs if we are part of the committee, or if slashing is enabled, or if we are configured to always reexecute.
-    // In fisherman mode, we always reexecute to validate proposals.
-    const { slashBroadcastedInvalidBlockPenalty, alwaysReexecuteBlockProposals, fishermanMode } = this.config;
-    const shouldReexecute =
-      fishermanMode ||
-      slashBroadcastedInvalidBlockPenalty > 0n ||
-      partOfCommittee ||
-      alwaysReexecuteBlockProposals ||
-      this.blobClient.canUpload();
-
-    const validationResult = await this.proposalHandler.handleBlockProposal(
-      proposal,
-      proposalSender,
-      !!shouldReexecute && !escapeHatchOpen,
-    );
+    // Reexecute outside the escape hatch so slashing observers can detect invalid proposals even when penalties are 0.
+    const validationResult = await this.proposalHandler.handleBlockProposal(proposal, proposalSender, !escapeHatchOpen);
 
     if (!validationResult.isValid) {
       const reason = validationResult.reason || 'unknown';
@@ -436,15 +507,18 @@ export class ValidatorClient extends (EventEmitter as new () => WatcherEmitter)
         this.metrics.incFailedAttestationsNodeIssue(1, reason, partOfCommittee);
       }
 
-      // Slash invalid block proposals (can happen even when not in committee)
       if (
         !escapeHatchOpen &&
         validationResult.reason &&
-        SLASHABLE_BLOCK_PROPOSAL_VALIDATION_RESULT.includes(validationResult.reason) &&
-        slashBroadcastedInvalidBlockPenalty > 0n
+        SLASHABLE_BLOCK_PROPOSAL_VALIDATION_RESULT.includes(validationResult.reason)
       ) {
-        this.log.warn(`Slashing proposer for invalid block proposal`, proposalInfo);
+        this.log.info(`Detected invalid block proposal offense`, {
+          ...proposalInfo,
+          amount: this.config.slashBroadcastedInvalidBlockPenalty,
+          offenseType: getOffenseTypeName(OffenseType.BROADCASTED_INVALID_BLOCK_PROPOSAL),
+        });
         this.slashInvalidBlock(proposal);
+        this.markInvalidProposalSlot(proposal.slotNumber);
       }
       return false;
     }
@@ -483,6 +557,11 @@ export class ValidatorClient extends (EventEmitter as new () => WatcherEmitter)
       return undefined;
     }
 
+    // Early-out for equivocation: refuses if we've already attested to a higher slot.
+    if (!this.shouldAttestToSlot(proposalSlotNumber)) {
+      return undefined;
+    }
+
     // Ignore proposals from ourselves (may happen in HA setups)
     if (proposer && this.getValidatorAddresses().some(addr => addr.equals(proposer))) {
       this.log.debug(`Ignoring block proposal from self for slot ${proposalSlotNumber}`, {
@@ -624,7 +703,7 @@ export class ValidatorClient extends (EventEmitter as new () => WatcherEmitter)
    */
   protected async uploadBlobsForCheckpoint(proposal: CheckpointProposalCore, proposalInfo: LogData): Promise<void> {
     try {
-      const lastBlockHeader = await this.blockSource.getBlockHeaderByArchive(proposal.archive);
+      const lastBlockHeader = (await this.blockSource.getBlockData({ archive: proposal.archive }))?.header;
       if (!lastBlockHeader) {
         this.log.warn(`Failed to get last block header for blob upload`, proposalInfo);
         return;
@@ -657,12 +736,6 @@ export class ValidatorClient extends (EventEmitter as new () => WatcherEmitter)
       return;
     }
 
-    // Trim the set if it's too big.
-    if (this.proposersOfInvalidBlocks.size > MAX_PROPOSERS_OF_INVALID_BLOCKS) {
-      // remove oldest proposer. `values` is guaranteed to be in insertion order.
-      this.proposersOfInvalidBlocks.delete(this.proposersOfInvalidBlocks.values().next().value!);
-    }
-
     this.proposersOfInvalidBlocks.add(proposer.toString());
 
     this.emit(WANT_TO_SLASH_EVENT, [
@@ -675,20 +748,115 @@ export class ValidatorClient extends (EventEmitter as new () => WatcherEmitter)
     ]);
   }
 
+  private handleInvalidCheckpointProposal(
+    proposal: CheckpointProposalCore,
+    result: CheckpointProposalValidationFailureResult,
+    proposalInfo: LogData,
+  ): void {
+    if (!SLASHABLE_CHECKPOINT_PROPOSAL_VALIDATION_RESULT[result.reason]) {
+      return;
+    }
+
+    this.markInvalidProposalSlot(proposal.slotNumber);
+
+    if (this.slashInvalidCheckpointProposal(proposal)) {
+      this.log.info(`Detected invalid checkpoint proposal offense`, {
+        ...proposalInfo,
+        reason: result.reason,
+        amount: this.config.slashBroadcastedInvalidCheckpointProposalPenalty,
+        offenseType: getOffenseTypeName(OffenseType.BROADCASTED_INVALID_CHECKPOINT_PROPOSAL),
+      });
+    }
+  }
+
+  private slashInvalidCheckpointProposal(proposal: CheckpointProposalCore): boolean {
+    const proposer = proposal.getSender();
+    if (!proposer) {
+      this.log.warn(`Cannot slash checkpoint proposal with invalid signature`, {
+        slotNumber: proposal.slotNumber,
+        archive: proposal.archive.toString(),
+      });
+      return false;
+    }
+
+    const offenseType = OffenseType.BROADCASTED_INVALID_CHECKPOINT_PROPOSAL;
+    const offenseKey = `${proposer.toString()}:${offenseType}:${proposal.slotNumber}`;
+    if (!this.invalidCheckpointProposalOffenseKeys.addIfAbsent(offenseKey)) {
+      return false;
+    }
+
+    this.emit(WANT_TO_SLASH_EVENT, [
+      {
+        validator: proposer,
+        amount: this.config.slashBroadcastedInvalidCheckpointProposalPenalty,
+        offenseType,
+        epochOrSlot: BigInt(proposal.slotNumber),
+      },
+    ]);
+    return true;
+  }
+
+  private markInvalidProposalSlot(slotNumber: SlotNumber): void {
+    this.slotsWithInvalidProposals.add(slotNumber);
+  }
+
+  private handleCheckpointAttestation(attestation: CheckpointAttestation): void {
+    const slotNumber = attestation.slotNumber;
+    if (!this.slotsWithInvalidProposals.has(slotNumber) || this.slotsWithProposalEquivocation.has(slotNumber)) {
+      return;
+    }
+
+    const attester = attestation.getSender();
+    if (!attester) {
+      this.log.warn(`Cannot slash checkpoint attestation with invalid signature`, {
+        slotNumber,
+        archive: attestation.archive.toString(),
+      });
+      return;
+    }
+
+    this.slashAttestedToInvalidCheckpointProposal(slotNumber, attester);
+  }
+
+  private slashAttestedToInvalidCheckpointProposal(slotNumber: SlotNumber, attester: EthAddress): void {
+    const offenseKey = `${attester.toString()}:${OffenseType.ATTESTED_TO_INVALID_CHECKPOINT_PROPOSAL}:${slotNumber}`;
+    if (!this.badAttestationOffenseKeys.addIfAbsent(offenseKey)) {
+      return;
+    }
+
+    this.log.info(`Detected attestation to invalid checkpoint proposal offense`, {
+      attester: attester.toString(),
+      slotNumber,
+      amount: this.config.slashAttestInvalidCheckpointProposalPenalty,
+      offenseType: getOffenseTypeName(OffenseType.ATTESTED_TO_INVALID_CHECKPOINT_PROPOSAL),
+    });
+
+    this.emit(WANT_TO_SLASH_EVENT, [
+      {
+        validator: attester,
+        amount: this.config.slashAttestInvalidCheckpointProposalPenalty,
+        offenseType: OffenseType.ATTESTED_TO_INVALID_CHECKPOINT_PROPOSAL,
+        epochOrSlot: BigInt(slotNumber),
+      },
+    ]);
+  }
+
   /**
    * Handle detection of a duplicate proposal (equivocation).
    * Emits a slash event when a proposer sends multiple proposals for the same position.
    */
   private handleDuplicateProposal(info: DuplicateProposalInfo): void {
     const { slot, proposer, type } = info;
+    this.slotsWithProposalEquivocation.add(slot);
 
-    this.log.warn(`Triggering slash event for duplicate ${type} proposal from ${proposer.toString()} at slot ${slot}`, {
+    this.log.info(`Detected duplicate ${type} proposal offense from ${proposer.toString()} at slot ${slot}`, {
       proposer: proposer.toString(),
       slot,
       type,
+      amount: this.config.slashDuplicateProposalPenalty,
+      offenseType: getOffenseTypeName(OffenseType.DUPLICATE_PROPOSAL),
     });
 
-    // Emit slash event
     this.emit(WANT_TO_SLASH_EVENT, [
       {
         validator: proposer,
@@ -697,6 +865,13 @@ export class ValidatorClient extends (EventEmitter as new () => WatcherEmitter)
         epochOrSlot: BigInt(slot),
       },
     ]);
+
+    this.emit(WANT_TO_CLEAR_SLASH_EVENT, [
+      {
+        offenseType: OffenseType.ATTESTED_TO_INVALID_CHECKPOINT_PROPOSAL,
+        epochOrSlot: BigInt(slot),
+      },
+    ]);
   }
 
   /**
@@ -706,9 +881,11 @@ export class ValidatorClient extends (EventEmitter as new () => WatcherEmitter)
   private handleDuplicateAttestation(info: DuplicateAttestationInfo): void {
     const { slot, attester } = info;
 
-    this.log.warn(`Triggering slash event for duplicate attestation from ${attester.toString()} at slot ${slot}`, {
+    this.log.info(`Detected duplicate attestation offense from ${attester.toString()} at slot ${slot}`, {
       attester: attester.toString(),
       slot,
+      amount: this.config.slashDuplicateAttestationPenalty,
+      offenseType: getOffenseTypeName(OffenseType.DUPLICATE_ATTESTATION),
     });
 
     this.emit(WANT_TO_SLASH_EVENT, [
@@ -758,7 +935,8 @@ export class ValidatorClient extends (EventEmitter as new () => WatcherEmitter)
       proposerAddress,
       {
         ...options,
-        broadcastInvalidBlockProposal: this.config.broadcastInvalidBlockProposal,
+        broadcastInvalidBlockProposal:
+          options.broadcastInvalidBlockProposal || this.config.broadcastInvalidBlockProposal,
       },
     );
     this.lastProposedBlock = newProposal;
@@ -798,6 +976,13 @@ export class ValidatorClient extends (EventEmitter as new () => WatcherEmitter)
       options,
     );
     this.lastProposedCheckpoint = newProposal;
+    // Self-record this slot's outcome on the re-execution tracker. Proposers don't run their
+    // own proposals through `handleCheckpointProposal`, so without this call the proposer's
+    // sentinel would see no outcome for slots it proposed and would mis-attribute itself as
+    // inactive. We pass the locally-computed `archive` (not `newProposal.archive`, which may
+    // be intentionally corrupted under test-only flags); from the proposer's local-view
+    // perspective the work it just completed is valid by definition.
+    this.proposalHandler.recordOwnCheckpointProposalAsValid(checkpointHeader.slotNumber, archive, checkpointNumber);
     return newProposal;
   }
 
@@ -860,31 +1045,21 @@ export class ValidatorClient extends (EventEmitter as new () => WatcherEmitter)
 
     await this.collectOwnAttestations(proposal, checkpointNumber);
 
-    const proposalId = proposal.archive.toString();
+    const proposalPayloadHash = proposal.getPayloadHash();
     const myAddresses = this.getValidatorAddresses();
 
     let attestations: CheckpointAttestation[] = [];
     while (true) {
-      // Filter out attestations with a mismatching archive. This should NOT happen since we have verified
-      // the proposer signature (ie our own) before accepting the attestation into the pool via the p2p client.
-      const collectedAttestations = (await this.p2pClient.getCheckpointAttestationsForSlot(slot, proposalId)).filter(
-        attestation => {
-          if (!attestation.archive.equals(proposal.archive)) {
-            this.log.warn(
-              `Received attestation for slot ${slot} with mismatched archive from ${attestation.getSender()?.toString()}`,
-              { attestationArchive: attestation.archive.toString(), proposalArchive: proposal.archive.toString() },
-            );
-            return false;
-          }
-          return true;
-        },
-      );
+      // The pool already filters by proposal payload hash; if any attestation slips through with a
+      // mismatched payload hash, drop it defensively. Equivocations are emitted as separate slash
+      // events from libp2p_service.
+      const collectedAttestations = await this.p2pClient.getCheckpointAttestationsForSlot(slot, proposalPayloadHash);
 
       // Log new attestations we collected
       const oldSenders = attestations.map(attestation => attestation.getSender());
       for (const collected of collectedAttestations) {
         const collectedSender = collected.getSender();
-        // Skip attestations with invalid signatures
+        // Skip attestations with invalid signatures. Should not happen as we don't add invalid attestations to our pool.
         if (!collectedSender) {
           this.log.warn(`Skipping attestation with invalid signature for slot ${slot}`);
           continue;