@aztec/validator-client 0.0.1-commit.2f68f620 → 0.0.1-commit.3100065

package/src/proposal_handler.ts

@@ -2,7 +2,7 @@ import type { Archiver } from '@aztec/archiver';
 import type { BlobClientInterface } from '@aztec/blob-client/client';
 import { type Blob, encodeCheckpointBlobDataFromBlocks, getBlobsPerL1Block } from '@aztec/blob-lib';
 import { INITIAL_L2_BLOCK_NUM } from '@aztec/constants';
-import { type EpochCache, PROPOSER_PIPELINING_SLOT_OFFSET } from '@aztec/epoch-cache';
+import type { EpochCache } from '@aztec/epoch-cache';
 import { validateFeeAssetPriceModifier } from '@aztec/ethereum/contracts';
 import {
   BlockNumber,
@@ -13,6 +13,7 @@ import {
 import { pick } from '@aztec/foundation/collection';
 import { Fr } from '@aztec/foundation/curves/bn254';
 import { TimeoutError } from '@aztec/foundation/error';
+import { FifoSet } from '@aztec/foundation/fifo-set';
 import type { LogData } from '@aztec/foundation/log';
 import { createLogger } from '@aztec/foundation/log';
 import { retryUntil } from '@aztec/foundation/retry';
@@ -22,7 +23,7 @@ import { BlockProposalValidator } from '@aztec/p2p/msg_validators';
 import type { BlockData, L2Block, L2BlockSink, L2BlockSource } from '@aztec/stdlib/block';
 import type { CheckpointReexecutionTracker, ReexecutionOutcome } from '@aztec/stdlib/checkpoint';
 import { getPreviousCheckpointOutHashes, validateCheckpoint } from '@aztec/stdlib/checkpoint';
-import { getEpochAtSlot, getLastL1SlotTimestampForL2Slot, getTimestampForSlot } from '@aztec/stdlib/epoch-helpers';
+import { getEpochAtSlot } from '@aztec/stdlib/epoch-helpers';
 import { Gas } from '@aztec/stdlib/gas';
 import type { ITxProvider, ValidatorClientFullConfig, WorldStateSynchronizer } from '@aztec/stdlib/interfaces/server';
 import {
@@ -31,6 +32,7 @@ import {
   computeInHashFromL1ToL2Messages,
 } from '@aztec/stdlib/messaging';
 import type { BlockProposal, CheckpointAttestation, CheckpointProposalCore } from '@aztec/stdlib/p2p';
+import type { ConsensusTimetable } from '@aztec/stdlib/timetable';
 import { MerkleTreeId } from '@aztec/stdlib/trees';
 import type { CheckpointGlobalVariables, FailedTx, Tx } from '@aztec/stdlib/tx';
 import {
@@ -151,7 +153,49 @@ type BlockProposalSlotValidationResult =
   | { isValid: true }
   | { isValid: false; reason: 'block_proposal_beyond_checkpoint' | 'checkpoint_proposal_equivocation' };
 
-/** Handles block and checkpoint proposals for both validator and non-validator nodes. */
+const MAX_TRACKED_INVALID_PROPOSAL_SLOTS = 1000;
+
+/** Block-proposal validation failures that constitute a slashable invalid-block offense. */
+export const SLASHABLE_BLOCK_PROPOSAL_VALIDATION_RESULT: BlockProposalValidationFailureReason[] = [
+  'state_mismatch',
+  'failed_txs',
+  'global_variables_mismatch',
+  'invalid_proposal',
+  'parent_block_wrong_slot',
+  'in_hash_mismatch',
+];
+
+/** Checkpoint-proposal validation failures that constitute a slashable invalid-checkpoint offense. */
+export const SLASHABLE_CHECKPOINT_PROPOSAL_VALIDATION_RESULT: Record<
+  CheckpointProposalValidationFailureReason,
+  boolean
+> = {
+  // enabled
+  ['invalid_fee_asset_price_modifier']: true,
+  ['checkpoint_header_mismatch']: true,
+  // These late mismatches should normally be caught by earlier checks, but if reached after validating the local
+  // checkpoint inputs, the proposer-signed payload disagrees with deterministic recomputation.
+  ['archive_mismatch']: true,
+  ['out_hash_mismatch']: true,
+  ['no_blocks_for_slot']: true,
+  ['too_many_blocks_in_checkpoint']: true,
+  ['checkpoint_validation_failed']: true,
+  ['last_block_archive_mismatch']: true,
+
+  // disabled
+  ['invalid_signature']: false,
+  ['last_block_not_found']: false,
+  ['block_fetch_error']: false,
+  ['checkpoint_already_published']: false,
+};
+
+/**
+ * Handles block and checkpoint proposals for both validator and non-validator nodes. Also tracks which slots
+ * had a slashable invalid proposal or a proposal equivocation, exposing them via the
+ * `InvalidProposalSlotSource` interface consumed by the attested-invalid-proposal slashing watcher. The
+ * tracking is populated as a side effect of validating/re-executing proposals, so any node that re-executes
+ * proposals (the default) can serve it — not only validators.
+ */
 export class ProposalHandler {
   public readonly tracer: Tracer;
 
@@ -164,7 +208,7 @@ export class ProposalHandler {
   };
 
   /** Archiver reference for setting proposed checkpoints (pipelining). Set via register(). */
-  private archiver?: Pick<Archiver, 'addProposedCheckpoint'>;
+  private archiver?: Pick<Archiver, 'addProposedCheckpoint' | 'getProposedCheckpointData'>;
 
   /** Returns current validator addresses for own-proposal detection. Set via register(). */
   private getOwnValidatorAddresses?: () => string[];
@@ -174,6 +218,12 @@ export class ProposalHandler {
 
   private checkpointProposalValidationFailureCallback?: CheckpointProposalValidationFailureCallback;
 
+  /** Slots at which a slashable invalid block or checkpoint proposal was observed. */
+  private readonly slotsWithInvalidProposals = FifoSet.withLimit<SlotNumber>(MAX_TRACKED_INVALID_PROPOSAL_SLOTS);
+
+  /** Slots at which a proposal equivocation was observed; suppresses attested-to-invalid-proposal slashing. */
+  private readonly slotsWithProposalEquivocation = FifoSet.withLimit<SlotNumber>(MAX_TRACKED_INVALID_PROPOSAL_SLOTS);
+
   constructor(
     private checkpointsBuilder: FullNodeCheckpointsBuilder,
     private worldState: WorldStateSynchronizer,
@@ -182,6 +232,7 @@ export class ProposalHandler {
     private txProvider: ITxProvider,
     private blockProposalValidator: BlockProposalValidator,
     private epochCache: EpochCache,
+    private timetable: ConsensusTimetable,
     private config: ValidatorClientFullConfig,
     private blobClient: BlobClientInterface,
     private reexecutionTracker: CheckpointReexecutionTracker,
@@ -219,6 +270,26 @@ export class ProposalHandler {
     this.reexecutionTracker.recordOutcome(slot, archive, 'valid', checkpointNumber);
   }
 
+  /** Whether a slashable invalid block or checkpoint proposal was observed at the given slot (InvalidProposalSlotSource). */
+  public hasInvalidProposals(slotNumber: SlotNumber): boolean {
+    return this.slotsWithInvalidProposals.has(slotNumber);
+  }
+
+  /** Whether a proposal equivocation was observed at the given slot (InvalidProposalSlotSource). */
+  public hasProposalEquivocation(slotNumber: SlotNumber): boolean {
+    return this.slotsWithProposalEquivocation.has(slotNumber);
+  }
+
+  /** Records a slot as having a slashable invalid proposal, for offense observers (sentinel/slasher watchers). */
+  public markInvalidProposalSlot(slotNumber: SlotNumber): void {
+    this.slotsWithInvalidProposals.add(slotNumber);
+  }
+
+  /** Records a slot as having a proposal equivocation, which suppresses attested-to-invalid-proposal slashing. */
+  public markProposalEquivocation(slotNumber: SlotNumber): void {
+    this.slotsWithProposalEquivocation.add(slotNumber);
+  }
+
   /**
    * Registers handlers for block and checkpoint proposals on the p2p client.
    * Records the p2p client so validation can inspect retained proposals.
@@ -230,7 +301,7 @@ export class ProposalHandler {
   register(
     p2pClient: P2P,
     shouldReexecute: boolean,
-    archiver?: Pick<Archiver, 'addProposedCheckpoint'>,
+    archiver?: Pick<Archiver, 'addProposedCheckpoint' | 'getProposedCheckpointData'>,
     getOwnValidatorAddresses?: () => string[],
   ): ProposalHandler {
     this.p2pClient = p2pClient;
@@ -254,6 +325,18 @@ export class ProposalHandler {
           });
           return true;
         } else {
+          // Track invalid proposals / equivocations so offense observers (the attested-invalid-proposal
+          // watcher) work on non-validator nodes too. Validators populate these via their own handlers.
+          // Skip invalid-proposal marking while the escape hatch is open, matching the validator path,
+          // which intentionally disables invalid-block slashing then.
+          if (result.reason === 'checkpoint_proposal_equivocation') {
+            this.markProposalEquivocation(slotNumber);
+          } else if (
+            SLASHABLE_BLOCK_PROPOSAL_VALIDATION_RESULT.includes(result.reason) &&
+            !(await this.epochCache.isEscapeHatchOpenAtSlot(slotNumber))
+          ) {
+            this.markInvalidProposalSlot(slotNumber);
+          }
           this.log.warn(
             `Non-validator block proposal ${blockNumber} at slot ${slotNumber} failed processing with ${result.reason}`,
             { blockNumber: result.blockNumber, slotNumber, reason: result.reason },
@@ -268,6 +351,11 @@ export class ProposalHandler {
 
     p2pClient.registerBlockProposalHandler(blockHandler);
 
+    // p2p detects duplicate (equivocated) proposals without routing them through the handlers above, so mark
+    // the slot as equivocated here. This suppresses false-positive attested-to-invalid-proposal slashing on
+    // non-validator offense collectors. Validators overwrite this with their own richer handler.
+    p2pClient.registerDuplicateProposalCallback(info => this.markProposalEquivocation(info.slot));
+
     // All-nodes checkpoint proposal handler: validates, caches, and sets proposed checkpoint for pipelining.
     // Runs for all nodes (validators and non-validators). Validators get the cached result in the
     // validator-specific callback (attestToCheckpointProposal) which runs after this one.
@@ -296,25 +384,35 @@ export class ProposalHandler {
           return undefined;
         }
 
-        // For own proposals, skip validation and return: the proposer already built and validated the
-        // checkpoint, and the sequencer's checkpoint proposal job pushed the proposed checkpoint to the
-        // archiver from local data before broadcasting. Gossipsub doesn't echo our own messages back, so
-        // this branch is normally unreachable — it remains as defense if an own proposal arrives by some
-        // other path.
+        // A proposal is "own" when it was signed by a validator key this node also owns. The true local
+        // proposer already built, validated, and stored this checkpoint before broadcasting, so a matching
+        // proposed checkpoint is already in its archiver — skip the redundant re-validation. An HA peer that
+        // shares the proposer's keys sees the same "own" proposal over gossip but never built it, so it has
+        // nothing stored; it falls through to the normal validate-and-persist path below to hydrate the
+        // proposed-checkpoint metadata it needs to build the next slot on top of this checkpoint.
         const proposer = proposal.getSender();
         const ownAddresses = this.getOwnValidatorAddresses?.();
         const isOwnProposal = proposer && ownAddresses?.some(addr => addr === proposer.toString());
 
         if (isOwnProposal) {
-          this.log.debug(`Skipping validation for own checkpoint proposal at slot ${proposal.slotNumber}`);
-          return undefined;
+          const existing = await this.archiver?.getProposedCheckpointData({ slot: proposal.slotNumber });
+          if (existing?.archive.root.equals(proposal.archive)) {
+            this.log.debug(`Skipping sync for existing own checkpoint proposal at slot ${proposal.slotNumber}`);
+            return undefined;
+          }
         }
 
         const result = await this.handleCheckpointProposal(proposal, proposalInfo);
         if (!result.isValid) {
+          // Track invalid checkpoint proposals so offense observers (the attested-invalid-proposal watcher)
+          // work on non-validator nodes too. This handler runs for all nodes; validators also mark via the
+          // failure callback below (idempotent).
+          if (SLASHABLE_CHECKPOINT_PROPOSAL_VALIDATION_RESULT[result.reason]) {
+            this.markInvalidProposalSlot(proposal.slotNumber);
+          }
           await this.checkpointProposalValidationFailureCallback?.(proposal, result, proposalInfo);
         } else if (this.archiver) {
-          const set = await this.setProposedCheckpointFromValidation(proposal);
+          const set = await this.setProposedCheckpoint(proposal);
           if (set) {
             this.metrics?.recordCheckpointProposalToPipelinedStateDuration(pipeliningTimer.ms());
           }
@@ -337,7 +435,6 @@ export class ProposalHandler {
   ): Promise<BlockProposalValidationResult> {
     const slotNumber = proposal.slotNumber;
     const proposer = proposal.getSender();
-    const config = this.checkpointsBuilder.getConfig();
 
     // Reject proposals with invalid signatures
     if (!proposer) {
@@ -405,8 +502,12 @@ export class ProposalHandler {
         : BlockNumber(parentBlock.header.getBlockNumber() + 1);
     proposalInfo.blockNumber = blockNumber;
 
-    // Check that this block number does not exist already
-    const existingBlock = await this.blockSource.getBlockData({ number: blockNumber });
+    // Check that this block number does not exist already. During a reorg the archiver can still hold a
+    // stale block at this number (a different archive, about to be pruned) while the proposal carries the
+    // rebuilt replacement; resolveExistingBlockAtNumber waits for the local prune in that case so the
+    // rebuilt block is processed in time to attest, rather than being permanently dropped on a bare
+    // number collision.
+    const existingBlock = await this.resolveExistingBlockAtNumber(blockNumber, proposal.archive, slotNumber);
     if (existingBlock) {
       this.log.warn(`Block number ${blockNumber} already exists, skipping processing`, proposalInfo);
       return { isValid: false, blockNumber, reason: 'block_number_already_exists' };
@@ -416,7 +517,7 @@ export class ProposalHandler {
     // and we do it even if we don't plan to re-execute the txs, so that we have them if another node needs them.
     const { txs, missingTxs } = await this.txProvider.getTxsForBlockProposal(proposal, blockNumber, {
       pinnedPeer: proposalSender,
-      deadline: this.getReexecutionDeadline(slotNumber, config),
+      deadline: this.getReexecutionDeadline(slotNumber),
     });
 
     // Record the tx-collection outcome on the re-execution tracker
@@ -490,7 +591,7 @@ export class ProposalHandler {
     }
 
     // If we succeeded, push this block into the archiver (unless disabled)
-    if (reexecutionResult?.block && this.config.skipPushProposedBlocksToArchiver === false) {
+    if (reexecutionResult?.block && !this.config.skipPushProposedBlocksToArchiver) {
       await this.blockSource.addBlock(reexecutionResult.block);
     }
 
@@ -524,16 +625,14 @@ export class ProposalHandler {
 
   private async getParentBlock(proposal: BlockProposal): Promise<'genesis' | BlockData | undefined> {
     const parentArchive = proposal.blockHeader.lastArchive.root;
-    const config = this.checkpointsBuilder.getConfig();
     const { genesisArchiveRoot } = await this.blockSource.getGenesisValues();
 
     if (parentArchive.equals(genesisArchiveRoot)) {
       return 'genesis';
     }
 
-    const deadline = this.getReexecutionDeadline(proposal.slotNumber, config);
-    const currentTime = this.dateProvider.now();
-    const timeoutDurationMs = deadline.getTime() - currentTime;
+    const deadline = this.getReexecutionDeadline(proposal.slotNumber);
+    const timeoutDurationMs = deadline.getTime() - this.dateProvider.now();
 
     try {
       return (
@@ -544,7 +643,7 @@ export class ProposalHandler {
               () =>
                 this.blockSource.syncImmediate().then(() => this.blockSource.getBlockData({ archive: parentArchive })),
               'force archiver sync',
-              timeoutDurationMs / 1000,
+              { deadline, dateProvider: this.dateProvider },
               0.5,
             ))
       );
@@ -558,6 +657,63 @@ export class ProposalHandler {
     }
   }
 
+  /**
+   * Resolves whether a block genuinely already exists at `blockNumber`. Returns the existing block only if
+   * it is a true duplicate of the proposal (matching archive). During a reorg the archiver can still hold a
+   * stale fork at this number (different archive) that is about to be pruned; in that case this forces L1
+   * sync and waits, bounded by the re-execution deadline, for the prune to land, then returns `undefined` so
+   * the rebuilt block proposal can be processed in time to attest. If the prune does not complete before the
+   * deadline it returns the stale block, so the caller falls back to the safe `block_number_already_exists`
+   * rejection.
+   */
+  private async resolveExistingBlockAtNumber(
+    blockNumber: BlockNumber,
+    proposalArchive: Fr,
+    slotNumber: SlotNumber,
+  ): Promise<BlockData | undefined> {
+    const existingBlock = await this.blockSource.getBlockData({ number: blockNumber });
+    if (!existingBlock || existingBlock.archive.root.equals(proposalArchive)) {
+      return existingBlock;
+    }
+
+    // A different block already occupies this number: it may be a stale fork being pruned during a reorg, not a
+    // genuine duplicate. Wait for the local prune rather than permanently rejecting the proposal.
+    const deadline = this.getReexecutionDeadline(slotNumber);
+    if (deadline.getTime() - this.dateProvider.now() <= 0) {
+      return existingBlock;
+    }
+
+    this.log.warn(`Block number ${blockNumber} already exists, awaiting potential prune`, {
+      blockNumber,
+      existingArchive: existingBlock.archive.root.toString(),
+      proposalArchive: proposalArchive.toString(),
+    });
+
+    try {
+      const { block } = await retryUntil(
+        async () => {
+          await this.blockSource.syncImmediate();
+          const block = await this.blockSource.getBlockData({ number: blockNumber });
+          // Resolve once the existing block is gone (pruned) or has been replaced by one matching the
+          // proposal — the same condition as the early return above. A matching block is returned so the
+          // caller still treats it as a genuine duplicate; an `undefined` (pruned) block lets the proposal
+          // be processed. Wrap in an object so the `undefined` case is still a truthy retry result.
+          return block === undefined || block.archive.root.equals(proposalArchive) ? { block } : undefined;
+        },
+        `prune of stale block ${blockNumber}`,
+        { deadline, dateProvider: this.dateProvider },
+        0.5,
+      );
+      return block;
+    } catch (err) {
+      if (err instanceof TimeoutError) {
+        this.log.warn(`Timed out waiting for stale block ${blockNumber} to be pruned`, { blockNumber });
+        return existingBlock;
+      }
+      throw err;
+    }
+  }
+
   private computeCheckpointNumber(
     proposal: BlockProposal,
     parentBlock: 'genesis' | BlockData,
@@ -681,15 +837,14 @@ export class ProposalHandler {
     return undefined;
   }
 
-  private getReexecutionDeadline(
-    slotNumber: SlotNumber,
-    config: { l1GenesisTime: bigint; slotDuration: number },
-  ): Date {
-    // Under proposer pipelining, the proposal slot may be ahead of wall clock time.
-    // Reexecution budgets should still be bounded by the current slot we are in now.
-    const wallclockSlot = slotNumber - PROPOSER_PIPELINING_SLOT_OFFSET;
-    const nextSlotTimestampSeconds = Number(getTimestampForSlot(SlotNumber(wallclockSlot + 1), config));
-    return new Date(nextSlotTimestampSeconds * 1000);
+  /**
+   * Hard re-execution/validation deadline for any block or checkpoint proposal targeting `slotNumber`:
+   * the single consensus `attestation_deadline` (`target_slot_start + S - 2E`). This is the latest the
+   * checkpoint can land on L1 in the target slot; all nodes agree on it. Loosened from the previous
+   * next-wall-clock-slot-boundary bound (see the timetable spec / refactor notes).
+   */
+  private getReexecutionDeadline(slotNumber: SlotNumber): Date {
+    return new Date(this.timetable.getAttestationDeadline(slotNumber) * 1000);
   }
 
   private getReexecuteFailureReason(err: any): BlockProposalValidationFailureReason {
@@ -769,7 +924,7 @@ export class ProposalHandler {
     );
 
     // Build the new block
-    const deadline = this.getReexecutionDeadline(slot, config);
+    const deadline = this.getReexecutionDeadline(slot);
     const maxBlockGas =
       this.config.validateMaxL2BlockGas !== undefined || this.config.validateMaxDABlockGas !== undefined
         ? new Gas(this.config.validateMaxDABlockGas ?? Infinity, this.config.validateMaxL2BlockGas ?? Infinity)
@@ -901,18 +1056,15 @@ export class ProposalHandler {
   ): Promise<CheckpointProposalValidationResult> {
     const slot = proposal.slotNumber;
 
-    // Block-sync deadline = the L1 publish deadline, i.e. the latest moment the proposer can submit
-    // this checkpoint and still have it land on L1 in the target slot. That is 12s (one Ethereum
-    // slot) before the last L1 block of the target slot, which is later than the target-slot start
-    // used for block re-execution. Keeping validation/attestation alive until then lets validators
-    // keep attesting right up to the proposer's real publish cutoff.
-    const l1Constants = this.epochCache.getL1Constants();
-    const publishDeadlineSeconds =
-      Number(getLastL1SlotTimestampForL2Slot(slot, l1Constants)) - l1Constants.ethereumSlotDuration;
-    const deadline = new Date(publishDeadlineSeconds * 1000);
-    const timeoutSeconds = Math.max(1, Math.floor((deadline.getTime() - this.dateProvider.now()) / 1000));
-
-    // Wait for last block to sync by archive
+    // Block-sync/validation deadline = the single consensus attestation_deadline (target_slot_start + S
+    // - 2E): the latest moment the proposer can submit this checkpoint and still have it land on L1 in
+    // the target slot. Keeping validation/attestation alive until then lets validators keep attesting
+    // right up to the proposer's real publish cutoff.
+    const deadline = this.getReexecutionDeadline(slot);
+
+    // Wait for last block to sync by archive. The deadline is passed to retryUntil as an absolute date so
+    // the remaining budget is derived from the date provider; a deadline already in the past times out
+    // after a single attempt instead of looping (the immediate-timeout semantics of the deadline overload).
     let lastBlockData;
     try {
       lastBlockData = await retryUntil(
@@ -921,7 +1073,7 @@ export class ProposalHandler {
           return await this.blockSource.getBlockData({ archive: proposal.archive });
         },
         `waiting for block with archive ${proposal.archive.toString()} for slot ${slot}`,
-        timeoutSeconds,
+        { deadline, dateProvider: this.dateProvider },
         0.5,
       );
     } catch (err) {
@@ -969,6 +1121,7 @@ export class ProposalHandler {
       };
     }
 
+    // Note this condition should never trigger, since we dont process block proposals that exceed indexWithinCheckpoint
     const maxBlocksPerCheckpoint = this.config.maxBlocksPerCheckpoint;
     if (maxBlocksPerCheckpoint !== undefined && blocks.length > maxBlocksPerCheckpoint) {
       this.log.warn(`Checkpoint proposal exceeds maxBlocksPerCheckpoint`, {
@@ -1132,11 +1285,11 @@ export class ProposalHandler {
   }
 
   /**
-   * Derives proposed checkpoint data from validated blocks and sets it on the archiver.
-   * Used after successful validation of a foreign proposal.
-   * Does not retry since we already waited for the block during validation.
+   * Derives proposed checkpoint data from validated blocks and sets it on the archiver, so this node can
+   * pipeline building on top of the checkpoint. Does not retry, since validation already waited for the
+   * last block to sync.
    */
-  private async setProposedCheckpointFromValidation(proposal: CheckpointProposalCore): Promise<boolean> {
+  private async setProposedCheckpoint(proposal: CheckpointProposalCore): Promise<boolean> {
     if (!this.archiver) {
       return false;
     }