@aztec/aztec-node 0.0.1-commit.8afd444 → 0.0.1-commit.934299a21

package/src/aztec-node/server.ts

@@ -1,6 +1,7 @@
 import { Archiver, createArchiver } from '@aztec/archiver';
 import { BBCircuitVerifier, QueuedIVCVerifier, TestCircuitVerifier } from '@aztec/bb-prover';
 import { type BlobClientInterface, createBlobClientWithFileStores } from '@aztec/blob-client/client';
+import { Blob } from '@aztec/blob-lib';
 import { ARCHIVE_HEIGHT, type L1_TO_L2_MSG_TREE_HEIGHT, type NOTE_HASH_TREE_HEIGHT } from '@aztec/constants';
 import { EpochCache, type EpochCacheInterface } from '@aztec/epoch-cache';
 import { createEthereumChain } from '@aztec/ethereum/chain';
@@ -8,7 +9,7 @@ import { getPublicClient } from '@aztec/ethereum/client';
 import { RegistryContract, RollupContract } from '@aztec/ethereum/contracts';
 import type { L1ContractAddresses } from '@aztec/ethereum/l1-contract-addresses';
 import { BlockNumber, CheckpointNumber, EpochNumber, SlotNumber } from '@aztec/foundation/branded-types';
-import { compactArray, pick } from '@aztec/foundation/collection';
+import { compactArray, pick, unique } from '@aztec/foundation/collection';
 import { Fr } from '@aztec/foundation/curves/bn254';
 import { EthAddress } from '@aztec/foundation/eth-address';
 import { BadRequestError } from '@aztec/foundation/json-rpc';
@@ -16,14 +17,19 @@ import { type Logger, createLogger } from '@aztec/foundation/log';
 import { count } from '@aztec/foundation/string';
 import { DateProvider, Timer } from '@aztec/foundation/timer';
 import { MembershipWitness, SiblingPath } from '@aztec/foundation/trees';
-import { KeystoreManager, loadKeystores, mergeKeystores } from '@aztec/node-keystore';
+import { type KeyStore, KeystoreManager, loadKeystores, mergeKeystores } from '@aztec/node-keystore';
 import { trySnapshotSync, uploadSnapshot } from '@aztec/node-lib/actions';
+import { createForwarderL1TxUtilsFromSigners, createL1TxUtilsFromSigners } from '@aztec/node-lib/factories';
 import {
-  createForwarderL1TxUtilsFromEthSigner,
-  createL1TxUtilsWithBlobsFromEthSigner,
-} from '@aztec/node-lib/factories';
-import { type P2P, type P2PClientDeps, createP2PClient, getDefaultAllowedSetupFunctions } from '@aztec/p2p';
+  type P2P,
+  type P2PClientDeps,
+  createP2PClient,
+  createTxValidatorForAcceptingTxsOverRPC,
+  getDefaultAllowedSetupFunctions,
+} from '@aztec/p2p';
 import { ProtocolContractAddress } from '@aztec/protocol-contracts';
+import { type ProverNode, type ProverNodeDeps, createProverNode } from '@aztec/prover-node';
+import { createKeyStoreForProver } from '@aztec/prover-node/config';
 import { GlobalVariableBuilder, SequencerClient, type SequencerPublisher } from '@aztec/sequencer-client';
 import { PublicProcessorFactory } from '@aztec/simulator/server';
 import {
@@ -35,7 +41,14 @@ import {
 } from '@aztec/slasher';
 import { CollectionLimitsConfig, PublicSimulatorConfig } from '@aztec/stdlib/avm';
 import { AztecAddress } from '@aztec/stdlib/aztec-address';
-import { BlockHash, type BlockParameter, type DataInBlock, L2Block, type L2BlockSource } from '@aztec/stdlib/block';
+import {
+  type BlockData,
+  BlockHash,
+  type BlockParameter,
+  type DataInBlock,
+  L2Block,
+  type L2BlockSource,
+} from '@aztec/stdlib/block';
 import type { PublishedCheckpoint } from '@aztec/stdlib/checkpoint';
 import type {
   ContractClassPublic,
@@ -63,7 +76,8 @@ import {
   type WorldStateSynchronizer,
   tryStop,
 } from '@aztec/stdlib/interfaces/server';
-import type { LogFilter, SiloedTag, Tag, TxScopedL2Log } from '@aztec/stdlib/logs';
+import type { DebugLogStore, LogFilter, SiloedTag, Tag, TxScopedL2Log } from '@aztec/stdlib/logs';
+import { InMemoryDebugLogStore, NullDebugLogStore } from '@aztec/stdlib/logs';
 import { InboxLeaf, type L1ToL2MessageSource } from '@aztec/stdlib/messaging';
 import { P2PClientType } from '@aztec/stdlib/p2p';
 import type { Offense, SlashPayloadRound } from '@aztec/stdlib/slashing';
@@ -97,7 +111,6 @@ import {
   ValidatorClient,
   createBlockProposalHandler,
   createValidatorClient,
-  createValidatorForAcceptingTxs,
 } from '@aztec/validator-client';
 import { createWorldStateSynchronizer } from '@aztec/world-state';
 
@@ -129,6 +142,7 @@ export class AztecNodeService implements AztecNode, AztecNodeAdmin, Traceable {
     protected readonly l1ToL2MessageSource: L1ToL2MessageSource,
     protected readonly worldStateSynchronizer: WorldStateSynchronizer,
     protected readonly sequencer: SequencerClient | undefined,
+    protected readonly proverNode: ProverNode | undefined,
     protected readonly slasherClient: SlasherClientInterface | undefined,
     protected readonly validatorsSentinel: Sentinel | undefined,
     protected readonly epochPruneWatcher: EpochPruneWatcher | undefined,
@@ -141,12 +155,22 @@ export class AztecNodeService implements AztecNode, AztecNodeAdmin, Traceable {
     private telemetry: TelemetryClient = getTelemetryClient(),
     private log = createLogger('node'),
     private blobClient?: BlobClientInterface,
+    private validatorClient?: ValidatorClient,
+    private keyStoreManager?: KeystoreManager,
+    private debugLogStore: DebugLogStore = new NullDebugLogStore(),
   ) {
     this.metrics = new NodeMetrics(telemetry, 'AztecNodeService');
     this.tracer = telemetry.getTracer('AztecNodeService');
 
     this.log.info(`Aztec Node version: ${this.packageVersion}`);
     this.log.info(`Aztec Node started on chain 0x${l1ChainId.toString(16)}`, config.l1Contracts);
+
+    // A defensive check that protects us against introducing a bug in the complex `createAndSync` function. We must
+    // never have debugLogStore enabled when not in test mode because then we would be accumulating debug logs in
+    // memory which could be a DoS vector on the sequencer (since no fees are paid for debug logs).
+    if (debugLogStore.isEnabled && config.realProofs) {
+      throw new Error('debugLogStore should never be enabled when realProofs are set');
+    }
   }
 
   public async getWorldStateSyncStatus(): Promise<WorldStateSyncStatus> {
@@ -171,10 +195,12 @@ export class AztecNodeService implements AztecNode, AztecNodeAdmin, Traceable {
       publisher?: SequencerPublisher;
       dateProvider?: DateProvider;
       p2pClientDeps?: P2PClientDeps<P2PClientType.Full>;
+      proverNodeDeps?: Partial<ProverNodeDeps>;
     } = {},
     options: {
       prefilledPublicData?: PublicDataTreeLeaf[];
       dontStartSequencer?: boolean;
+      dontStartProverNode?: boolean;
     } = {},
   ): Promise<AztecNodeService> {
     const config = { ...inputConfig }; // Copy the config so we dont mutate the input object
@@ -184,16 +210,29 @@ export class AztecNodeService implements AztecNode, AztecNodeAdmin, Traceable {
     const dateProvider = deps.dateProvider ?? new DateProvider();
     const ethereumChain = createEthereumChain(config.l1RpcUrls, config.l1ChainId);
 
-    // Build a key store from file if given or from environment otherwise
+    // Build a key store from file if given or from environment otherwise.
+    // We keep the raw KeyStore available so we can merge with prover keys if enableProverNode is set.
     let keyStoreManager: KeystoreManager | undefined;
     const keyStoreProvided = config.keyStoreDirectory !== undefined && config.keyStoreDirectory.length > 0;
     if (keyStoreProvided) {
       const keyStores = loadKeystores(config.keyStoreDirectory!);
       keyStoreManager = new KeystoreManager(mergeKeystores(keyStores));
     } else {
-      const keyStore = createKeyStoreForValidator(config);
-      if (keyStore) {
-        keyStoreManager = new KeystoreManager(keyStore);
+      const rawKeyStores: KeyStore[] = [];
+      const validatorKeyStore = createKeyStoreForValidator(config);
+      if (validatorKeyStore) {
+        rawKeyStores.push(validatorKeyStore);
+      }
+      if (config.enableProverNode) {
+        const proverKeyStore = createKeyStoreForProver(config);
+        if (proverKeyStore) {
+          rawKeyStores.push(proverKeyStore);
+        }
+      }
+      if (rawKeyStores.length > 0) {
+        keyStoreManager = new KeystoreManager(
+          rawKeyStores.length === 1 ? rawKeyStores[0] : mergeKeystores(rawKeyStores),
+        );
       }
     }
 
@@ -204,10 +243,8 @@ export class AztecNodeService implements AztecNode, AztecNodeAdmin, Traceable {
       if (keyStoreManager === undefined) {
         throw new Error('Failed to create key store, a requirement for running a validator');
       }
-      if (!keyStoreProvided) {
-        log.warn(
-          'KEY STORE CREATED FROM ENVIRONMENT, IT IS RECOMMENDED TO USE A FILE-BASED KEY STORE IN PRODUCTION ENVIRONMENTS',
-        );
+      if (!keyStoreProvided && process.env.NODE_ENV !== 'test') {
+        log.warn("Keystore created from env: it's recommended to use a file-based key store for production");
       }
       ValidatorClient.validateKeyStoreConfiguration(keyStoreManager, log);
     }
@@ -249,7 +286,7 @@ export class AztecNodeService implements AztecNode, AztecNodeAdmin, Traceable {
       );
     }
 
-    const blobClient = await createBlobClientWithFileStores(config, createLogger('node:blob-client:client'));
+    const blobClient = await createBlobClientWithFileStores(config, log.createChild('blob-client'));
 
     // attempt snapshot sync if possible
     await trySnapshotSync(config, log);
@@ -273,9 +310,19 @@ export class AztecNodeService implements AztecNode, AztecNodeAdmin, Traceable {
       config.realProofs || config.debugForceTxProofVerification
         ? await BBCircuitVerifier.new(config)
         : new TestCircuitVerifier(config.proverTestVerificationDelayMs);
+
+    let debugLogStore: DebugLogStore;
     if (!config.realProofs) {
       log.warn(`Aztec node is accepting fake proofs`);
+
+      debugLogStore = new InMemoryDebugLogStore();
+      log.info(
+        'Aztec node started in test mode (realProofs set to false) hence debug logs from public functions will be collected and served',
+      );
+    } else {
+      debugLogStore = new NullDebugLogStore();
     }
+
     const proofVerifier = new QueuedIVCVerifier(config, circuitVerifier);
 
     // create the tx pool and the p2p client, which will need the l2 block source
@@ -412,19 +459,19 @@ export class AztecNodeService implements AztecNode, AztecNodeAdmin, Traceable {
       );
       await slasherClient.start();
 
-      const l1TxUtils = config.publisherForwarderAddress
-        ? await createForwarderL1TxUtilsFromEthSigner(
+      const l1TxUtils = config.sequencerPublisherForwarderAddress
+        ? await createForwarderL1TxUtilsFromSigners(
             publicClient,
             keyStoreManager!.createAllValidatorPublisherSigners(),
-            config.publisherForwarderAddress,
+            config.sequencerPublisherForwarderAddress,
             { ...config, scope: 'sequencer' },
-            { telemetry, logger: log.createChild('l1-tx-utils'), dateProvider },
+            { telemetry, logger: log.createChild('l1-tx-utils'), dateProvider, kzg: Blob.getViemKzgInstance() },
           )
-        : await createL1TxUtilsWithBlobsFromEthSigner(
+        : await createL1TxUtilsFromSigners(
             publicClient,
             keyStoreManager!.createAllValidatorPublisherSigners(),
             { ...config, scope: 'sequencer' },
-            { telemetry, logger: log.createChild('l1-tx-utils'), dateProvider },
+            { telemetry, logger: log.createChild('l1-tx-utils'), dateProvider, kzg: Blob.getViemKzgInstance() },
           );
 
       // Create and start the sequencer client
@@ -434,6 +481,7 @@ export class AztecNodeService implements AztecNode, AztecNodeAdmin, Traceable {
         archiver,
         dateProvider,
         telemetry,
+        debugLogStore,
       );
 
       sequencer = await SequencerClient.new(config, {
@@ -461,6 +509,29 @@ export class AztecNodeService implements AztecNode, AztecNodeAdmin, Traceable {
       log.warn(`Sequencer created but not started`);
     }
 
+    // Create prover node subsystem if enabled
+    let proverNode: ProverNode | undefined;
+    if (config.enableProverNode) {
+      proverNode = await createProverNode(config, {
+        ...deps.proverNodeDeps,
+        telemetry,
+        dateProvider,
+        archiver,
+        worldStateSynchronizer,
+        p2pClient,
+        epochCache,
+        blobClient,
+        keyStoreManager,
+      });
+
+      if (!options.dontStartProverNode) {
+        await proverNode.start();
+        log.info(`Prover node subsystem started`);
+      } else {
+        log.info(`Prover node subsystem created but not started`);
+      }
+    }
+
     const globalVariableBuilder = new GlobalVariableBuilder({
       ...config,
       rollupVersion: BigInt(config.rollupVersion),
@@ -468,7 +539,7 @@ export class AztecNodeService implements AztecNode, AztecNodeAdmin, Traceable {
       slotDuration: Number(slotDuration),
     });
 
-    return new AztecNodeService(
+    const node = new AztecNodeService(
       config,
       p2pClient,
       archiver,
@@ -477,6 +548,7 @@ export class AztecNodeService implements AztecNode, AztecNodeAdmin, Traceable {
       archiver,
       worldStateSynchronizer,
       sequencer,
+      proverNode,
       slasherClient,
       validatorsSentinel,
       epochPruneWatcher,
@@ -489,7 +561,12 @@ export class AztecNodeService implements AztecNode, AztecNodeAdmin, Traceable {
       telemetry,
       log,
       blobClient,
+      validatorClient,
+      keyStoreManager,
+      debugLogStore,
     );
+
+    return node;
   }
 
   /**
@@ -500,6 +577,11 @@ export class AztecNodeService implements AztecNode, AztecNodeAdmin, Traceable {
     return this.sequencer;
   }
 
+  /** Returns the prover node subsystem, if enabled. */
+  public getProverNode(): ProverNode | undefined {
+    return this.proverNode;
+  }
+
   public getBlockSource(): L2BlockSource {
     return this.blockSource;
   }
@@ -553,6 +635,7 @@ export class AztecNodeService implements AztecNode, AztecNodeAdmin, Traceable {
       enr,
       l1ContractAddresses: contractAddresses,
       protocolContractAddresses: protocolContractAddresses,
+      realProofs: !!this.config.realProofs,
     };
 
     return nodeInfo;
@@ -774,18 +857,22 @@ export class AztecNodeService implements AztecNode, AztecNodeAdmin, Traceable {
     // Then get the actual tx from the archiver, which tracks every tx in a mined block.
     const settledTxReceipt = await this.blockSource.getSettledTxReceipt(txHash);
 
+    let receipt: TxReceipt;
     if (settledTxReceipt) {
-      // If the archiver has the receipt then return it.
-      return settledTxReceipt;
+      receipt = settledTxReceipt;
     } else if (isKnownToPool) {
       // If the tx is in the pool but not in the archiver, it's pending.
       // This handles race conditions between archiver and p2p, where the archiver
       // has pruned the block in which a tx was mined, but p2p has not caught up yet.
-      return new TxReceipt(txHash, TxStatus.PENDING, undefined, undefined);
+      receipt = new TxReceipt(txHash, TxStatus.PENDING, undefined, undefined);
     } else {
       // Otherwise, if we don't know the tx, we consider it dropped.
-      return new TxReceipt(txHash, TxStatus.DROPPED, undefined, 'Tx dropped by P2P node');
+      receipt = new TxReceipt(txHash, TxStatus.DROPPED, undefined, 'Tx dropped by P2P node');
     }
+
+    this.debugLogStore.decorateReceiptWithLogs(txHash.toString(), receipt);
+
+    return receipt;
   }
 
   public getTxEffect(txHash: TxHash): Promise<IndexedTxEffect | undefined> {
@@ -802,6 +889,7 @@ export class AztecNodeService implements AztecNode, AztecNodeAdmin, Traceable {
     await tryStop(this.slasherClient);
     await tryStop(this.proofVerifier);
     await tryStop(this.sequencer);
+    await tryStop(this.proverNode);
     await tryStop(this.p2pClient);
     await tryStop(this.worldStateSynchronizer);
     await tryStop(this.blockSource);
@@ -1105,6 +1193,14 @@ export class AztecNodeService implements AztecNode, AztecNodeAdmin, Traceable {
     return await this.blockSource.getBlockHeaderByArchive(archive);
   }
 
+  public getBlockData(number: BlockNumber): Promise<BlockData | undefined> {
+    return this.blockSource.getBlockData(number);
+  }
+
+  public getBlockDataByArchive(archive: Fr): Promise<BlockData | undefined> {
+    return this.blockSource.getBlockDataByArchive(archive);
+  }
+
   /**
    * Simulates the public part of a transaction with the current state.
    * @param tx - The transaction to simulate.
@@ -1128,7 +1224,8 @@ export class AztecNodeService implements AztecNode, AztecNodeAdmin, Traceable {
     }
 
     const txHash = tx.getTxHash();
-    const blockNumber = BlockNumber((await this.blockSource.getBlockNumber()) + 1);
+    const latestBlockNumber = await this.blockSource.getBlockNumber();
+    const blockNumber = BlockNumber.add(latestBlockNumber, 1);
 
     // If sequencer is not initialized, we just set these values to zero for simulation.
     const coinbase = EthAddress.ZERO;
@@ -1152,6 +1249,8 @@ export class AztecNodeService implements AztecNode, AztecNodeAdmin, Traceable {
       blockNumber,
     });
 
+    // Ensure world-state has caught up with the latest block we loaded from the archiver
+    await this.worldStateSynchronizer.syncImmediate(latestBlockNumber);
     const merkleTreeFork = await this.worldStateSynchronizer.fork();
     try {
       const config = PublicSimulatorConfig.from({
@@ -1167,7 +1266,7 @@ export class AztecNodeService implements AztecNode, AztecNodeAdmin, Traceable {
       const processor = publicProcessorFactory.create(merkleTreeFork, newGlobalVariables, config);
 
       // REFACTOR: Consider merging ProcessReturnValues into ProcessedTx
-      const [processedTxs, failedTxs, _usedTxs, returns] = await processor.process([tx]);
+      const [processedTxs, failedTxs, _usedTxs, returns, _blobFields, debugLogs] = await processor.process([tx]);
       // REFACTOR: Consider returning the error rather than throwing
       if (failedTxs.length) {
         this.log.warn(`Simulated tx ${txHash} fails: ${failedTxs[0].error}`, { txHash });
@@ -1181,6 +1280,7 @@ export class AztecNodeService implements AztecNode, AztecNodeAdmin, Traceable {
         processedTx.txEffect,
         returns,
         processedTx.gasUsed,
+        debugLogs,
       );
     } finally {
       await merkleTreeFork.close();
@@ -1194,10 +1294,10 @@ export class AztecNodeService implements AztecNode, AztecNodeAdmin, Traceable {
     const db = this.worldStateSynchronizer.getCommitted();
     const verifier = isSimulation ? undefined : this.proofVerifier;
 
-    // We accept transactions if they are not expired by the next slot (checked based on the IncludeByTimestamp field)
+    // We accept transactions if they are not expired by the next slot (checked based on the ExpirationTimestamp field)
     const { ts: nextSlotTimestamp } = this.epochCache.getEpochAndSlotInNextL1Slot();
     const blockNumber = BlockNumber((await this.blockSource.getBlockNumber()) + 1);
-    const validator = createValidatorForAcceptingTxs(
+    const validator = createTxValidatorForAcceptingTxsOverRPC(
       db,
       this.contractDataSource,
       verifier,
@@ -1376,6 +1476,94 @@ export class AztecNodeService implements AztecNode, AztecNodeAdmin, Traceable {
     }
   }
 
+  public async reloadKeystore(): Promise<void> {
+    if (!this.config.keyStoreDirectory?.length) {
+      throw new BadRequestError(
+        'Cannot reload keystore: node is not using a file-based keystore. ' +
+          'Set KEY_STORE_DIRECTORY to use file-based keystores.',
+      );
+    }
+    if (!this.validatorClient) {
+      throw new BadRequestError('Cannot reload keystore: validator is not enabled.');
+    }
+
+    this.log.info('Reloading keystore from disk');
+
+    // Re-read and validate keystore files
+    const keyStores = loadKeystores(this.config.keyStoreDirectory);
+    const newManager = new KeystoreManager(mergeKeystores(keyStores));
+    await newManager.validateSigners();
+    ValidatorClient.validateKeyStoreConfiguration(newManager, this.log);
+
+    // Validate that every validator's publisher keys overlap with the L1 signers
+    // that were initialized at startup. Publishers cannot be hot-reloaded, so a
+    // validator with a publisher key that doesn't match any existing L1 signer
+    // would silently fail on every proposer slot.
+    if (this.keyStoreManager && this.sequencer) {
+      const oldAdapter = NodeKeystoreAdapter.fromKeyStoreManager(this.keyStoreManager);
+      const availablePublishers = new Set(
+        oldAdapter
+          .getAttesterAddresses()
+          .flatMap(a => oldAdapter.getPublisherAddresses(a).map(p => p.toString().toLowerCase())),
+      );
+
+      const newAdapter = NodeKeystoreAdapter.fromKeyStoreManager(newManager);
+      for (const attester of newAdapter.getAttesterAddresses()) {
+        const pubs = newAdapter.getPublisherAddresses(attester);
+        if (pubs.length > 0 && !pubs.some(p => availablePublishers.has(p.toString().toLowerCase()))) {
+          throw new BadRequestError(
+            `Cannot reload keystore: validator ${attester} has publisher keys ` +
+              `[${pubs.map(p => p.toString()).join(', ')}] but none match the L1 signers initialized at startup ` +
+              `[${[...availablePublishers].join(', ')}]. Publishers cannot be hot-reloaded — ` +
+              `use an existing publisher key or restart the node.`,
+          );
+        }
+      }
+    }
+
+    // Build adapters for old and new keystores to compute diff
+    const newAdapter = NodeKeystoreAdapter.fromKeyStoreManager(newManager);
+    const newAddresses = newAdapter.getAttesterAddresses();
+    const oldAddresses = this.keyStoreManager
+      ? NodeKeystoreAdapter.fromKeyStoreManager(this.keyStoreManager).getAttesterAddresses()
+      : [];
+
+    const oldSet = new Set(oldAddresses.map(a => a.toString()));
+    const newSet = new Set(newAddresses.map(a => a.toString()));
+    const added = newAddresses.filter(a => !oldSet.has(a.toString()));
+    const removed = oldAddresses.filter(a => !newSet.has(a.toString()));
+
+    if (added.length > 0) {
+      this.log.info(`Keystore reload: adding attester keys: ${added.map(a => a.toString()).join(', ')}`);
+    }
+    if (removed.length > 0) {
+      this.log.info(`Keystore reload: removing attester keys: ${removed.map(a => a.toString()).join(', ')}`);
+    }
+    if (added.length === 0 && removed.length === 0) {
+      this.log.info('Keystore reload: attester keys unchanged');
+    }
+
+    // Update the validator client (coinbase, feeRecipient, attester keys)
+    this.validatorClient.reloadKeystore(newManager);
+
+    // Update the publisher factory's keystore so newly-added validators
+    // can be matched to existing publisher keys when proposing blocks.
+    if (this.sequencer) {
+      this.sequencer.updatePublisherNodeKeyStore(newAdapter);
+    }
+
+    // Update slasher's "don't-slash-self" list with new validator addresses
+    if (this.slasherClient && !this.config.slashSelfAllowed) {
+      const slashValidatorsNever = unique(
+        [...(this.config.slashValidatorsNever ?? []), ...newAddresses].map(a => a.toString()),
+      ).map(EthAddress.fromString);
+      this.slasherClient.updateConfig({ slashValidatorsNever });
+    }
+
+    this.keyStoreManager = newManager;
+    this.log.info('Keystore reloaded: coinbase, feeRecipient, and attester keys updated');
+  }
+
   #getInitialHeaderHash(): Promise<BlockHash> {
     if (!this.initialHeaderHashPromise) {
       this.initialHeaderHashPromise = this.worldStateSynchronizer.getCommitted().getInitialHeader().hash();

package/src/sentinel/sentinel.ts

@@ -36,6 +36,17 @@ import EventEmitter from 'node:events';
 
 import { SentinelStore } from './store.js';
 
+/** Maps a validator status to its category: proposer or attestation. */
+function statusToCategory(status: ValidatorStatusInSlot): ValidatorStatusType {
+  switch (status) {
+    case 'attestation-sent':
+    case 'attestation-missed':
+      return 'attestation';
+    default:
+      return 'proposer';
+  }
+}
+
 export class Sentinel extends (EventEmitter as new () => WatcherEmitter) implements L2BlockStreamEventHandler, Watcher {
   protected runningPromise: RunningPromise;
   protected blockStream!: L2BlockStream;
@@ -128,15 +139,15 @@ export class Sentinel extends (EventEmitter as new () => WatcherEmitter) impleme
       return;
     }
     const blockNumber = event.block.number;
-    const block = await this.archiver.getL2Block(blockNumber);
-    if (!block) {
-      this.logger.error(`Failed to get block ${blockNumber}`, { block });
+    const header = await this.archiver.getBlockHeader(blockNumber);
+    if (!header) {
+      this.logger.error(`Failed to get block header ${blockNumber}`);
       return;
     }
 
     // TODO(palla/slash): We should only be computing proven performance if this is
     // a full proof epoch and not a partial one, otherwise we'll end up with skewed stats.
-    const epoch = getEpochAtSlot(block.header.getSlot(), this.epochCache.getL1Constants());
+    const epoch = getEpochAtSlot(header.getSlot(), this.epochCache.getL1Constants());
     this.logger.debug(`Computing proven performance for epoch ${epoch}`);
     const performance = await this.computeProvenPerformance(epoch);
     this.logger.info(`Computed proven performance for epoch ${epoch}`, performance);
@@ -147,7 +158,11 @@ export class Sentinel extends (EventEmitter as new () => WatcherEmitter) impleme
 
   protected async computeProvenPerformance(epoch: EpochNumber): Promise<ValidatorsEpochPerformance> {
     const [fromSlot, toSlot] = getSlotRangeForEpoch(epoch, this.epochCache.getL1Constants());
-    const { committee } = await this.epochCache.getCommittee(fromSlot);
+    const { committee, isEscapeHatchOpen } = await this.epochCache.getCommittee(fromSlot);
+    if (isEscapeHatchOpen) {
+      this.logger.info(`Skipping proven performance for epoch ${epoch} - escape hatch is open`);
+      return {};
+    }
     if (!committee) {
       this.logger.trace(`No committee found for slot ${fromSlot}`);
       return {};
@@ -316,7 +331,12 @@ export class Sentinel extends (EventEmitter as new () => WatcherEmitter) impleme
    * and updates overall stats.
    */
   protected async processSlot(slot: SlotNumber) {
-    const { epoch, seed, committee } = await this.epochCache.getCommittee(slot);
+    const { epoch, seed, committee, isEscapeHatchOpen } = await this.epochCache.getCommittee(slot);
+    if (isEscapeHatchOpen) {
+      this.logger.info(`Skipping slot ${slot} at epoch ${epoch} - escape hatch is open`);
+      this.lastProcessedSlot = slot;
+      return;
+    }
     if (!committee || committee.length === 0) {
       this.logger.trace(`No committee found for slot ${slot} at epoch ${epoch}`);
       this.lastProcessedSlot = slot;
@@ -336,16 +356,16 @@ export class Sentinel extends (EventEmitter as new () => WatcherEmitter) impleme
 
     // Check if there is an L2 block in L1 for this L2 slot
 
-    // Here we get all attestations for the block mined at the given slot,
-    // or all attestations for all proposals in the slot if no block was mined.
+    // Here we get all checkpoint attestations for the checkpoint at the given slot,
+    // or all checkpoint attestations for all proposals in the slot if no checkpoint was mined.
     // We gather from both p2p (contains the ones seen on the p2p layer) and archiver
-    // (contains the ones synced from mined blocks, which we may have missed from p2p).
-    const block = this.slotNumberToCheckpoint.get(slot);
-    const p2pAttested = await this.p2p.getCheckpointAttestationsForSlot(slot, block?.archive);
+    // (contains the ones synced from mined checkpoints, which we may have missed from p2p).
+    const checkpoint = this.slotNumberToCheckpoint.get(slot);
+    const p2pAttested = await this.p2p.getCheckpointAttestationsForSlot(slot, checkpoint?.archive);
     // Filter out attestations with invalid signatures
     const p2pAttestors = p2pAttested.map(a => a.getSender()).filter((s): s is EthAddress => s !== undefined);
     const attestors = new Set(
-      [...p2pAttestors.map(a => a.toString()), ...(block?.attestors.map(a => a.toString()) ?? [])].filter(
+      [...p2pAttestors.map(a => a.toString()), ...(checkpoint?.attestors.map(a => a.toString()) ?? [])].filter(
         addr => proposer.toString() !== addr, // Exclude the proposer from the attestors
       ),
     );
@@ -356,20 +376,29 @@ export class Sentinel extends (EventEmitter as new () => WatcherEmitter) impleme
     // But we'll leave that corner case out to reduce pressure on the node.
     // TODO(palla/slash): This breaks if a given node has more than one validator in the current committee,
     // since they will attest to their own proposal it even if it's not re-executable.
-    const blockStatus = block ? 'mined' : attestors.size > 0 ? 'proposed' : 'missed';
-    this.logger.debug(`Block for slot ${slot} was ${blockStatus}`, { ...block, slot });
+    let status: 'checkpoint-mined' | 'checkpoint-proposed' | 'checkpoint-missed' | 'blocks-missed';
+    if (checkpoint) {
+      status = 'checkpoint-mined';
+    } else if (attestors.size > 0) {
+      status = 'checkpoint-proposed';
+    } else {
+      // No checkpoint on L1 and no checkpoint attestations seen. Check if block proposals were sent for this slot.
+      const hasBlockProposals = await this.p2p.hasBlockProposalsForSlot(slot);
+      status = hasBlockProposals ? 'checkpoint-missed' : 'blocks-missed';
+    }
+    this.logger.debug(`Checkpoint status for slot ${slot}: ${status}`, { ...checkpoint, slot });
 
-    // Get attestors that failed their duties for this block, but only if there was a block proposed
+    // Get attestors that failed their checkpoint attestation duties, but only if there was a checkpoint proposed or mined
     const missedAttestors = new Set(
-      blockStatus === 'missed'
+      status === 'blocks-missed' || status === 'checkpoint-missed'
         ? []
         : committee.filter(v => !attestors.has(v.toString()) && !proposer.equals(v)).map(v => v.toString()),
     );
 
     this.logger.debug(`Retrieved ${attestors.size} attestors out of ${committee.length} for slot ${slot}`, {
-      blockStatus,
+      status,
       proposer: proposer.toString(),
-      ...block,
+      ...checkpoint,
       slot,
       attestors: [...attestors],
       missedAttestors: [...missedAttestors],
@@ -379,7 +408,7 @@ export class Sentinel extends (EventEmitter as new () => WatcherEmitter) impleme
     // Compute the status for each validator in the committee
     const statusFor = (who: `0x${string}`): ValidatorStatusInSlot | undefined => {
       if (who === proposer.toString()) {
-        return `block-${blockStatus}`;
+        return status;
       } else if (attestors.has(who)) {
         return 'attestation-sent';
       } else if (missedAttestors.has(who)) {
@@ -472,14 +501,16 @@ export class Sentinel extends (EventEmitter as new () => WatcherEmitter) impleme
   ): ValidatorStats {
     let history = fromSlot ? allHistory.filter(h => BigInt(h.slot) >= fromSlot) : allHistory;
     history = toSlot ? history.filter(h => BigInt(h.slot) <= toSlot) : history;
-    const lastProposal = history.filter(h => h.status === 'block-proposed' || h.status === 'block-mined').at(-1);
+    const lastProposal = history
+      .filter(h => h.status === 'checkpoint-proposed' || h.status === 'checkpoint-mined')
+      .at(-1);
     const lastAttestation = history.filter(h => h.status === 'attestation-sent').at(-1);
     return {
       address: EthAddress.fromString(address),
       lastProposal: this.computeFromSlot(lastProposal?.slot),
       lastAttestation: this.computeFromSlot(lastAttestation?.slot),
       totalSlots: history.length,
-      missedProposals: this.computeMissed(history, 'block', ['block-missed']),
+      missedProposals: this.computeMissed(history, 'proposer', ['checkpoint-missed', 'blocks-missed']),
       missedAttestations: this.computeMissed(history, 'attestation', ['attestation-missed']),
       history,
     };
@@ -487,10 +518,12 @@ export class Sentinel extends (EventEmitter as new () => WatcherEmitter) impleme
 
   protected computeMissed(
     history: ValidatorStatusHistory,
-    computeOverPrefix: ValidatorStatusType | undefined,
+    computeOverCategory: ValidatorStatusType | undefined,
     filter: ValidatorStatusInSlot[],
   ) {
-    const relevantHistory = history.filter(h => !computeOverPrefix || h.status.startsWith(computeOverPrefix));
+    const relevantHistory = history.filter(
+      h => !computeOverCategory || statusToCategory(h.status) === computeOverCategory,
+    );
     const filteredHistory = relevantHistory.filter(h => filter.includes(h.status));
     return {
       currentStreak: countWhile([...relevantHistory].reverse(), h => filter.includes(h.status)),