@aztec/archiver 0.0.1-commit.3e3d0c9cd → 0.0.1-commit.3f5453c7b

package/src/l1/calldata_retriever.ts

@@ -1,6 +1,7 @@
 import { MULTI_CALL_3_ADDRESS, type ViemCommitteeAttestations, type ViemHeader } from '@aztec/ethereum/contracts';
 import type { ViemPublicClient, ViemPublicDebugClient } from '@aztec/ethereum/types';
 import { CheckpointNumber } from '@aztec/foundation/branded-types';
+import { LruSet } from '@aztec/foundation/collection';
 import { Fr } from '@aztec/foundation/curves/bn254';
 import { EthAddress } from '@aztec/foundation/eth-address';
 import type { Logger } from '@aztec/foundation/log';
@@ -44,7 +45,7 @@ type CheckpointData = {
  */
 export class CalldataRetriever {
   /** Tx hashes we've already logged for trace+debug failure (log once per tx per process). */
-  private static readonly traceFailureWarnedTxHashes = new Set<string>();
+  private static readonly traceFailureWarnedTxHashes = new LruSet<string>(1000);
 
   /** Clears the trace-failure warned set. For testing only. */
   static resetTraceFailureWarnedForTesting(): void {

package/src/l1/data_retrieval.ts

@@ -144,7 +144,7 @@ export async function retrievedToPublishedCheckpoint({
  * @param blobClient - The blob client client for fetching blob data.
  * @param searchStartBlock - The block number to use for starting the search.
  * @param searchEndBlock - The highest block number that we should search up to.
- * @param contractAddresses - The contract addresses (governanceProposerAddress, slashFactoryAddress, slashingProposerAddress).
+ * @param contractAddresses - The contract addresses (governanceProposerAddress, slashingProposerAddress).
  * @param instrumentation - The archiver instrumentation instance.
  * @param logger - The logger instance.
  * @param isHistoricalSync - Whether this is a historical sync.
@@ -344,14 +344,10 @@ export async function getCheckpointBlobDataFromBlobs(
 /** Given an L1 to L2 message, retrieves its corresponding event from the Inbox within a specific block range. */
 export async function retrieveL1ToL2Message(
   inbox: InboxContract,
-  leaf: Fr,
-  fromBlock: bigint,
-  toBlock: bigint,
+  message: InboxMessage,
 ): Promise<InboxMessage | undefined> {
-  const logs = await inbox.getMessageSentEventByHash(leaf.toString(), fromBlock, toBlock);
-
-  const messages = mapLogsInboxMessage(logs);
-  return messages.length > 0 ? messages[0] : undefined;
+  const log = await inbox.getMessageSentEventByHash(message.leaf.toString(), message.l1BlockHash.toString());
+  return log && mapLogInboxMessage(log);
 }
 
 /**
@@ -374,22 +370,22 @@ export async function retrieveL1ToL2Messages(
       break;
     }
 
-    retrievedL1ToL2Messages.push(...mapLogsInboxMessage(messageSentLogs));
+    retrievedL1ToL2Messages.push(...messageSentLogs.map(mapLogInboxMessage));
     searchStartBlock = messageSentLogs.at(-1)!.l1BlockNumber + 1n;
   }
 
   return retrievedL1ToL2Messages;
 }
 
-function mapLogsInboxMessage(logs: MessageSentLog[]): InboxMessage[] {
-  return logs.map(log => ({
+function mapLogInboxMessage(log: MessageSentLog): InboxMessage {
+  return {
     index: log.args.index,
     leaf: log.args.leaf,
     l1BlockNumber: log.l1BlockNumber,
     l1BlockHash: log.l1BlockHash,
     checkpointNumber: log.args.checkpointNumber,
     rollingHash: log.args.rollingHash,
-  }));
+  };
 }
 
 /** Retrieves L2ProofVerified events from the rollup contract. */

package/src/modules/data_source_base.ts

@@ -6,7 +6,13 @@ import { isDefined } from '@aztec/foundation/types';
 import type { FunctionSelector } from '@aztec/stdlib/abi';
 import type { AztecAddress } from '@aztec/stdlib/aztec-address';
 import { type BlockData, type BlockHash, CheckpointedL2Block, L2Block, type L2Tips } from '@aztec/stdlib/block';
-import { Checkpoint, type CheckpointData, PublishedCheckpoint } from '@aztec/stdlib/checkpoint';
+import {
+  Checkpoint,
+  type CheckpointData,
+  type CommonCheckpointData,
+  type ProposedCheckpointData,
+  PublishedCheckpoint,
+} from '@aztec/stdlib/checkpoint';
 import type { ContractClassPublic, ContractDataSource, ContractInstanceWithAddress } from '@aztec/stdlib/contract';
 import { type L1RollupConstants, getSlotRangeForEpoch } from '@aztec/stdlib/epoch-helpers';
 import type { GetContractClassLogsResponse, GetPublicLogsResponse } from '@aztec/stdlib/interfaces/client';
@@ -157,6 +163,14 @@ export abstract class ArchiverDataSourceBase
     return this.store.getSettledTxReceipt(txHash, this.l1Constants);
   }
 
+  public getProposedCheckpoint(): Promise<CommonCheckpointData | undefined> {
+    return this.store.getProposedCheckpoint();
+  }
+
+  public getProposedCheckpointOnly(): Promise<ProposedCheckpointData | undefined> {
+    return this.store.getProposedCheckpointOnly();
+  }
+
   public isPendingChainInvalid(): Promise<boolean> {
     return this.getPendingChainValidationStatus().then(status => !status.valid);
   }

package/src/modules/data_store_updater.ts

@@ -1,29 +1,21 @@
 import { BlockNumber, CheckpointNumber } from '@aztec/foundation/branded-types';
-import { Fr } from '@aztec/foundation/curves/bn254';
+import { filterAsync } from '@aztec/foundation/collection';
 import { createLogger } from '@aztec/foundation/log';
-import {
-  ContractClassPublishedEvent,
-  PrivateFunctionBroadcastedEvent,
-  UtilityFunctionBroadcastedEvent,
-} from '@aztec/protocol-contracts/class-registry';
+import { ContractClassPublishedEvent } from '@aztec/protocol-contracts/class-registry';
 import {
   ContractInstancePublishedEvent,
   ContractInstanceUpdatedEvent,
 } from '@aztec/protocol-contracts/instance-registry';
 import type { L2Block, ValidateCheckpointResult } from '@aztec/stdlib/block';
-import { type PublishedCheckpoint, validateCheckpoint } from '@aztec/stdlib/checkpoint';
+import { type ProposedCheckpointInput, type PublishedCheckpoint, validateCheckpoint } from '@aztec/stdlib/checkpoint';
 import {
-  type ExecutablePrivateFunctionWithMembershipProof,
-  type UtilityFunctionWithMembershipProof,
-  computePublicBytecodeCommitment,
-  isValidPrivateFunctionMembershipProof,
-  isValidUtilityFunctionMembershipProof,
+  type ContractClassPublicWithCommitment,
+  computeContractAddressFromInstance,
+  computeContractClassId,
 } from '@aztec/stdlib/contract';
 import type { ContractClassLog, PrivateLog, PublicLog } from '@aztec/stdlib/logs';
 import type { UInt64 } from '@aztec/stdlib/types';
 
-import groupBy from 'lodash.groupby';
-
 import type { KVArchiverDataStore } from '../store/kv_archiver_store.js';
 import type { L2TipsCache } from '../store/l2_tips_cache.js';
 
@@ -54,8 +46,7 @@ export class ArchiverDataStoreUpdater {
   /**
    * Adds a proposed block to the store with contract class/instance extraction from logs.
    * This is an uncheckpointed block that has been proposed by the sequencer but not yet included in a checkpoint on L1.
-   * Extracts ContractClassPublished, ContractInstancePublished, ContractInstanceUpdated events,
-   * and individually broadcasted functions from the block logs.
+   * Extracts ContractClassPublished, ContractInstancePublished, ContractInstanceUpdated events from the block logs.
    *
    * @param block - The proposed L2 block to add.
    * @param pendingChainValidationStatus - Optional validation status to set.
@@ -87,8 +78,7 @@ export class ArchiverDataStoreUpdater {
    * Reconciles local blocks with incoming checkpoints from L1.
    * Adds new checkpoints to the store with contract class/instance extraction from logs.
    * Prunes any local blocks that conflict with checkpoint data (by comparing archive roots).
-   * Extracts ContractClassPublished, ContractInstancePublished, ContractInstanceUpdated events,
-   * and individually broadcasted functions from the checkpoint block logs.
+   * Extracts ContractClassPublished, ContractInstancePublished, ContractInstanceUpdated events from the checkpoint block logs.
    *
    * @param checkpoints - The published checkpoints to add.
    * @param pendingChainValidationStatus - Optional validation status to set.
@@ -128,6 +118,15 @@ export class ArchiverDataStoreUpdater {
     return result;
   }
 
+  public async setProposedCheckpoint(proposedCheckpoint: ProposedCheckpointInput) {
+    const result = await this.store.transactionAsync(async () => {
+      await this.store.setProposedCheckpoint(proposedCheckpoint);
+      await this.l2TipsCache?.refresh();
+    });
+
+    return result;
+  }
+
   /**
    * Checks for local proposed blocks that do not match the ones to be checkpointed and prunes them.
    * This method handles multiple checkpoints but returns after pruning the first conflict found.
@@ -221,6 +220,10 @@ export class ArchiverDataStoreUpdater {
       }
 
       const result = await this.removeBlocksAfter(blockNumber);
+
+      // Clear the proposed checkpoint if it exists, since its blocks have been pruned
+      await this.store.deleteProposedCheckpoint();
+
       await this.l2TipsCache?.refresh();
       return result;
     });
@@ -313,9 +316,6 @@ export class ArchiverDataStoreUpdater {
         this.updatePublishedContractClasses(contractClassLogs, block.number, operation),
         this.updateDeployedContractInstances(privateLogs, block.number, operation),
         this.updateUpdatedContractInstances(publicLogs, block.header.globalVariables.timestamp, operation),
-        operation === Operation.Store
-          ? this.storeBroadcastedIndividualFunctions(contractClassLogs, block.number)
-          : Promise.resolve(true),
       ])
     ).every(Boolean);
   }
@@ -332,18 +332,37 @@ export class ArchiverDataStoreUpdater {
       .filter(log => ContractClassPublishedEvent.isContractClassPublishedEvent(log))
       .map(log => ContractClassPublishedEvent.fromLog(log));
 
-    const contractClasses = await Promise.all(contractClassPublishedEvents.map(e => e.toContractClassPublic()));
-    if (contractClasses.length > 0) {
-      contractClasses.forEach(c => this.log.verbose(`${Operation[operation]} contract class ${c.id.toString()}`));
-      if (operation == Operation.Store) {
-        // TODO: Will probably want to create some worker threads to compute these bytecode commitments as they are expensive
-        const commitments = await Promise.all(
-          contractClasses.map(c => computePublicBytecodeCommitment(c.packedBytecode)),
-        );
-        return await this.store.addContractClasses(contractClasses, commitments, blockNum);
-      } else if (operation == Operation.Delete) {
+    if (operation == Operation.Delete) {
+      const contractClasses = contractClassPublishedEvents.map(e => e.toContractClassPublic());
+      if (contractClasses.length > 0) {
+        contractClasses.forEach(c => this.log.verbose(`${Operation[operation]} contract class ${c.id.toString()}`));
         return await this.store.deleteContractClasses(contractClasses, blockNum);
       }
+      return true;
+    }
+
+    // Compute bytecode commitments and validate class IDs in a single pass.
+    const contractClasses: ContractClassPublicWithCommitment[] = [];
+    for (const event of contractClassPublishedEvents) {
+      const contractClass = await event.toContractClassPublicWithBytecodeCommitment();
+      const computedClassId = await computeContractClassId({
+        artifactHash: contractClass.artifactHash,
+        privateFunctionsRoot: contractClass.privateFunctionsRoot,
+        publicBytecodeCommitment: contractClass.publicBytecodeCommitment,
+      });
+      if (!computedClassId.equals(contractClass.id)) {
+        this.log.warn(
+          `Skipping contract class with mismatched id at block ${blockNum}. Claimed ${contractClass.id}, computed ${computedClassId}`,
+          { blockNum, contractClassId: event.contractClassId.toString() },
+        );
+        continue;
+      }
+      contractClasses.push(contractClass);
+    }
+
+    if (contractClasses.length > 0) {
+      contractClasses.forEach(c => this.log.verbose(`${Operation[operation]} contract class ${c.id.toString()}`));
+      return await this.store.addContractClasses(contractClasses, blockNum);
     }
     return true;
   }
@@ -356,10 +375,27 @@ export class ArchiverDataStoreUpdater {
     blockNum: BlockNumber,
     operation: Operation,
   ): Promise<boolean> {
-    const contractInstances = allLogs
+    const allInstances = allLogs
       .filter(log => ContractInstancePublishedEvent.isContractInstancePublishedEvent(log))
       .map(log => ContractInstancePublishedEvent.fromLog(log))
       .map(e => e.toContractInstance());
+
+    // Verify that each instance's address matches the one derived from its fields if we're adding
+    const contractInstances =
+      operation === Operation.Delete
+        ? allInstances
+        : await filterAsync(allInstances, async instance => {
+            const computedAddress = await computeContractAddressFromInstance(instance);
+            if (!computedAddress.equals(instance.address)) {
+              this.log.warn(
+                `Found contract instance with mismatched address at block ${blockNum}. Claimed ${instance.address} but computed ${computedAddress}.`,
+                { instanceAddress: instance.address.toString(), computedAddress: computedAddress.toString(), blockNum },
+              );
+              return false;
+            }
+            return true;
+          });
+
     if (contractInstances.length > 0) {
       contractInstances.forEach(c =>
         this.log.verbose(`${Operation[operation]} contract instance at ${c.address.toString()}`),
@@ -398,67 +434,4 @@ export class ArchiverDataStoreUpdater {
     }
     return true;
   }
-
-  /**
-   * Stores the functions that were broadcasted individually.
-   *
-   * @dev Beware that there is not a delete variant of this, since they are added to contract classes
-   *      and will be deleted as part of the class if needed.
-   */
-  private async storeBroadcastedIndividualFunctions(
-    allLogs: ContractClassLog[],
-    _blockNum: BlockNumber,
-  ): Promise<boolean> {
-    // Filter out private and utility function broadcast events
-    const privateFnEvents = allLogs
-      .filter(log => PrivateFunctionBroadcastedEvent.isPrivateFunctionBroadcastedEvent(log))
-      .map(log => PrivateFunctionBroadcastedEvent.fromLog(log));
-    const utilityFnEvents = allLogs
-      .filter(log => UtilityFunctionBroadcastedEvent.isUtilityFunctionBroadcastedEvent(log))
-      .map(log => UtilityFunctionBroadcastedEvent.fromLog(log));
-
-    // Group all events by contract class id
-    for (const [classIdString, classEvents] of Object.entries(
-      groupBy([...privateFnEvents, ...utilityFnEvents], e => e.contractClassId.toString()),
-    )) {
-      const contractClassId = Fr.fromHexString(classIdString);
-      const contractClass = await this.store.getContractClass(contractClassId);
-      if (!contractClass) {
-        this.log.warn(`Skipping broadcasted functions as contract class ${contractClassId.toString()} was not found`);
-        continue;
-      }
-
-      // Split private and utility functions, and filter out invalid ones
-      const allFns = classEvents.map(e => e.toFunctionWithMembershipProof());
-      const privateFns = allFns.filter(
-        (fn): fn is ExecutablePrivateFunctionWithMembershipProof => 'utilityFunctionsTreeRoot' in fn,
-      );
-      const utilityFns = allFns.filter(
-        (fn): fn is UtilityFunctionWithMembershipProof => 'privateFunctionsArtifactTreeRoot' in fn,
-      );
-
-      const privateFunctionsWithValidity = await Promise.all(
-        privateFns.map(async fn => ({ fn, valid: await isValidPrivateFunctionMembershipProof(fn, contractClass) })),
-      );
-      const validPrivateFns = privateFunctionsWithValidity.filter(({ valid }) => valid).map(({ fn }) => fn);
-      const utilityFunctionsWithValidity = await Promise.all(
-        utilityFns.map(async fn => ({
-          fn,
-          valid: await isValidUtilityFunctionMembershipProof(fn, contractClass),
-        })),
-      );
-      const validUtilityFns = utilityFunctionsWithValidity.filter(({ valid }) => valid).map(({ fn }) => fn);
-      const validFnCount = validPrivateFns.length + validUtilityFns.length;
-      if (validFnCount !== allFns.length) {
-        this.log.warn(`Skipping ${allFns.length - validFnCount} invalid functions`);
-      }
-
-      // Store the functions in the contract class in a single operation
-      if (validFnCount > 0) {
-        this.log.verbose(`Storing ${validFnCount} functions for contract class ${contractClassId.toString()}`);
-      }
-      return await this.store.addFunctions(contractClassId, validPrivateFns, validUtilityFns);
-    }
-    return true;
-  }
 }

package/src/modules/instrumentation.ts

@@ -32,6 +32,7 @@ export class ArchiverInstrumentation {
   private pruneCount: UpDownCounter;
 
   private syncDurationPerBlock: Histogram;
+  private syncDurationPerCheckpoint: Histogram;
   private syncBlockCount: UpDownCounter;
   private manaPerBlock: Histogram;
   private txsPerBlock: Histogram;
@@ -68,6 +69,8 @@ export class ArchiverInstrumentation {
 
     this.syncDurationPerBlock = meter.createHistogram(Metrics.ARCHIVER_SYNC_PER_BLOCK);
 
+    this.syncDurationPerCheckpoint = meter.createHistogram(Metrics.ARCHIVER_SYNC_PER_CHECKPOINT);
+
     this.syncBlockCount = createUpDownCounterWithDefault(meter, Metrics.ARCHIVER_SYNC_BLOCK_COUNT);
 
     this.manaPerBlock = meter.createHistogram(Metrics.ARCHIVER_MANA_PER_BLOCK);
@@ -113,17 +116,26 @@ export class ArchiverInstrumentation {
     return this.telemetry.isEnabled();
   }
 
-  public processNewBlocks(syncTimePerBlock: number, blocks: L2Block[]) {
+  public processNewProposedBlock(syncTimePerBlock: number, block: L2Block) {
+    const attrs = { [Attributes.STATUS]: 'proposed' };
+    this.blockHeight.record(block.number, attrs);
     this.syncDurationPerBlock.record(Math.ceil(syncTimePerBlock));
+
+    // Per block metrics
+    this.txCount.add(block.body.txEffects.length);
+    this.txsPerBlock.record(block.body.txEffects.length);
+    this.manaPerBlock.record(block.header.totalManaUsed.toNumber() / 1e6);
+  }
+
+  public processNewCheckpointedBlocks(syncTimePerCheckpoint: number, blocks: L2Block[]) {
+    if (blocks.length === 0) {
+      return;
+    }
+
+    this.syncDurationPerCheckpoint.record(Math.ceil(syncTimePerCheckpoint));
     this.blockHeight.record(Math.max(...blocks.map(b => b.number)));
     this.checkpointHeight.record(Math.max(...blocks.map(b => b.checkpointNumber)));
     this.syncBlockCount.add(blocks.length);
-
-    for (const block of blocks) {
-      this.txCount.add(block.body.txEffects.length);
-      this.txsPerBlock.record(block.body.txEffects.length);
-      this.manaPerBlock.record(block.header.totalManaUsed.toNumber() / 1e6);
-    }
   }
 
   public processNewMessages(count: number, syncPerMessageMs: number) {