@aztec/archiver 0.86.0 → 0.87.0

package/src/archiver/data_retrieval.ts

@@ -1,18 +1,17 @@
 import { Blob, BlobDeserializationError } from '@aztec/blob-lib';
 import type { BlobSinkClientInterface } from '@aztec/blob-sink/client';
-import type { EpochProofPublicInputArgs, ViemPublicClient } from '@aztec/ethereum';
+import type { EpochProofPublicInputArgs, ViemClient, ViemPublicClient } from '@aztec/ethereum';
 import { asyncPool } from '@aztec/foundation/async-pool';
+import { Buffer16, Buffer32 } from '@aztec/foundation/buffer';
 import type { EthAddress } from '@aztec/foundation/eth-address';
 import { Signature, type ViemSignature } from '@aztec/foundation/eth-signature';
 import { Fr } from '@aztec/foundation/fields';
 import { type Logger, createLogger } from '@aztec/foundation/log';
-import { numToUInt32BE } from '@aztec/foundation/serialize';
 import { ForwarderAbi, type InboxAbi, RollupAbi } from '@aztec/l1-artifacts';
 import { Body, L2Block } from '@aztec/stdlib/block';
-import { InboxLeaf } from '@aztec/stdlib/messaging';
 import { Proof } from '@aztec/stdlib/proofs';
 import { AppendOnlyTreeSnapshot } from '@aztec/stdlib/trees';
-import { BlockHeader } from '@aztec/stdlib/tx';
+import { BlockHeader, GlobalVariables, ProposedBlockHeader, StateReference } from '@aztec/stdlib/tx';
 
 import {
   type GetContractEventsReturnType,
@@ -25,8 +24,68 @@ import {
 
 import { NoBlobBodiesFoundError } from './errors.js';
 import type { DataRetrieval } from './structs/data_retrieval.js';
+import type { InboxMessage } from './structs/inbox_message.js';
 import type { L1PublishedData, PublishedL2Block } from './structs/published.js';
 
+export type RetrievedL2Block = {
+  l2BlockNumber: bigint;
+  archiveRoot: Fr;
+  stateReference: StateReference;
+  header: ProposedBlockHeader;
+  body: Body;
+  l1: L1PublishedData;
+  chainId: Fr;
+  version: Fr;
+  signatures: Signature[];
+};
+
+export function retrievedBlockToPublishedL2Block(retrievedBlock: RetrievedL2Block): PublishedL2Block {
+  const {
+    l2BlockNumber,
+    archiveRoot,
+    stateReference,
+    header: proposedHeader,
+    body,
+    l1,
+    chainId,
+    version,
+    signatures,
+  } = retrievedBlock;
+
+  const archive = new AppendOnlyTreeSnapshot(
+    archiveRoot,
+    Number(l2BlockNumber + 1n), // nextAvailableLeafIndex
+  );
+
+  const globalVariables = GlobalVariables.from({
+    chainId,
+    version,
+    blockNumber: new Fr(l2BlockNumber),
+    slotNumber: proposedHeader.slotNumber,
+    timestamp: new Fr(proposedHeader.timestamp),
+    coinbase: proposedHeader.coinbase,
+    feeRecipient: proposedHeader.feeRecipient,
+    gasFees: proposedHeader.gasFees,
+  });
+
+  const header = BlockHeader.from({
+    lastArchive: new AppendOnlyTreeSnapshot(proposedHeader.lastArchiveRoot, Number(l2BlockNumber)),
+    contentCommitment: proposedHeader.contentCommitment,
+    state: stateReference,
+    globalVariables,
+    totalFees: body.txEffects.reduce((accum, txEffect) => accum.add(txEffect.transactionFee), Fr.ZERO),
+    totalManaUsed: proposedHeader.totalManaUsed,
+  });
+
+  const block = new L2Block(archive, header, body);
+
+  return {
+    block,
+    l1,
+    signatures,
+  };
+}
+
 /**
  * Fetches new L2 blocks.
  * @param publicClient - The viem public client to use for transaction retrieval.
@@ -43,8 +102,11 @@ export async function retrieveBlocksFromRollup(
   searchStartBlock: bigint,
   searchEndBlock: bigint,
   logger: Logger = createLogger('archiver'),
-): Promise<PublishedL2Block[]> {
-  const retrievedBlocks: PublishedL2Block[] = [];
+): Promise<RetrievedL2Block[]> {
+  const retrievedBlocks: RetrievedL2Block[] = [];
+
+  let rollupConstants: { chainId: Fr; version: Fr } | undefined;
+
   do {
     if (searchStartBlock > searchEndBlock) {
       break;
@@ -68,11 +130,17 @@ export async function retrieveBlocksFromRollup(
       `Got ${l2BlockProposedLogs.length} L2 block processed logs for L2 blocks ${l2BlockProposedLogs[0].args.blockNumber}-${lastLog.args.blockNumber} between L1 blocks ${searchStartBlock}-${searchEndBlock}`,
     );
 
+    if (rollupConstants === undefined) {
+      const [chainId, version] = await Promise.all([publicClient.getChainId(), rollup.read.getVersion()]);
+      rollupConstants = { chainId: new Fr(chainId), version: new Fr(version) };
+    }
+
     const newBlocks = await processL2BlockProposedLogs(
       rollup,
       publicClient,
       blobSinkClient,
       l2BlockProposedLogs,
+      rollupConstants,
       logger,
     );
     retrievedBlocks.push(...newBlocks);
@@ -90,14 +158,15 @@ export async function retrieveBlocksFromRollup(
  * @param logs - L2BlockProposed logs.
  * @returns - An array blocks.
  */
-export async function processL2BlockProposedLogs(
+async function processL2BlockProposedLogs(
   rollup: GetContractReturnType<typeof RollupAbi, ViemPublicClient>,
   publicClient: ViemPublicClient,
   blobSinkClient: BlobSinkClientInterface,
   logs: GetContractEventsReturnType<typeof RollupAbi, 'L2BlockProposed'>,
+  { chainId, version }: { chainId: Fr; version: Fr },
   logger: Logger,
-): Promise<PublishedL2Block[]> {
-  const retrievedBlocks: PublishedL2Block[] = [];
+): Promise<RetrievedL2Block[]> {
+  const retrievedBlocks: RetrievedL2Block[] = [];
   await asyncPool(10, logs, async log => {
     const l2BlockNumber = log.args.blockNumber!;
     const archive = log.args.archive!;
@@ -122,7 +191,7 @@ export async function processL2BlockProposedLogs(
         timestamp: await getL1BlockTime(publicClient, log.blockNumber),
       };
 
-      retrievedBlocks.push({ ...block, l1 });
+      retrievedBlocks.push({ ...block, l1, chainId, version });
       logger.trace(`Retrieved L2 block ${l2BlockNumber} from L1 tx ${log.transactionHash}`, {
         l1BlockNumber: log.blockNumber,
         l2BlockNumber,
@@ -187,7 +256,7 @@ function extractRollupProposeCalldata(forwarderData: Hex, rollupAddress: Hex): H
       if (rollupFunctionName === 'propose') {
         return callData;
       }
-    } catch (err) {
+    } catch {
       // Skip invalid function data
       continue;
     }
@@ -202,7 +271,7 @@ function extractRollupProposeCalldata(forwarderData: Hex, rollupAddress: Hex): H
  * TODO: Add retries and error management.
  * @param publicClient - The viem public client to use for transaction retrieval.
  * @param txHash - Hash of the tx that published it.
- * @param l2BlockNum - L2 block number.
+ * @param l2BlockNumber - L2 block number.
  * @returns L2 block from the calldata, deserialized
  */
 async function getBlockFromRollupTx(
@@ -210,10 +279,10 @@ async function getBlockFromRollupTx(
   blobSinkClient: BlobSinkClientInterface,
   txHash: `0x${string}`,
   blobHashes: Buffer[], // TODO(md): buffer32?
-  l2BlockNum: bigint,
+  l2BlockNumber: bigint,
   rollupAddress: Hex,
   logger: Logger,
-): Promise<Omit<PublishedL2Block, 'l1'>> {
+): Promise<Omit<RetrievedL2Block, 'l1' | 'chainId' | 'version'>> {
   const { input: forwarderData, blockHash } = await publicClient.getTransaction({ hash: txHash });
 
   const rollupData = extractRollupProposeCalldata(forwarderData, rollupAddress);
@@ -226,10 +295,11 @@ async function getBlockFromRollupTx(
     throw new Error(`Unexpected rollup method called ${rollupFunctionName}`);
   }
 
-  const [decodedArgs, signatures] = rollupArgs! as readonly [
+  const [decodedArgs, signatures, _blobInput] = rollupArgs! as readonly [
     {
       header: Hex;
       archive: Hex;
+      stateReference: Hex;
       blockHash: Hex;
       oracleInput: {
         feeAssetPriceModifier: bigint;
@@ -240,10 +310,10 @@ async function getBlockFromRollupTx(
     Hex,
   ];
 
-  const header = BlockHeader.fromBuffer(Buffer.from(hexToBytes(decodedArgs.header)));
+  const header = ProposedBlockHeader.fromBuffer(Buffer.from(hexToBytes(decodedArgs.header)));
   const blobBodies = await blobSinkClient.getBlobSidecar(blockHash, blobHashes);
   if (blobBodies.length === 0) {
-    throw new NoBlobBodiesFoundError(Number(l2BlockNum));
+    throw new NoBlobBodiesFoundError(Number(l2BlockNumber));
   }
 
   let blockFields: Fr[];
@@ -259,23 +329,30 @@ async function getBlockFromRollupTx(
   }
 
   // The blob source gives us blockFields, and we must construct the body from them:
-  const blockBody = Body.fromBlobFields(blockFields);
+  const body = Body.fromBlobFields(blockFields);
 
-  const blockNumberFromHeader = header.globalVariables.blockNumber.toBigInt();
+  const archiveRoot = new Fr(Buffer.from(hexToBytes(decodedArgs.archive)));
 
-  if (blockNumberFromHeader !== l2BlockNum) {
-    throw new Error(`Block number mismatch: expected ${l2BlockNum} but got ${blockNumberFromHeader}`);
-  }
+  const stateReference = StateReference.fromBuffer(Buffer.from(hexToBytes(decodedArgs.stateReference)));
 
-  const archive = AppendOnlyTreeSnapshot.fromBuffer(
-    Buffer.concat([
-      Buffer.from(hexToBytes(decodedArgs.archive)), // L2Block.archive.root
-      numToUInt32BE(Number(l2BlockNum + 1n)), // L2Block.archive.nextAvailableLeafIndex
-    ]),
-  );
+  return {
+    l2BlockNumber,
+    archiveRoot,
+    stateReference,
+    header,
+    body,
+    signatures: signatures.map(Signature.fromViemSignature),
+  };
+}
 
-  const block = new L2Block(archive, header, blockBody);
-  return { block, signatures: signatures.map(Signature.fromViemSignature) };
+/** Given an L1 to L2 message, retrieves its corresponding event from the Inbox. */
+export async function retrieveL1ToL2Message(
+  inbox: GetContractReturnType<typeof InboxAbi, ViemClient>,
+  leaf: Fr,
+  fromBlock: bigint,
+): Promise<InboxMessage | undefined> {
+  const logs = await inbox.getEvents.MessageSent({ hash: leaf.toString() }, { fromBlock });
+  return mapLogsInboxMessage(logs)[0];
 }
 
 /**
@@ -288,39 +365,39 @@ async function getBlockFromRollupTx(
  * @returns An array of InboxLeaf and next eth block to search from.
  */
 export async function retrieveL1ToL2Messages(
-  inbox: GetContractReturnType<typeof InboxAbi, ViemPublicClient>,
+  inbox: GetContractReturnType<typeof InboxAbi, ViemClient>,
   searchStartBlock: bigint,
   searchEndBlock: bigint,
-): Promise<DataRetrieval<InboxLeaf>> {
-  const retrievedL1ToL2Messages: InboxLeaf[] = [];
-  do {
-    if (searchStartBlock > searchEndBlock) {
-      break;
-    }
-
+): Promise<InboxMessage[]> {
+  const retrievedL1ToL2Messages: InboxMessage[] = [];
+  while (searchStartBlock <= searchEndBlock) {
     const messageSentLogs = (
-      await inbox.getEvents.MessageSent(
-        {},
-        {
-          fromBlock: searchStartBlock,
-          toBlock: searchEndBlock,
-        },
-      )
+      await inbox.getEvents.MessageSent({}, { fromBlock: searchStartBlock, toBlock: searchEndBlock })
     ).filter(log => log.blockNumber! >= searchStartBlock && log.blockNumber! <= searchEndBlock);
 
     if (messageSentLogs.length === 0) {
       break;
     }
 
-    for (const log of messageSentLogs) {
-      const { index, hash } = log.args;
-      retrievedL1ToL2Messages.push(new InboxLeaf(index!, Fr.fromHexString(hash!)));
-    }
+    retrievedL1ToL2Messages.push(...mapLogsInboxMessage(messageSentLogs));
+    searchStartBlock = messageSentLogs.at(-1)!.blockNumber + 1n;
+  }
 
-    // handles the case when there are no new messages:
-    searchStartBlock = (messageSentLogs.findLast(msgLog => !!msgLog)?.blockNumber || searchStartBlock) + 1n;
-  } while (searchStartBlock <= searchEndBlock);
-  return { lastProcessedL1BlockNumber: searchStartBlock - 1n, retrievedData: retrievedL1ToL2Messages };
+  return retrievedL1ToL2Messages;
+}
+
+function mapLogsInboxMessage(logs: GetContractEventsReturnType<typeof InboxAbi, 'MessageSent'>): InboxMessage[] {
+  return logs.map(log => {
+    const { index, hash, l2BlockNumber, rollingHash } = log.args;
+    return {
+      index: index!,
+      leaf: Fr.fromHexString(hash!),
+      l1BlockNumber: log.blockNumber,
+      l1BlockHash: Buffer32.fromString(log.blockHash),
+      l2BlockNumber: l2BlockNumber!,
+      rollingHash: Buffer16.fromString(rollingHash!),
+    };
+  });
 }
 
 /** Retrieves L2ProofVerified events from the rollup contract. */

package/src/archiver/errors.ts

@@ -3,3 +3,24 @@ export class NoBlobBodiesFoundError extends Error {
     super(`No blob bodies found for block ${l2BlockNum}`);
   }
 }
+
+export class InitialBlockNumberNotSequentialError extends Error {
+  constructor(
+    public readonly newBlockNumber: number,
+    public readonly previousBlockNumber: number | undefined,
+  ) {
+    super(
+      `Cannot insert new block ${newBlockNumber} given previous block number in store is ${
+        previousBlockNumber ?? 'undefined'
+      }`,
+    );
+  }
+}
+
+export class BlockNumberNotSequentialError extends Error {
+  constructor(newBlockNumber: number, previous: number | undefined) {
+    super(
+      `Cannot insert new block ${newBlockNumber} given previous block number in batch is ${previous ?? 'undefined'}`,
+    );
+  }
+}

package/src/archiver/instrumentation.ts

@@ -28,7 +28,10 @@ export class ArchiverInstrumentation {
 
   private log = createLogger('archiver:instrumentation');
 
-  private constructor(private telemetry: TelemetryClient, lmdbStats?: LmdbStatsCallback) {
+  private constructor(
+    private telemetry: TelemetryClient,
+    lmdbStats?: LmdbStatsCallback,
+  ) {
     this.tracer = telemetry.getTracer('Archiver');
     const meter = telemetry.getMeter('Archiver');
     this.blockHeight = meter.createGauge(Metrics.ARCHIVER_BLOCK_HEIGHT, {

package/src/archiver/kv_archiver_store/block_store.ts

@@ -8,6 +8,7 @@ import { Body, L2Block, L2BlockHash } from '@aztec/stdlib/block';
 import { AppendOnlyTreeSnapshot } from '@aztec/stdlib/trees';
 import { BlockHeader, type IndexedTxEffect, TxHash, TxReceipt } from '@aztec/stdlib/tx';
 
+import { BlockNumberNotSequentialError, InitialBlockNumberNotSequentialError } from '../errors.js';
 import type { L1PublishedData, PublishedL2Block } from '../structs/published.js';
 
 export { TxReceipt, type TxEffect, type TxHash } from '@aztec/stdlib/tx';
@@ -37,9 +38,6 @@ export class BlockStore {
   /** Stores l2 block number of the last proven block */
   #lastProvenL2Block: AztecAsyncSingleton<number>;
 
-  /** Stores l2 epoch number of the last proven epoch */
-  #lastProvenL2Epoch: AztecAsyncSingleton<number>;
-
   /** Index mapping transaction hash (as a string) to its location in a block */
   #txIndex: AztecAsyncMap<string, BlockIndexValue>;
 
@@ -55,7 +53,6 @@ export class BlockStore {
     this.#contractIndex = db.openMap('archiver_contract_index');
     this.#lastSynchedL1Block = db.openSingleton('archiver_last_synched_l1_block');
     this.#lastProvenL2Block = db.openSingleton('archiver_last_proven_l2_block');
-    this.#lastProvenL2Epoch = db.openSingleton('archiver_last_proven_l2_epoch');
   }
 
   /**
@@ -63,13 +60,32 @@ export class BlockStore {
    * @param blocks - The L2 blocks to be added to the store.
    * @returns True if the operation is successful.
    */
-  async addBlocks(blocks: PublishedL2Block[]): Promise<boolean> {
+  async addBlocks(blocks: PublishedL2Block[], opts: { force?: boolean } = {}): Promise<boolean> {
     if (blocks.length === 0) {
       return true;
     }
 
     return await this.db.transactionAsync(async () => {
+      // Check that the block immediately before the first block to be added is present in the store.
+      const firstBlockNumber = blocks[0].block.number;
+      const [previousBlockNumber] = await toArray(
+        this.#blocks.keysAsync({ reverse: true, limit: 1, end: firstBlockNumber - 1 }),
+      );
+      const hasPreviousBlock =
+        firstBlockNumber === INITIAL_L2_BLOCK_NUM ||
+        (previousBlockNumber !== undefined && previousBlockNumber === firstBlockNumber - 1);
+      if (!opts.force && !hasPreviousBlock) {
+        throw new InitialBlockNumberNotSequentialError(firstBlockNumber, previousBlockNumber);
+      }
+
+      // Iterate over blocks array and insert them, checking that the block numbers are sequential.
+      let previousBlock: PublishedL2Block | undefined = undefined;
       for (const block of blocks) {
+        if (!opts.force && previousBlock && previousBlock.block.number + 1 !== block.block.number) {
+          throw new BlockNumberNotSequentialError(block.block.number, previousBlock.block.number);
+        }
+        previousBlock = block;
+
         await this.#blocks.set(block.block.number, {
           header: block.block.header.toBuffer(),
           archive: block.block.archive.toBuffer(),
@@ -104,6 +120,11 @@ export class BlockStore {
         throw new Error(`Can only unwind blocks from the tip (requested ${from} but current tip is ${last})`);
       }
 
+      const proven = await this.getProvenL2BlockNumber();
+      if (from - blocksToUnwind < proven) {
+        await this.setProvenL2BlockNumber(from - blocksToUnwind);
+      }
+
       for (let i = 0; i < blocksToUnwind; i++) {
         const blockNumber = from - i;
         const block = await this.getBlock(blockNumber);
@@ -130,7 +151,7 @@ export class BlockStore {
    * @returns The requested L2 blocks
    */
   async *getBlocks(start: number, limit: number): AsyncIterableIterator<PublishedL2Block> {
-    for await (const [blockNumber, blockStorage] of this.#blocks.entriesAsync(this.#computeBlockRange(start, limit))) {
+    for await (const [blockNumber, blockStorage] of this.getBlockStorages(start, limit)) {
       const block = await this.getBlockFromBlockStorage(blockNumber, blockStorage);
       if (block) {
         yield block;
@@ -158,7 +179,7 @@ export class BlockStore {
    * @returns The requested L2 block headers
    */
   async *getBlockHeaders(start: number, limit: number): AsyncIterableIterator<BlockHeader> {
-    for await (const [blockNumber, blockStorage] of this.#blocks.entriesAsync(this.#computeBlockRange(start, limit))) {
+    for await (const [blockNumber, blockStorage] of this.getBlockStorages(start, limit)) {
       const header = BlockHeader.fromBuffer(blockStorage.header);
       if (header.getBlockNumber() !== blockNumber) {
         throw new Error(
@@ -169,6 +190,19 @@ export class BlockStore {
     }
   }
 
+  private async *getBlockStorages(start: number, limit: number) {
+    let expectedBlockNumber = start;
+    for await (const [blockNumber, blockStorage] of this.#blocks.entriesAsync(this.#computeBlockRange(start, limit))) {
+      if (blockNumber !== expectedBlockNumber) {
+        throw new Error(
+          `Block number mismatch when iterating blocks from archive (expected ${expectedBlockNumber} but got ${blockNumber})`,
+        );
+      }
+      expectedBlockNumber++;
+      yield [blockNumber, blockStorage] as const;
+    }
+  }
+
   private async getBlockFromBlockStorage(blockNumber: number, blockStorage: BlockStorage) {
     const header = BlockHeader.fromBuffer(blockStorage.header);
     const archive = AppendOnlyTreeSnapshot.fromBuffer(blockStorage.archive);
@@ -283,7 +317,11 @@ export class BlockStore {
   }
 
   async getProvenL2BlockNumber(): Promise<number> {
-    return (await this.#lastProvenL2Block.getAsync()) ?? 0;
+    const [latestBlockNumber, provenBlockNumber] = await Promise.all([
+      this.getSynchedL2BlockNumber(),
+      this.#lastProvenL2Block.getAsync(),
+    ]);
+    return (provenBlockNumber ?? 0) > latestBlockNumber ? latestBlockNumber : (provenBlockNumber ?? 0);
   }
 
   setProvenL2BlockNumber(blockNumber: number) {

package/src/archiver/kv_archiver_store/contract_instance_store.ts

@@ -15,22 +15,30 @@ type ContractInstanceUpdateKey = [string, number] | [string, number, number];
  */
 export class ContractInstanceStore {
   #contractInstances: AztecAsyncMap<string, Buffer>;
+  #contractInstanceDeployedAt: AztecAsyncMap<string, number>;
   #contractInstanceUpdates: AztecAsyncMap<ContractInstanceUpdateKey, Buffer>;
 
-  constructor(db: AztecAsyncKVStore) {
+  constructor(private db: AztecAsyncKVStore) {
     this.#contractInstances = db.openMap('archiver_contract_instances');
+    this.#contractInstanceDeployedAt = db.openMap('archiver_contract_instances_deployment_block_number');
     this.#contractInstanceUpdates = db.openMap('archiver_contract_instance_updates');
   }
 
-  addContractInstance(contractInstance: ContractInstanceWithAddress): Promise<void> {
-    return this.#contractInstances.set(
-      contractInstance.address.toString(),
-      new SerializableContractInstance(contractInstance).toBuffer(),
-    );
+  addContractInstance(contractInstance: ContractInstanceWithAddress, blockNumber: number): Promise<void> {
+    return this.db.transactionAsync(async () => {
+      await this.#contractInstances.set(
+        contractInstance.address.toString(),
+        new SerializableContractInstance(contractInstance).toBuffer(),
+      );
+      await this.#contractInstanceDeployedAt.set(contractInstance.address.toString(), blockNumber);
+    });
   }
 
   deleteContractInstance(contractInstance: ContractInstanceWithAddress): Promise<void> {
-    return this.#contractInstances.delete(contractInstance.address.toString());
+    return this.db.transactionAsync(async () => {
+      await this.#contractInstances.delete(contractInstance.address.toString());
+      await this.#contractInstanceDeployedAt.delete(contractInstance.address.toString());
+    });
   }
 
   getUpdateKey(contractAddress: AztecAddress, blockNumber: number, logIndex?: number): ContractInstanceUpdateKey {
@@ -71,6 +79,7 @@ export class ContractInstanceStore {
     const queryResult = await this.#contractInstanceUpdates
       .valuesAsync({
         reverse: true,
+        start: this.getUpdateKey(address, 0), // Make sure we only look at updates for this contract
         end: this.getUpdateKey(address, blockNumber + 1), // No update can match this key since it doesn't have a log index. We want the highest key <= blockNumber
         limit: 1,
       })
@@ -104,4 +113,8 @@ export class ContractInstanceStore {
     );
     return instance;
   }
+
+  getContractInstanceDeploymentBlockNumber(address: AztecAddress): Promise<number | undefined> {
+    return this.#contractInstanceDeployedAt.getAsync(address.toString());
+  }
 }