@aztec/archiver 0.0.0-test.1 → 0.0.1-commit.1142ef1

package/src/archiver/data_retrieval.ts

@@ -1,422 +0,0 @@
-import { Blob, BlobDeserializationError } from '@aztec/blob-lib';
-import type { BlobSinkClientInterface } from '@aztec/blob-sink/client';
-import type { EpochProofPublicInputArgs, ViemPublicClient } from '@aztec/ethereum';
-import { asyncPool } from '@aztec/foundation/async-pool';
-import type { EthAddress } from '@aztec/foundation/eth-address';
-import type { ViemSignature } from '@aztec/foundation/eth-signature';
-import { Fr } from '@aztec/foundation/fields';
-import { type Logger, createLogger } from '@aztec/foundation/log';
-import { numToUInt32BE } from '@aztec/foundation/serialize';
-import { ForwarderAbi, type InboxAbi, RollupAbi } from '@aztec/l1-artifacts';
-import { Body, L2Block } from '@aztec/stdlib/block';
-import { InboxLeaf } from '@aztec/stdlib/messaging';
-import { Proof } from '@aztec/stdlib/proofs';
-import { AppendOnlyTreeSnapshot } from '@aztec/stdlib/trees';
-import { BlockHeader } from '@aztec/stdlib/tx';
-
-import {
-  type GetContractEventsReturnType,
-  type GetContractReturnType,
-  type Hex,
-  decodeFunctionData,
-  getAbiItem,
-  hexToBytes,
-} from 'viem';
-
-import { NoBlobBodiesFoundError } from './errors.js';
-import type { DataRetrieval } from './structs/data_retrieval.js';
-import type { L1Published, L1PublishedData } from './structs/published.js';
-
-/**
- * Fetches new L2 blocks.
- * @param publicClient - The viem public client to use for transaction retrieval.
- * @param rollupAddress - The address of the rollup contract.
- * @param searchStartBlock - The block number to use for starting the search.
- * @param searchEndBlock - The highest block number that we should search up to.
- * @param expectedNextL2BlockNum - The next L2 block number that we expect to find.
- * @returns An array of block; as well as the next eth block to search from.
- */
-export async function retrieveBlocksFromRollup(
-  rollup: GetContractReturnType<typeof RollupAbi, ViemPublicClient>,
-  publicClient: ViemPublicClient,
-  blobSinkClient: BlobSinkClientInterface,
-  searchStartBlock: bigint,
-  searchEndBlock: bigint,
-  logger: Logger = createLogger('archiver'),
-): Promise<L1Published<L2Block>[]> {
-  const retrievedBlocks: L1Published<L2Block>[] = [];
-  do {
-    if (searchStartBlock > searchEndBlock) {
-      break;
-    }
-    const l2BlockProposedLogs = (
-      await rollup.getEvents.L2BlockProposed(
-        {},
-        {
-          fromBlock: searchStartBlock,
-          toBlock: searchEndBlock,
-        },
-      )
-    ).filter(log => log.blockNumber! >= searchStartBlock && log.blockNumber! <= searchEndBlock);
-
-    if (l2BlockProposedLogs.length === 0) {
-      break;
-    }
-
-    const lastLog = l2BlockProposedLogs[l2BlockProposedLogs.length - 1];
-    logger.debug(
-      `Got ${l2BlockProposedLogs.length} L2 block processed logs for L2 blocks ${l2BlockProposedLogs[0].args.blockNumber}-${lastLog.args.blockNumber} between L1 blocks ${searchStartBlock}-${searchEndBlock}`,
-    );
-
-    const newBlocks = await processL2BlockProposedLogs(
-      rollup,
-      publicClient,
-      blobSinkClient,
-      l2BlockProposedLogs,
-      logger,
-    );
-    retrievedBlocks.push(...newBlocks);
-    searchStartBlock = lastLog.blockNumber! + 1n;
-  } while (searchStartBlock <= searchEndBlock);
-
-  // The asyncpool from processL2BlockProposedLogs will not necessarily return the blocks in order, so we sort them before returning.
-  return retrievedBlocks.sort((a, b) => Number(a.l1.blockNumber - b.l1.blockNumber));
-}
-
-/**
- * Processes newly received L2BlockProposed logs.
- * @param rollup - The rollup contract
- * @param publicClient - The viem public client to use for transaction retrieval.
- * @param logs - L2BlockProposed logs.
- * @returns - An array blocks.
- */
-export async function processL2BlockProposedLogs(
-  rollup: GetContractReturnType<typeof RollupAbi, ViemPublicClient>,
-  publicClient: ViemPublicClient,
-  blobSinkClient: BlobSinkClientInterface,
-  logs: GetContractEventsReturnType<typeof RollupAbi, 'L2BlockProposed'>,
-  logger: Logger,
-): Promise<L1Published<L2Block>[]> {
-  const retrievedBlocks: L1Published<L2Block>[] = [];
-  await asyncPool(10, logs, async log => {
-    const l2BlockNumber = log.args.blockNumber!;
-    const archive = log.args.archive!;
-    const archiveFromChain = await rollup.read.archiveAt([l2BlockNumber]);
-    const blobHashes = log.args.versionedBlobHashes!.map(blobHash => Buffer.from(blobHash.slice(2), 'hex'));
-
-    // The value from the event and contract will match only if the block is in the chain.
-    if (archive === archiveFromChain) {
-      const block = await getBlockFromRollupTx(
-        publicClient,
-        blobSinkClient,
-        log.transactionHash!,
-        blobHashes,
-        l2BlockNumber,
-        rollup.address,
-        logger,
-      );
-
-      const l1: L1PublishedData = {
-        blockNumber: log.blockNumber,
-        blockHash: log.blockHash,
-        timestamp: await getL1BlockTime(publicClient, log.blockNumber),
-      };
-
-      retrievedBlocks.push({ data: block, l1 });
-    } else {
-      logger.warn(`Ignoring L2 block ${l2BlockNumber} due to archive root mismatch`, {
-        actual: archive,
-        expected: archiveFromChain,
-      });
-    }
-  });
-
-  return retrievedBlocks;
-}
-
-export async function getL1BlockTime(publicClient: ViemPublicClient, blockNumber: bigint): Promise<bigint> {
-  const block = await publicClient.getBlock({ blockNumber, includeTransactions: false });
-  return block.timestamp;
-}
-
-/**
- * Extracts the first 'propose' method calldata from a forwarder transaction's data.
- * @param forwarderData - The forwarder transaction input data
- * @param rollupAddress - The address of the rollup contract
- * @returns The calldata for the first 'propose' method call to the rollup contract
- */
-function extractRollupProposeCalldata(forwarderData: Hex, rollupAddress: Hex): Hex {
-  // TODO(#11451): custom forwarders
-  const { functionName: forwarderFunctionName, args: forwarderArgs } = decodeFunctionData({
-    abi: ForwarderAbi,
-    data: forwarderData,
-  });
-
-  if (forwarderFunctionName !== 'forward') {
-    throw new Error(`Unexpected forwarder method called ${forwarderFunctionName}`);
-  }
-
-  if (forwarderArgs.length !== 2) {
-    throw new Error(`Unexpected number of arguments for forwarder`);
-  }
-
-  const [to, data] = forwarderArgs;
-
-  // Find all rollup calls
-  const rollupAddressLower = rollupAddress.toLowerCase();
-
-  for (let i = 0; i < to.length; i++) {
-    const addr = to[i];
-    if (addr.toLowerCase() !== rollupAddressLower) {
-      continue;
-    }
-    const callData = data[i];
-
-    try {
-      const { functionName: rollupFunctionName } = decodeFunctionData({
-        abi: RollupAbi,
-        data: callData,
-      });
-
-      if (rollupFunctionName === 'propose') {
-        return callData;
-      }
-    } catch (err) {
-      // Skip invalid function data
-      continue;
-    }
-  }
-
-  throw new Error(`Rollup address not found in forwarder args`);
-}
-
-/**
- * Gets block from the calldata of an L1 transaction.
- * Assumes that the block was published from an EOA.
- * TODO: Add retries and error management.
- * @param publicClient - The viem public client to use for transaction retrieval.
- * @param txHash - Hash of the tx that published it.
- * @param l2BlockNum - L2 block number.
- * @returns L2 block from the calldata, deserialized
- */
-async function getBlockFromRollupTx(
-  publicClient: ViemPublicClient,
-  blobSinkClient: BlobSinkClientInterface,
-  txHash: `0x${string}`,
-  blobHashes: Buffer[], // WORKTODO(md): buffer32?
-  l2BlockNum: bigint,
-  rollupAddress: Hex,
-  logger: Logger,
-): Promise<L2Block> {
-  const { input: forwarderData, blockHash } = await publicClient.getTransaction({ hash: txHash });
-
-  const rollupData = extractRollupProposeCalldata(forwarderData, rollupAddress);
-  const { functionName: rollupFunctionName, args: rollupArgs } = decodeFunctionData({
-    abi: RollupAbi,
-    data: rollupData,
-  });
-
-  if (rollupFunctionName !== 'propose') {
-    throw new Error(`Unexpected rollup method called ${rollupFunctionName}`);
-  }
-
-  const [decodedArgs, ,] = rollupArgs! as readonly [
-    {
-      header: Hex;
-      archive: Hex;
-      blockHash: Hex;
-      oracleInput: {
-        feeAssetPriceModifier: bigint;
-      };
-      txHashes: Hex[];
-    },
-    ViemSignature[],
-    Hex,
-  ];
-
-  const header = BlockHeader.fromBuffer(Buffer.from(hexToBytes(decodedArgs.header)));
-  const blobBodies = await blobSinkClient.getBlobSidecar(blockHash, blobHashes);
-  if (blobBodies.length === 0) {
-    throw new NoBlobBodiesFoundError(Number(l2BlockNum));
-  }
-
-  let blockFields: Fr[];
-  try {
-    blockFields = Blob.toEncodedFields(blobBodies);
-  } catch (err: any) {
-    if (err instanceof BlobDeserializationError) {
-      logger.fatal(err.message);
-    } else {
-      logger.fatal('Unable to sync: failed to decode fetched blob, this blob was likely not created by us');
-    }
-    throw err;
-  }
-
-  // The blob source gives us blockFields, and we must construct the body from them:
-  const blockBody = Body.fromBlobFields(blockFields);
-
-  const blockNumberFromHeader = header.globalVariables.blockNumber.toBigInt();
-
-  if (blockNumberFromHeader !== l2BlockNum) {
-    throw new Error(`Block number mismatch: expected ${l2BlockNum} but got ${blockNumberFromHeader}`);
-  }
-
-  const archive = AppendOnlyTreeSnapshot.fromBuffer(
-    Buffer.concat([
-      Buffer.from(hexToBytes(decodedArgs.archive)), // L2Block.archive.root
-      numToUInt32BE(Number(l2BlockNum + 1n)), // L2Block.archive.nextAvailableLeafIndex
-    ]),
-  );
-
-  return new L2Block(archive, header, blockBody);
-}
-
-/**
- * Fetch L1 to L2 messages.
- * @param publicClient - The viem public client to use for transaction retrieval.
- * @param inboxAddress - The address of the inbox contract to fetch messages from.
- * @param blockUntilSynced - If true, blocks until the archiver has fully synced.
- * @param searchStartBlock - The block number to use for starting the search.
- * @param searchEndBlock - The highest block number that we should search up to.
- * @returns An array of InboxLeaf and next eth block to search from.
- */
-export async function retrieveL1ToL2Messages(
-  inbox: GetContractReturnType<typeof InboxAbi, ViemPublicClient>,
-  searchStartBlock: bigint,
-  searchEndBlock: bigint,
-): Promise<DataRetrieval<InboxLeaf>> {
-  const retrievedL1ToL2Messages: InboxLeaf[] = [];
-  do {
-    if (searchStartBlock > searchEndBlock) {
-      break;
-    }
-
-    const messageSentLogs = (
-      await inbox.getEvents.MessageSent(
-        {},
-        {
-          fromBlock: searchStartBlock,
-          toBlock: searchEndBlock,
-        },
-      )
-    ).filter(log => log.blockNumber! >= searchStartBlock && log.blockNumber! <= searchEndBlock);
-
-    if (messageSentLogs.length === 0) {
-      break;
-    }
-
-    for (const log of messageSentLogs) {
-      const { index, hash } = log.args;
-      retrievedL1ToL2Messages.push(new InboxLeaf(index!, Fr.fromHexString(hash!)));
-    }
-
-    // handles the case when there are no new messages:
-    searchStartBlock = (messageSentLogs.findLast(msgLog => !!msgLog)?.blockNumber || searchStartBlock) + 1n;
-  } while (searchStartBlock <= searchEndBlock);
-  return { lastProcessedL1BlockNumber: searchStartBlock - 1n, retrievedData: retrievedL1ToL2Messages };
-}
-
-/** Retrieves L2ProofVerified events from the rollup contract. */
-export async function retrieveL2ProofVerifiedEvents(
-  publicClient: ViemPublicClient,
-  rollupAddress: EthAddress,
-  searchStartBlock: bigint,
-  searchEndBlock?: bigint,
-): Promise<{ l1BlockNumber: bigint; l2BlockNumber: bigint; proverId: Fr; txHash: Hex }[]> {
-  const logs = await publicClient.getLogs({
-    address: rollupAddress.toString(),
-    fromBlock: searchStartBlock,
-    toBlock: searchEndBlock ? searchEndBlock : undefined,
-    strict: true,
-    event: getAbiItem({ abi: RollupAbi, name: 'L2ProofVerified' }),
-  });
-
-  return logs.map(log => ({
-    l1BlockNumber: log.blockNumber,
-    l2BlockNumber: log.args.blockNumber,
-    proverId: Fr.fromHexString(log.args.proverId),
-    txHash: log.transactionHash,
-  }));
-}
-
-/** Retrieve submitted proofs from the rollup contract */
-export async function retrieveL2ProofsFromRollup(
-  publicClient: ViemPublicClient,
-  rollupAddress: EthAddress,
-  searchStartBlock: bigint,
-  searchEndBlock?: bigint,
-): Promise<DataRetrieval<{ proof: Proof; proverId: Fr; l2BlockNumber: bigint; txHash: `0x${string}` }>> {
-  const logs = await retrieveL2ProofVerifiedEvents(publicClient, rollupAddress, searchStartBlock, searchEndBlock);
-  const retrievedData: { proof: Proof; proverId: Fr; l2BlockNumber: bigint; txHash: `0x${string}` }[] = [];
-  const lastProcessedL1BlockNumber = logs.length > 0 ? logs.at(-1)!.l1BlockNumber : searchStartBlock - 1n;
-
-  for (const { txHash, proverId, l2BlockNumber } of logs) {
-    const proofData = await getProofFromSubmitProofTx(publicClient, txHash, proverId);
-    retrievedData.push({ proof: proofData.proof, proverId: proofData.proverId, l2BlockNumber, txHash });
-  }
-  return {
-    retrievedData,
-    lastProcessedL1BlockNumber,
-  };
-}
-
-export type SubmitBlockProof = {
-  archiveRoot: Fr;
-  proverId: Fr;
-  aggregationObject: Buffer;
-  proof: Proof;
-};
-
-/**
- * Gets block metadata (header and archive snapshot) from the calldata of an L1 transaction.
- * Assumes that the block was published from an EOA.
- * TODO: Add retries and error management.
- * @param publicClient - The viem public client to use for transaction retrieval.
- * @param txHash - Hash of the tx that published it.
- * @param l2BlockNum - L2 block number.
- * @returns L2 block metadata (header and archive) from the calldata, deserialized
- */
-export async function getProofFromSubmitProofTx(
-  publicClient: ViemPublicClient,
-  txHash: `0x${string}`,
-  expectedProverId: Fr,
-): Promise<SubmitBlockProof> {
-  const { input: data } = await publicClient.getTransaction({ hash: txHash });
-  const { functionName, args } = decodeFunctionData({ abi: RollupAbi, data });
-
-  let proverId: Fr;
-  let archiveRoot: Fr;
-  let aggregationObject: Buffer;
-  let proof: Proof;
-
-  if (functionName === 'submitEpochRootProof') {
-    const [decodedArgs] = args as readonly [
-      {
-        start: bigint;
-        end: bigint;
-        args: EpochProofPublicInputArgs;
-        fees: readonly Hex[];
-        aggregationObject: Hex;
-        proof: Hex;
-      },
-    ];
-
-    aggregationObject = Buffer.from(hexToBytes(decodedArgs.aggregationObject));
-    proverId = Fr.fromHexString(decodedArgs.args.proverId);
-    archiveRoot = Fr.fromHexString(decodedArgs.args.endArchive);
-    proof = Proof.fromBuffer(Buffer.from(hexToBytes(decodedArgs.proof)));
-  } else {
-    throw new Error(`Unexpected proof method called ${functionName}`);
-  }
-
-  if (!proverId.equals(expectedProverId)) {
-    throw new Error(`Prover ID mismatch: expected ${expectedProverId} but got ${proverId}`);
-  }
-
-  return {
-    proverId,
-    aggregationObject,
-    archiveRoot,
-    proof,
-  };
-}

package/src/archiver/kv_archiver_store/nullifier_store.ts

@@ -1,97 +0,0 @@
-import { MAX_NULLIFIERS_PER_TX } from '@aztec/constants';
-import type { Fr } from '@aztec/foundation/fields';
-import { createLogger } from '@aztec/foundation/log';
-import type { AztecAsyncKVStore, AztecAsyncMap } from '@aztec/kv-store';
-import type { InBlock, L2Block } from '@aztec/stdlib/block';
-
-export class NullifierStore {
-  #nullifiersToBlockNumber: AztecAsyncMap<string, number>;
-  #nullifiersToBlockHash: AztecAsyncMap<string, string>;
-  #nullifiersToIndex: AztecAsyncMap<string, number>;
-  #log = createLogger('archiver:log_store');
-
-  constructor(private db: AztecAsyncKVStore) {
-    this.#nullifiersToBlockNumber = db.openMap('archiver_nullifiers_to_block_number');
-    this.#nullifiersToBlockHash = db.openMap('archiver_nullifiers_to_block_hash');
-    this.#nullifiersToIndex = db.openMap('archiver_nullifiers_to_index');
-  }
-
-  async addNullifiers(blocks: L2Block[]): Promise<boolean> {
-    const blockHashes = await Promise.all(blocks.map(block => block.hash()));
-    await this.db.transactionAsync(async () => {
-      await Promise.all(
-        blocks.map((block, i) => {
-          const dataStartIndexForBlock =
-            block.header.state.partial.nullifierTree.nextAvailableLeafIndex -
-            block.body.txEffects.length * MAX_NULLIFIERS_PER_TX;
-          return Promise.all(
-            block.body.txEffects.map((txEffects, txIndex) => {
-              const dataStartIndexForTx = dataStartIndexForBlock + txIndex * MAX_NULLIFIERS_PER_TX;
-              return Promise.all(
-                txEffects.nullifiers.map(async (nullifier, nullifierIndex) => {
-                  await this.#nullifiersToBlockNumber.set(nullifier.toString(), block.number);
-                  await this.#nullifiersToBlockHash.set(nullifier.toString(), blockHashes[i].toString());
-                  await this.#nullifiersToIndex.set(nullifier.toString(), dataStartIndexForTx + nullifierIndex);
-                }),
-              );
-            }),
-          );
-        }),
-      );
-    });
-    return true;
-  }
-
-  async deleteNullifiers(blocks: L2Block[]): Promise<boolean> {
-    await this.db.transactionAsync(async () => {
-      for (const block of blocks) {
-        for (const nullifier of block.body.txEffects.flatMap(tx => tx.nullifiers)) {
-          await Promise.all([
-            this.#nullifiersToBlockNumber.delete(nullifier.toString()),
-            this.#nullifiersToBlockHash.delete(nullifier.toString()),
-            this.#nullifiersToIndex.delete(nullifier.toString()),
-          ]);
-        }
-      }
-    });
-    return true;
-  }
-
-  async findNullifiersIndexesWithBlock(
-    blockNumber: number,
-    nullifiers: Fr[],
-  ): Promise<(InBlock<bigint> | undefined)[]> {
-    const asStrings = nullifiers.map(x => x.toString());
-
-    const maybeNullifiers = await Promise.all(
-      asStrings.map(async nullifier => {
-        const [data, l2BlockNumber, l2BlockHash] = await Promise.all([
-          this.#nullifiersToIndex.getAsync(nullifier),
-          this.#nullifiersToBlockNumber.getAsync(nullifier),
-          this.#nullifiersToBlockHash.getAsync(nullifier),
-        ]);
-        return {
-          data,
-          l2BlockNumber,
-          l2BlockHash,
-        };
-      }),
-    );
-    return maybeNullifiers.map(({ data, l2BlockNumber, l2BlockHash }) => {
-      if (
-        data === undefined ||
-        l2BlockNumber === undefined ||
-        l2BlockHash === undefined ||
-        l2BlockNumber > blockNumber
-      ) {
-        return undefined;
-      } else {
-        return {
-          data: BigInt(data),
-          l2BlockNumber,
-          l2BlockHash,
-        } as InBlock<bigint>;
-      }
-    });
-  }
-}

package/src/archiver/memory_archiver_store/l1_to_l2_message_store.ts

@@ -1,61 +0,0 @@
-import { L1_TO_L2_MSG_SUBTREE_HEIGHT } from '@aztec/constants';
-import type { Fr } from '@aztec/foundation/fields';
-import { InboxLeaf } from '@aztec/stdlib/messaging';
-
-/**
- * A simple in-memory implementation of an L1 to L2 message store.
- */
-export class L1ToL2MessageStore {
-  /**
-   * A map pointing from a key in a "messageIndex" format to the corresponding L1 to L2 message hash.
-   */
-  protected store: Map<string, Fr> = new Map();
-
-  #l1ToL2MessagesSubtreeSize = 2 ** L1_TO_L2_MSG_SUBTREE_HEIGHT;
-
-  constructor() {}
-
-  getTotalL1ToL2MessageCount(): bigint {
-    return BigInt(this.store.size);
-  }
-
-  addMessage(message: InboxLeaf) {
-    this.store.set(`${message.index}`, message.leaf);
-  }
-
-  getMessages(blockNumber: bigint): Fr[] {
-    const messages: Fr[] = [];
-    let undefinedMessageFound = false;
-    const startIndex = Number(InboxLeaf.smallestIndexFromL2Block(blockNumber));
-
-    for (let i = startIndex; i < startIndex + this.#l1ToL2MessagesSubtreeSize; i++) {
-      // This is inefficient but probably fine for now.
-      const message = this.store.get(`${i}`);
-      if (message) {
-        if (undefinedMessageFound) {
-          throw new Error(`L1 to L2 message gap found in block ${blockNumber}`);
-        }
-        messages.push(message);
-      } else {
-        undefinedMessageFound = true;
-        // We continue iterating over messages here to verify that there are no more messages after the undefined one.
-        // --> If this was the case this would imply there is some issue with log fetching.
-      }
-    }
-    return messages;
-  }
-
-  /**
-   * Gets the L1 to L2 message index in the L1 to L2 message tree.
-   * @param l1ToL2Message - The L1 to L2 message.
-   * @returns The index of the L1 to L2 message in the L1 to L2 message tree (undefined if not found).
-   */
-  getMessageIndex(l1ToL2Message: Fr): bigint | undefined {
-    for (const [key, message] of this.store.entries()) {
-      if (message.equals(l1ToL2Message)) {
-        return BigInt(key);
-      }
-    }
-    return undefined;
-  }
-}