@aztec/archiver 0.0.1-commit.9593d84 → 0.0.1-commit.96dac018d

package/src/factory.ts

@@ -1,35 +1,52 @@
-import { createLogger } from '@aztec/foundation/log';
+import { EpochCache } from '@aztec/epoch-cache';
+import { createEthereumChain } from '@aztec/ethereum/chain';
+import { InboxContract, RollupContract } from '@aztec/ethereum/contracts';
+import type { ViemPublicDebugClient } from '@aztec/ethereum/types';
+import { BlockNumber } from '@aztec/foundation/branded-types';
+import { Buffer32 } from '@aztec/foundation/buffer';
+import { merge } from '@aztec/foundation/collection';
+import { Fr } from '@aztec/foundation/curves/bn254';
+import { DateProvider } from '@aztec/foundation/timer';
 import type { DataStoreConfig } from '@aztec/kv-store/config';
 import { createStore } from '@aztec/kv-store/lmdb-v2';
 import { protocolContractNames } from '@aztec/protocol-contracts';
 import { BundledProtocolContractsProvider } from '@aztec/protocol-contracts/providers/bundle';
 import { FunctionType, decodeFunctionSignature } from '@aztec/stdlib/abi';
+import type { ArchiverEmitter } from '@aztec/stdlib/block';
 import { type ContractClassPublic, computePublicBytecodeCommitment } from '@aztec/stdlib/contract';
+import type { L1RollupConstants } from '@aztec/stdlib/epoch-helpers';
+import { getTelemetryClient } from '@aztec/telemetry-client';
 
-import { Archiver, type ArchiverDeps } from './archiver/archiver.js';
-import type { ArchiverConfig } from './archiver/config.js';
-import { ARCHIVER_DB_VERSION, KVArchiverDataStore } from './archiver/kv_archiver_store/kv_archiver_store.js';
+import { EventEmitter } from 'events';
+import { createPublicClient, fallback, http } from 'viem';
+
+import { Archiver, type ArchiverDeps } from './archiver.js';
+import { type ArchiverConfig, mapArchiverConfig } from './config.js';
+import { ArchiverInstrumentation } from './modules/instrumentation.js';
+import { ArchiverL1Synchronizer } from './modules/l1_synchronizer.js';
+import { ARCHIVER_DB_VERSION, KVArchiverDataStore } from './store/kv_archiver_store.js';
+import { L2TipsCache } from './store/l2_tips_cache.js';
 
 export const ARCHIVER_STORE_NAME = 'archiver';
 
 /** Creates an archiver store. */
 export async function createArchiverStore(
   userConfig: Pick<ArchiverConfig, 'archiverStoreMapSizeKb' | 'maxLogs'> & DataStoreConfig,
+  l1Constants: Pick<L1RollupConstants, 'epochDuration'>,
 ) {
   const config = {
     ...userConfig,
     dataStoreMapSizeKb: userConfig.archiverStoreMapSizeKb ?? userConfig.dataStoreMapSizeKb,
   };
-  const store = await createStore(ARCHIVER_STORE_NAME, ARCHIVER_DB_VERSION, config, createLogger('archiver:lmdb'));
-  return new KVArchiverDataStore(store, config.maxLogs);
+  const store = await createStore(ARCHIVER_STORE_NAME, ARCHIVER_DB_VERSION, config);
+  return new KVArchiverDataStore(store, config.maxLogs, l1Constants);
 }
 
 /**
  * Creates a local archiver.
  * @param config - The archiver configuration.
- * @param blobSinkClient - The blob sink client.
+ * @param deps - The archiver dependencies (blobClient, epochCache, dateProvider, telemetry).
  * @param opts - The options.
- * @param telemetry - The telemetry client.
  * @returns The local archiver.
  */
 export async function createArchiver(
@@ -37,12 +54,124 @@ export async function createArchiver(
   deps: ArchiverDeps,
   opts: { blockUntilSync: boolean } = { blockUntilSync: true },
 ): Promise<Archiver> {
-  const archiverStore = await createArchiverStore(config);
+  const archiverStore = await createArchiverStore(config, { epochDuration: config.aztecEpochDuration });
   await registerProtocolContracts(archiverStore);
-  return Archiver.createAndSync(config, archiverStore, deps, opts.blockUntilSync);
+
+  // Create Ethereum clients
+  const chain = createEthereumChain(config.l1RpcUrls, config.l1ChainId);
+  const publicClient = createPublicClient({
+    chain: chain.chainInfo,
+    transport: fallback(config.l1RpcUrls.map(url => http(url, { batch: false }))),
+    pollingInterval: config.viemPollingIntervalMS,
+  });
+
+  // Create debug client using debug RPC URLs if available, otherwise fall back to regular RPC URLs
+  const debugRpcUrls = config.l1DebugRpcUrls.length > 0 ? config.l1DebugRpcUrls : config.l1RpcUrls;
+  const debugClient = createPublicClient({
+    chain: chain.chainInfo,
+    transport: fallback(debugRpcUrls.map(url => http(url, { batch: false }))),
+    pollingInterval: config.viemPollingIntervalMS,
+  }) as ViemPublicDebugClient;
+
+  // Create L1 contract instances
+  const rollup = new RollupContract(publicClient, config.l1Contracts.rollupAddress);
+  const inbox = new InboxContract(publicClient, config.l1Contracts.inboxAddress);
+
+  // Fetch L1 constants from rollup contract
+  const [
+    l1StartBlock,
+    l1GenesisTime,
+    proofSubmissionEpochs,
+    genesisArchiveRoot,
+    slashingProposerAddress,
+    targetCommitteeSize,
+  ] = await Promise.all([
+    rollup.getL1StartBlock(),
+    rollup.getL1GenesisTime(),
+    rollup.getProofSubmissionEpochs(),
+    rollup.getGenesisArchiveTreeRoot(),
+    rollup.getSlashingProposerAddress(),
+    rollup.getTargetCommitteeSize(),
+  ] as const);
+
+  const l1StartBlockHash = await publicClient
+    .getBlock({ blockNumber: l1StartBlock, includeTransactions: false })
+    .then(block => Buffer32.fromString(block.hash));
+
+  const { aztecEpochDuration: epochDuration, aztecSlotDuration: slotDuration, ethereumSlotDuration } = config;
+
+  const l1Constants = {
+    l1StartBlockHash,
+    l1StartBlock,
+    l1GenesisTime,
+    epochDuration,
+    slotDuration,
+    ethereumSlotDuration,
+    proofSubmissionEpochs: Number(proofSubmissionEpochs),
+    targetCommitteeSize,
+    genesisArchiveRoot: Fr.fromString(genesisArchiveRoot.toString()),
+  };
+
+  const archiverConfig = merge(
+    {
+      pollingIntervalMs: 10_000,
+      batchSize: 100,
+      maxAllowedEthClientDriftSeconds: 300,
+      ethereumAllowNoDebugHosts: false,
+    },
+    mapArchiverConfig(config),
+  );
+
+  const epochCache = deps.epochCache ?? (await EpochCache.create(config.l1Contracts.rollupAddress, config, deps));
+  const telemetry = deps.telemetry ?? getTelemetryClient();
+  const instrumentation = await ArchiverInstrumentation.new(telemetry, () => archiverStore.estimateSize());
+
+  // Create the event emitter that will be shared by archiver and synchronizer
+  const events = new EventEmitter() as ArchiverEmitter;
+
+  // Create L2 tips cache shared by archiver and synchronizer
+  const l2TipsCache = new L2TipsCache(archiverStore.blockStore);
+
+  // Create the L1 synchronizer
+  const synchronizer = new ArchiverL1Synchronizer(
+    publicClient,
+    debugClient,
+    rollup,
+    inbox,
+    archiverStore,
+    archiverConfig,
+    deps.blobClient,
+    epochCache,
+    deps.dateProvider ?? new DateProvider(),
+    instrumentation,
+    l1Constants,
+    events,
+    instrumentation.tracer,
+    l2TipsCache,
+    undefined, // log (use default)
+  );
+
+  const archiver = new Archiver(
+    publicClient,
+    debugClient,
+    rollup,
+    { ...config.l1Contracts, slashingProposerAddress },
+    archiverStore,
+    archiverConfig,
+    deps.blobClient,
+    instrumentation,
+    l1Constants,
+    synchronizer,
+    events,
+    l2TipsCache,
+  );
+
+  await archiver.start(opts.blockUntilSync);
+  return archiver;
 }
 
-async function registerProtocolContracts(store: KVArchiverDataStore) {
+/** Registers protocol contracts in the archiver store. */
+export async function registerProtocolContracts(store: KVArchiverDataStore) {
   const blockNumber = 0;
   for (const name of protocolContractNames) {
     const provider = new BundledProtocolContractsProvider();
@@ -59,7 +188,7 @@ async function registerProtocolContracts(store: KVArchiverDataStore) {
 
     await store.registerContractFunctionSignatures(publicFunctionSignatures);
     const bytecodeCommitment = await computePublicBytecodeCommitment(contractClassPublic.packedBytecode);
-    await store.addContractClasses([contractClassPublic], [bytecodeCommitment], blockNumber);
-    await store.addContractInstances([contract.instance], blockNumber);
+    await store.addContractClasses([contractClassPublic], [bytecodeCommitment], BlockNumber(blockNumber));
+    await store.addContractInstances([contract.instance], BlockNumber(blockNumber));
   }
 }

package/src/index.ts

@@ -1,5 +1,13 @@
-export * from './archiver/index.js';
 export * from './factory.js';
-export * from './rpc/index.js';
+export * from './interfaces.js';
+export * from './archiver.js';
+export * from './modules/data_source_base.js';
+export * from './modules/data_store_updater.js';
+export * from './config.js';
 
-export { retrieveL2ProofVerifiedEvents } from './archiver/data_retrieval.js';
+export { type L1PublishedData } from './structs/published.js';
+export { KVArchiverDataStore, ARCHIVER_DB_VERSION } from './store/kv_archiver_store.js';
+export { ContractInstanceStore } from './store/contract_instance_store.js';
+export { L2TipsCache } from './store/l2_tips_cache.js';
+
+export { retrieveCheckpointsFromRollup, retrieveL2ProofVerifiedEvents } from './l1/data_retrieval.js';

package/src/interfaces.ts

@@ -0,0 +1,9 @@
+import type { L2BlockSource } from '@aztec/stdlib/block';
+import type { ContractDataSource } from '@aztec/stdlib/contract';
+import type { L2LogsSource } from '@aztec/stdlib/interfaces/server';
+import type { L1ToL2MessageSource } from '@aztec/stdlib/messaging';
+
+/**
+ * Helper interface to combine all sources this archiver implementation provides.
+ */
+export type ArchiverDataSource = L2BlockSource & L2LogsSource & ContractDataSource & L1ToL2MessageSource;

package/src/l1/README.md

@@ -0,0 +1,55 @@
+# Archiver L1 Data Retrieval
+
+Modules and classes to handle data retrieval from L1 for the archiver.
+
+## Calldata Retriever
+
+The sequencer publisher bundles multiple operations into a single multicall3 transaction for gas
+efficiency. The archiver needs to extract the `propose` calldata from these bundled transactions
+to reconstruct L2 blocks.
+
+The retriever uses hash matching against `attestationsHash` and `payloadDigest` from the
+`CheckpointProposed` L1 event to verify it has found the correct propose calldata. These hashes
+are always required.
+
+### Multicall3 Decoding with Hash Matching
+
+First attempt to decode the transaction as a multicall3 `aggregate3` call:
+
+- Check if transaction is to multicall3 address (`0xcA11bde05977b3631167028862bE2a173976CA11`)
+- Decode as `aggregate3(Call3[] calldata calls)`
+- Find all calls matching the rollup contract address and the `propose` function selector
+- Verify each candidate by computing `attestationsHash` (keccak256 of ABI-encoded attestations)
+  and `payloadDigest` (keccak256 of the consensus payload signing hash) and comparing against
+  expected values from the `CheckpointProposed` event
+- Return the verified candidate (if multiple verify, return the first with a warning)
+
+This approach works regardless of what other calls are in the multicall3 bundle, because hash
+matching identifies the correct propose call without needing an allowlist.
+
+### Direct Propose Call
+
+Second attempt to decode the transaction as a direct `propose` call to the rollup contract:
+
+- Check if transaction is to the rollup address
+- Decode as `propose` function call
+- Verify against expected hashes
+- Return the transaction input as the propose calldata
+
+### Spire Proposer Call
+
+Given existing attempts to route the call via the Spire proposer, we also check if the tx is
+`to` the proposer known address. If so, we extract all wrapped calls and try each as either
+a multicall3 or direct propose call, using hash matching to find and verify the correct one.
+
+Since the Spire proposer is upgradeable, we check that the implementation has not changed in
+order to decode. Any validation failure triggers fallback to the next step.
+
+### Debug and Trace Transaction Fallback
+
+Last, we use L1 node's trace/debug RPC methods to definitively identify the one successful
+`propose` call within the tx. We can then extract the exact calldata that hit the `propose`
+function in the rollup contract.
+
+This approach requires access to a debug-enabled L1 node, which may be more resource-intensive,
+so we only use it as a fallback when earlier steps fail, which should be rare in practice.

package/src/l1/bin/retrieve-calldata.ts

@@ -0,0 +1,194 @@
+#!/usr/bin/env node
+import type { ViemPublicClient, ViemPublicDebugClient } from '@aztec/ethereum/types';
+import { CheckpointNumber } from '@aztec/foundation/branded-types';
+import { EthAddress } from '@aztec/foundation/eth-address';
+import { createLogger } from '@aztec/foundation/log';
+import { RollupAbi } from '@aztec/l1-artifacts/RollupAbi';
+
+import { type Hex, createPublicClient, decodeEventLog, getAbiItem, http, toEventSelector } from 'viem';
+import { mainnet } from 'viem/chains';
+
+import { CalldataRetriever } from '../calldata_retriever.js';
+
+const logger = createLogger('archiver:calldata-test');
+
+interface ScriptArgs {
+  rollupAddress: EthAddress;
+  txHash: Hex;
+  rpcUrl: string;
+  targetCommitteeSize: number;
+}
+
+function parseArgs(): ScriptArgs {
+  const args = process.argv.slice(2);
+
+  if (args.length < 2) {
+    // eslint-disable-next-line no-console
+    console.error('Usage: node index.js <rollup-address> <tx-hash> [target-committee-size]');
+    // eslint-disable-next-line no-console
+    console.error('');
+    // eslint-disable-next-line no-console
+    console.error('Environment variables:');
+    // eslint-disable-next-line no-console
+    console.error('  ETHEREUM_HOST or RPC_URL - Ethereum RPC endpoint');
+    // eslint-disable-next-line no-console
+    console.error('');
+    // eslint-disable-next-line no-console
+    console.error('Example:');
+    // eslint-disable-next-line no-console
+    console.error('  RPC_URL=https://eth-mainnet.g.alchemy.com/v2/YOUR-API-KEY \\');
+    // eslint-disable-next-line no-console
+    console.error('  node index.js 0x1234... 0xabcd... 32');
+    process.exit(1);
+  }
+
+  const rollupAddress = EthAddress.fromString(args[0]);
+  const txHash = args[1] as Hex;
+  const targetCommitteeSize = args[2] ? parseInt(args[2], 10) : 24;
+
+  const rpcUrl = process.env.ETHEREUM_HOST || process.env.RPC_URL;
+  if (!rpcUrl) {
+    // eslint-disable-next-line no-console
+    console.error('Error: ETHEREUM_HOST or RPC_URL environment variable must be set');
+    process.exit(1);
+  }
+
+  if (targetCommitteeSize <= 0 || targetCommitteeSize > 256) {
+    // eslint-disable-next-line no-console
+    console.error('Error: target-committee-size must be between 1 and 256');
+    process.exit(1);
+  }
+
+  return { rollupAddress, txHash, rpcUrl, targetCommitteeSize };
+}
+
+async function main() {
+  const { rollupAddress, txHash, rpcUrl, targetCommitteeSize } = parseArgs();
+
+  logger.info('Calldata Retriever Test Script');
+  logger.info('===============================');
+  logger.info(`Rollup Address: ${rollupAddress.toString()}`);
+  logger.info(`Transaction Hash: ${txHash}`);
+  logger.info(`RPC URL: ${rpcUrl}`);
+  logger.info(`Target Committee Size: ${targetCommitteeSize}`);
+  logger.info('');
+
+  try {
+    // Create viem public client
+    const publicClient = createPublicClient({
+      chain: mainnet,
+      transport: http(rpcUrl, { batch: false }),
+    });
+
+    logger.info('Fetching transaction...');
+    const tx = await publicClient.getTransaction({ hash: txHash });
+
+    if (!tx) {
+      throw new Error(`Transaction ${txHash} not found`);
+    }
+
+    logger.info(`Transaction found in block ${tx.blockNumber}`);
+
+    // Create CalldataRetriever
+    const retriever = new CalldataRetriever(
+      publicClient as unknown as ViemPublicClient,
+      publicClient as unknown as ViemPublicDebugClient,
+      targetCommitteeSize,
+      undefined,
+      logger,
+      rollupAddress,
+    );
+
+    // Extract checkpoint number and hashes from transaction logs
+    logger.info('Decoding transaction to extract checkpoint number and hashes...');
+    const receipt = await publicClient.getTransactionReceipt({ hash: txHash });
+
+    // Look for CheckpointProposed event
+    const checkpointProposedEventAbi = getAbiItem({ abi: RollupAbi, name: 'CheckpointProposed' });
+    const checkpointProposedLog = receipt.logs.find(log => {
+      try {
+        return (
+          log.address.toLowerCase() === rollupAddress.toString().toLowerCase() &&
+          log.topics[0] === toEventSelector(checkpointProposedEventAbi)
+        );
+      } catch {
+        return false;
+      }
+    });
+
+    if (!checkpointProposedLog || checkpointProposedLog.topics[1] === undefined) {
+      throw new Error(`Checkpoint proposed event not found`);
+    }
+
+    const checkpointNumber = CheckpointNumber.fromBigInt(BigInt(checkpointProposedLog.topics[1]));
+
+    // Decode the full event to extract attestationsHash and payloadDigest
+    const decodedEvent = decodeEventLog({
+      abi: RollupAbi,
+      data: checkpointProposedLog.data,
+      topics: checkpointProposedLog.topics,
+    });
+
+    const eventArgs = decodedEvent.args as {
+      checkpointNumber: bigint;
+      archive: Hex;
+      versionedBlobHashes: Hex[];
+      attestationsHash: Hex;
+      payloadDigest: Hex;
+    };
+
+    if (!eventArgs.attestationsHash || !eventArgs.payloadDigest) {
+      throw new Error(`CheckpointProposed event missing attestationsHash or payloadDigest`);
+    }
+
+    const expectedHashes = {
+      attestationsHash: eventArgs.attestationsHash,
+      payloadDigest: eventArgs.payloadDigest,
+    };
+
+    logger.info(`Checkpoint Number: ${checkpointNumber}`);
+    logger.info(`Attestations Hash: ${expectedHashes.attestationsHash}`);
+    logger.info(`Payload Digest: ${expectedHashes.payloadDigest}`);
+
+    logger.info('');
+    logger.info('Retrieving checkpoint from rollup transaction...');
+    logger.info('');
+
+    const result = await retriever.getCheckpointFromRollupTx(txHash, [], checkpointNumber, expectedHashes);
+
+    logger.info(' Successfully retrieved block header!');
+    logger.info('');
+    logger.info('Block Header Details:');
+    logger.info('====================');
+    logger.info(`Checkpoint Number: ${result.checkpointNumber}`);
+    logger.info(`Block Hash: ${result.blockHash}`);
+    logger.info(`Archive Root: ${result.archiveRoot.toString()}`);
+    logger.info('');
+    logger.info('Header:');
+    logger.info(`  Slot Number: ${result.header.slotNumber.toString()}`);
+    logger.info(`  Timestamp: ${result.header.timestamp.toString()}`);
+    logger.info(`  Coinbase: ${result.header.coinbase.toString()}`);
+    logger.info(`  Fee Recipient: ${result.header.feeRecipient.toString()}`);
+    logger.info(`  Total Mana Used: ${result.header.totalManaUsed.toString()}`);
+    logger.info('');
+    logger.info('Attestations:');
+    logger.info(`  Count: ${result.attestations.length}`);
+    logger.info(`  Non-empty attestations: ${result.attestations.filter((a: any) => !a.signature.isEmpty()).length}`);
+
+    process.exit(0);
+  } catch (error) {
+    logger.error('Error retrieving block header:');
+    logger.error(error instanceof Error ? error.message : String(error));
+
+    if (error instanceof Error && error.stack) {
+      logger.debug(error.stack);
+    }
+
+    process.exit(1);
+  }
+}
+
+// Only run if this is the main module
+if (import.meta.url === `file://${process.argv[1]}`) {
+  void main();
+}