@aztec/archiver 0.0.1-commit.1142ef1 → 0.0.1-commit.11bf3dd6e

package/src/errors.ts

@@ -0,0 +1,146 @@
+import type { Fr } from '@aztec/foundation/schemas';
+
+export class NoBlobBodiesFoundError extends Error {
+  constructor(l2BlockNum: number) {
+    super(`No blob bodies found for block ${l2BlockNum}`);
+  }
+}
+
+export class BlockNumberNotSequentialError extends Error {
+  constructor(newBlockNumber: number, previous: number | undefined) {
+    super(`Cannot insert new block ${newBlockNumber} given previous block number is ${previous ?? 'undefined'}`);
+  }
+}
+
+export class InitialCheckpointNumberNotSequentialError extends Error {
+  constructor(
+    public readonly newCheckpointNumber: number,
+    public readonly previousCheckpointNumber: number | undefined,
+  ) {
+    super(
+      `Cannot insert new checkpoint ${newCheckpointNumber} given previous checkpoint number in store is ${
+        previousCheckpointNumber ?? 'undefined'
+      }`,
+    );
+  }
+}
+
+export class CheckpointNumberNotSequentialError extends Error {
+  constructor(
+    newCheckpointNumber: number,
+    previous: number | undefined,
+    source: 'confirmed' | 'proposed' = 'confirmed',
+  ) {
+    super(
+      `Cannot insert new checkpoint ${newCheckpointNumber} given previous ${source} checkpoint number is ${previous ?? 'undefined'}`,
+    );
+  }
+}
+
+export class BlockIndexNotSequentialError extends Error {
+  constructor(newBlockIndex: number, previousBlockIndex: number | undefined) {
+    super(
+      `Cannot insert new block at checkpoint index ${newBlockIndex} given previous block index is ${previousBlockIndex ?? 'undefined'}`,
+    );
+  }
+}
+
+export class BlockArchiveNotConsistentError extends Error {
+  constructor(
+    newBlockNumber: number,
+    previousBlockNumber: number | undefined,
+    newBlockArchive: Fr,
+    previousBlockArchive: Fr,
+  ) {
+    super(
+      `Cannot insert new block number ${newBlockNumber} with archive ${newBlockArchive.toString()} previous block number is ${previousBlockNumber ?? 'undefined'}, previous archive is ${previousBlockArchive?.toString() ?? 'undefined'}`,
+    );
+  }
+}
+
+export class CheckpointNotFoundError extends Error {
+  constructor(checkpointNumber: number) {
+    super(`Failed to find expected checkpoint number ${checkpointNumber}`);
+  }
+}
+
+export class BlockNotFoundError extends Error {
+  constructor(blockNumber: number) {
+    super(`Failed to find expected block number ${blockNumber}`);
+  }
+}
+
+/** Thrown when a proposed block matches a block that was already checkpointed. This is expected for late proposals. */
+export class BlockAlreadyCheckpointedError extends Error {
+  constructor(public readonly blockNumber: number) {
+    super(`Block ${blockNumber} has already been checkpointed with the same content`);
+    this.name = 'BlockAlreadyCheckpointedError';
+  }
+}
+
+/** Thrown when logs are added for a tag whose last stored log has a higher block number than the new log. */
+export class OutOfOrderLogInsertionError extends Error {
+  constructor(
+    public readonly logType: 'private' | 'public',
+    public readonly tag: string,
+    public readonly lastBlockNumber: number,
+    public readonly newBlockNumber: number,
+  ) {
+    super(
+      `Out-of-order ${logType} log insertion for tag ${tag}: ` +
+        `last existing log is from block ${lastBlockNumber} but new log is from block ${newBlockNumber}`,
+    );
+    this.name = 'OutOfOrderLogInsertionError';
+  }
+}
+
+/** Thrown when L1 to L2 messages are requested for a checkpoint whose message tree hasn't been sealed yet. */
+export class L1ToL2MessagesNotReadyError extends Error {
+  constructor(
+    public readonly checkpointNumber: number,
+    public readonly inboxTreeInProgress: bigint,
+  ) {
+    super(
+      `Cannot get L1 to L2 messages for checkpoint ${checkpointNumber}: ` +
+        `inbox tree in progress is ${inboxTreeInProgress}, messages not yet sealed`,
+    );
+    this.name = 'L1ToL2MessagesNotReadyError';
+  }
+}
+
+/** Thrown when a proposed checkpoint number is stale (already processed). */
+export class ProposedCheckpointStaleError extends Error {
+  constructor(
+    public readonly proposedCheckpointNumber: number,
+    public readonly currentProposedNumber: number,
+  ) {
+    super(`Stale proposed checkpoint ${proposedCheckpointNumber}: current proposed is ${currentProposedNumber}`);
+    this.name = 'ProposedCheckpointStaleError';
+  }
+}
+
+/** Thrown when a proposed checkpoint number is not the expected confirmed + 1. */
+export class ProposedCheckpointNotSequentialError extends Error {
+  constructor(
+    public readonly proposedCheckpointNumber: number,
+    public readonly confirmedCheckpointNumber: number,
+  ) {
+    super(
+      `Proposed checkpoint ${proposedCheckpointNumber} is not sequential: expected ${confirmedCheckpointNumber + 1} (confirmed + 1)`,
+    );
+    this.name = 'ProposedCheckpointNotSequentialError';
+  }
+}
+
+/** Thrown when a proposed block conflicts with an already checkpointed block (different content). */
+export class CannotOverwriteCheckpointedBlockError extends Error {
+  constructor(
+    public readonly blockNumber: number,
+    public readonly lastCheckpointedBlock: number,
+  ) {
+    super(
+      `Cannot add block ${blockNumber}: would overwrite checkpointed data (checkpointed up to block ${lastCheckpointedBlock})`,
+    );
+    this.name = 'CannotOverwriteCheckpointedBlockError';
+  }
+}

package/src/factory.ts

@@ -1,15 +1,31 @@
+import { EpochCache } from '@aztec/epoch-cache';
+import { createEthereumChain } from '@aztec/ethereum/chain';
+import { makeL1HttpTransport } from '@aztec/ethereum/client';
+import { InboxContract, RollupContract } from '@aztec/ethereum/contracts';
+import type { ViemPublicDebugClient } from '@aztec/ethereum/types';
 import { BlockNumber } from '@aztec/foundation/branded-types';
-import { createLogger } from '@aztec/foundation/log';
-import type { DataStoreConfig } from '@aztec/kv-store/config';
+import { Buffer32 } from '@aztec/foundation/buffer';
+import { merge } from '@aztec/foundation/collection';
+import { Fr } from '@aztec/foundation/curves/bn254';
+import { DateProvider } from '@aztec/foundation/timer';
 import { createStore } from '@aztec/kv-store/lmdb-v2';
 import { protocolContractNames } from '@aztec/protocol-contracts';
 import { BundledProtocolContractsProvider } from '@aztec/protocol-contracts/providers/bundle';
 import { FunctionType, decodeFunctionSignature } from '@aztec/stdlib/abi';
-import { type ContractClassPublic, computePublicBytecodeCommitment } from '@aztec/stdlib/contract';
+import type { ArchiverEmitter } from '@aztec/stdlib/block';
+import { type ContractClassPublicWithCommitment, computePublicBytecodeCommitment } from '@aztec/stdlib/contract';
+import type { DataStoreConfig } from '@aztec/stdlib/kv-store';
+import { getTelemetryClient } from '@aztec/telemetry-client';
 
-import { Archiver, type ArchiverDeps } from './archiver/archiver.js';
-import type { ArchiverConfig } from './archiver/config.js';
-import { ARCHIVER_DB_VERSION, KVArchiverDataStore } from './archiver/kv_archiver_store/kv_archiver_store.js';
+import { EventEmitter } from 'events';
+import { createPublicClient } from 'viem';
+
+import { Archiver, type ArchiverDeps } from './archiver.js';
+import { type ArchiverConfig, mapArchiverConfig } from './config.js';
+import { ArchiverInstrumentation } from './modules/instrumentation.js';
+import { ArchiverL1Synchronizer } from './modules/l1_synchronizer.js';
+import { ARCHIVER_DB_VERSION, KVArchiverDataStore } from './store/kv_archiver_store.js';
+import { L2TipsCache } from './store/l2_tips_cache.js';
 
 export const ARCHIVER_STORE_NAME = 'archiver';
 
@@ -21,16 +37,15 @@ export async function createArchiverStore(
     ...userConfig,
     dataStoreMapSizeKb: userConfig.archiverStoreMapSizeKb ?? userConfig.dataStoreMapSizeKb,
   };
-  const store = await createStore(ARCHIVER_STORE_NAME, ARCHIVER_DB_VERSION, config, createLogger('archiver:lmdb'));
+  const store = await createStore(ARCHIVER_STORE_NAME, ARCHIVER_DB_VERSION, config);
   return new KVArchiverDataStore(store, config.maxLogs);
 }
 
 /**
  * Creates a local archiver.
  * @param config - The archiver configuration.
- * @param blobClient - The blob client client.
+ * @param deps - The archiver dependencies (blobClient, epochCache, dateProvider, telemetry).
  * @param opts - The options.
- * @param telemetry - The telemetry client.
  * @returns The local archiver.
  */
 export async function createArchiver(
@@ -40,18 +55,140 @@ export async function createArchiver(
 ): Promise<Archiver> {
   const archiverStore = await createArchiverStore(config);
   await registerProtocolContracts(archiverStore);
-  return Archiver.createAndSync(config, archiverStore, deps, opts.blockUntilSync);
+
+  // Create Ethereum clients
+  const chain = createEthereumChain(config.l1RpcUrls, config.l1ChainId);
+  const httpTimeout = config.l1HttpTimeoutMS;
+  const publicClient = createPublicClient({
+    chain: chain.chainInfo,
+    transport: makeL1HttpTransport(config.l1RpcUrls, { timeout: httpTimeout }),
+    pollingInterval: config.viemPollingIntervalMS,
+  });
+
+  // Create debug client using debug RPC URLs if available, otherwise fall back to regular RPC URLs
+  const debugRpcUrls = config.l1DebugRpcUrls.length > 0 ? config.l1DebugRpcUrls : config.l1RpcUrls;
+  const debugClient = createPublicClient({
+    chain: chain.chainInfo,
+    transport: makeL1HttpTransport(debugRpcUrls, { timeout: httpTimeout }),
+    pollingInterval: config.viemPollingIntervalMS,
+  }) as ViemPublicDebugClient;
+
+  // Create L1 contract instances
+  const rollup = new RollupContract(publicClient, config.l1Contracts.rollupAddress);
+  const inbox = new InboxContract(publicClient, config.l1Contracts.inboxAddress);
+
+  // Fetch L1 constants from rollup contract
+  const [
+    l1StartBlock,
+    l1GenesisTime,
+    proofSubmissionEpochs,
+    genesisArchiveRoot,
+    slashingProposerAddress,
+    targetCommitteeSize,
+    rollupManaLimit,
+  ] = await Promise.all([
+    rollup.getL1StartBlock(),
+    rollup.getL1GenesisTime(),
+    rollup.getProofSubmissionEpochs(),
+    rollup.getGenesisArchiveTreeRoot(),
+    rollup.getSlashingProposerAddress(),
+    rollup.getTargetCommitteeSize(),
+    rollup.getManaLimit(),
+  ] as const);
+
+  const l1StartBlockHash = await publicClient
+    .getBlock({ blockNumber: l1StartBlock, includeTransactions: false })
+    .then(block => Buffer32.fromString(block.hash));
+
+  const { aztecEpochDuration: epochDuration, aztecSlotDuration: slotDuration, ethereumSlotDuration } = config;
+
+  const l1Constants = {
+    l1StartBlockHash,
+    l1StartBlock,
+    l1GenesisTime,
+    epochDuration,
+    slotDuration,
+    ethereumSlotDuration,
+    proofSubmissionEpochs: Number(proofSubmissionEpochs),
+    targetCommitteeSize,
+    genesisArchiveRoot: Fr.fromString(genesisArchiveRoot.toString()),
+    rollupManaLimit: Number(rollupManaLimit),
+  };
+
+  const archiverConfig = merge(
+    {
+      pollingIntervalMs: 10_000,
+      batchSize: 100,
+      maxAllowedEthClientDriftSeconds: 300,
+      ethereumAllowNoDebugHosts: false,
+    },
+    mapArchiverConfig(config),
+  );
+
+  const epochCache = deps.epochCache ?? (await EpochCache.create(config.l1Contracts.rollupAddress, config, deps));
+  const telemetry = deps.telemetry ?? getTelemetryClient();
+  const instrumentation = await ArchiverInstrumentation.new(telemetry, () => archiverStore.estimateSize());
+
+  // Create the event emitter that will be shared by archiver and synchronizer
+  const events = new EventEmitter() as ArchiverEmitter;
+
+  // Create L2 tips cache shared by archiver and synchronizer
+  const l2TipsCache = new L2TipsCache(archiverStore.blockStore);
+
+  // Create the L1 synchronizer
+  const synchronizer = new ArchiverL1Synchronizer(
+    publicClient,
+    debugClient,
+    rollup,
+    inbox,
+    archiverStore,
+    archiverConfig,
+    deps.blobClient,
+    epochCache,
+    deps.dateProvider ?? new DateProvider(),
+    instrumentation,
+    l1Constants,
+    events,
+    instrumentation.tracer,
+    l2TipsCache,
+    undefined, // log (use default)
+  );
+
+  const archiver = new Archiver(
+    publicClient,
+    debugClient,
+    rollup,
+    { ...config.l1Contracts, slashingProposerAddress },
+    archiverStore,
+    archiverConfig,
+    deps.blobClient,
+    instrumentation,
+    l1Constants,
+    synchronizer,
+    events,
+    l2TipsCache,
+  );
+
+  await archiver.start(opts.blockUntilSync);
+  return archiver;
 }
 
-async function registerProtocolContracts(store: KVArchiverDataStore) {
+/** Registers protocol contracts in the archiver store. Idempotent — skips contracts that already exist (e.g. on node restart). */
+export async function registerProtocolContracts(store: KVArchiverDataStore) {
   const blockNumber = 0;
   for (const name of protocolContractNames) {
     const provider = new BundledProtocolContractsProvider();
     const contract = await provider.getProtocolContractArtifact(name);
-    const contractClassPublic: ContractClassPublic = {
+
+    // Skip if already registered (happens on node restart with a persisted store).
+    if (await store.getContractClass(contract.contractClass.id)) {
+      continue;
+    }
+
+    const publicBytecodeCommitment = await computePublicBytecodeCommitment(contract.contractClass.packedBytecode);
+    const contractClassPublic: ContractClassPublicWithCommitment = {
       ...contract.contractClass,
-      privateFunctions: [],
-      utilityFunctions: [],
+      publicBytecodeCommitment,
     };
 
     const publicFunctionSignatures = contract.artifact.functions
@@ -59,8 +196,7 @@ async function registerProtocolContracts(store: KVArchiverDataStore) {
       .map(fn => decodeFunctionSignature(fn.name, fn.parameters));
 
     await store.registerContractFunctionSignatures(publicFunctionSignatures);
-    const bytecodeCommitment = await computePublicBytecodeCommitment(contractClassPublic.packedBytecode);
-    await store.addContractClasses([contractClassPublic], [bytecodeCommitment], BlockNumber(blockNumber));
+    await store.addContractClasses([contractClassPublic], BlockNumber(blockNumber));
     await store.addContractInstances([contract.instance], BlockNumber(blockNumber));
   }
 }

package/src/index.ts

@@ -1,5 +1,13 @@
-export * from './archiver/index.js';
 export * from './factory.js';
-export * from './rpc/index.js';
+export * from './interfaces.js';
+export * from './archiver.js';
+export * from './modules/data_source_base.js';
+export * from './modules/data_store_updater.js';
+export * from './config.js';
 
-export { retrieveCheckpointsFromRollup, retrieveL2ProofVerifiedEvents } from './archiver/l1/data_retrieval.js';
+export { type L1PublishedData } from './structs/published.js';
+export { KVArchiverDataStore, ARCHIVER_DB_VERSION } from './store/kv_archiver_store.js';
+export { ContractInstanceStore } from './store/contract_instance_store.js';
+export { L2TipsCache } from './store/l2_tips_cache.js';
+
+export { retrieveCheckpointsFromRollup, retrieveL2ProofVerifiedEvents } from './l1/data_retrieval.js';

package/src/interfaces.ts

@@ -0,0 +1,9 @@
+import type { L2BlockSource } from '@aztec/stdlib/block';
+import type { ContractDataSource } from '@aztec/stdlib/contract';
+import type { L2LogsSource } from '@aztec/stdlib/interfaces/server';
+import type { L1ToL2MessageSource } from '@aztec/stdlib/messaging';
+
+/**
+ * Helper interface to combine all sources this archiver implementation provides.
+ */
+export type ArchiverDataSource = L2BlockSource & L2LogsSource & ContractDataSource & L1ToL2MessageSource;

package/src/l1/README.md

@@ -0,0 +1,55 @@
+# Archiver L1 Data Retrieval
+
+Modules and classes to handle data retrieval from L1 for the archiver.
+
+## Calldata Retriever
+
+The sequencer publisher bundles multiple operations into a single multicall3 transaction for gas
+efficiency. The archiver needs to extract the `propose` calldata from these bundled transactions
+to reconstruct L2 blocks.
+
+The retriever uses hash matching against `attestationsHash` and `payloadDigest` from the
+`CheckpointProposed` L1 event to verify it has found the correct propose calldata. These hashes
+are always required.
+
+### Multicall3 Decoding with Hash Matching
+
+First attempt to decode the transaction as a multicall3 `aggregate3` call:
+
+- Check if transaction is to multicall3 address (`0xcA11bde05977b3631167028862bE2a173976CA11`)
+- Decode as `aggregate3(Call3[] calldata calls)`
+- Find all calls matching the rollup contract address and the `propose` function selector
+- Verify each candidate by computing `attestationsHash` (keccak256 of ABI-encoded attestations)
+  and `payloadDigest` (keccak256 of the consensus payload signing hash) and comparing against
+  expected values from the `CheckpointProposed` event
+- Return the verified candidate (if multiple verify, return the first with a warning)
+
+This approach works regardless of what other calls are in the multicall3 bundle, because hash
+matching identifies the correct propose call without needing an allowlist.
+
+### Direct Propose Call
+
+Second attempt to decode the transaction as a direct `propose` call to the rollup contract:
+
+- Check if transaction is to the rollup address
+- Decode as `propose` function call
+- Verify against expected hashes
+- Return the transaction input as the propose calldata
+
+### Spire Proposer Call
+
+Given existing attempts to route the call via the Spire proposer, we also check if the tx is
+`to` the proposer known address. If so, we extract all wrapped calls and try each as either
+a multicall3 or direct propose call, using hash matching to find and verify the correct one.
+
+Since the Spire proposer is upgradeable, we check that the implementation has not changed in
+order to decode. Any validation failure triggers fallback to the next step.
+
+### Debug and Trace Transaction Fallback
+
+Last, we use L1 node's trace/debug RPC methods to definitively identify the one successful
+`propose` call within the tx. We can then extract the exact calldata that hit the `propose`
+function in the rollup contract.
+
+This approach requires access to a debug-enabled L1 node, which may be more resource-intensive,
+so we only use it as a fallback when earlier steps fail, which should be rare in practice.

package/src/{archiver/l1 → l1}/bin/retrieve-calldata.ts

@@ -3,8 +3,9 @@ import type { ViemPublicClient, ViemPublicDebugClient } from '@aztec/ethereum/ty
 import { CheckpointNumber } from '@aztec/foundation/branded-types';
 import { EthAddress } from '@aztec/foundation/eth-address';
 import { createLogger } from '@aztec/foundation/log';
+import { RollupAbi } from '@aztec/l1-artifacts/RollupAbi';
 
-import { type Hex, createPublicClient, http } from 'viem';
+import { type Hex, createPublicClient, decodeEventLog, getAbiItem, http, toEventSelector } from 'viem';
 import { mainnet } from 'viem/chains';
 
 import { CalldataRetriever } from '../calldata_retriever.js';
@@ -88,14 +89,6 @@ async function main() {
 
     logger.info(`Transaction found in block ${tx.blockNumber}`);
 
-    // For simplicity, use zero addresses for optional contract addresses
-    // In production, these would be fetched from the rollup contract or configuration
-    const slashingProposerAddress = EthAddress.ZERO;
-    const governanceProposerAddress = EthAddress.ZERO;
-    const slashFactoryAddress = undefined;
-
-    logger.info('Using zero addresses for governance/slashing (can be configured if needed)');
-
     // Create CalldataRetriever
     const retriever = new CalldataRetriever(
       publicClient as unknown as ViemPublicClient,
@@ -103,48 +96,67 @@ async function main() {
       targetCommitteeSize,
       undefined,
       logger,
-      {
-        rollupAddress,
-        governanceProposerAddress,
-        slashingProposerAddress,
-        slashFactoryAddress,
-      },
+      rollupAddress,
     );
 
-    // Extract L2 block number from transaction logs
-    logger.info('Decoding transaction to extract L2 block number...');
+    // Extract checkpoint number and hashes from transaction logs
+    logger.info('Decoding transaction to extract checkpoint number and hashes...');
     const receipt = await publicClient.getTransactionReceipt({ hash: txHash });
-    const l2BlockProposedEvent = receipt.logs.find(log => {
+
+    // Look for CheckpointProposed event
+    const checkpointProposedEventAbi = getAbiItem({ abi: RollupAbi, name: 'CheckpointProposed' });
+    const checkpointProposedLog = receipt.logs.find(log => {
       try {
-        // Try to match the L2BlockProposed event
         return (
           log.address.toLowerCase() === rollupAddress.toString().toLowerCase() &&
-          log.topics[0] === '0x2f1d0e696fa5186494a2f2f89a0e0bcbb15d607f6c5eac4637e07e1e5e7d3c00' // L2BlockProposed event signature
+          log.topics[0] === toEventSelector(checkpointProposedEventAbi)
         );
       } catch {
         return false;
       }
     });
 
-    let l2BlockNumber: number;
-    if (l2BlockProposedEvent && l2BlockProposedEvent.topics[1]) {
-      // L2 block number is typically the first indexed parameter
-      l2BlockNumber = Number(BigInt(l2BlockProposedEvent.topics[1]));
-      logger.info(`L2 Block Number (from event): ${l2BlockNumber}`);
-    } else {
-      // Fallback: try to extract from transaction data or use a default
-      logger.warn('Could not extract L2 block number from event, using block number as fallback');
-      l2BlockNumber = Number(tx.blockNumber);
+    if (!checkpointProposedLog || checkpointProposedLog.topics[1] === undefined) {
+      throw new Error(`Checkpoint proposed event not found`);
     }
 
+    const checkpointNumber = CheckpointNumber.fromBigInt(BigInt(checkpointProposedLog.topics[1]));
+
+    // Decode the full event to extract attestationsHash and payloadDigest
+    const decodedEvent = decodeEventLog({
+      abi: RollupAbi,
+      data: checkpointProposedLog.data,
+      topics: checkpointProposedLog.topics,
+    });
+
+    const eventArgs = decodedEvent.args as {
+      checkpointNumber: bigint;
+      archive: Hex;
+      versionedBlobHashes: Hex[];
+      attestationsHash: Hex;
+      payloadDigest: Hex;
+    };
+
+    if (!eventArgs.attestationsHash || !eventArgs.payloadDigest) {
+      throw new Error(`CheckpointProposed event missing attestationsHash or payloadDigest`);
+    }
+
+    const expectedHashes = {
+      attestationsHash: eventArgs.attestationsHash,
+      payloadDigest: eventArgs.payloadDigest,
+    };
+
+    logger.info(`Checkpoint Number: ${checkpointNumber}`);
+    logger.info(`Attestations Hash: ${expectedHashes.attestationsHash}`);
+    logger.info(`Payload Digest: ${expectedHashes.payloadDigest}`);
+
     logger.info('');
-    logger.info('Retrieving block header from rollup transaction...');
+    logger.info('Retrieving checkpoint from rollup transaction...');
     logger.info('');
 
-    // For this script, we don't have blob hashes or expected hashes, so pass empty arrays/objects
-    const result = await retriever.getCheckpointFromRollupTx(txHash, [], CheckpointNumber(l2BlockNumber), {});
+    const result = await retriever.getCheckpointFromRollupTx(txHash, [], checkpointNumber, expectedHashes);
 
-    logger.info(' Successfully retrieved block header!');
+    logger.info(' Successfully retrieved block header!');
     logger.info('');
     logger.info('Block Header Details:');
     logger.info('====================');