@aztec/archiver 0.0.1-commit.5daedc8 → 0.0.1-commit.6201a7b05

package/src/errors.ts

@@ -0,0 +1,213 @@
+import type { BlockNumber, CheckpointNumber } from '@aztec/foundation/branded-types';
+import type { Fr } from '@aztec/foundation/schemas';
+
+export class NoBlobBodiesFoundError extends Error {
+  constructor(l2BlockNum: number) {
+    super(`No blob bodies found for block ${l2BlockNum}`);
+    this.name = 'NoBlobBodiesFoundError';
+  }
+}
+
+export class BlockNumberNotSequentialError extends Error {
+  constructor(newBlockNumber: number, previous: number | undefined) {
+    super(`Cannot insert new block ${newBlockNumber} given previous block number is ${previous ?? 'undefined'}`);
+    this.name = 'BlockNumberNotSequentialError';
+  }
+}
+
+export class InitialCheckpointNumberNotSequentialError extends Error {
+  constructor(
+    public readonly newCheckpointNumber: number,
+    public readonly previousCheckpointNumber: number | undefined,
+  ) {
+    super(
+      `Cannot insert new checkpoint ${newCheckpointNumber} given previous checkpoint number in store is ${
+        previousCheckpointNumber ?? 'undefined'
+      }`,
+    );
+    this.name = 'InitialCheckpointNumberNotSequentialError';
+  }
+}
+
+export class BlockCheckpointNumberNotSequentialError extends Error {
+  constructor(
+    blockNumber: BlockNumber,
+    blockCheckpointNumber: CheckpointNumber,
+    previous: CheckpointNumber | undefined,
+  ) {
+    super(
+      `Cannot insert new block ${blockNumber} for checkpoint ${blockCheckpointNumber} given previous checkpoint number is ${previous ?? 'undefined'}`,
+    );
+    this.name = 'BlockCheckpointNumberNotSequentialError';
+  }
+}
+
+export class CheckpointNumberNotSequentialError extends Error {
+  constructor(newCheckpointNumber: CheckpointNumber, previous: CheckpointNumber | undefined) {
+    super(
+      `Cannot insert new checkpoint ${newCheckpointNumber} given previous checkpoint number is ${previous ?? 'undefined'}`,
+    );
+    this.name = 'CheckpointNumberNotSequentialError';
+  }
+}
+
+export class BlockIndexNotSequentialError extends Error {
+  constructor(newBlockIndex: number, previousBlockIndex: number | undefined) {
+    super(
+      `Cannot insert new block at checkpoint index ${newBlockIndex} given previous block index is ${previousBlockIndex ?? 'undefined'}`,
+    );
+    this.name = 'BlockIndexNotSequentialError';
+  }
+}
+
+export class BlockArchiveNotConsistentError extends Error {
+  constructor(
+    newBlockNumber: number,
+    previousBlockNumber: number | undefined,
+    newBlockArchive: Fr,
+    previousBlockArchive: Fr,
+  ) {
+    super(
+      `Cannot insert new block number ${newBlockNumber} with archive ${newBlockArchive.toString()} previous block number is ${previousBlockNumber ?? 'undefined'}, previous archive is ${previousBlockArchive?.toString() ?? 'undefined'}`,
+    );
+    this.name = 'BlockArchiveNotConsistentError';
+  }
+}
+
+export class CheckpointNotFoundError extends Error {
+  constructor(checkpointNumber: number) {
+    super(`Failed to find expected checkpoint number ${checkpointNumber}`);
+    this.name = 'CheckpointNotFoundError';
+  }
+}
+
+export class BlockNotFoundError extends Error {
+  constructor(blockNumber: number) {
+    super(`Failed to find expected block number ${blockNumber}`);
+    this.name = 'BlockNotFoundError';
+  }
+}
+
+/** Thrown when a proposed block matches a block that was already checkpointed. This is expected for late proposals. */
+export class BlockAlreadyCheckpointedError extends Error {
+  constructor(public readonly blockNumber: number) {
+    super(`Block ${blockNumber} has already been checkpointed with the same content`);
+    this.name = 'BlockAlreadyCheckpointedError';
+  }
+}
+
+/** Thrown when logs are added for a tag whose last stored log has a higher block number than the new log. */
+export class OutOfOrderLogInsertionError extends Error {
+  constructor(
+    public readonly logType: 'private' | 'public',
+    public readonly tag: string,
+    public readonly lastBlockNumber: number,
+    public readonly newBlockNumber: number,
+  ) {
+    super(
+      `Out-of-order ${logType} log insertion for tag ${tag}: ` +
+        `last existing log is from block ${lastBlockNumber} but new log is from block ${newBlockNumber}`,
+    );
+    this.name = 'OutOfOrderLogInsertionError';
+  }
+}
+
+/** Thrown when L1 to L2 messages are requested for a checkpoint whose message tree hasn't been sealed yet. */
+export class L1ToL2MessagesNotReadyError extends Error {
+  constructor(
+    public readonly checkpointNumber: number,
+    public readonly inboxTreeInProgress: bigint,
+  ) {
+    super(
+      `Cannot get L1 to L2 messages for checkpoint ${checkpointNumber}: ` +
+        `inbox tree in progress is ${inboxTreeInProgress}, messages not yet sealed`,
+    );
+    this.name = 'L1ToL2MessagesNotReadyError';
+  }
+}
+
+/** Thrown when a proposed checkpoint number is stale (already processed). */
+export class ProposedCheckpointStaleError extends Error {
+  constructor(
+    public readonly proposedCheckpointNumber: number,
+    public readonly currentProposedNumber: number,
+  ) {
+    super(`Stale proposed checkpoint ${proposedCheckpointNumber}: current proposed is ${currentProposedNumber}`);
+    this.name = 'ProposedCheckpointStaleError';
+  }
+}
+
+/** Thrown when a proposed checkpoint number is not the expected latestTip + 1. */
+export class ProposedCheckpointNotSequentialError extends Error {
+  constructor(
+    public readonly proposedCheckpointNumber: number,
+    public readonly latestTipNumber: number,
+  ) {
+    super(
+      `Proposed checkpoint ${proposedCheckpointNumber} is not sequential: expected ${latestTipNumber + 1} (latest tip + 1, where tip is highest of confirmed or pending)`,
+    );
+    this.name = 'ProposedCheckpointNotSequentialError';
+  }
+}
+
+/** Thrown when a proposed checkpoint or block L2 slot has already expired on L1. */
+export class BlockOrCheckpointSlotExpiredError extends Error {
+  constructor(
+    public readonly slot: number,
+    public readonly nextSlotStart: bigint,
+    public readonly l1TimestampSynced: bigint | undefined,
+  ) {
+    super(
+      `Checkpoint or block for slot ${slot} is expired: L1 synced to ${l1TimestampSynced} which is past the next slot start ${nextSlotStart}. ` +
+        `If the checkpoint still lands via a late L1 tx, the archiver will pick it up via normal L1-sync (not the pending-queue shortcut).`,
+    );
+    this.name = 'BlockOrCheckpointSlotExpiredError';
+  }
+}
+
+/** Thrown when attempting to promote a proposed checkpoint but no proposed checkpoint exists in the store. */
+export class NoProposedCheckpointToPromoteError extends Error {
+  constructor() {
+    super('Cannot promote proposed checkpoint: no proposed checkpoint exists');
+    this.name = 'NoProposedCheckpointToPromoteError';
+  }
+}
+
+/** Thrown when the archive root of the proposed checkpoint does not match the expected one. */
+export class ProposedCheckpointArchiveRootMismatchError extends Error {
+  constructor(
+    public readonly expectedArchiveRoot: Fr,
+    public readonly actualArchiveRoot: Fr,
+  ) {
+    super(
+      `Cannot promote proposed checkpoint: archive root mismatch (expected ${expectedArchiveRoot}, got ${actualArchiveRoot})`,
+    );
+    this.name = 'ProposedCheckpointArchiveRootMismatchError';
+  }
+}
+
+/** Thrown when the proposed checkpoint does not directly follow the latest confirmed checkpoint. */
+export class ProposedCheckpointPromotionNotSequentialError extends Error {
+  constructor(
+    public readonly proposedCheckpointNumber: number,
+    public readonly latestCheckpointNumber: number,
+  ) {
+    super(
+      `Cannot promote proposed checkpoint: not sequential (latest ${latestCheckpointNumber}, proposed ${proposedCheckpointNumber})`,
+    );
+    this.name = 'ProposedCheckpointPromotionNotSequentialError';
+  }
+}
+
+/** Thrown when a proposed block conflicts with an already checkpointed block (different content). */
+export class CannotOverwriteCheckpointedBlockError extends Error {
+  constructor(
+    public readonly blockNumber: number,
+    public readonly lastCheckpointedBlock: number,
+  ) {
+    super(
+      `Cannot add block ${blockNumber}: would overwrite checkpointed data (checkpointed up to block ${lastCheckpointedBlock})`,
+    );
+    this.name = 'CannotOverwriteCheckpointedBlockError';
+  }
+}

package/src/factory.ts

@@ -1,14 +1,31 @@
-import { createLogger } from '@aztec/foundation/log';
-import type { DataStoreConfig } from '@aztec/kv-store/config';
+import { EpochCache } from '@aztec/epoch-cache';
+import { createEthereumChain } from '@aztec/ethereum/chain';
+import { makeL1HttpTransport } from '@aztec/ethereum/client';
+import { InboxContract, RollupContract } from '@aztec/ethereum/contracts';
+import type { ViemPublicDebugClient } from '@aztec/ethereum/types';
+import { BlockNumber } from '@aztec/foundation/branded-types';
+import { Buffer32 } from '@aztec/foundation/buffer';
+import { merge } from '@aztec/foundation/collection';
+import { Fr } from '@aztec/foundation/curves/bn254';
+import { DateProvider } from '@aztec/foundation/timer';
 import { createStore } from '@aztec/kv-store/lmdb-v2';
 import { protocolContractNames } from '@aztec/protocol-contracts';
 import { BundledProtocolContractsProvider } from '@aztec/protocol-contracts/providers/bundle';
 import { FunctionType, decodeFunctionSignature } from '@aztec/stdlib/abi';
-import { type ContractClassPublic, computePublicBytecodeCommitment } from '@aztec/stdlib/contract';
+import type { ArchiverEmitter } from '@aztec/stdlib/block';
+import { type ContractClassPublicWithCommitment, computePublicBytecodeCommitment } from '@aztec/stdlib/contract';
+import type { DataStoreConfig } from '@aztec/stdlib/kv-store';
+import { getTelemetryClient } from '@aztec/telemetry-client';
 
-import { Archiver, type ArchiverDeps } from './archiver/archiver.js';
-import type { ArchiverConfig } from './archiver/config.js';
-import { ARCHIVER_DB_VERSION, KVArchiverDataStore } from './archiver/kv_archiver_store/kv_archiver_store.js';
+import { EventEmitter } from 'events';
+import { createPublicClient } from 'viem';
+
+import { Archiver, type ArchiverDeps } from './archiver.js';
+import { type ArchiverConfig, mapArchiverConfig } from './config.js';
+import { ArchiverInstrumentation } from './modules/instrumentation.js';
+import { ArchiverL1Synchronizer } from './modules/l1_synchronizer.js';
+import { ARCHIVER_DB_VERSION, KVArchiverDataStore } from './store/kv_archiver_store.js';
+import { L2TipsCache } from './store/l2_tips_cache.js';
 
 export const ARCHIVER_STORE_NAME = 'archiver';
 
@@ -20,16 +37,15 @@ export async function createArchiverStore(
     ...userConfig,
     dataStoreMapSizeKb: userConfig.archiverStoreMapSizeKb ?? userConfig.dataStoreMapSizeKb,
   };
-  const store = await createStore(ARCHIVER_STORE_NAME, ARCHIVER_DB_VERSION, config, createLogger('archiver:lmdb'));
+  const store = await createStore(ARCHIVER_STORE_NAME, ARCHIVER_DB_VERSION, config);
   return new KVArchiverDataStore(store, config.maxLogs);
 }
 
 /**
  * Creates a local archiver.
  * @param config - The archiver configuration.
- * @param blobSinkClient - The blob sink client.
+ * @param deps - The archiver dependencies (blobClient, epochCache, dateProvider, telemetry).
  * @param opts - The options.
- * @param telemetry - The telemetry client.
  * @returns The local archiver.
  */
 export async function createArchiver(
@@ -39,18 +55,141 @@ export async function createArchiver(
 ): Promise<Archiver> {
   const archiverStore = await createArchiverStore(config);
   await registerProtocolContracts(archiverStore);
-  return Archiver.createAndSync(config, archiverStore, deps, opts.blockUntilSync);
+
+  // Create Ethereum clients
+  const chain = createEthereumChain(config.l1RpcUrls, config.l1ChainId);
+  const httpTimeout = config.l1HttpTimeoutMS;
+  const publicClient = createPublicClient({
+    chain: chain.chainInfo,
+    transport: makeL1HttpTransport(config.l1RpcUrls, { timeout: httpTimeout }),
+    pollingInterval: config.viemPollingIntervalMS,
+  });
+
+  // Create debug client using debug RPC URLs if available, otherwise fall back to regular RPC URLs
+  const debugRpcUrls = config.l1DebugRpcUrls.length > 0 ? config.l1DebugRpcUrls : config.l1RpcUrls;
+  const debugClient = createPublicClient({
+    chain: chain.chainInfo,
+    transport: makeL1HttpTransport(debugRpcUrls, { timeout: httpTimeout }),
+    pollingInterval: config.viemPollingIntervalMS,
+  }) as ViemPublicDebugClient;
+
+  // Create L1 contract instances
+  const rollup = new RollupContract(publicClient, config.l1Contracts.rollupAddress);
+  const inbox = new InboxContract(publicClient, config.l1Contracts.inboxAddress);
+
+  // Fetch L1 constants from rollup contract
+  const [
+    l1StartBlock,
+    l1GenesisTime,
+    proofSubmissionEpochs,
+    genesisArchiveRoot,
+    slashingProposerAddress,
+    targetCommitteeSize,
+    rollupManaLimit,
+  ] = await Promise.all([
+    rollup.getL1StartBlock(),
+    rollup.getL1GenesisTime(),
+    rollup.getProofSubmissionEpochs(),
+    rollup.getGenesisArchiveTreeRoot(),
+    rollup.getSlashingProposerAddress(),
+    rollup.getTargetCommitteeSize(),
+    rollup.getManaLimit(),
+  ] as const);
+
+  const l1StartBlockHash = await publicClient
+    .getBlock({ blockNumber: l1StartBlock, includeTransactions: false })
+    .then(block => Buffer32.fromString(block.hash));
+
+  const { aztecEpochDuration: epochDuration, aztecSlotDuration: slotDuration, ethereumSlotDuration } = config;
+
+  const l1Constants = {
+    l1StartBlockHash,
+    l1StartBlock,
+    l1GenesisTime,
+    epochDuration,
+    slotDuration,
+    ethereumSlotDuration,
+    proofSubmissionEpochs: Number(proofSubmissionEpochs),
+    targetCommitteeSize,
+    genesisArchiveRoot: Fr.fromString(genesisArchiveRoot.toString()),
+    rollupManaLimit: Number(rollupManaLimit),
+  };
+
+  const archiverConfig = merge(
+    {
+      pollingIntervalMs: 10_000,
+      batchSize: 100,
+      maxAllowedEthClientDriftSeconds: 300,
+      ethereumAllowNoDebugHosts: false,
+      skipHistoricalLogsCheck: false,
+    },
+    mapArchiverConfig(config),
+  );
+
+  const epochCache = deps.epochCache ?? (await EpochCache.create(config.l1Contracts.rollupAddress, config, deps));
+  const telemetry = deps.telemetry ?? getTelemetryClient();
+  const instrumentation = await ArchiverInstrumentation.new(telemetry, () => archiverStore.estimateSize());
+
+  // Create the event emitter that will be shared by archiver and synchronizer
+  const events = new EventEmitter() as ArchiverEmitter;
+
+  // Create L2 tips cache shared by archiver and synchronizer
+  const l2TipsCache = new L2TipsCache(archiverStore.blockStore);
+
+  // Create the L1 synchronizer
+  const synchronizer = new ArchiverL1Synchronizer(
+    publicClient,
+    debugClient,
+    rollup,
+    inbox,
+    archiverStore,
+    archiverConfig,
+    deps.blobClient,
+    epochCache,
+    deps.dateProvider ?? new DateProvider(),
+    instrumentation,
+    l1Constants,
+    events,
+    instrumentation.tracer,
+    l2TipsCache,
+    undefined, // log (use default)
+  );
+
+  const archiver = new Archiver(
+    publicClient,
+    debugClient,
+    rollup,
+    { ...config.l1Contracts, slashingProposerAddress },
+    archiverStore,
+    archiverConfig,
+    deps.blobClient,
+    instrumentation,
+    l1Constants,
+    synchronizer,
+    events,
+    l2TipsCache,
+  );
+
+  await archiver.start(opts.blockUntilSync);
+  return archiver;
 }
 
-async function registerProtocolContracts(store: KVArchiverDataStore) {
+/** Registers protocol contracts in the archiver store. Idempotent — skips contracts that already exist (e.g. on node restart). */
+export async function registerProtocolContracts(store: KVArchiverDataStore) {
   const blockNumber = 0;
   for (const name of protocolContractNames) {
     const provider = new BundledProtocolContractsProvider();
     const contract = await provider.getProtocolContractArtifact(name);
-    const contractClassPublic: ContractClassPublic = {
+
+    // Skip if already registered (happens on node restart with a persisted store).
+    if (await store.getContractClass(contract.contractClass.id)) {
+      continue;
+    }
+
+    const publicBytecodeCommitment = await computePublicBytecodeCommitment(contract.contractClass.packedBytecode);
+    const contractClassPublic: ContractClassPublicWithCommitment = {
       ...contract.contractClass,
-      privateFunctions: [],
-      utilityFunctions: [],
+      publicBytecodeCommitment,
     };
 
     const publicFunctionSignatures = contract.artifact.functions
@@ -58,8 +197,7 @@ async function registerProtocolContracts(store: KVArchiverDataStore) {
       .map(fn => decodeFunctionSignature(fn.name, fn.parameters));
 
     await store.registerContractFunctionSignatures(publicFunctionSignatures);
-    const bytecodeCommitment = await computePublicBytecodeCommitment(contractClassPublic.packedBytecode);
-    await store.addContractClasses([contractClassPublic], [bytecodeCommitment], blockNumber);
-    await store.addContractInstances([contract.instance], blockNumber);
+    await store.addContractClasses([contractClassPublic], BlockNumber(blockNumber));
+    await store.addContractInstances([contract.instance], BlockNumber(blockNumber));
   }
 }

package/src/index.ts

@@ -1,5 +1,14 @@
-export * from './archiver/index.js';
 export * from './factory.js';
-export * from './rpc/index.js';
+export * from './interfaces.js';
+export * from './archiver.js';
+export * from './modules/data_source_base.js';
+export * from './modules/data_store_updater.js';
+export * from './config.js';
 
-export { retrieveL2ProofVerifiedEvents } from './archiver/data_retrieval.js';
+export { type L1PublishedData } from './structs/published.js';
+export { KVArchiverDataStore, ARCHIVER_DB_VERSION } from './store/kv_archiver_store.js';
+export { ContractInstanceStore } from './store/contract_instance_store.js';
+export { L2TipsCache } from './store/l2_tips_cache.js';
+
+export { retrieveL2ProofVerifiedEvents } from './l1/data_retrieval.js';
+export { CalldataRetriever } from './l1/calldata_retriever.js';

package/src/interfaces.ts

@@ -0,0 +1,9 @@
+import type { L2BlockSource } from '@aztec/stdlib/block';
+import type { ContractDataSource } from '@aztec/stdlib/contract';
+import type { L2LogsSource } from '@aztec/stdlib/interfaces/server';
+import type { L1ToL2MessageSource } from '@aztec/stdlib/messaging';
+
+/**
+ * Helper interface to combine all sources this archiver implementation provides.
+ */
+export type ArchiverDataSource = L2BlockSource & L2LogsSource & ContractDataSource & L1ToL2MessageSource;

package/src/l1/README.md

@@ -0,0 +1,55 @@
+# Archiver L1 Data Retrieval
+
+Modules and classes to handle data retrieval from L1 for the archiver.
+
+## Calldata Retriever
+
+The sequencer publisher bundles multiple operations into a single multicall3 transaction for gas
+efficiency. The archiver needs to extract the `propose` calldata from these bundled transactions
+to reconstruct L2 blocks.
+
+The retriever uses hash matching against `attestationsHash` and `payloadDigest` from the
+`CheckpointProposed` L1 event to verify it has found the correct propose calldata. These hashes
+are always required.
+
+### Multicall3 Decoding with Hash Matching
+
+First attempt to decode the transaction as a multicall3 `aggregate3` call:
+
+- Check if transaction is to multicall3 address (`0xcA11bde05977b3631167028862bE2a173976CA11`)
+- Decode as `aggregate3(Call3[] calldata calls)`
+- Find all calls matching the rollup contract address and the `propose` function selector
+- Verify each candidate by computing `attestationsHash` (keccak256 of ABI-encoded attestations)
+  and `payloadDigest` (keccak256 of the consensus payload signing hash) and comparing against
+  expected values from the `CheckpointProposed` event
+- Return the verified candidate (if multiple verify, return the first with a warning)
+
+This approach works regardless of what other calls are in the multicall3 bundle, because hash
+matching identifies the correct propose call without needing an allowlist.
+
+### Direct Propose Call
+
+Second attempt to decode the transaction as a direct `propose` call to the rollup contract:
+
+- Check if transaction is to the rollup address
+- Decode as `propose` function call
+- Verify against expected hashes
+- Return the transaction input as the propose calldata
+
+### Spire Proposer Call
+
+Given existing attempts to route the call via the Spire proposer, we also check if the tx is
+`to` the proposer known address. If so, we extract all wrapped calls and try each as either
+a multicall3 or direct propose call, using hash matching to find and verify the correct one.
+
+Since the Spire proposer is upgradeable, we check that the implementation has not changed in
+order to decode. Any validation failure triggers fallback to the next step.
+
+### Debug and Trace Transaction Fallback
+
+Last, we use L1 node's trace/debug RPC methods to definitively identify the one successful
+`propose` call within the tx. We can then extract the exact calldata that hit the `propose`
+function in the rollup contract.
+
+This approach requires access to a debug-enabled L1 node, which may be more resource-intensive,
+so we only use it as a fallback when earlier steps fail, which should be rare in practice.