@aztec/archiver 0.0.1-commit.29c6b1a3 → 0.0.1-commit.2c0ee1788

package/src/store/contract_class_store.ts

@@ -2,14 +2,11 @@ import { Fr } from '@aztec/foundation/curves/bn254';
 import { toArray } from '@aztec/foundation/iterable';
 import { BufferReader, numToUInt8, serializeToBuffer } from '@aztec/foundation/serialize';
 import type { AztecAsyncKVStore, AztecAsyncMap } from '@aztec/kv-store';
-import { FunctionSelector } from '@aztec/stdlib/abi';
 import type {
   ContractClassPublic,
   ContractClassPublicWithBlockNumber,
-  ExecutablePrivateFunctionWithMembershipProof,
-  UtilityFunctionWithMembershipProof,
+  ContractClassPublicWithCommitment,
 } from '@aztec/stdlib/contract';
-import { Vector } from '@aztec/stdlib/types';
 
 /**
  * LMDB-based contract class storage for the archiver.
@@ -23,23 +20,53 @@ export class ContractClassStore {
     this.#bytecodeCommitments = db.openMap('archiver_bytecode_commitments');
   }
 
+  /**
+   * Adds multiple contract classes to the store.
+   * @param data - Contract classes (with bytecode commitments) to add.
+   * @param blockNumber - L2 block number where the classes were registered.
+   * @returns True if every insert succeeded.
+   */
+  async addContractClasses(data: ContractClassPublicWithCommitment[], blockNumber: number): Promise<boolean> {
+    return (await Promise.all(data.map(c => this.addContractClass(c, c.publicBytecodeCommitment, blockNumber)))).every(
+      Boolean,
+    );
+  }
+
+  /**
+   * Removes multiple contract classes from the store, but only if they were registered at or after the given block.
+   * @param data - Contract classes to delete.
+   * @param blockNumber - Lower bound on the block number at which the classes were registered.
+   * @returns True if every delete succeeded.
+   */
+  async deleteContractClasses(data: ContractClassPublic[], blockNumber: number): Promise<boolean> {
+    return (await Promise.all(data.map(c => this.deleteContractClass(c, blockNumber)))).every(Boolean);
+  }
+
   async addContractClass(
     contractClass: ContractClassPublic,
     bytecodeCommitment: Fr,
     blockNumber: number,
   ): Promise<void> {
-    await this.#contractClasses.setIfNotExists(
-      contractClass.id.toString(),
-      serializeContractClassPublic({ ...contractClass, l2BlockNumber: blockNumber }),
-    );
-    await this.#bytecodeCommitments.setIfNotExists(contractClass.id.toString(), bytecodeCommitment.toBuffer());
+    await this.db.transactionAsync(async () => {
+      const key = contractClass.id.toString();
+      if (await this.#contractClasses.hasAsync(key)) {
+        throw new Error(`Contract class ${key} already exists, cannot add again at block ${blockNumber}`);
+      }
+      await this.#contractClasses.set(
+        key,
+        serializeContractClassPublic({ ...contractClass, l2BlockNumber: blockNumber }),
+      );
+      await this.#bytecodeCommitments.set(key, bytecodeCommitment.toBuffer());
+    });
   }
 
-  async deleteContractClasses(contractClass: ContractClassPublic, blockNumber: number): Promise<void> {
+  async deleteContractClass(contractClass: ContractClassPublic, blockNumber: number): Promise<void> {
     const restoredContractClass = await this.#contractClasses.getAsync(contractClass.id.toString());
     if (restoredContractClass && deserializeContractClassPublic(restoredContractClass).l2BlockNumber >= blockNumber) {
-      await this.#contractClasses.delete(contractClass.id.toString());
-      await this.#bytecodeCommitments.delete(contractClass.id.toString());
+      await this.db.transactionAsync(async () => {
+        await this.#contractClasses.delete(contractClass.id.toString());
+        await this.#bytecodeCommitments.delete(contractClass.id.toString());
+      });
     }
   }
 
@@ -56,37 +83,6 @@ export class ContractClassStore {
   async getContractClassIds(): Promise<Fr[]> {
     return (await toArray(this.#contractClasses.keysAsync())).map(key => Fr.fromHexString(key));
   }
-
-  async addFunctions(
-    contractClassId: Fr,
-    newPrivateFunctions: ExecutablePrivateFunctionWithMembershipProof[],
-    newUtilityFunctions: UtilityFunctionWithMembershipProof[],
-  ): Promise<boolean> {
-    await this.db.transactionAsync(async () => {
-      const existingClassBuffer = await this.#contractClasses.getAsync(contractClassId.toString());
-      if (!existingClassBuffer) {
-        throw new Error(`Unknown contract class ${contractClassId} when adding private functions to store`);
-      }
-
-      const existingClass = deserializeContractClassPublic(existingClassBuffer);
-      const { privateFunctions: existingPrivateFns, utilityFunctions: existingUtilityFns } = existingClass;
-
-      const updatedClass: Omit<ContractClassPublicWithBlockNumber, 'id'> = {
-        ...existingClass,
-        privateFunctions: [
-          ...existingPrivateFns,
-          ...newPrivateFunctions.filter(newFn => !existingPrivateFns.some(f => f.selector.equals(newFn.selector))),
-        ],
-        utilityFunctions: [
-          ...existingUtilityFns,
-          ...newUtilityFunctions.filter(newFn => !existingUtilityFns.some(f => f.selector.equals(newFn.selector))),
-        ],
-      };
-      await this.#contractClasses.set(contractClassId.toString(), serializeContractClassPublic(updatedClass));
-    });
-
-    return true;
-  }
 }
 
 function serializeContractClassPublic(contractClass: Omit<ContractClassPublicWithBlockNumber, 'id'>): Buffer {
@@ -94,83 +90,19 @@ function serializeContractClassPublic(contractClass: Omit<ContractClassPublicWit
     contractClass.l2BlockNumber,
     numToUInt8(contractClass.version),
     contractClass.artifactHash,
-    contractClass.privateFunctions.length,
-    contractClass.privateFunctions.map(serializePrivateFunction),
-    contractClass.utilityFunctions.length,
-    contractClass.utilityFunctions.map(serializeUtilityFunction),
     contractClass.packedBytecode.length,
     contractClass.packedBytecode,
     contractClass.privateFunctionsRoot,
   );
 }
 
-function serializePrivateFunction(fn: ExecutablePrivateFunctionWithMembershipProof): Buffer {
-  return serializeToBuffer(
-    fn.selector,
-    fn.vkHash,
-    fn.bytecode.length,
-    fn.bytecode,
-    fn.functionMetadataHash,
-    fn.artifactMetadataHash,
-    fn.utilityFunctionsTreeRoot,
-    new Vector(fn.privateFunctionTreeSiblingPath),
-    fn.privateFunctionTreeLeafIndex,
-    new Vector(fn.artifactTreeSiblingPath),
-    fn.artifactTreeLeafIndex,
-  );
-}
-
-function serializeUtilityFunction(fn: UtilityFunctionWithMembershipProof): Buffer {
-  return serializeToBuffer(
-    fn.selector,
-    fn.bytecode.length,
-    fn.bytecode,
-    fn.functionMetadataHash,
-    fn.artifactMetadataHash,
-    fn.privateFunctionsArtifactTreeRoot,
-    new Vector(fn.artifactTreeSiblingPath),
-    fn.artifactTreeLeafIndex,
-  );
-}
-
 function deserializeContractClassPublic(buffer: Buffer): Omit<ContractClassPublicWithBlockNumber, 'id'> {
   const reader = BufferReader.asReader(buffer);
   return {
     l2BlockNumber: reader.readNumber(),
     version: reader.readUInt8() as 1,
     artifactHash: reader.readObject(Fr),
-    privateFunctions: reader.readVector({ fromBuffer: deserializePrivateFunction }),
-    utilityFunctions: reader.readVector({ fromBuffer: deserializeUtilityFunction }),
     packedBytecode: reader.readBuffer(),
     privateFunctionsRoot: reader.readObject(Fr),
   };
 }
-
-function deserializePrivateFunction(buffer: Buffer | BufferReader): ExecutablePrivateFunctionWithMembershipProof {
-  const reader = BufferReader.asReader(buffer);
-  return {
-    selector: reader.readObject(FunctionSelector),
-    vkHash: reader.readObject(Fr),
-    bytecode: reader.readBuffer(),
-    functionMetadataHash: reader.readObject(Fr),
-    artifactMetadataHash: reader.readObject(Fr),
-    utilityFunctionsTreeRoot: reader.readObject(Fr),
-    privateFunctionTreeSiblingPath: reader.readVector(Fr),
-    privateFunctionTreeLeafIndex: reader.readNumber(),
-    artifactTreeSiblingPath: reader.readVector(Fr),
-    artifactTreeLeafIndex: reader.readNumber(),
-  };
-}
-
-function deserializeUtilityFunction(buffer: Buffer | BufferReader): UtilityFunctionWithMembershipProof {
-  const reader = BufferReader.asReader(buffer);
-  return {
-    selector: reader.readObject(FunctionSelector),
-    bytecode: reader.readBuffer(),
-    functionMetadataHash: reader.readObject(Fr),
-    artifactMetadataHash: reader.readObject(Fr),
-    privateFunctionsArtifactTreeRoot: reader.readObject(Fr),
-    artifactTreeSiblingPath: reader.readVector(Fr),
-    artifactTreeLeafIndex: reader.readNumber(),
-  };
-}

package/src/store/contract_instance_store.ts

@@ -25,13 +25,59 @@ export class ContractInstanceStore {
     this.#contractInstanceUpdates = db.openMap('archiver_contract_instance_updates');
   }
 
+  /**
+   * Adds multiple contract instances to the store.
+   * @param data - Contract instances to add.
+   * @param blockNumber - L2 block number where the instances were deployed.
+   * @returns True if every insert succeeded.
+   */
+  async addContractInstances(data: ContractInstanceWithAddress[], blockNumber: number): Promise<boolean> {
+    return (await Promise.all(data.map(c => this.addContractInstance(c, blockNumber)))).every(Boolean);
+  }
+
+  /**
+   * Removes multiple contract instances from the store.
+   * @param data - Contract instances to delete.
+   * @returns True if every delete succeeded.
+   */
+  async deleteContractInstances(data: ContractInstanceWithAddress[]): Promise<boolean> {
+    return (await Promise.all(data.map(c => this.deleteContractInstance(c)))).every(Boolean);
+  }
+
+  /**
+   * Adds multiple contract instance updates to the store.
+   * @param data - Contract instance updates to add.
+   * @param timestamp - Timestamp at which the updates were scheduled.
+   * @returns True if every insert succeeded.
+   */
+  async addContractInstanceUpdates(data: ContractInstanceUpdateWithAddress[], timestamp: UInt64): Promise<boolean> {
+    return (
+      await Promise.all(data.map((update, logIndex) => this.addContractInstanceUpdate(update, timestamp, logIndex)))
+    ).every(Boolean);
+  }
+
+  /**
+   * Removes multiple contract instance updates from the store.
+   * @param data - Contract instance updates to delete.
+   * @param timestamp - Timestamp at which the updates were scheduled.
+   * @returns True if every delete succeeded.
+   */
+  async deleteContractInstanceUpdates(data: ContractInstanceUpdateWithAddress[], timestamp: UInt64): Promise<boolean> {
+    return (
+      await Promise.all(data.map((update, logIndex) => this.deleteContractInstanceUpdate(update, timestamp, logIndex)))
+    ).every(Boolean);
+  }
+
   addContractInstance(contractInstance: ContractInstanceWithAddress, blockNumber: number): Promise<void> {
     return this.db.transactionAsync(async () => {
-      await this.#contractInstances.set(
-        contractInstance.address.toString(),
-        new SerializableContractInstance(contractInstance).toBuffer(),
-      );
-      await this.#contractInstancePublishedAt.set(contractInstance.address.toString(), blockNumber);
+      const key = contractInstance.address.toString();
+      if (await this.#contractInstances.hasAsync(key)) {
+        throw new Error(
+          `Contract instance at ${key} already exists (deployed at block ${await this.#contractInstancePublishedAt.getAsync(key)}), cannot add again at block ${blockNumber}`,
+        );
+      }
+      await this.#contractInstances.set(key, new SerializableContractInstance(contractInstance).toBuffer());
+      await this.#contractInstancePublishedAt.set(key, blockNumber);
     });
   }
 

package/src/store/data_stores.ts

@@ -0,0 +1,108 @@
+import type { L1BlockId } from '@aztec/ethereum/l1-types';
+import type { AztecAsyncKVStore } from '@aztec/kv-store';
+import type { ContractDataSource } from '@aztec/stdlib/contract';
+
+import { join } from 'path';
+
+import { ArchiverContractDataSourceAdapter } from '../modules/contract_data_source_adapter.js';
+import { BlockStore } from './block_store.js';
+import { ContractClassStore } from './contract_class_store.js';
+import { ContractInstanceStore } from './contract_instance_store.js';
+import { FunctionNamesCache } from './function_names_cache.js';
+import { LogStore } from './log_store.js';
+import { MessageStore } from './message_store.js';
+
+export const ARCHIVER_DB_VERSION = 6;
+
+/**
+ * Represents the latest L1 block processed by the archiver for various objects in L2.
+ */
+export type ArchiverL1SynchPoint = {
+  /** Number of the last L1 block that added a new L2 checkpoint metadata.  */
+  blocksSynchedTo?: bigint;
+  /** Last L1 block checked for L1 to L2 messages. */
+  messagesSynchedTo?: L1BlockId;
+};
+
+/**
+ * Bundle of archiver-owned LMDB substores plus the in-memory caches that span them.
+ *
+ * Replaces the former `KVArchiverDataStore` pass-through wrapper. Callers reach into
+ * the relevant substore directly (e.g. `stores.blocks.getBlock`) and use
+ * {@link createArchiverDataStores} to wire them up against a shared KV store.
+ */
+export type ArchiverDataStores = {
+  /** The underlying key-value store. Use {@link AztecAsyncKVStore.transactionAsync} to compose updates atomically. */
+  db: AztecAsyncKVStore;
+  /** Blocks, checkpoints, tx effects, proven/finalized state. */
+  blocks: BlockStore;
+  /** Public, private and contract class logs. */
+  logs: LogStore;
+  /** L1 to L2 messages and message sync state. */
+  messages: MessageStore;
+  /** Contract classes (with bytecode commitments). */
+  contractClasses: ContractClassStore;
+  /** Contract instances and contract instance updates. */
+  contractInstances: ContractInstanceStore;
+  /** In-memory cache of public function selectors -> names. */
+  functionNames: FunctionNamesCache;
+};
+
+/** Options used by {@link createArchiverDataStores}. */
+export type CreateArchiverDataStoresOptions = {
+  /** Maximum number of logs returned per page when paginating tagged log queries. */
+  logsMaxPageSize?: number;
+};
+
+/**
+ * Wires up the archiver substores against a shared KV store and returns the
+ * {@link ArchiverDataStores} bundle.
+ */
+export function createArchiverDataStores(
+  db: AztecAsyncKVStore,
+  opts: CreateArchiverDataStoresOptions = {},
+): ArchiverDataStores {
+  const blocks = new BlockStore(db);
+  return {
+    db,
+    blocks,
+    logs: new LogStore(db, blocks, opts.logsMaxPageSize ?? 1000),
+    messages: new MessageStore(db),
+    contractClasses: new ContractClassStore(db),
+    contractInstances: new ContractInstanceStore(db),
+    functionNames: new FunctionNamesCache(),
+  };
+}
+
+/**
+ * Returns the L1 sync point of the archiver, combining the block sync point from {@link BlockStore}
+ * and the message sync point from {@link MessageStore}.
+ */
+export async function getArchiverSynchPoint(stores: ArchiverDataStores): Promise<ArchiverL1SynchPoint> {
+  const [blocksSynchedTo, messagesSynchedTo] = await Promise.all([
+    stores.blocks.getSynchedL1BlockNumber(),
+    stores.messages.getSynchedL1Block(),
+  ]);
+  return { blocksSynchedTo, messagesSynchedTo };
+}
+
+/**
+ * Backs up the underlying KV store to the given folder. Returns the path to the resulting db file.
+ */
+export async function backupArchiverDataStores(
+  stores: ArchiverDataStores,
+  path: string,
+  compress = true,
+): Promise<string> {
+  await stores.db.backupTo(path, compress);
+  return join(path, 'data.mdb');
+}
+
+/**
+ * Returns a {@link ContractDataSource} adapter over {@link ArchiverDataStores}.
+ * Used by contexts (e.g. offline epoch re-prover tools) that need a ContractDataSource
+ * but do not need a full archiver instance.
+ */
+export function createContractDataSource(stores: ArchiverDataStores): ContractDataSource {
+  return new ArchiverContractDataSourceAdapter(stores);
+}

package/src/store/function_names_cache.ts

@@ -0,0 +1,37 @@
+import { createLogger } from '@aztec/foundation/log';
+import { FunctionSelector } from '@aztec/stdlib/abi';
+
+const MAX_FUNCTION_SIGNATURES = 1000;
+const MAX_FUNCTION_NAME_LEN = 256;
+
+/**
+ * In-memory cache mapping public function selectors to function names.
+ *
+ * Populated opportunistically (e.g. by PXE registering signatures from artifacts) so the
+ * archiver can attach human-readable names to logs and traces. Bounded by
+ * {@link MAX_FUNCTION_SIGNATURES} to avoid unbounded growth from untrusted callers.
+ */
+export class FunctionNamesCache {
+  private readonly log = createLogger('archiver:data-stores');
+  private readonly names: Map<string, string> = new Map();
+
+  /** Adds the given public function signatures to the cache. */
+  public async register(signatures: string[]): Promise<void> {
+    for (const sig of signatures) {
+      if (this.names.size > MAX_FUNCTION_SIGNATURES) {
+        return;
+      }
+      try {
+        const selector = await FunctionSelector.fromSignature(sig);
+        this.names.set(selector.toString(), sig.slice(0, sig.indexOf('(')).slice(0, MAX_FUNCTION_NAME_LEN));
+      } catch {
+        this.log.warn(`Failed to parse signature: ${sig}. Ignoring`);
+      }
+    }
+  }
+
+  /** Looks up a function name for the given selector, or returns undefined if not registered. */
+  public get(selector: FunctionSelector): string | undefined {
+    return this.names.get(selector.toString());
+  }
+}

package/src/store/l2_tips_cache.ts

@@ -0,0 +1,134 @@
+import { INITIAL_L2_BLOCK_NUM } from '@aztec/constants';
+import { BlockNumber, CheckpointNumber } from '@aztec/foundation/branded-types';
+import {
+  type BlockData,
+  type CheckpointId,
+  GENESIS_BLOCK_HEADER_HASH,
+  GENESIS_CHECKPOINT_HEADER_HASH,
+  type L2Tips,
+} from '@aztec/stdlib/block';
+
+import type { BlockStore } from './block_store.js';
+
+/**
+ * In-memory cache for L2 chain tips (proposed, checkpointed, proven, finalized).
+ * Populated from the BlockStore on first access, then kept up-to-date by the ArchiverDataStoreUpdater.
+ * Refresh calls should happen within the store transaction that mutates block data to ensure consistency.
+ */
+export class L2TipsCache {
+  #tipsPromise: Promise<L2Tips> | undefined;
+
+  constructor(private blockStore: BlockStore) {}
+
+  /** Returns the cached L2 tips. Loads from the block store on first call. */
+  public getL2Tips(): Promise<L2Tips> {
+    return (this.#tipsPromise ??= this.loadFromStore());
+  }
+
+  /** Reloads the L2 tips from the block store. Should be called within the store transaction that mutates data. */
+  public async refresh(): Promise<void> {
+    this.#tipsPromise = this.loadFromStore();
+    await this.#tipsPromise;
+  }
+
+  private async loadFromStore(): Promise<L2Tips> {
+    const [
+      latestBlockNumber,
+      provenBlockNumber,
+      proposedCheckpointBlockNumber,
+      checkpointedBlockNumber,
+      finalizedBlockNumber,
+    ] = await Promise.all([
+      this.blockStore.getLatestL2BlockNumber(),
+      this.blockStore.getProvenBlockNumber(),
+      this.blockStore.getProposedCheckpointL2BlockNumber(),
+      this.blockStore.getCheckpointedL2BlockNumber(),
+      this.blockStore.getFinalizedL2BlockNumber(),
+    ]);
+
+    const genesisBlockHeader = {
+      blockHash: GENESIS_BLOCK_HEADER_HASH,
+      checkpointNumber: CheckpointNumber.ZERO,
+    } as const;
+    const beforeInitialBlockNumber = BlockNumber(INITIAL_L2_BLOCK_NUM - 1);
+
+    const getBlockData = (blockNumber: BlockNumber) =>
+      blockNumber > beforeInitialBlockNumber ? this.blockStore.getBlockData(blockNumber) : genesisBlockHeader;
+
+    const [latestBlockData, provenBlockData, proposedCheckpointBlockData, checkpointedBlockData, finalizedBlockData] =
+      await Promise.all(
+        [
+          latestBlockNumber,
+          provenBlockNumber,
+          proposedCheckpointBlockNumber,
+          checkpointedBlockNumber,
+          finalizedBlockNumber,
+        ].map(getBlockData),
+      );
+
+    if (
+      !latestBlockData ||
+      !provenBlockData ||
+      !finalizedBlockData ||
+      !checkpointedBlockData ||
+      !proposedCheckpointBlockData
+    ) {
+      throw new Error('Failed to load block data for L2 tips');
+    }
+
+    const [provenCheckpointId, finalizedCheckpointId, proposedCheckpointId, checkpointedCheckpointId] =
+      await Promise.all([
+        this.getCheckpointIdForBlock(provenBlockData),
+        this.getCheckpointIdForBlock(finalizedBlockData),
+        this.getCheckpointIdForProposedCheckpoint(checkpointedBlockData),
+        this.getCheckpointIdForBlock(checkpointedBlockData),
+      ]);
+
+    return {
+      proposed: { number: latestBlockNumber, hash: latestBlockData.blockHash.toString() },
+      proven: {
+        block: { number: provenBlockNumber, hash: provenBlockData.blockHash.toString() },
+        checkpoint: provenCheckpointId,
+      },
+      proposedCheckpoint: {
+        block: { number: proposedCheckpointBlockNumber, hash: proposedCheckpointBlockData.blockHash.toString() },
+        checkpoint: proposedCheckpointId,
+      },
+      finalized: {
+        block: { number: finalizedBlockNumber, hash: finalizedBlockData.blockHash.toString() },
+        checkpoint: finalizedCheckpointId,
+      },
+      checkpointed: {
+        block: { number: checkpointedBlockNumber, hash: checkpointedBlockData.blockHash.toString() },
+        checkpoint: checkpointedCheckpointId,
+      },
+    };
+  }
+
+  private async getCheckpointIdForProposedCheckpoint(
+    checkpointedBlockData: Pick<BlockData, 'checkpointNumber'>,
+  ): Promise<CheckpointId> {
+    const checkpointData = await this.blockStore.getLastProposedCheckpoint();
+    if (!checkpointData) {
+      return this.getCheckpointIdForBlock(checkpointedBlockData);
+    }
+    return {
+      number: checkpointData.checkpointNumber,
+      hash: checkpointData.header.hash().toString(),
+    };
+  }
+
+  private async getCheckpointIdForBlock(blockData: Pick<BlockData, 'checkpointNumber'>): Promise<CheckpointId> {
+    const checkpointData = await this.blockStore.getCheckpointData(blockData.checkpointNumber);
+    if (!checkpointData) {
+      return {
+        number: CheckpointNumber.ZERO,
+        hash: GENESIS_CHECKPOINT_HEADER_HASH.toString(),
+      };
+    }
+    return {
+      number: checkpointData.checkpointNumber,
+      hash: checkpointData.header.hash().toString(),
+    };
+  }
+}