@aztec/archiver 0.0.1-commit.f2ce05ee → 0.0.1-commit.f5a9928

package/src/store/contract_class_store.ts

@@ -2,14 +2,12 @@ import { Fr } from '@aztec/foundation/curves/bn254';
 import { toArray } from '@aztec/foundation/iterable';
 import { BufferReader, numToUInt8, serializeToBuffer } from '@aztec/foundation/serialize';
 import type { AztecAsyncKVStore, AztecAsyncMap } from '@aztec/kv-store';
-import { FunctionSelector } from '@aztec/stdlib/abi';
+import { isProtocolContractClass } from '@aztec/protocol-contracts';
 import type {
   ContractClassPublic,
   ContractClassPublicWithBlockNumber,
-  ExecutablePrivateFunctionWithMembershipProof,
-  UtilityFunctionWithMembershipProof,
+  ContractClassPublicWithCommitment,
 } from '@aztec/stdlib/contract';
-import { Vector } from '@aztec/stdlib/types';
 
 /**
  * LMDB-based contract class storage for the archiver.
@@ -23,21 +21,64 @@ export class ContractClassStore {
     this.#bytecodeCommitments = db.openMap('archiver_bytecode_commitments');
   }
 
+  /**
+   * Adds multiple contract classes to the store.
+   * @param data - Contract classes (with bytecode commitments) to add.
+   * @param blockNumber - L2 block number where the classes were registered.
+   * @returns True if every insert succeeded.
+   */
+  async addContractClasses(data: ContractClassPublicWithCommitment[], blockNumber: number): Promise<boolean> {
+    return (await Promise.all(data.map(c => this.addContractClass(c, c.publicBytecodeCommitment, blockNumber)))).every(
+      Boolean,
+    );
+  }
+
+  /**
+   * Removes multiple contract classes from the store, but only if they were registered at or after the given block.
+   * @param data - Contract classes to delete.
+   * @param blockNumber - Lower bound on the block number at which the classes were registered.
+   * @returns True if every delete succeeded.
+   */
+  async deleteContractClasses(data: ContractClassPublic[], blockNumber: number): Promise<boolean> {
+    return (await Promise.all(data.map(c => this.deleteContractClass(c, blockNumber)))).every(Boolean);
+  }
+
   async addContractClass(
     contractClass: ContractClassPublic,
     bytecodeCommitment: Fr,
     blockNumber: number,
   ): Promise<void> {
     await this.db.transactionAsync(async () => {
-      await this.#contractClasses.setIfNotExists(
-        contractClass.id.toString(),
+      const key = contractClass.id.toString();
+      const existing = await this.#contractClasses.getAsync(key);
+      if (existing !== undefined) {
+        // Protocol contracts are preloaded at block 0, so a later on-chain (re-)publish of a bundled
+        // protocol class id is valid and must be a no-op. Keep the existing block-0 entry untouched
+        // (do not bump its block number) so it survives reorgs of the publishing block.
+        if (isProtocolContractClass(contractClass.id)) {
+          return;
+        }
+        // An L1 reorg can re-present an already-stored checkpoint, replaying this class at the same
+        // block; treat that as a no-op. A duplicate at a different block still signals double-processing.
+        if (deserializeContractClassPublic(existing).l2BlockNumber === blockNumber) {
+          return;
+        }
+        throw new Error(`Contract class ${key} already exists, cannot add again at block ${blockNumber}`);
+      }
+      await this.#contractClasses.set(
+        key,
         serializeContractClassPublic({ ...contractClass, l2BlockNumber: blockNumber }),
       );
-      await this.#bytecodeCommitments.setIfNotExists(contractClass.id.toString(), bytecodeCommitment.toBuffer());
+      await this.#bytecodeCommitments.set(key, bytecodeCommitment.toBuffer());
     });
   }
 
-  async deleteContractClasses(contractClass: ContractClassPublic, blockNumber: number): Promise<void> {
+  async deleteContractClass(contractClass: ContractClassPublic, blockNumber: number): Promise<void> {
+    // Protocol contracts are preloaded at block 0 and must never be deleted, even when the block that
+    // (re-)published them on-chain is unwound by a reorg.
+    if (isProtocolContractClass(contractClass.id)) {
+      return;
+    }
     const restoredContractClass = await this.#contractClasses.getAsync(contractClass.id.toString());
     if (restoredContractClass && deserializeContractClassPublic(restoredContractClass).l2BlockNumber >= blockNumber) {
       await this.db.transactionAsync(async () => {
@@ -60,37 +101,6 @@ export class ContractClassStore {
   async getContractClassIds(): Promise<Fr[]> {
     return (await toArray(this.#contractClasses.keysAsync())).map(key => Fr.fromHexString(key));
   }
-
-  async addFunctions(
-    contractClassId: Fr,
-    newPrivateFunctions: ExecutablePrivateFunctionWithMembershipProof[],
-    newUtilityFunctions: UtilityFunctionWithMembershipProof[],
-  ): Promise<boolean> {
-    await this.db.transactionAsync(async () => {
-      const existingClassBuffer = await this.#contractClasses.getAsync(contractClassId.toString());
-      if (!existingClassBuffer) {
-        throw new Error(`Unknown contract class ${contractClassId} when adding private functions to store`);
-      }
-
-      const existingClass = deserializeContractClassPublic(existingClassBuffer);
-      const { privateFunctions: existingPrivateFns, utilityFunctions: existingUtilityFns } = existingClass;
-
-      const updatedClass: Omit<ContractClassPublicWithBlockNumber, 'id'> = {
-        ...existingClass,
-        privateFunctions: [
-          ...existingPrivateFns,
-          ...newPrivateFunctions.filter(newFn => !existingPrivateFns.some(f => f.selector.equals(newFn.selector))),
-        ],
-        utilityFunctions: [
-          ...existingUtilityFns,
-          ...newUtilityFunctions.filter(newFn => !existingUtilityFns.some(f => f.selector.equals(newFn.selector))),
-        ],
-      };
-      await this.#contractClasses.set(contractClassId.toString(), serializeContractClassPublic(updatedClass));
-    });
-
-    return true;
-  }
 }
 
 function serializeContractClassPublic(contractClass: Omit<ContractClassPublicWithBlockNumber, 'id'>): Buffer {
@@ -98,83 +108,19 @@ function serializeContractClassPublic(contractClass: Omit<ContractClassPublicWit
     contractClass.l2BlockNumber,
     numToUInt8(contractClass.version),
     contractClass.artifactHash,
-    contractClass.privateFunctions.length,
-    contractClass.privateFunctions.map(serializePrivateFunction),
-    contractClass.utilityFunctions.length,
-    contractClass.utilityFunctions.map(serializeUtilityFunction),
     contractClass.packedBytecode.length,
     contractClass.packedBytecode,
     contractClass.privateFunctionsRoot,
   );
 }
 
-function serializePrivateFunction(fn: ExecutablePrivateFunctionWithMembershipProof): Buffer {
-  return serializeToBuffer(
-    fn.selector,
-    fn.vkHash,
-    fn.bytecode.length,
-    fn.bytecode,
-    fn.functionMetadataHash,
-    fn.artifactMetadataHash,
-    fn.utilityFunctionsTreeRoot,
-    new Vector(fn.privateFunctionTreeSiblingPath),
-    fn.privateFunctionTreeLeafIndex,
-    new Vector(fn.artifactTreeSiblingPath),
-    fn.artifactTreeLeafIndex,
-  );
-}
-
-function serializeUtilityFunction(fn: UtilityFunctionWithMembershipProof): Buffer {
-  return serializeToBuffer(
-    fn.selector,
-    fn.bytecode.length,
-    fn.bytecode,
-    fn.functionMetadataHash,
-    fn.artifactMetadataHash,
-    fn.privateFunctionsArtifactTreeRoot,
-    new Vector(fn.artifactTreeSiblingPath),
-    fn.artifactTreeLeafIndex,
-  );
-}
-
 function deserializeContractClassPublic(buffer: Buffer): Omit<ContractClassPublicWithBlockNumber, 'id'> {
   const reader = BufferReader.asReader(buffer);
   return {
     l2BlockNumber: reader.readNumber(),
     version: reader.readUInt8() as 1,
     artifactHash: reader.readObject(Fr),
-    privateFunctions: reader.readVector({ fromBuffer: deserializePrivateFunction }),
-    utilityFunctions: reader.readVector({ fromBuffer: deserializeUtilityFunction }),
     packedBytecode: reader.readBuffer(),
     privateFunctionsRoot: reader.readObject(Fr),
   };
 }
-
-function deserializePrivateFunction(buffer: Buffer | BufferReader): ExecutablePrivateFunctionWithMembershipProof {
-  const reader = BufferReader.asReader(buffer);
-  return {
-    selector: reader.readObject(FunctionSelector),
-    vkHash: reader.readObject(Fr),
-    bytecode: reader.readBuffer(),
-    functionMetadataHash: reader.readObject(Fr),
-    artifactMetadataHash: reader.readObject(Fr),
-    utilityFunctionsTreeRoot: reader.readObject(Fr),
-    privateFunctionTreeSiblingPath: reader.readVector(Fr),
-    privateFunctionTreeLeafIndex: reader.readNumber(),
-    artifactTreeSiblingPath: reader.readVector(Fr),
-    artifactTreeLeafIndex: reader.readNumber(),
-  };
-}
-
-function deserializeUtilityFunction(buffer: Buffer | BufferReader): UtilityFunctionWithMembershipProof {
-  const reader = BufferReader.asReader(buffer);
-  return {
-    selector: reader.readObject(FunctionSelector),
-    bytecode: reader.readBuffer(),
-    functionMetadataHash: reader.readObject(Fr),
-    artifactMetadataHash: reader.readObject(Fr),
-    privateFunctionsArtifactTreeRoot: reader.readObject(Fr),
-    artifactTreeSiblingPath: reader.readVector(Fr),
-    artifactTreeLeafIndex: reader.readNumber(),
-  };
-}

package/src/store/contract_instance_store.ts

@@ -1,5 +1,7 @@
 import type { Fr } from '@aztec/foundation/curves/bn254';
+import { first } from '@aztec/foundation/iterable';
 import type { AztecAsyncKVStore, AztecAsyncMap } from '@aztec/kv-store';
+import { isProtocolContract } from '@aztec/protocol-contracts';
 import type { AztecAddress } from '@aztec/stdlib/aztec-address';
 import {
   type ContractInstanceUpdateWithAddress,
@@ -25,17 +27,79 @@ export class ContractInstanceStore {
     this.#contractInstanceUpdates = db.openMap('archiver_contract_instance_updates');
   }
 
+  /**
+   * Adds multiple contract instances to the store.
+   * @param data - Contract instances to add.
+   * @param blockNumber - L2 block number where the instances were deployed.
+   * @returns True if every insert succeeded.
+   */
+  async addContractInstances(data: ContractInstanceWithAddress[], blockNumber: number): Promise<boolean> {
+    return (await Promise.all(data.map(c => this.addContractInstance(c, blockNumber)))).every(Boolean);
+  }
+
+  /**
+   * Removes multiple contract instances from the store.
+   * @param data - Contract instances to delete.
+   * @returns True if every delete succeeded.
+   */
+  async deleteContractInstances(data: ContractInstanceWithAddress[]): Promise<boolean> {
+    return (await Promise.all(data.map(c => this.deleteContractInstance(c)))).every(Boolean);
+  }
+
+  /**
+   * Adds multiple contract instance updates to the store.
+   * @param data - Contract instance updates to add.
+   * @param timestamp - Timestamp at which the updates were scheduled.
+   * @returns True if every insert succeeded.
+   */
+  async addContractInstanceUpdates(data: ContractInstanceUpdateWithAddress[], timestamp: UInt64): Promise<boolean> {
+    return (
+      await Promise.all(data.map((update, logIndex) => this.addContractInstanceUpdate(update, timestamp, logIndex)))
+    ).every(Boolean);
+  }
+
+  /**
+   * Removes multiple contract instance updates from the store.
+   * @param data - Contract instance updates to delete.
+   * @param timestamp - Timestamp at which the updates were scheduled.
+   * @returns True if every delete succeeded.
+   */
+  async deleteContractInstanceUpdates(data: ContractInstanceUpdateWithAddress[], timestamp: UInt64): Promise<boolean> {
+    return (
+      await Promise.all(data.map((update, logIndex) => this.deleteContractInstanceUpdate(update, timestamp, logIndex)))
+    ).every(Boolean);
+  }
+
   addContractInstance(contractInstance: ContractInstanceWithAddress, blockNumber: number): Promise<void> {
     return this.db.transactionAsync(async () => {
-      await this.#contractInstances.set(
-        contractInstance.address.toString(),
-        new SerializableContractInstance(contractInstance).toBuffer(),
-      );
-      await this.#contractInstancePublishedAt.set(contractInstance.address.toString(), blockNumber);
+      const key = contractInstance.address.toString();
+      if (await this.#contractInstances.hasAsync(key)) {
+        // Protocol contracts are preloaded at block 0, so a later on-chain (re-)publish of a bundled
+        // protocol instance is valid and must be a no-op. Keep the existing block-0 entry untouched.
+        if (isProtocolContract(contractInstance.address)) {
+          return;
+        }
+        const existingBlockNumber = await this.#contractInstancePublishedAt.getAsync(key);
+        // An L1 reorg can re-present an already-stored checkpoint, replaying this instance at the same
+        // block; treat that as a no-op. A duplicate at a different block still signals double-processing.
+        if (existingBlockNumber === blockNumber) {
+          return;
+        }
+        throw new Error(
+          `Contract instance at ${key} already exists (deployed at block ${existingBlockNumber}), cannot add again at block ${blockNumber}`,
+        );
+      }
+      await this.#contractInstances.set(key, new SerializableContractInstance(contractInstance).toBuffer());
+      await this.#contractInstancePublishedAt.set(key, blockNumber);
     });
   }
 
   deleteContractInstance(contractInstance: ContractInstanceWithAddress): Promise<void> {
+    // Protocol contracts are preloaded at block 0 and must never be deleted, even when the block that
+    // (re-)published them on-chain is unwound by a reorg.
+    if (isProtocolContract(contractInstance.address)) {
+      return Promise.resolve();
+    }
     return this.db.transactionAsync(async () => {
       await this.#contractInstances.delete(contractInstance.address.toString());
       await this.#contractInstancePublishedAt.delete(contractInstance.address.toString());
@@ -71,19 +135,18 @@ export class ContractInstanceStore {
 
   async getCurrentContractInstanceClassId(address: AztecAddress, timestamp: UInt64, originalClassId: Fr): Promise<Fr> {
     // We need to find the last update before the given timestamp
-    const queryResult = await this.#contractInstanceUpdates
-      .valuesAsync({
+    const serializedUpdate = await first(
+      this.#contractInstanceUpdates.valuesAsync({
         reverse: true,
         start: this.getUpdateKey(address, 0n), // Make sure we only look at updates for this contract
         end: this.getUpdateKey(address, timestamp + 1n), // No update can match this key since it doesn't have a log index. We want the highest key <= timestamp
         limit: 1,
-      })
-      .next();
-    if (queryResult.done) {
+      }),
+    );
+    if (serializedUpdate === undefined) {
       return originalClassId;
     }
 
-    const serializedUpdate = queryResult.value;
     const update = SerializableContractInstanceUpdate.fromBuffer(serializedUpdate);
     if (timestamp < update.timestampOfChange) {
       return update.prevContractClassId.isZero() ? originalClassId : update.prevContractClassId;

package/src/store/data_stores.ts

@@ -0,0 +1,104 @@
+import type { L1BlockId } from '@aztec/ethereum/l1-types';
+import type { AztecAsyncKVStore } from '@aztec/kv-store';
+import type { BlockHash } from '@aztec/stdlib/block';
+import type { ContractDataSource } from '@aztec/stdlib/contract';
+
+import { join } from 'path';
+
+import { ArchiverContractDataSourceAdapter } from '../modules/contract_data_source_adapter.js';
+import { BlockStore } from './block_store.js';
+import { ContractClassStore } from './contract_class_store.js';
+import { ContractInstanceStore } from './contract_instance_store.js';
+import { FunctionNamesCache } from './function_names_cache.js';
+import { LogStore } from './log_store.js';
+import { MessageStore } from './message_store.js';
+
+export const ARCHIVER_DB_VERSION = 7;
+
+/**
+ * Represents the latest L1 block processed by the archiver for various objects in L2.
+ */
+export type ArchiverL1SynchPoint = {
+  /** Number of the last L1 block that added a new L2 checkpoint metadata.  */
+  blocksSynchedTo?: bigint;
+  /** Last L1 block checked for L1 to L2 messages. */
+  messagesSynchedTo?: L1BlockId;
+};
+
+/**
+ * Bundle of archiver-owned LMDB substores plus the in-memory caches that span them.
+ *
+ * Replaces the former `KVArchiverDataStore` pass-through wrapper. Callers reach into
+ * the relevant substore directly (e.g. `stores.blocks.getBlock`) and use
+ * {@link createArchiverDataStores} to wire them up against a shared KV store.
+ */
+export type ArchiverDataStores = {
+  /** The underlying key-value store. Use {@link AztecAsyncKVStore.transactionAsync} to compose updates atomically. */
+  db: AztecAsyncKVStore;
+  /** Blocks, checkpoints, tx effects, proven/finalized state. */
+  blocks: BlockStore;
+  /** Public, private and contract class logs. */
+  logs: LogStore;
+  /** L1 to L2 messages and message sync state. */
+  messages: MessageStore;
+  /** Contract classes (with bytecode commitments). */
+  contractClasses: ContractClassStore;
+  /** Contract instances and contract instance updates. */
+  contractInstances: ContractInstanceStore;
+  /** In-memory cache of public function selectors -> names. */
+  functionNames: FunctionNamesCache;
+};
+
+/**
+ * Wires up the archiver substores against a shared KV store and returns the
+ * {@link ArchiverDataStores} bundle.
+ *
+ * @param genesisBlockHash - Hash of the synthetic genesis block, forwarded to the {@link LogStore} so it
+ *   can resolve a genesis `referenceBlock` (used by the PXE during early sync) instead of treating it as a
+ *   reorg.
+ */
+export function createArchiverDataStores(db: AztecAsyncKVStore, genesisBlockHash: BlockHash): ArchiverDataStores {
+  const blocks = new BlockStore(db);
+  return {
+    db,
+    blocks,
+    logs: new LogStore(db, blocks, genesisBlockHash),
+    messages: new MessageStore(db),
+    contractClasses: new ContractClassStore(db),
+    contractInstances: new ContractInstanceStore(db),
+    functionNames: new FunctionNamesCache(),
+  };
+}
+
+/**
+ * Returns the L1 sync point of the archiver, combining the block sync point from {@link BlockStore}
+ * and the message sync point from {@link MessageStore}.
+ */
+export async function getArchiverSynchPoint(stores: ArchiverDataStores): Promise<ArchiverL1SynchPoint> {
+  const [blocksSynchedTo, messagesSynchedTo] = await Promise.all([
+    stores.blocks.getSynchedL1BlockNumber(),
+    stores.messages.getSynchedL1Block(),
+  ]);
+  return { blocksSynchedTo, messagesSynchedTo };
+}
+
+/**
+ * Backs up the underlying KV store to the given folder. Returns the path to the resulting db file.
+ */
+export async function backupArchiverDataStores(
+  stores: ArchiverDataStores,
+  path: string,
+  compress = true,
+): Promise<string> {
+  await stores.db.backupTo(path, compress);
+  return join(path, 'data.mdb');
+}
+
+/**
+ * Returns a {@link ContractDataSource} adapter over {@link ArchiverDataStores}.
+ * Used by contexts (e.g. offline epoch re-prover tools) that need a ContractDataSource
+ * but do not need a full archiver instance.
+ */
+export function createContractDataSource(stores: ArchiverDataStores): ContractDataSource {
+  return new ArchiverContractDataSourceAdapter(stores);
+}

package/src/store/function_names_cache.ts

@@ -0,0 +1,37 @@
+import { createLogger } from '@aztec/foundation/log';
+import { FunctionSelector } from '@aztec/stdlib/abi';
+
+const MAX_FUNCTION_SIGNATURES = 1000;
+const MAX_FUNCTION_NAME_LEN = 256;
+
+/**
+ * In-memory cache mapping public function selectors to function names.
+ *
+ * Populated opportunistically (e.g. by PXE registering signatures from artifacts) so the
+ * archiver can attach human-readable names to logs and traces. Bounded by
+ * {@link MAX_FUNCTION_SIGNATURES} to avoid unbounded growth from untrusted callers.
+ */
+export class FunctionNamesCache {
+  private readonly log = createLogger('archiver:data-stores');
+  private readonly names: Map<string, string> = new Map();
+
+  /** Adds the given public function signatures to the cache. */
+  public async register(signatures: string[]): Promise<void> {
+    for (const sig of signatures) {
+      if (this.names.size > MAX_FUNCTION_SIGNATURES) {
+        return;
+      }
+      try {
+        const selector = await FunctionSelector.fromSignature(sig);
+        this.names.set(selector.toString(), sig.slice(0, sig.indexOf('(')).slice(0, MAX_FUNCTION_NAME_LEN));
+      } catch {
+        this.log.warn(`Failed to parse signature: ${sig}. Ignoring`);
+      }
+    }
+  }
+
+  /** Looks up a function name for the given selector, or returns undefined if not registered. */
+  public get(selector: FunctionSelector): string | undefined {
+    return this.names.get(selector.toString());
+  }
+}

package/src/store/l2_tips_cache.ts

@@ -0,0 +1,35 @@
+import type { BlockHash, L2Tips } from '@aztec/stdlib/block';
+
+import type { BlockStore } from './block_store.js';
+
+/**
+ * In-memory cache for L2 chain tips (proposed, checkpointed, proven, finalized).
+ * Populated from the BlockStore on first access, then kept up-to-date by the ArchiverDataStoreUpdater.
+ * Refresh calls should happen *after* the store transaction that mutates block data has committed,
+ * so the cache loads from committed state and is never replaced if the writer aborts.
+ */
+export class L2TipsCache {
+  #tipsPromise: Promise<L2Tips> | undefined;
+
+  /**
+   * The genesis block hash is dynamic — derived from the injected initial header, which depends on
+   * `genesisTimestamp` and any prefilled state — so it is supplied here rather than read from store.
+   * The genesis checkpoint hash, by contrast, is the static protocol constant and is resolved
+   * inside the block store.
+   */
+  constructor(
+    private blockStore: BlockStore,
+    private readonly initialBlockHash: BlockHash,
+  ) {}
+
+  /** Returns the cached L2 tips. Loads from the block store on first call. */
+  public getL2Tips(): Promise<L2Tips> {
+    return (this.#tipsPromise ??= this.blockStore.getL2TipsData(this.initialBlockHash));
+  }
+
+  /** Reloads the L2 tips from the block store. Should be called after the writer transaction has committed. */
+  public async refresh(): Promise<void> {
+    this.#tipsPromise = this.blockStore.getL2TipsData(this.initialBlockHash);
+    await this.#tipsPromise;
+  }
+}