@aztec/archiver 0.0.1-commit.cd76b27 → 0.0.1-commit.ce4f8c4f2

package/src/store/kv_archiver_store.ts

@@ -16,11 +16,10 @@ import {
 import type { CheckpointData, PublishedCheckpoint } from '@aztec/stdlib/checkpoint';
 import type {
   ContractClassPublic,
+  ContractClassPublicWithCommitment,
   ContractDataSource,
   ContractInstanceUpdateWithAddress,
   ContractInstanceWithAddress,
-  ExecutablePrivateFunctionWithMembershipProof,
-  UtilityFunctionWithMembershipProof,
 } from '@aztec/stdlib/contract';
 import type { L1RollupConstants } from '@aztec/stdlib/epoch-helpers';
 import type { GetContractClassLogsResponse, GetPublicLogsResponse } from '@aztec/stdlib/interfaces/client';
@@ -37,7 +36,7 @@ import { ContractInstanceStore } from './contract_instance_store.js';
 import { LogStore } from './log_store.js';
 import { MessageStore } from './message_store.js';
 
-export const ARCHIVER_DB_VERSION = 5;
+export const ARCHIVER_DB_VERSION = 6;
 export const MAX_FUNCTION_SIGNATURES = 1000;
 export const MAX_FUNCTION_NAME_LEN = 256;
 
@@ -71,9 +70,8 @@ export class KVArchiverDataStore implements ContractDataSource {
   constructor(
     private db: AztecAsyncKVStore,
     logsMaxPageSize: number = 1000,
-    l1Constants: Pick<L1RollupConstants, 'epochDuration'>,
   ) {
-    this.#blockStore = new BlockStore(db, l1Constants);
+    this.#blockStore = new BlockStore(db);
     this.#logStore = new LogStore(db, this.#blockStore, logsMaxPageSize);
     this.#messageStore = new MessageStore(db);
     this.#contractClassStore = new ContractClassStore(db);
@@ -168,19 +166,14 @@ export class KVArchiverDataStore implements ContractDataSource {
 
   /**
    * Add new contract classes from an L2 block to the store's list.
-   * @param data - List of contract classes to be added.
-   * @param bytecodeCommitments - Bytecode commitments for the contract classes.
+   * @param data - List of contract classes (with bytecode commitments) to be added.
    * @param blockNumber - Number of the L2 block the contracts were registered in.
    * @returns True if the operation is successful.
    */
-  async addContractClasses(
-    data: ContractClassPublic[],
-    bytecodeCommitments: Fr[],
-    blockNumber: BlockNumber,
-  ): Promise<boolean> {
+  async addContractClasses(data: ContractClassPublicWithCommitment[], blockNumber: BlockNumber): Promise<boolean> {
     return (
       await Promise.all(
-        data.map((c, i) => this.#contractClassStore.addContractClass(c, bytecodeCommitments[i], blockNumber)),
+        data.map(c => this.#contractClassStore.addContractClass(c, c.publicBytecodeCommitment, blockNumber)),
       )
     ).every(Boolean);
   }
@@ -195,15 +188,6 @@ export class KVArchiverDataStore implements ContractDataSource {
     return this.#contractClassStore.getBytecodeCommitment(contractClassId);
   }
 
-  /** Adds private functions to a contract class. */
-  addFunctions(
-    contractClassId: Fr,
-    privateFunctions: ExecutablePrivateFunctionWithMembershipProof[],
-    utilityFunctions: UtilityFunctionWithMembershipProof[],
-  ): Promise<boolean> {
-    return this.#contractClassStore.addFunctions(contractClassId, privateFunctions, utilityFunctions);
-  }
-
   /**
    * Add new contract instances from an L2 block to the store's list.
    * @param data - List of contract instances to be added.
@@ -246,14 +230,14 @@ export class KVArchiverDataStore implements ContractDataSource {
   }
 
   /**
-   * Append new proposed blocks to the store's list.
-   * These are uncheckpointed blocks that have been proposed by the sequencer but not yet included in a checkpoint on L1.
+   * Append a new proposed block to the store.
+   * This is an uncheckpointed block that has been proposed by the sequencer but not yet included in a checkpoint on L1.
    * For checkpointed blocks (already published to L1), use addCheckpoints() instead.
-   * @param blocks - The proposed L2 blocks to be added to the store.
+   * @param block - The proposed L2 block to be added to the store.
    * @returns True if the operation is successful.
    */
-  addProposedBlocks(blocks: L2Block[], opts: { force?: boolean; checkpointNumber?: number } = {}): Promise<boolean> {
-    return this.#blockStore.addProposedBlocks(blocks, opts);
+  addProposedBlock(block: L2Block, opts: { force?: boolean } = {}): Promise<boolean> {
+    return this.#blockStore.addProposedBlock(block, opts);
   }
 
   /**
@@ -410,8 +394,11 @@ export class KVArchiverDataStore implements ContractDataSource {
    * @param txHash - The hash of a tx we try to get the receipt for.
    * @returns The requested tx receipt (or undefined if not found).
    */
-  getSettledTxReceipt(txHash: TxHash): Promise<TxReceipt | undefined> {
-    return this.#blockStore.getSettledTxReceipt(txHash);
+  getSettledTxReceipt(
+    txHash: TxHash,
+    l1Constants?: Pick<L1RollupConstants, 'epochDuration'>,
+  ): Promise<TxReceipt | undefined> {
+    return this.#blockStore.getSettledTxReceipt(txHash, l1Constants);
   }
 
   /**
@@ -472,10 +459,11 @@ export class KVArchiverDataStore implements ContractDataSource {
    * array implies no logs match that tag.
    * @param tags - The tags to search for.
    * @param page - The page number (0-indexed) for pagination. Returns at most 10 logs per tag per page.
+   * @param upToBlockNumber - If set, only return logs from blocks up to and including this block number.
    */
-  getPrivateLogsByTags(tags: SiloedTag[], page?: number): Promise<TxScopedL2Log[][]> {
+  getPrivateLogsByTags(tags: SiloedTag[], page?: number, upToBlockNumber?: BlockNumber): Promise<TxScopedL2Log[][]> {
     try {
-      return this.#logStore.getPrivateLogsByTags(tags, page);
+      return this.#logStore.getPrivateLogsByTags(tags, page, upToBlockNumber);
     } catch (err) {
       return Promise.reject(err);
     }
@@ -487,14 +475,16 @@ export class KVArchiverDataStore implements ContractDataSource {
    * @param contractAddress - The contract address to search logs for.
    * @param tags - The tags to search for.
    * @param page - The page number (0-indexed) for pagination. Returns at most 10 logs per tag per page.
+   * @param upToBlockNumber - If set, only return logs from blocks up to and including this block number.
    */
   getPublicLogsByTagsFromContract(
     contractAddress: AztecAddress,
     tags: Tag[],
     page?: number,
+    upToBlockNumber?: BlockNumber,
   ): Promise<TxScopedL2Log[][]> {
     try {
-      return this.#logStore.getPublicLogsByTagsFromContract(contractAddress, tags, page);
+      return this.#logStore.getPublicLogsByTagsFromContract(contractAddress, tags, page, upToBlockNumber);
     } catch (err) {
       return Promise.reject(err);
     }
@@ -542,6 +532,22 @@ export class KVArchiverDataStore implements ContractDataSource {
     await this.#blockStore.setProvenCheckpointNumber(checkpointNumber);
   }
 
+  /**
+   * Gets the number of the latest finalized checkpoint processed.
+   * @returns The number of the latest finalized checkpoint processed.
+   */
+  getFinalizedCheckpointNumber(): Promise<CheckpointNumber> {
+    return this.#blockStore.getFinalizedCheckpointNumber();
+  }
+
+  /**
+   * Stores the number of the latest finalized checkpoint processed.
+   * @param checkpointNumber - The number of the latest finalized checkpoint processed.
+   */
+  async setFinalizedCheckpointNumber(checkpointNumber: CheckpointNumber) {
+    await this.#blockStore.setFinalizedCheckpointNumber(checkpointNumber);
+  }
+
   async setBlockSynchedL1BlockNumber(l1BlockNumber: bigint) {
     await this.#blockStore.setSynchedL1BlockNumber(l1BlockNumber);
   }
@@ -585,6 +591,11 @@ export class KVArchiverDataStore implements ContractDataSource {
     return this.#messageStore.rollbackL1ToL2MessagesToCheckpoint(targetCheckpointNumber);
   }
 
+  /** Persists the inbox tree-in-progress checkpoint number from L1 state. */
+  public setInboxTreeInProgress(value: bigint): Promise<void> {
+    return this.#messageStore.setInboxTreeInProgress(value);
+  }
+
   /** Returns an async iterator to all L1 to L2 messages on the range. */
   public iterateL1ToL2Messages(range: CustomRange<bigint> = {}): AsyncIterableIterator<InboxMessage> {
     return this.#messageStore.iterateL1ToL2Messages(range);

package/src/store/log_store.ts

@@ -1,6 +1,6 @@
 import { INITIAL_L2_BLOCK_NUM } from '@aztec/constants';
 import { BlockNumber } from '@aztec/foundation/branded-types';
-import { filterAsync } from '@aztec/foundation/collection';
+import { compactArray, filterAsync } from '@aztec/foundation/collection';
 import { Fr } from '@aztec/foundation/curves/bn254';
 import { createLogger } from '@aztec/foundation/log';
 import { BufferReader, numToUInt32BE } from '@aztec/foundation/serialize';
@@ -22,6 +22,7 @@ import {
 } from '@aztec/stdlib/logs';
 import { TxHash } from '@aztec/stdlib/tx';
 
+import { OutOfOrderLogInsertionError } from '../errors.js';
 import type { BlockStore } from './block_store.js';
 
 /**
@@ -165,10 +166,21 @@ export class LogStore {
 
     for (const taggedLogBuffer of currentPrivateTaggedLogs) {
       if (taggedLogBuffer.logBuffers && taggedLogBuffer.logBuffers.length > 0) {
-        privateTaggedLogs.set(
-          taggedLogBuffer.tag,
-          taggedLogBuffer.logBuffers!.concat(privateTaggedLogs.get(taggedLogBuffer.tag)!),
-        );
+        const newLogs = privateTaggedLogs.get(taggedLogBuffer.tag)!;
+        if (newLogs.length === 0) {
+          continue;
+        }
+        const lastExisting = TxScopedL2Log.fromBuffer(taggedLogBuffer.logBuffers.at(-1)!);
+        const firstNew = TxScopedL2Log.fromBuffer(newLogs[0]);
+        if (lastExisting.blockNumber > firstNew.blockNumber) {
+          throw new OutOfOrderLogInsertionError(
+            'private',
+            taggedLogBuffer.tag,
+            lastExisting.blockNumber,
+            firstNew.blockNumber,
+          );
+        }
+        privateTaggedLogs.set(taggedLogBuffer.tag, taggedLogBuffer.logBuffers.concat(newLogs));
       }
     }
 
@@ -200,10 +212,21 @@ export class LogStore {
 
     for (const taggedLogBuffer of currentPublicTaggedLogs) {
       if (taggedLogBuffer.logBuffers && taggedLogBuffer.logBuffers.length > 0) {
-        publicTaggedLogs.set(
-          taggedLogBuffer.tag,
-          taggedLogBuffer.logBuffers!.concat(publicTaggedLogs.get(taggedLogBuffer.tag)!),
-        );
+        const newLogs = publicTaggedLogs.get(taggedLogBuffer.tag)!;
+        if (newLogs.length === 0) {
+          continue;
+        }
+        const lastExisting = TxScopedL2Log.fromBuffer(taggedLogBuffer.logBuffers.at(-1)!);
+        const firstNew = TxScopedL2Log.fromBuffer(newLogs[0]);
+        if (lastExisting.blockNumber > firstNew.blockNumber) {
+          throw new OutOfOrderLogInsertionError(
+            'public',
+            taggedLogBuffer.tag,
+            lastExisting.blockNumber,
+            firstNew.blockNumber,
+          );
+        }
+        publicTaggedLogs.set(taggedLogBuffer.tag, taggedLogBuffer.logBuffers.concat(newLogs));
       }
     }
 
@@ -290,18 +313,49 @@ export class LogStore {
 
   deleteLogs(blocks: L2Block[]): Promise<boolean> {
     return this.db.transactionAsync(async () => {
-      await Promise.all(
-        blocks.map(async block => {
-          // Delete private logs
-          const privateKeys = (await this.#privateLogKeysByBlock.getAsync(block.number)) ?? [];
-          await Promise.all(privateKeys.map(tag => this.#privateLogsByTag.delete(tag)));
-
-          // Delete public logs
-          const publicKeys = (await this.#publicLogKeysByBlock.getAsync(block.number)) ?? [];
-          await Promise.all(publicKeys.map(key => this.#publicLogsByContractAndTag.delete(key)));
-        }),
+      const blockNumbers = new Set(blocks.map(block => block.number));
+      const firstBlockToDelete = Math.min(...blockNumbers);
+
+      // Collect all unique private tags across all blocks being deleted
+      const allPrivateTags = new Set(
+        compactArray(await Promise.all(blocks.map(block => this.#privateLogKeysByBlock.getAsync(block.number)))).flat(),
       );
 
+      // Trim private logs: for each tag, delete all instances including and after the first block being deleted.
+      // This hinges on the invariant that logs for a given tag are always inserted in order of block number, which is enforced in #addPrivateLogs.
+      for (const tag of allPrivateTags) {
+        const existing = await this.#privateLogsByTag.getAsync(tag);
+        if (existing === undefined || existing.length === 0) {
+          continue;
+        }
+        const lastIndexToKeep = existing.findLastIndex(
+          buf => TxScopedL2Log.getBlockNumberFromBuffer(buf) < firstBlockToDelete,
+        );
+        const remaining = existing.slice(0, lastIndexToKeep + 1);
+        await (remaining.length > 0 ? this.#privateLogsByTag.set(tag, remaining) : this.#privateLogsByTag.delete(tag));
+      }
+
+      // Collect all unique public keys across all blocks being deleted
+      const allPublicKeys = new Set(
+        compactArray(await Promise.all(blocks.map(block => this.#publicLogKeysByBlock.getAsync(block.number)))).flat(),
+      );
+
+      // And do the same as we did with private logs
+      for (const key of allPublicKeys) {
+        const existing = await this.#publicLogsByContractAndTag.getAsync(key);
+        if (existing === undefined || existing.length === 0) {
+          continue;
+        }
+        const lastIndexToKeep = existing.findLastIndex(
+          buf => TxScopedL2Log.getBlockNumberFromBuffer(buf) < firstBlockToDelete,
+        );
+        const remaining = existing.slice(0, lastIndexToKeep + 1);
+        await (remaining.length > 0
+          ? this.#publicLogsByContractAndTag.set(key, remaining)
+          : this.#publicLogsByContractAndTag.delete(key));
+      }
+
+      // After trimming the tagged logs, we can delete the block-level keys that track which tags are in which blocks.
       await Promise.all(
         blocks.map(block =>
           Promise.all([
@@ -322,17 +376,30 @@ export class LogStore {
    * array implies no logs match that tag.
    * @param tags - The tags to search for.
    * @param page - The page number (0-indexed) for pagination.
+   * @param upToBlockNumber - If set, only return logs from blocks up to and including this block number.
    * @returns An array of log arrays, one per tag. Returns at most MAX_LOGS_PER_TAG logs per tag per page. If
    * MAX_LOGS_PER_TAG logs are returned for a tag, the caller should fetch the next page to check for more logs.
    */
-  async getPrivateLogsByTags(tags: SiloedTag[], page: number = 0): Promise<TxScopedL2Log[][]> {
+  async getPrivateLogsByTags(
+    tags: SiloedTag[],
+    page: number = 0,
+    upToBlockNumber?: BlockNumber,
+  ): Promise<TxScopedL2Log[][]> {
     const logs = await Promise.all(tags.map(tag => this.#privateLogsByTag.getAsync(tag.toString())));
+
     const start = page * MAX_LOGS_PER_TAG;
     const end = start + MAX_LOGS_PER_TAG;
 
-    return logs.map(
-      logBuffers => logBuffers?.slice(start, end).map(logBuffer => TxScopedL2Log.fromBuffer(logBuffer)) ?? [],
-    );
+    return logs.map(logBuffers => {
+      const deserialized = logBuffers?.slice(start, end).map(buf => TxScopedL2Log.fromBuffer(buf)) ?? [];
+      if (upToBlockNumber !== undefined) {
+        const cutoff = deserialized.findIndex(log => log.blockNumber > upToBlockNumber);
+        if (cutoff !== -1) {
+          return deserialized.slice(0, cutoff);
+        }
+      }
+      return deserialized;
+    });
   }
 
   /**
@@ -341,6 +408,7 @@ export class LogStore {
    * @param contractAddress - The contract address to search logs for.
    * @param tags - The tags to search for.
    * @param page - The page number (0-indexed) for pagination.
+   * @param upToBlockNumber - If set, only return logs from blocks up to and including this block number.
    * @returns An array of log arrays, one per tag. Returns at most MAX_LOGS_PER_TAG logs per tag per page. If
    * MAX_LOGS_PER_TAG logs are returned for a tag, the caller should fetch the next page to check for more logs.
    */
@@ -348,6 +416,7 @@ export class LogStore {
     contractAddress: AztecAddress,
     tags: Tag[],
     page: number = 0,
+    upToBlockNumber?: BlockNumber,
   ): Promise<TxScopedL2Log[][]> {
     const logs = await Promise.all(
       tags.map(tag => {
@@ -358,9 +427,16 @@ export class LogStore {
     const start = page * MAX_LOGS_PER_TAG;
     const end = start + MAX_LOGS_PER_TAG;
 
-    return logs.map(
-      logBuffers => logBuffers?.slice(start, end).map(logBuffer => TxScopedL2Log.fromBuffer(logBuffer)) ?? [],
-    );
+    return logs.map(logBuffers => {
+      const deserialized = logBuffers?.slice(start, end).map(buf => TxScopedL2Log.fromBuffer(buf)) ?? [];
+      if (upToBlockNumber !== undefined) {
+        const cutoff = deserialized.findIndex(log => log.blockNumber > upToBlockNumber);
+        if (cutoff !== -1) {
+          return deserialized.slice(0, cutoff);
+        }
+      }
+      return deserialized;
+    });
   }
 
   /**
@@ -588,11 +664,24 @@ export class LogStore {
     txLogs: PublicLog[],
     filter: LogFilter = {},
   ): boolean {
+    if (filter.fromBlock && blockNumber < filter.fromBlock) {
+      return false;
+    }
+    if (filter.toBlock && blockNumber >= filter.toBlock) {
+      return false;
+    }
+    if (filter.txHash && !txHash.equals(filter.txHash)) {
+      return false;
+    }
+
     let maxLogsHit = false;
     let logIndex = typeof filter.afterLog?.logIndex === 'number' ? filter.afterLog.logIndex + 1 : 0;
     for (; logIndex < txLogs.length; logIndex++) {
       const log = txLogs[logIndex];
-      if (!filter.contractAddress || log.contractAddress.equals(filter.contractAddress)) {
+      if (
+        (!filter.contractAddress || log.contractAddress.equals(filter.contractAddress)) &&
+        (!filter.tag || log.fields[0]?.equals(filter.tag))
+      ) {
         results.push(
           new ExtendedPublicLog(new LogId(BlockNumber(blockNumber), blockHash, txHash, txIndex, logIndex), log),
         );
@@ -616,6 +705,16 @@ export class LogStore {
     txLogs: ContractClassLog[],
     filter: LogFilter = {},
   ): boolean {
+    if (filter.fromBlock && blockNumber < filter.fromBlock) {
+      return false;
+    }
+    if (filter.toBlock && blockNumber >= filter.toBlock) {
+      return false;
+    }
+    if (filter.txHash && !txHash.equals(filter.txHash)) {
+      return false;
+    }
+
     let maxLogsHit = false;
     let logIndex = typeof filter.afterLog?.logIndex === 'number' ? filter.afterLog.logIndex + 1 : 0;
     for (; logIndex < txLogs.length; logIndex++) {

package/src/store/message_store.ts

@@ -14,6 +14,7 @@ import {
 } from '@aztec/kv-store';
 import { InboxLeaf } from '@aztec/stdlib/messaging';
 
+import { L1ToL2MessagesNotReadyError } from '../errors.js';
 import {
   type InboxMessage,
   deserializeInboxMessage,
@@ -40,6 +41,8 @@ export class MessageStore {
   #lastSynchedL1Block: AztecAsyncSingleton<Buffer>;
   /** Stores total messages stored */
   #totalMessageCount: AztecAsyncSingleton<bigint>;
+  /** Stores the checkpoint number whose message tree is currently being filled on L1. */
+  #inboxTreeInProgress: AztecAsyncSingleton<bigint>;
 
   #log = createLogger('archiver:message_store');
 
@@ -48,6 +51,7 @@ export class MessageStore {
     this.#l1ToL2MessageIndices = db.openMap('archiver_l1_to_l2_message_indices');
     this.#lastSynchedL1Block = db.openSingleton('archiver_last_l1_block_id');
     this.#totalMessageCount = db.openSingleton('archiver_l1_to_l2_message_count');
+    this.#inboxTreeInProgress = db.openSingleton('archiver_inbox_tree_in_progress');
   }
 
   public async getTotalL1ToL2MessageCount(): Promise<bigint> {
@@ -137,7 +141,7 @@ export class MessageStore {
           );
         }
 
-        // Check the first message in a block has the correct index.
+        // Check the first message in a checkpoint has the correct index.
         if (
           (!lastMessage || message.checkpointNumber > lastMessage.checkpointNumber) &&
           message.index !== expectedStart
@@ -185,7 +189,22 @@ export class MessageStore {
     return msg ? deserializeInboxMessage(msg) : undefined;
   }
 
+  /** Returns the inbox tree-in-progress checkpoint number from L1, or undefined if not yet set. */
+  public getInboxTreeInProgress(): Promise<bigint | undefined> {
+    return this.#inboxTreeInProgress.getAsync();
+  }
+
+  /** Persists the inbox tree-in-progress checkpoint number from L1 state. */
+  public async setInboxTreeInProgress(value: bigint): Promise<void> {
+    await this.#inboxTreeInProgress.set(value);
+  }
+
   public async getL1ToL2Messages(checkpointNumber: CheckpointNumber): Promise<Fr[]> {
+    const treeInProgress = await this.#inboxTreeInProgress.getAsync();
+    if (treeInProgress !== undefined && BigInt(checkpointNumber) >= treeInProgress) {
+      throw new L1ToL2MessagesNotReadyError(checkpointNumber, treeInProgress);
+    }
+
     const messages: Fr[] = [];
 
     const [startIndex, endIndex] = InboxLeaf.indexRangeForCheckpoint(checkpointNumber);