@aztec/archiver 0.56.0 → 0.58.0

package/src/archiver/memory_archiver_store/memory_archiver_store.ts

@@ -15,18 +15,21 @@ import {
   TxReceipt,
   type UnencryptedL2BlockL2Logs,
 } from '@aztec/circuit-types';
-import { Fr, INITIAL_L2_BLOCK_NUM } from '@aztec/circuits.js';
-import { type ContractArtifact } from '@aztec/foundation/abi';
-import { type AztecAddress } from '@aztec/foundation/aztec-address';
 import {
   type ContractClassPublic,
+  type ContractClassPublicWithBlockNumber,
   type ContractInstanceWithAddress,
   type ExecutablePrivateFunctionWithMembershipProof,
+  Fr,
+  type Header,
+  INITIAL_L2_BLOCK_NUM,
   type UnconstrainedFunctionWithMembershipProof,
-} from '@aztec/types/contracts';
+} from '@aztec/circuits.js';
+import { type ContractArtifact } from '@aztec/foundation/abi';
+import { type AztecAddress } from '@aztec/foundation/aztec-address';
 
 import { type ArchiverDataStore, type ArchiverL1SynchPoint } from '../archiver_store.js';
-import { type DataRetrieval, type SingletonDataRetrieval } from '../structs/data_retrieval.js';
+import { type DataRetrieval } from '../structs/data_retrieval.js';
 import { type L1Published } from '../structs/published.js';
 import { L1ToL2MessageStore } from './l1_to_l2_message_store.js';
 
@@ -44,23 +47,11 @@ export class MemoryArchiverStore implements ArchiverDataStore {
    */
   private txEffects: TxEffect[] = [];
 
-  /**
-   * An array containing all the encrypted logs that have been fetched so far.
-   * Note: Index in the "outer" array equals to (corresponding L2 block's number - INITIAL_L2_BLOCK_NUM).
-   */
-  private noteEncryptedLogsPerBlock: EncryptedNoteL2BlockL2Logs[] = [];
+  private noteEncryptedLogsPerBlock: Map<number, EncryptedNoteL2BlockL2Logs> = new Map();
 
-  /**
-   * An array containing all the encrypted logs that have been fetched so far.
-   * Note: Index in the "outer" array equals to (corresponding L2 block's number - INITIAL_L2_BLOCK_NUM).
-   */
-  private encryptedLogsPerBlock: EncryptedL2BlockL2Logs[] = [];
+  private encryptedLogsPerBlock: Map<number, EncryptedL2BlockL2Logs> = new Map();
 
-  /**
-   * An array containing all the unencrypted logs that have been fetched so far.
-   * Note: Index in the "outer" array equals to (corresponding L2 block's number - INITIAL_L2_BLOCK_NUM).
-   */
-  private unencryptedLogsPerBlock: UnencryptedL2BlockL2Logs[] = [];
+  private unencryptedLogsPerBlock: Map<number, UnencryptedL2BlockL2Logs> = new Map();
 
   /**
    * Contains all L1 to L2 messages.
@@ -69,7 +60,7 @@ export class MemoryArchiverStore implements ArchiverDataStore {
 
   private contractArtifacts: Map<string, ContractArtifact> = new Map();
 
-  private contractClasses: Map<string, ContractClassPublic> = new Map();
+  private contractClasses: Map<string, ContractClassPublicWithBlockNumber> = new Map();
 
   private privateFunctions: Map<string, ExecutablePrivateFunctionWithMembershipProof[]> = new Map();
 
@@ -79,9 +70,9 @@ export class MemoryArchiverStore implements ArchiverDataStore {
 
   private lastL1BlockNewBlocks: bigint | undefined = undefined;
   private lastL1BlockNewMessages: bigint | undefined = undefined;
-  private lastL1BlockNewProvenLogs: bigint | undefined = undefined;
 
   private lastProvenL2BlockNumber: number = 0;
+  private lastProvenL2EpochNumber: number = 0;
 
   constructor(
     /** The max number of logs that can be obtained in 1 "getUnencryptedLogs" call. */
@@ -129,9 +120,24 @@ export class MemoryArchiverStore implements ArchiverDataStore {
     return Promise.resolve(true);
   }
 
-  public addContractClasses(data: ContractClassPublic[], _blockNumber: number): Promise<boolean> {
+  public addContractClasses(data: ContractClassPublic[], blockNumber: number): Promise<boolean> {
     for (const contractClass of data) {
-      this.contractClasses.set(contractClass.id.toString(), contractClass);
+      if (!this.contractClasses.has(contractClass.id.toString())) {
+        this.contractClasses.set(contractClass.id.toString(), {
+          ...contractClass,
+          l2BlockNumber: blockNumber,
+        });
+      }
+    }
+    return Promise.resolve(true);
+  }
+
+  public deleteContractClasses(data: ContractClassPublic[], blockNumber: number): Promise<boolean> {
+    for (const contractClass of data) {
+      const restored = this.contractClasses.get(contractClass.id.toString());
+      if (restored && restored.l2BlockNumber >= blockNumber) {
+        this.contractClasses.delete(contractClass.id.toString());
+      }
     }
     return Promise.resolve(true);
   }
@@ -143,6 +149,13 @@ export class MemoryArchiverStore implements ArchiverDataStore {
     return Promise.resolve(true);
   }
 
+  public deleteContractInstances(data: ContractInstanceWithAddress[], _blockNumber: number): Promise<boolean> {
+    for (const contractInstance of data) {
+      this.contractInstances.delete(contractInstance.address.toString());
+    }
+    return Promise.resolve(true);
+  }
+
   /**
    * Append new blocks to the store's list.
    * @param blocks - The L2 blocks to be added to the store and the last processed L1 block.
@@ -161,33 +174,58 @@ export class MemoryArchiverStore implements ArchiverDataStore {
   }
 
   /**
-   * Append new logs to the store's list.
-   * @param encryptedLogs - The encrypted logs to be added to the store.
-   * @param unencryptedLogs - The unencrypted logs to be added to the store.
-   * @param blockNumber - The block for which to add the logs.
-   * @returns True if the operation is successful.
+   * Unwinds blocks from the database
+   * @param from -  The tip of the chain, passed for verification purposes,
+   *                ensuring that we don't end up deleting something we did not intend
+   * @param blocksToUnwind - The number of blocks we are to unwind
+   * @returns True if the operation is successful
    */
-  addLogs(
-    noteEncryptedLogs: EncryptedNoteL2BlockL2Logs,
-    encryptedLogs: EncryptedL2BlockL2Logs,
-    unencryptedLogs: UnencryptedL2BlockL2Logs,
-    blockNumber: number,
-  ): Promise<boolean> {
-    if (noteEncryptedLogs) {
-      this.noteEncryptedLogsPerBlock[blockNumber - INITIAL_L2_BLOCK_NUM] = noteEncryptedLogs;
+  public async unwindBlocks(from: number, blocksToUnwind: number): Promise<boolean> {
+    const last = await this.getSynchedL2BlockNumber();
+    if (from != last) {
+      throw new Error(`Can only remove the tip`);
     }
 
-    if (encryptedLogs) {
-      this.encryptedLogsPerBlock[blockNumber - INITIAL_L2_BLOCK_NUM] = encryptedLogs;
+    const stopAt = from - blocksToUnwind;
+    while ((await this.getSynchedL2BlockNumber()) > stopAt) {
+      const block = this.l2Blocks.pop();
+      if (block == undefined) {
+        break;
+      }
+      block.data.body.txEffects.forEach(() => this.txEffects.pop());
     }
 
-    if (unencryptedLogs) {
-      this.unencryptedLogsPerBlock[blockNumber - INITIAL_L2_BLOCK_NUM] = unencryptedLogs;
-    }
+    return Promise.resolve(true);
+  }
+
+  /**
+   * Append new logs to the store's list.
+   * @param block - The block for which to add the logs.
+   * @returns True if the operation is successful.
+   */
+  addLogs(blocks: L2Block[]): Promise<boolean> {
+    blocks.forEach(block => {
+      this.noteEncryptedLogsPerBlock.set(block.number, block.body.noteEncryptedLogs);
+      this.encryptedLogsPerBlock.set(block.number, block.body.encryptedLogs);
+      this.unencryptedLogsPerBlock.set(block.number, block.body.unencryptedLogs);
+    });
+    return Promise.resolve(true);
+  }
+
+  deleteLogs(blocks: L2Block[]): Promise<boolean> {
+    blocks.forEach(block => {
+      this.encryptedLogsPerBlock.delete(block.number);
+      this.noteEncryptedLogsPerBlock.delete(block.number);
+      this.unencryptedLogsPerBlock.delete(block.number);
+    });
 
     return Promise.resolve(true);
   }
 
+  getTotalL1ToL2MessageCount(): Promise<bigint> {
+    return Promise.resolve(this.l1ToL2Messages.getTotalL1ToL2MessageCount());
+  }
+
   /**
    * Append L1 to L2 messages to the store.
    * @param messages - The L1 to L2 messages to be added to the store and the last processed L1 block.
@@ -226,12 +264,15 @@ export class MemoryArchiverStore implements ArchiverDataStore {
    * @remarks When "from" is smaller than genesis block number, blocks from the beginning are returned.
    */
   public getBlocks(from: number, limit: number): Promise<L1Published<L2Block>[]> {
-    // Return an empty array if we are outside of range
     if (limit < 1) {
       return Promise.reject(new Error(`Invalid limit: ${limit}`));
     }
 
-    const fromIndex = Math.max(from - INITIAL_L2_BLOCK_NUM, 0);
+    if (from < INITIAL_L2_BLOCK_NUM) {
+      return Promise.reject(new Error(`Invalid start: ${from}`));
+    }
+
+    const fromIndex = from - INITIAL_L2_BLOCK_NUM;
     if (fromIndex >= this.l2Blocks.length) {
       return Promise.resolve([]);
     }
@@ -240,6 +281,11 @@ export class MemoryArchiverStore implements ArchiverDataStore {
     return Promise.resolve(this.l2Blocks.slice(fromIndex, toIndex));
   }
 
+  public async getBlockHeaders(from: number, limit: number): Promise<Header[]> {
+    const blocks = await this.getBlocks(from, limit);
+    return blocks.map(block => block.data.header);
+  }
+
   /**
    * Gets a tx effect.
    * @param txHash - The txHash of the tx effect.
@@ -297,9 +343,14 @@ export class MemoryArchiverStore implements ArchiverDataStore {
     logType: TLogType,
   ): Promise<L2BlockL2Logs<FromLogType<TLogType>>[]> {
     if (from < INITIAL_L2_BLOCK_NUM || limit < 1) {
-      throw new Error(`Invalid limit: ${limit}`);
+      return Promise.resolve([]);
     }
-    const logs = (() => {
+
+    if (from > this.l2Blocks.length) {
+      return Promise.resolve([]);
+    }
+
+    const logMap = (() => {
       switch (logType) {
         case LogType.ENCRYPTED:
           return this.encryptedLogsPerBlock;
@@ -309,14 +360,24 @@ export class MemoryArchiverStore implements ArchiverDataStore {
         default:
           return this.unencryptedLogsPerBlock;
       }
-    })() as L2BlockL2Logs<FromLogType<TLogType>>[];
+    })() as Map<number, L2BlockL2Logs<FromLogType<TLogType>>>;
 
-    if (from > logs.length) {
-      return Promise.resolve([]);
-    }
-    const startIndex = from - INITIAL_L2_BLOCK_NUM;
+    const startIndex = from;
     const endIndex = startIndex + limit;
-    return Promise.resolve(logs.slice(startIndex, endIndex));
+    const upper = Math.min(endIndex, this.l2Blocks.length + INITIAL_L2_BLOCK_NUM);
+
+    const l = [];
+    for (let i = startIndex; i < upper; i++) {
+      const log = logMap.get(i);
+      if (log) {
+        l.push(log);
+      } else {
+        // I hate typescript sometimes
+        l.push(undefined as unknown as L2BlockL2Logs<FromLogType<TLogType>>);
+      }
+    }
+
+    return Promise.resolve(l);
   }
 
   /**
@@ -327,37 +388,37 @@ export class MemoryArchiverStore implements ArchiverDataStore {
    */
   getUnencryptedLogs(filter: LogFilter): Promise<GetUnencryptedLogsResponse> {
     let txHash: TxHash | undefined;
-    let fromBlockIndex = 0;
-    let toBlockIndex = this.unencryptedLogsPerBlock.length;
+    let fromBlock = 0;
+    let toBlock = this.l2Blocks.length + INITIAL_L2_BLOCK_NUM;
     let txIndexInBlock = 0;
     let logIndexInTx = 0;
 
     if (filter.afterLog) {
       // Continuation parameter is set --> tx hash is ignored
       if (filter.fromBlock == undefined || filter.fromBlock <= filter.afterLog.blockNumber) {
-        fromBlockIndex = filter.afterLog.blockNumber - INITIAL_L2_BLOCK_NUM;
+        fromBlock = filter.afterLog.blockNumber;
         txIndexInBlock = filter.afterLog.txIndex;
         logIndexInTx = filter.afterLog.logIndex + 1; // We want to start from the next log
       } else {
-        fromBlockIndex = filter.fromBlock - INITIAL_L2_BLOCK_NUM;
+        fromBlock = filter.fromBlock;
       }
     } else {
       txHash = filter.txHash;
 
       if (filter.fromBlock !== undefined) {
-        fromBlockIndex = filter.fromBlock - INITIAL_L2_BLOCK_NUM;
+        fromBlock = filter.fromBlock;
       }
     }
 
     if (filter.toBlock !== undefined) {
-      toBlockIndex = filter.toBlock - INITIAL_L2_BLOCK_NUM;
+      toBlock = filter.toBlock;
     }
 
     // Ensure the indices are within block array bounds
-    fromBlockIndex = Math.max(fromBlockIndex, 0);
-    toBlockIndex = Math.min(toBlockIndex, this.unencryptedLogsPerBlock.length);
+    fromBlock = Math.max(fromBlock, INITIAL_L2_BLOCK_NUM);
+    toBlock = Math.min(toBlock, this.l2Blocks.length + INITIAL_L2_BLOCK_NUM);
 
-    if (fromBlockIndex > this.unencryptedLogsPerBlock.length || toBlockIndex < fromBlockIndex || toBlockIndex <= 0) {
+    if (fromBlock > this.l2Blocks.length || toBlock < fromBlock || toBlock <= 0) {
       return Promise.resolve({
         logs: [],
         maxLogsHit: false,
@@ -368,27 +429,30 @@ export class MemoryArchiverStore implements ArchiverDataStore {
 
     const logs: ExtendedUnencryptedL2Log[] = [];
 
-    for (; fromBlockIndex < toBlockIndex; fromBlockIndex++) {
-      const block = this.l2Blocks[fromBlockIndex];
-      const blockLogs = this.unencryptedLogsPerBlock[fromBlockIndex];
-      for (; txIndexInBlock < blockLogs.txLogs.length; txIndexInBlock++) {
-        const txLogs = blockLogs.txLogs[txIndexInBlock].unrollLogs();
-        for (; logIndexInTx < txLogs.length; logIndexInTx++) {
-          const log = txLogs[logIndexInTx];
-          if (
-            (!txHash || block.data.body.txEffects[txIndexInBlock].txHash.equals(txHash)) &&
-            (!contractAddress || log.contractAddress.equals(contractAddress))
-          ) {
-            logs.push(new ExtendedUnencryptedL2Log(new LogId(block.data.number, txIndexInBlock, logIndexInTx), log));
-            if (logs.length === this.maxLogs) {
-              return Promise.resolve({
-                logs,
-                maxLogsHit: true,
-              });
+    for (; fromBlock < toBlock; fromBlock++) {
+      const block = this.l2Blocks[fromBlock - INITIAL_L2_BLOCK_NUM];
+      const blockLogs = this.unencryptedLogsPerBlock.get(fromBlock);
+
+      if (blockLogs) {
+        for (; txIndexInBlock < blockLogs.txLogs.length; txIndexInBlock++) {
+          const txLogs = blockLogs.txLogs[txIndexInBlock].unrollLogs();
+          for (; logIndexInTx < txLogs.length; logIndexInTx++) {
+            const log = txLogs[logIndexInTx];
+            if (
+              (!txHash || block.data.body.txEffects[txIndexInBlock].txHash.equals(txHash)) &&
+              (!contractAddress || log.contractAddress.equals(contractAddress))
+            ) {
+              logs.push(new ExtendedUnencryptedL2Log(new LogId(block.data.number, txIndexInBlock, logIndexInTx), log));
+              if (logs.length === this.maxLogs) {
+                return Promise.resolve({
+                  logs,
+                  maxLogsHit: true,
+                });
+              }
             }
           }
+          logIndexInTx = 0;
         }
-        logIndexInTx = 0;
       }
       txIndexInBlock = 0;
     }
@@ -414,9 +478,17 @@ export class MemoryArchiverStore implements ArchiverDataStore {
     return Promise.resolve(this.lastProvenL2BlockNumber);
   }
 
-  public setProvenL2BlockNumber(l2BlockNumber: SingletonDataRetrieval<number>): Promise<void> {
-    this.lastProvenL2BlockNumber = l2BlockNumber.retrievedData;
-    this.lastL1BlockNewProvenLogs = l2BlockNumber.lastProcessedL1BlockNumber;
+  public getProvenL2EpochNumber(): Promise<number | undefined> {
+    return Promise.resolve(this.lastProvenL2EpochNumber);
+  }
+
+  public setProvenL2BlockNumber(l2BlockNumber: number): Promise<void> {
+    this.lastProvenL2BlockNumber = l2BlockNumber;
+    return Promise.resolve();
+  }
+
+  public setProvenL2EpochNumber(l2EpochNumber: number): Promise<void> {
+    this.lastProvenL2EpochNumber = l2EpochNumber;
     return Promise.resolve();
   }
 
@@ -434,7 +506,6 @@ export class MemoryArchiverStore implements ArchiverDataStore {
     return Promise.resolve({
       blocksSynchedTo: this.lastL1BlockNewBlocks,
       messagesSynchedTo: this.lastL1BlockNewMessages,
-      provenLogsSynchedTo: this.lastL1BlockNewProvenLogs,
     });
   }
 

package/src/factory.ts

@@ -1,5 +1,7 @@
+import { type ContractClassPublic } from '@aztec/circuits.js';
 import { createDebugLogger } from '@aztec/foundation/log';
 import { createStore } from '@aztec/kv-store/utils';
+import { getCanonicalProtocolContract, protocolContractNames } from '@aztec/protocol-contracts';
 import { type TelemetryClient } from '@aztec/telemetry-client';
 import { NoopTelemetryClient } from '@aztec/telemetry-client/noop';
 
@@ -16,8 +18,24 @@ export async function createArchiver(
   if (!config.archiverUrl) {
     const store = await createStore('archiver', config, createDebugLogger('aztec:archiver:lmdb'));
     const archiverStore = new KVArchiverDataStore(store, config.maxLogs);
+    await initWithProtocolContracts(archiverStore);
     return Archiver.createAndSync(config, archiverStore, telemetry, opts.blockUntilSync);
   } else {
     return createArchiverClient(config.archiverUrl);
   }
 }
+
+async function initWithProtocolContracts(store: KVArchiverDataStore) {
+  const blockNumber = 0;
+  for (const name of protocolContractNames) {
+    const contract = getCanonicalProtocolContract(name);
+    const contractClassPublic: ContractClassPublic = {
+      ...contract.contractClass,
+      privateFunctions: [],
+      unconstrainedFunctions: [],
+    };
+    await store.addContractArtifact(contract.address, contract.artifact);
+    await store.addContractClasses([contractClassPublic], blockNumber);
+    await store.addContractInstances([contract.instance], blockNumber);
+  }
+}

package/src/index.ts

@@ -25,6 +25,7 @@ async function main() {
   const config = getArchiverConfigFromEnv();
   const { l1RpcUrl: rpcUrl, l1Contracts } = config;
 
+  log.info(`Starting archiver in main(): ${JSON.stringify(config)}`);
   const publicClient = createPublicClient({
     chain: localhost,
     transport: http(rpcUrl),

package/src/test/index.ts

@@ -0,0 +1,3 @@
+export * from './mock_l2_block_source.js';
+export * from './mock_l1_to_l2_message_source.js';
+export * from './mock_archiver.js';

package/src/test/mock_archiver.ts

@@ -0,0 +1,55 @@
+import { type L1ToL2MessageSource, type L2Block, type L2BlockSource } from '@aztec/circuit-types';
+import { type Fr } from '@aztec/circuits.js';
+
+import { MockL1ToL2MessageSource } from './mock_l1_to_l2_message_source.js';
+import { MockL2BlockSource } from './mock_l2_block_source.js';
+
+/**
+ * A mocked implementation of the archiver that implements L2BlockSource and L1ToL2MessageSource.
+ */
+export class MockArchiver extends MockL2BlockSource implements L2BlockSource, L1ToL2MessageSource {
+  private messageSource = new MockL1ToL2MessageSource(0);
+
+  public setL1ToL2Messages(blockNumber: number, msgs: Fr[]) {
+    this.messageSource.setL1ToL2Messages(blockNumber, msgs);
+  }
+
+  getL1ToL2Messages(blockNumber: bigint): Promise<Fr[]> {
+    return this.messageSource.getL1ToL2Messages(blockNumber);
+  }
+
+  getL1ToL2MessageIndex(_l1ToL2Message: Fr, _startIndex: bigint): Promise<bigint | undefined> {
+    return this.messageSource.getL1ToL2MessageIndex(_l1ToL2Message, _startIndex);
+  }
+}
+
+/**
+ * A mocked implementation of the archiver with a set of precomputed blocks and messages.
+ */
+export class MockPrefilledArchiver extends MockArchiver {
+  private precomputed: L2Block[];
+
+  constructor(precomputed: L2Block[], messages: Fr[][]) {
+    super();
+    this.precomputed = precomputed.slice();
+    messages.forEach((msgs, i) => this.setL1ToL2Messages(i + 1, msgs));
+  }
+
+  public setPrefilledBlocks(blocks: L2Block[], messages: Fr[][]) {
+    for (const block of blocks) {
+      this.precomputed[block.number - 1] = block;
+    }
+    messages.forEach((msgs, i) => this.setL1ToL2Messages(blocks[i].number, msgs));
+  }
+
+  public override createBlocks(numBlocks: number) {
+    if (this.l2Blocks.length + numBlocks > this.precomputed.length) {
+      throw new Error(
+        `Not enough precomputed blocks to create ${numBlocks} more blocks (already at ${this.l2Blocks.length})`,
+      );
+    }
+
+    const fromBlock = this.l2Blocks.length;
+    this.addBlocks(this.precomputed.slice(fromBlock, fromBlock + numBlocks));
+  }
+}

package/src/test/mock_l1_to_l2_message_source.ts

@@ -0,0 +1,31 @@
+import { type L1ToL2MessageSource } from '@aztec/circuit-types';
+import { type Fr } from '@aztec/circuits.js';
+
+/**
+ * A mocked implementation of L1ToL2MessageSource to be used in tests.
+ */
+export class MockL1ToL2MessageSource implements L1ToL2MessageSource {
+  private messagesPerBlock = new Map<number, Fr[]>();
+
+  constructor(private blockNumber: number) {}
+
+  public setL1ToL2Messages(blockNumber: number, msgs: Fr[]) {
+    this.messagesPerBlock.set(blockNumber, msgs);
+  }
+
+  public setBlockNumber(blockNumber: number) {
+    this.blockNumber = blockNumber;
+  }
+
+  getL1ToL2Messages(blockNumber: bigint): Promise<Fr[]> {
+    return Promise.resolve(this.messagesPerBlock.get(Number(blockNumber)) ?? []);
+  }
+
+  getL1ToL2MessageIndex(_l1ToL2Message: Fr, _startIndex: bigint): Promise<bigint | undefined> {
+    throw new Error('Method not implemented.');
+  }
+
+  getBlockNumber(): Promise<number> {
+    return Promise.resolve(this.blockNumber);
+  }
+}