@aztec/archiver 0.61.0 → 0.63.0

package/src/archiver/epoch_helpers.ts

@@ -1,26 +1,30 @@
-import { AZTEC_EPOCH_DURATION, AZTEC_SLOT_DURATION } from '@aztec/circuits.js';
+type TimeConstants = {
+  l1GenesisTime: bigint;
+  epochDuration: number;
+  slotDuration: number;
+};
 
 /** Returns the slot number for a given timestamp. */
-export function getSlotAtTimestamp(ts: bigint, constants: { l1GenesisTime: bigint }) {
-  return ts < constants.l1GenesisTime ? 0n : (ts - constants.l1GenesisTime) / BigInt(AZTEC_SLOT_DURATION);
+export function getSlotAtTimestamp(ts: bigint, constants: Pick<TimeConstants, 'l1GenesisTime' | 'slotDuration'>) {
+  return ts < constants.l1GenesisTime ? 0n : (ts - constants.l1GenesisTime) / BigInt(constants.slotDuration);
 }
 
 /** Returns the epoch number for a given timestamp. */
-export function getEpochNumberAtTimestamp(ts: bigint, constants: { l1GenesisTime: bigint }) {
-  return getSlotAtTimestamp(ts, constants) / BigInt(AZTEC_EPOCH_DURATION);
+export function getEpochNumberAtTimestamp(ts: bigint, constants: TimeConstants) {
+  return getSlotAtTimestamp(ts, constants) / BigInt(constants.epochDuration);
 }
 
 /** Returns the range of slots (inclusive) for a given epoch number. */
-export function getSlotRangeForEpoch(epochNumber: bigint) {
-  const startSlot = epochNumber * BigInt(AZTEC_EPOCH_DURATION);
-  return [startSlot, startSlot + BigInt(AZTEC_EPOCH_DURATION) - 1n];
+export function getSlotRangeForEpoch(epochNumber: bigint, constants: Pick<TimeConstants, 'epochDuration'>) {
+  const startSlot = epochNumber * BigInt(constants.epochDuration);
+  return [startSlot, startSlot + BigInt(constants.epochDuration) - 1n];
 }
 
 /** Returns the range of L1 timestamps (inclusive) for a given epoch number. */
-export function getTimestampRangeForEpoch(epochNumber: bigint, constants: { l1GenesisTime: bigint }) {
-  const [startSlot, endSlot] = getSlotRangeForEpoch(epochNumber);
+export function getTimestampRangeForEpoch(epochNumber: bigint, constants: TimeConstants) {
+  const [startSlot, endSlot] = getSlotRangeForEpoch(epochNumber, constants);
   return [
-    constants.l1GenesisTime + startSlot * BigInt(AZTEC_SLOT_DURATION),
-    constants.l1GenesisTime + endSlot * BigInt(AZTEC_SLOT_DURATION),
+    constants.l1GenesisTime + startSlot * BigInt(constants.slotDuration),
+    constants.l1GenesisTime + endSlot * BigInt(constants.slotDuration),
   ];
 }

package/src/archiver/kv_archiver_store/kv_archiver_store.ts

@@ -1,5 +1,4 @@
 import {
-  type EncryptedL2NoteLog,
   type FromLogType,
   type GetUnencryptedLogsResponse,
   type InboxLeaf,
@@ -10,6 +9,7 @@ import {
   type TxEffect,
   type TxHash,
   type TxReceipt,
+  type TxScopedL2Log,
 } from '@aztec/circuit-types';
 import {
   type ContractClassPublic,
@@ -199,13 +199,12 @@ export class KVArchiverDataStore implements ArchiverDataStore {
   }
 
   /**
-   * Gets the first L1 to L2 message index in the L1 to L2 message tree which is greater than or equal to `startIndex`.
+   * Gets the L1 to L2 message index in the L1 to L2 message tree.
    * @param l1ToL2Message - The L1 to L2 message.
-   * @param startIndex - The index to start searching from.
    * @returns The index of the L1 to L2 message in the L1 to L2 message tree (undefined if not found).
    */
-  getL1ToL2MessageIndex(l1ToL2Message: Fr, startIndex: bigint): Promise<bigint | undefined> {
-    return Promise.resolve(this.#messageStore.getL1ToL2MessageIndex(l1ToL2Message, startIndex));
+  getL1ToL2MessageIndex(l1ToL2Message: Fr): Promise<bigint | undefined> {
+    return Promise.resolve(this.#messageStore.getL1ToL2MessageIndex(l1ToL2Message));
   }
 
   /**
@@ -246,7 +245,7 @@ export class KVArchiverDataStore implements ArchiverDataStore {
    * @returns For each received tag, an array of matching logs is returned. An empty array implies no logs match
    * that tag.
    */
-  getLogsByTags(tags: Fr[]): Promise<EncryptedL2NoteLog[][]> {
+  getLogsByTags(tags: Fr[]): Promise<TxScopedL2Log[][]> {
     try {
       return this.#logStore.getLogsByTags(tags);
     } catch (err) {
@@ -267,6 +266,19 @@ export class KVArchiverDataStore implements ArchiverDataStore {
     }
   }
 
+  /**
+   * Gets contract class logs based on the provided filter.
+   * @param filter - The filter to apply to the logs.
+   * @returns The requested logs.
+   */
+  getContractClassLogs(filter: LogFilter): Promise<GetUnencryptedLogsResponse> {
+    try {
+      return Promise.resolve(this.#logStore.getContractClassLogs(filter));
+    } catch (err) {
+      return Promise.reject(err);
+    }
+  }
+
   /**
    * Gets the number of the latest L2 block processed.
    * @returns The number of the latest L2 block processed.

package/src/archiver/kv_archiver_store/log_store.ts

@@ -1,6 +1,7 @@
 import {
+  type Body,
+  ContractClass2BlockL2Logs,
   EncryptedL2BlockL2Logs,
-  EncryptedL2NoteLog,
   EncryptedNoteL2BlockL2Logs,
   ExtendedUnencryptedL2Log,
   type FromLogType,
@@ -10,13 +11,14 @@ import {
   type LogFilter,
   LogId,
   LogType,
+  TxScopedL2Log,
   UnencryptedL2BlockL2Logs,
   type UnencryptedL2Log,
 } from '@aztec/circuit-types';
 import { Fr } from '@aztec/circuits.js';
-import { INITIAL_L2_BLOCK_NUM } from '@aztec/circuits.js/constants';
+import { INITIAL_L2_BLOCK_NUM, MAX_NOTE_HASHES_PER_TX } from '@aztec/circuits.js/constants';
 import { createDebugLogger } from '@aztec/foundation/log';
-import { type AztecKVStore, type AztecMap, type AztecMultiMap } from '@aztec/kv-store';
+import { type AztecKVStore, type AztecMap } from '@aztec/kv-store';
 
 import { type BlockStore } from './block_store.js';
 
@@ -25,58 +27,123 @@ import { type BlockStore } from './block_store.js';
  */
 export class LogStore {
   #noteEncryptedLogsByBlock: AztecMap<number, Buffer>;
-  #noteEncryptedLogsByHash: AztecMap<string, Buffer>;
-  #noteEncryptedLogHashesByTag: AztecMultiMap<string, string>;
-  #noteEncryptedLogTagsByBlock: AztecMultiMap<number, string>;
+  #logsByTag: AztecMap<string, Buffer[]>;
+  #logTagsByBlock: AztecMap<number, string[]>;
   #encryptedLogsByBlock: AztecMap<number, Buffer>;
   #unencryptedLogsByBlock: AztecMap<number, Buffer>;
+  #contractClassLogsByBlock: AztecMap<number, Buffer>;
   #logsMaxPageSize: number;
   #log = createDebugLogger('aztec:archiver:log_store');
 
   constructor(private db: AztecKVStore, private blockStore: BlockStore, logsMaxPageSize: number = 1000) {
     this.#noteEncryptedLogsByBlock = db.openMap('archiver_note_encrypted_logs_by_block');
-    this.#noteEncryptedLogsByHash = db.openMap('archiver_note_encrypted_logs_by_hash');
-    this.#noteEncryptedLogHashesByTag = db.openMultiMap('archiver_tagged_note_encrypted_log_hashes_by_tag');
-    this.#noteEncryptedLogTagsByBlock = db.openMultiMap('archiver_note_encrypted_log_tags_by_block');
+    this.#logsByTag = db.openMap('archiver_tagged_logs_by_tag');
+    this.#logTagsByBlock = db.openMap('archiver_log_tags_by_block');
     this.#encryptedLogsByBlock = db.openMap('archiver_encrypted_logs_by_block');
     this.#unencryptedLogsByBlock = db.openMap('archiver_unencrypted_logs_by_block');
+    this.#contractClassLogsByBlock = db.openMap('archiver_contract_class_logs_by_block');
 
     this.#logsMaxPageSize = logsMaxPageSize;
   }
 
+  #extractTaggedLogs(block: L2Block, logType: keyof Pick<Body, 'noteEncryptedLogs' | 'unencryptedLogs'>) {
+    const taggedLogs = new Map<string, Buffer[]>();
+    const dataStartIndexForBlock =
+      block.header.state.partial.noteHashTree.nextAvailableLeafIndex -
+      block.body.numberOfTxsIncludingPadded * MAX_NOTE_HASHES_PER_TX;
+    block.body[logType].txLogs.forEach((txLogs, txIndex) => {
+      const txHash = block.body.txEffects[txIndex].txHash;
+      const dataStartIndexForTx = dataStartIndexForBlock + txIndex * MAX_NOTE_HASHES_PER_TX;
+      const logs = txLogs.unrollLogs();
+      logs.forEach(log => {
+        if (
+          (logType == 'noteEncryptedLogs' && log.data.length < 32) ||
+          // TODO remove when #9835 and #9836 are fixed
+          (logType === 'unencryptedLogs' && log.data.length < 32 * 33)
+        ) {
+          this.#log.warn(`Skipping log (${logType}) with invalid data length: ${log.data.length}`);
+          return;
+        }
+        try {
+          let tag = Fr.ZERO;
+          // TODO remove when #9835 and #9836 are fixed. The partial note logs are emitted as bytes, but encoded as Fields.
+          // This means that for every 32 bytes of payload, we only have 1 byte of data.
+          // Also, the tag is not stored in the first 32 bytes of the log, (that's the length of public fields now) but in the next 32.
+          if (logType === 'unencryptedLogs') {
+            const correctedBuffer = Buffer.alloc(32);
+            const initialOffset = 32;
+            for (let i = 0; i < 32; i++) {
+              const byte = Fr.fromBuffer(
+                log.data.subarray(i * 32 + initialOffset, i * 32 + 32 + initialOffset),
+              ).toNumber();
+              correctedBuffer.writeUInt8(byte, i);
+            }
+            tag = new Fr(correctedBuffer);
+          } else {
+            tag = new Fr(log.data.subarray(0, 32));
+          }
+          this.#log.verbose(`Found tagged (${logType}) log with tag ${tag.toString()} in block ${block.number}`);
+          const currentLogs = taggedLogs.get(tag.toString()) ?? [];
+          currentLogs.push(
+            new TxScopedL2Log(
+              txHash,
+              dataStartIndexForTx,
+              block.number,
+              logType === 'unencryptedLogs',
+              log.data,
+            ).toBuffer(),
+          );
+          taggedLogs.set(tag.toString(), currentLogs);
+        } catch (err) {
+          this.#log.warn(`Failed to add tagged log to store: ${err}`);
+        }
+      });
+    });
+    return taggedLogs;
+  }
+
   /**
    * Append new logs to the store's list.
    * @param blocks - The blocks for which to add the logs.
    * @returns True if the operation is successful.
    */
-  addLogs(blocks: L2Block[]): Promise<boolean> {
+  async addLogs(blocks: L2Block[]): Promise<boolean> {
+    const taggedLogsToAdd = blocks
+      .flatMap(block => [
+        this.#extractTaggedLogs(block, 'noteEncryptedLogs'),
+        this.#extractTaggedLogs(block, 'unencryptedLogs'),
+      ])
+      .reduce((acc, val) => {
+        for (const [tag, logs] of val.entries()) {
+          const currentLogs = acc.get(tag) ?? [];
+          acc.set(tag, currentLogs.concat(logs));
+        }
+        return acc;
+      });
+    const tagsToUpdate = Array.from(taggedLogsToAdd.keys());
+    const currentTaggedLogs = await this.db.transaction(() =>
+      tagsToUpdate.map(tag => ({ tag, logBuffers: this.#logsByTag.get(tag) })),
+    );
+    currentTaggedLogs.forEach(taggedLogBuffer => {
+      if (taggedLogBuffer.logBuffers && taggedLogBuffer.logBuffers.length > 0) {
+        taggedLogsToAdd.set(
+          taggedLogBuffer.tag,
+          taggedLogBuffer.logBuffers!.concat(taggedLogsToAdd.get(taggedLogBuffer.tag)!),
+        );
+      }
+    });
     return this.db.transaction(() => {
       blocks.forEach(block => {
+        const tagsInBlock = [];
+        for (const [tag, logs] of taggedLogsToAdd.entries()) {
+          void this.#logsByTag.set(tag, logs);
+          tagsInBlock.push(tag);
+        }
+        void this.#logTagsByBlock.set(block.number, tagsInBlock);
         void this.#noteEncryptedLogsByBlock.set(block.number, block.body.noteEncryptedLogs.toBuffer());
-        block.body.noteEncryptedLogs.txLogs.forEach(txLogs => {
-          const noteLogs = txLogs.unrollLogs();
-          noteLogs.forEach(noteLog => {
-            if (noteLog.data.length < 32) {
-              this.#log.warn(`Skipping note log with invalid data length: ${noteLog.data.length}`);
-              return;
-            }
-            try {
-              const tag = new Fr(noteLog.data.subarray(0, 32));
-              const hexHash = noteLog.hash().toString('hex');
-              // Ideally we'd store all of the logs for a matching tag in an AztecMultiMap, but this type doesn't doesn't
-              // handle storing buffers well. The 'ordered-binary' encoding returns an error trying to decode buffers
-              // ('the number <> cannot be converted to a BigInt because it is not an integer'). We therefore store
-              // instead the hashes of the logs.
-              void this.#noteEncryptedLogHashesByTag.set(tag.toString(), hexHash);
-              void this.#noteEncryptedLogsByHash.set(hexHash, noteLog.toBuffer());
-              void this.#noteEncryptedLogTagsByBlock.set(block.number, tag.toString());
-            } catch (err) {
-              this.#log.warn(`Failed to add tagged note log to store: ${err}`);
-            }
-          });
-        });
         void this.#encryptedLogsByBlock.set(block.number, block.body.encryptedLogs.toBuffer());
         void this.#unencryptedLogsByBlock.set(block.number, block.body.unencryptedLogs.toBuffer());
+        void this.#contractClassLogsByBlock.set(block.number, block.body.contractClassLogs.toBuffer());
       });
 
       return true;
@@ -84,26 +151,19 @@ export class LogStore {
   }
 
   async deleteLogs(blocks: L2Block[]): Promise<boolean> {
-    const noteTagsToDelete = await this.db.transaction(() => {
-      return blocks.flatMap(block => Array.from(this.#noteEncryptedLogTagsByBlock.getValues(block.number)));
-    });
-    const noteLogHashesToDelete = await this.db.transaction(() => {
-      return noteTagsToDelete.flatMap(tag => Array.from(this.#noteEncryptedLogHashesByTag.getValues(tag)));
+    const tagsToDelete = await this.db.transaction(() => {
+      return blocks.flatMap(block => this.#logTagsByBlock.get(block.number)?.map(tag => tag.toString()) ?? []);
     });
     return this.db.transaction(() => {
       blocks.forEach(block => {
         void this.#noteEncryptedLogsByBlock.delete(block.number);
         void this.#encryptedLogsByBlock.delete(block.number);
         void this.#unencryptedLogsByBlock.delete(block.number);
-        void this.#noteEncryptedLogTagsByBlock.delete(block.number);
-      });
-
-      noteTagsToDelete.forEach(tag => {
-        void this.#noteEncryptedLogHashesByTag.delete(tag.toString());
+        void this.#logTagsByBlock.delete(block.number);
       });
 
-      noteLogHashesToDelete.forEach(hash => {
-        void this.#noteEncryptedLogsByHash.delete(hash);
+      tagsToDelete.forEach(tag => {
+        void this.#logsByTag.delete(tag.toString());
       });
 
       return true;
@@ -156,20 +216,12 @@ export class LogStore {
    * @returns For each received tag, an array of matching logs is returned. An empty array implies no logs match
    * that tag.
    */
-  getLogsByTags(tags: Fr[]): Promise<EncryptedL2NoteLog[][]> {
-    return this.db.transaction(() => {
-      return tags.map(tag => {
-        const logHashes = Array.from(this.#noteEncryptedLogHashesByTag.getValues(tag.toString()));
-        return (
-          logHashes
-            .map(hash => this.#noteEncryptedLogsByHash.get(hash))
-            // addLogs should ensure that we never have undefined logs, but we filter them out regardless to protect
-            // ourselves from database corruption
-            .filter(noteLogBuffer => noteLogBuffer != undefined)
-            .map(noteLogBuffer => EncryptedL2NoteLog.fromBuffer(noteLogBuffer!))
-        );
-      });
-    });
+  getLogsByTags(tags: Fr[]): Promise<TxScopedL2Log[][]> {
+    return this.db.transaction(() =>
+      tags
+        .map(tag => this.#logsByTag.get(tag.toString()))
+        .map(noteLogBuffers => noteLogBuffers?.map(noteLogBuffer => TxScopedL2Log.fromBuffer(noteLogBuffer)) ?? []),
+    );
   }
 
   /**
@@ -236,6 +288,72 @@ export class LogStore {
     return { logs, maxLogsHit };
   }
 
+  /**
+   * Gets contract class logs based on the provided filter.
+   * @param filter - The filter to apply to the logs.
+   * @returns The requested logs.
+   */
+  getContractClassLogs(filter: LogFilter): GetUnencryptedLogsResponse {
+    if (filter.afterLog) {
+      return this.#filterContractClassLogsBetweenBlocks(filter);
+    } else if (filter.txHash) {
+      return this.#filterContractClassLogsOfTx(filter);
+    } else {
+      return this.#filterContractClassLogsBetweenBlocks(filter);
+    }
+  }
+
+  #filterContractClassLogsOfTx(filter: LogFilter): GetUnencryptedLogsResponse {
+    if (!filter.txHash) {
+      throw new Error('Missing txHash');
+    }
+
+    const [blockNumber, txIndex] = this.blockStore.getTxLocation(filter.txHash) ?? [];
+    if (typeof blockNumber !== 'number' || typeof txIndex !== 'number') {
+      return { logs: [], maxLogsHit: false };
+    }
+    const contractClassLogsBuffer = this.#contractClassLogsByBlock.get(blockNumber);
+    const contractClassLogsInBlock = contractClassLogsBuffer
+      ? ContractClass2BlockL2Logs.fromBuffer(contractClassLogsBuffer)
+      : new ContractClass2BlockL2Logs([]);
+    const txLogs = contractClassLogsInBlock.txLogs[txIndex].unrollLogs();
+
+    const logs: ExtendedUnencryptedL2Log[] = [];
+    const maxLogsHit = this.#accumulateLogs(logs, blockNumber, txIndex, txLogs, filter);
+
+    return { logs, maxLogsHit };
+  }
+
+  #filterContractClassLogsBetweenBlocks(filter: LogFilter): GetUnencryptedLogsResponse {
+    const start =
+      filter.afterLog?.blockNumber ?? Math.max(filter.fromBlock ?? INITIAL_L2_BLOCK_NUM, INITIAL_L2_BLOCK_NUM);
+    const end = filter.toBlock;
+
+    if (typeof end === 'number' && end < start) {
+      return {
+        logs: [],
+        maxLogsHit: true,
+      };
+    }
+
+    const logs: ExtendedUnencryptedL2Log[] = [];
+
+    let maxLogsHit = false;
+    loopOverBlocks: for (const [blockNumber, logBuffer] of this.#contractClassLogsByBlock.entries({ start, end })) {
+      const contractClassLogsInBlock = ContractClass2BlockL2Logs.fromBuffer(logBuffer);
+      for (let txIndex = filter.afterLog?.txIndex ?? 0; txIndex < contractClassLogsInBlock.txLogs.length; txIndex++) {
+        const txLogs = contractClassLogsInBlock.txLogs[txIndex].unrollLogs();
+        maxLogsHit = this.#accumulateLogs(logs, blockNumber, txIndex, txLogs, filter);
+        if (maxLogsHit) {
+          this.#log.debug(`Max logs hit at block ${blockNumber}`);
+          break loopOverBlocks;
+        }
+      }
+    }
+
+    return { logs, maxLogsHit };
+  }
+
   #accumulateLogs(
     results: ExtendedUnencryptedL2Log[],
     blockNumber: number,

package/src/archiver/kv_archiver_store/message_store.ts

@@ -10,7 +10,7 @@ import { type DataRetrieval } from '../structs/data_retrieval.js';
  */
 export class MessageStore {
   #l1ToL2Messages: AztecMap<string, Buffer>;
-  #l1ToL2MessageIndices: AztecMap<string, bigint[]>; // We store array of bigints here because there can be duplicate messages
+  #l1ToL2MessageIndices: AztecMap<string, bigint>;
   #lastSynchedL1Block: AztecSingleton<bigint>;
   #totalMessageCount: AztecSingleton<bigint>;
 
@@ -57,11 +57,8 @@ export class MessageStore {
 
       for (const message of messages.retrievedData) {
         const key = `${message.index}`;
-        void this.#l1ToL2Messages.setIfNotExists(key, message.leaf.toBuffer());
-
-        const indices = this.#l1ToL2MessageIndices.get(message.leaf.toString()) ?? [];
-        indices.push(message.index);
-        void this.#l1ToL2MessageIndices.set(message.leaf.toString(), indices);
+        void this.#l1ToL2Messages.set(key, message.leaf.toBuffer());
+        void this.#l1ToL2MessageIndices.set(message.leaf.toString(), message.index);
       }
 
       const lastTotalMessageCount = this.getTotalL1ToL2MessageCount();
@@ -72,15 +69,12 @@ export class MessageStore {
   }
 
   /**
-   * Gets the first L1 to L2 message index in the L1 to L2 message tree which is greater than or equal to `startIndex`.
+   * Gets the L1 to L2 message index in the L1 to L2 message tree.
    * @param l1ToL2Message - The L1 to L2 message.
-   * @param startIndex - The index to start searching from.
    * @returns The index of the L1 to L2 message in the L1 to L2 message tree (undefined if not found).
    */
-  getL1ToL2MessageIndex(l1ToL2Message: Fr, startIndex: bigint): Promise<bigint | undefined> {
-    const indices = this.#l1ToL2MessageIndices.get(l1ToL2Message.toString()) ?? [];
-    const index = indices.find(i => i >= startIndex);
-    return Promise.resolve(index);
+  getL1ToL2MessageIndex(l1ToL2Message: Fr): Promise<bigint | undefined> {
+    return Promise.resolve(this.#l1ToL2MessageIndices.get(l1ToL2Message.toString()));
   }
 
   getL1ToL2Messages(blockNumber: bigint): Fr[] {

package/src/archiver/memory_archiver_store/l1_to_l2_message_store.ts

@@ -46,19 +46,14 @@ export class L1ToL2MessageStore {
   }
 
   /**
-   * Gets the first L1 to L2 message index in the L1 to L2 message tree which is greater than or equal to `startIndex`.
+   * Gets the L1 to L2 message index in the L1 to L2 message tree.
    * @param l1ToL2Message - The L1 to L2 message.
-   * @param startIndex - The index to start searching from.
    * @returns The index of the L1 to L2 message in the L1 to L2 message tree (undefined if not found).
    */
-  getMessageIndex(l1ToL2Message: Fr, startIndex: bigint): bigint | undefined {
+  getMessageIndex(l1ToL2Message: Fr): bigint | undefined {
     for (const [key, message] of this.store.entries()) {
       if (message.equals(l1ToL2Message)) {
-        const indexInTheWholeTree = BigInt(key);
-        if (indexInTheWholeTree < startIndex) {
-          continue;
-        }
-        return indexInTheWholeTree;
+        return BigInt(key);
       }
     }
     return undefined;