@aztec/archiver 0.22.0 → 0.24.0

package/src/archiver/kv_archiver_store/log_store.ts

@@ -0,0 +1,174 @@
+import {
+  ExtendedUnencryptedL2Log,
+  GetUnencryptedLogsResponse,
+  INITIAL_L2_BLOCK_NUM,
+  L2BlockL2Logs,
+  LogFilter,
+  LogId,
+  LogType,
+  UnencryptedL2Log,
+} from '@aztec/circuit-types';
+import { createDebugLogger } from '@aztec/foundation/log';
+import { AztecKVStore, AztecMap } from '@aztec/kv-store';
+
+import { BlockStore } from './block_store.js';
+
+/**
+ * A store for logs
+ */
+export class LogStore {
+  #encryptedLogs: AztecMap<number, Buffer>;
+  #unencryptedLogs: AztecMap<number, Buffer>;
+  #logsMaxPageSize: number;
+  #log = createDebugLogger('aztec:archiver:log_store');
+
+  constructor(private db: AztecKVStore, private blockStore: BlockStore, logsMaxPageSize: number = 1000) {
+    this.#encryptedLogs = db.openMap('archiver_encrypted_logs');
+    this.#unencryptedLogs = db.openMap('archiver_unencrypted_logs');
+
+    this.#logsMaxPageSize = logsMaxPageSize;
+  }
+
+  /**
+   * Append new logs to the store's list.
+   * @param encryptedLogs - The logs to be added to the store.
+   * @param unencryptedLogs - The type of the logs to be added to the store.
+   * @param blockNumber - The block for which to add the logs.
+   * @returns True if the operation is successful.
+   */
+  addLogs(
+    encryptedLogs: L2BlockL2Logs | undefined,
+    unencryptedLogs: L2BlockL2Logs | undefined,
+    blockNumber: number,
+  ): Promise<boolean> {
+    return this.db.transaction(() => {
+      if (encryptedLogs) {
+        void this.#encryptedLogs.set(blockNumber, encryptedLogs.toBuffer());
+      }
+
+      if (unencryptedLogs) {
+        void this.#unencryptedLogs.set(blockNumber, unencryptedLogs.toBuffer());
+      }
+
+      return true;
+    });
+  }
+
+  /**
+   * Gets up to `limit` amount of logs starting from `from`.
+   * @param start - Number of the L2 block to which corresponds the first logs to be returned.
+   * @param limit - The number of logs to return.
+   * @param logType - Specifies whether to return encrypted or unencrypted logs.
+   * @returns The requested logs.
+   */
+  *getLogs(start: number, limit: number, logType: LogType): IterableIterator<L2BlockL2Logs> {
+    const logMap = logType === LogType.ENCRYPTED ? this.#encryptedLogs : this.#unencryptedLogs;
+    for (const buffer of logMap.values({ start, limit })) {
+      yield L2BlockL2Logs.fromBuffer(buffer);
+    }
+  }
+
+  /**
+   * Gets unencrypted logs based on the provided filter.
+   * @param filter - The filter to apply to the logs.
+   * @returns The requested logs.
+   */
+  getUnencryptedLogs(filter: LogFilter): GetUnencryptedLogsResponse {
+    if (filter.afterLog) {
+      return this.#filterUnencryptedLogsBetweenBlocks(filter);
+    } else if (filter.txHash) {
+      return this.#filterUnencryptedLogsOfTx(filter);
+    } else {
+      return this.#filterUnencryptedLogsBetweenBlocks(filter);
+    }
+  }
+
+  #filterUnencryptedLogsOfTx(filter: LogFilter): GetUnencryptedLogsResponse {
+    if (!filter.txHash) {
+      throw new Error('Missing txHash');
+    }
+
+    const [blockNumber, txIndex] = this.blockStore.getL2TxLocation(filter.txHash) ?? [];
+    if (typeof blockNumber !== 'number' || typeof txIndex !== 'number') {
+      return { logs: [], maxLogsHit: false };
+    }
+
+    const unencryptedLogsInBlock = this.#getBlockLogs(blockNumber, LogType.UNENCRYPTED);
+    const txLogs = unencryptedLogsInBlock.txLogs[txIndex].unrollLogs().map(log => UnencryptedL2Log.fromBuffer(log));
+
+    const logs: ExtendedUnencryptedL2Log[] = [];
+    const maxLogsHit = this.#accumulateLogs(logs, blockNumber, txIndex, txLogs, filter);
+
+    return { logs, maxLogsHit };
+  }
+
+  #filterUnencryptedLogsBetweenBlocks(filter: LogFilter): GetUnencryptedLogsResponse {
+    const start =
+      filter.afterLog?.blockNumber ?? Math.max(filter.fromBlock ?? INITIAL_L2_BLOCK_NUM, INITIAL_L2_BLOCK_NUM);
+    const end = filter.toBlock;
+
+    if (typeof end === 'number' && end < start) {
+      return {
+        logs: [],
+        maxLogsHit: true,
+      };
+    }
+
+    const logs: ExtendedUnencryptedL2Log[] = [];
+
+    let maxLogsHit = false;
+    loopOverBlocks: for (const [blockNumber, logBuffer] of this.#unencryptedLogs.entries({ start, end })) {
+      const unencryptedLogsInBlock = L2BlockL2Logs.fromBuffer(logBuffer);
+      for (let txIndex = filter.afterLog?.txIndex ?? 0; txIndex < unencryptedLogsInBlock.txLogs.length; txIndex++) {
+        const txLogs = unencryptedLogsInBlock.txLogs[txIndex].unrollLogs().map(log => UnencryptedL2Log.fromBuffer(log));
+        maxLogsHit = this.#accumulateLogs(logs, blockNumber, txIndex, txLogs, filter);
+        if (maxLogsHit) {
+          this.#log(`Max logs hit at block ${blockNumber}`);
+          break loopOverBlocks;
+        }
+      }
+    }
+
+    return { logs, maxLogsHit };
+  }
+
+  #accumulateLogs(
+    results: ExtendedUnencryptedL2Log[],
+    blockNumber: number,
+    txIndex: number,
+    txLogs: UnencryptedL2Log[],
+    filter: LogFilter,
+  ): boolean {
+    let maxLogsHit = false;
+    let logIndex = typeof filter.afterLog?.logIndex === 'number' ? filter.afterLog.logIndex + 1 : 0;
+    for (; logIndex < txLogs.length; logIndex++) {
+      const log = txLogs[logIndex];
+      if (filter.contractAddress && !log.contractAddress.equals(filter.contractAddress)) {
+        continue;
+      }
+
+      if (filter.selector && !log.selector.equals(filter.selector)) {
+        continue;
+      }
+
+      results.push(new ExtendedUnencryptedL2Log(new LogId(blockNumber, txIndex, logIndex), log));
+      if (results.length >= this.#logsMaxPageSize) {
+        maxLogsHit = true;
+        break;
+      }
+    }
+
+    return maxLogsHit;
+  }
+
+  #getBlockLogs(blockNumber: number, logType: LogType): L2BlockL2Logs {
+    const logMap = logType === LogType.ENCRYPTED ? this.#encryptedLogs : this.#unencryptedLogs;
+    const buffer = logMap.get(blockNumber);
+
+    if (!buffer) {
+      return new L2BlockL2Logs([]);
+    }
+
+    return L2BlockL2Logs.fromBuffer(buffer);
+  }
+}

package/src/archiver/kv_archiver_store/message_store.ts

@@ -0,0 +1,170 @@
+import { L1ToL2Message } from '@aztec/circuit-types';
+import { Fr } from '@aztec/circuits.js';
+import { createDebugLogger } from '@aztec/foundation/log';
+import { AztecCounter, AztecKVStore, AztecMap, AztecSingleton } from '@aztec/kv-store';
+
+/**
+ * A message stored in the database
+ */
+type Message = {
+  /** The L1ToL2Message */
+  message: Buffer;
+  /** The message's fee */
+  fee: number;
+  /** Has it _ever_ been confirmed? */
+  confirmed: boolean;
+};
+
+/**
+ * LMDB implementation of the ArchiverDataStore interface.
+ */
+export class MessageStore {
+  #messages: AztecMap<string, Message>;
+  #pendingMessagesByFee: AztecCounter<[number, string]>;
+  #lastL1BlockAddingMessages: AztecSingleton<bigint>;
+  #lastL1BlockCancellingMessages: AztecSingleton<bigint>;
+
+  #log = createDebugLogger('aztec:archiver:message_store');
+
+  constructor(private db: AztecKVStore) {
+    this.#messages = db.openMap('archiver_l1_to_l2_messages');
+    this.#pendingMessagesByFee = db.openCounter('archiver_messages_by_fee');
+    this.#lastL1BlockAddingMessages = db.openSingleton('archiver_last_l1_block_adding_messages');
+    this.#lastL1BlockCancellingMessages = db.openSingleton('archiver_last_l1_block_cancelling_messages');
+  }
+
+  /**
+   * Gets the last L1 block number that emitted new messages and the block that cancelled messages.
+   * @returns The last L1 block number processed
+   */
+  getL1BlockNumber() {
+    return {
+      addedMessages: this.#lastL1BlockAddingMessages.get() ?? 0n,
+      cancelledMessages: this.#lastL1BlockCancellingMessages.get() ?? 0n,
+    };
+  }
+
+  /**
+   * Append new pending L1 to L2 messages to the store.
+   * @param messages - The L1 to L2 messages to be added to the store.
+   * @param l1BlockNumber - The L1 block number for which to add the messages.
+   * @returns True if the operation is successful.
+   */
+  addPendingMessages(messages: L1ToL2Message[], l1BlockNumber: bigint): Promise<boolean> {
+    return this.db.transaction(() => {
+      const lastL1BlockNumber = this.#lastL1BlockAddingMessages.get() ?? 0n;
+      if (lastL1BlockNumber >= l1BlockNumber) {
+        return false;
+      }
+
+      void this.#lastL1BlockAddingMessages.set(l1BlockNumber);
+
+      for (const message of messages) {
+        const messageKey = message.entryKey?.toString();
+        if (!messageKey) {
+          throw new Error('Message does not have an entry key');
+        }
+
+        void this.#messages.setIfNotExists(messageKey, {
+          message: message.toBuffer(),
+          fee: message.fee,
+          confirmed: false,
+        });
+
+        void this.#pendingMessagesByFee.update([message.fee, messageKey], 1);
+      }
+
+      return true;
+    });
+  }
+
+  /**
+   * Remove pending L1 to L2 messages from the store (if they were cancelled).
+   * @param messageKeys - The message keys to be removed from the store.
+   * @param l1BlockNumber - The L1 block number for which to remove the messages.
+   * @returns True if the operation is successful.
+   */
+  cancelPendingMessages(messageKeys: Fr[], l1BlockNumber: bigint): Promise<boolean> {
+    return this.db.transaction(() => {
+      const lastL1BlockNumber = this.#lastL1BlockCancellingMessages.get() ?? 0n;
+      if (lastL1BlockNumber >= l1BlockNumber) {
+        return false;
+      }
+
+      void this.#lastL1BlockCancellingMessages.set(l1BlockNumber);
+
+      for (const messageKey of messageKeys) {
+        const messageCtx = this.#messages.get(messageKey.toString());
+        if (!messageCtx) {
+          throw new Error(`Message ${messageKey.toString()} not found`);
+        }
+
+        void this.#pendingMessagesByFee.update([messageCtx.fee, messageKey.toString()], -1);
+      }
+
+      return true;
+    });
+  }
+
+  /**
+   * Messages that have been published in an L2 block are confirmed.
+   * Add them to the confirmed store, also remove them from the pending store.
+   * @param messageKeys - The message keys to be removed from the store.
+   * @returns True if the operation is successful.
+   */
+  confirmPendingMessages(messageKeys: Fr[]): Promise<boolean> {
+    return this.db.transaction(() => {
+      for (const messageKey of messageKeys) {
+        const messageCtx = this.#messages.get(messageKey.toString());
+        if (!messageCtx) {
+          throw new Error(`Message ${messageKey.toString()} not found`);
+        }
+        messageCtx.confirmed = true;
+
+        void this.#messages.set(messageKey.toString(), messageCtx);
+        void this.#pendingMessagesByFee.update([messageCtx.fee, messageKey.toString()], -1);
+      }
+
+      return true;
+    });
+  }
+
+  /**
+   * Gets the confirmed L1 to L2 message corresponding to the given message key.
+   * @param messageKey - The message key to look up.
+   * @returns The requested L1 to L2 message or throws if not found.
+   */
+  getConfirmedMessage(messageKey: Fr): L1ToL2Message {
+    const messageCtx = this.#messages.get(messageKey.toString());
+    if (!messageCtx) {
+      throw new Error(`Message ${messageKey.toString()} not found`);
+    }
+
+    if (!messageCtx.confirmed) {
+      throw new Error(`Message ${messageKey.toString()} not confirmed`);
+    }
+
+    return L1ToL2Message.fromBuffer(messageCtx.message);
+  }
+
+  /**
+   * Gets up to `limit` amount of pending L1 to L2 messages, sorted by fee
+   * @param limit - The number of messages to return (by default NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP).
+   * @returns The requested L1 to L2 message keys.
+   */
+  getPendingMessageKeysByFee(limit: number): Fr[] {
+    const messageKeys: Fr[] = [];
+
+    for (const [[_, messageKey], count] of this.#pendingMessagesByFee.entries({
+      reverse: true,
+    })) {
+      // put `count` copies of this message in the result list
+      messageKeys.push(...Array(count).fill(Fr.fromString(messageKey)));
+      if (messageKeys.length >= limit) {
+        break;
+      }
+    }
+
+    return messageKeys;
+  }
+}

package/src/archiver/memory_archiver_store/l1_to_l2_message_store.ts

@@ -0,0 +1,91 @@
+import { L1ToL2Message } from '@aztec/circuit-types';
+import { Fr } from '@aztec/foundation/fields';
+
+/**
+ * A simple in-memory implementation of an L1 to L2 message store
+ * that handles message duplication.
+ */
+export class L1ToL2MessageStore {
+  /**
+   * A map containing the message key to the corresponding L1 to L2
+   * messages (and the number of times the message has been seen).
+   */
+  protected store: Map<bigint, L1ToL2MessageAndCount> = new Map();
+
+  constructor() {}
+
+  addMessage(messageKey: Fr, message: L1ToL2Message) {
+    const messageKeyBigInt = messageKey.toBigInt();
+    const msgAndCount = this.store.get(messageKeyBigInt);
+    if (msgAndCount) {
+      msgAndCount.count++;
+    } else {
+      this.store.set(messageKeyBigInt, { message, count: 1 });
+    }
+  }
+
+  getMessage(messageKey: Fr): L1ToL2Message | undefined {
+    return this.store.get(messageKey.value)?.message;
+  }
+
+  getMessageAndCount(messageKey: Fr): L1ToL2MessageAndCount | undefined {
+    return this.store.get(messageKey.value);
+  }
+}
+
+/**
+ * Specifically for the store that will hold pending messages
+ * for removing messages or fetching multiple messages.
+ */
+export class PendingL1ToL2MessageStore extends L1ToL2MessageStore {
+  getMessageKeys(limit: number): Fr[] {
+    if (limit < 1) {
+      return [];
+    }
+    // fetch `limit` number of messages from the store with the highest fee.
+    // Note the store has multiple of the same message. So if a message has count 2, include both of them in the result:
+    const messages: Fr[] = [];
+    const sortedMessages = Array.from(this.store.values()).sort((a, b) => b.message.fee - a.message.fee);
+    for (const messageAndCount of sortedMessages) {
+      for (let i = 0; i < messageAndCount.count; i++) {
+        messages.push(messageAndCount.message.entryKey!);
+        if (messages.length === limit) {
+          return messages;
+        }
+      }
+    }
+    return messages;
+  }
+
+  removeMessage(messageKey: Fr) {
+    // ignore 0 - messageKey is a hash, so a 0 can probabilistically never occur. It is best to skip it.
+    if (messageKey.equals(Fr.ZERO)) {
+      return;
+    }
+
+    const messageKeyBigInt = messageKey.value;
+    const msgAndCount = this.store.get(messageKeyBigInt);
+    if (!msgAndCount) {
+      throw new Error(`Unable to remove message: L1 to L2 Message with key ${messageKeyBigInt} not found in store`);
+    }
+    if (msgAndCount.count > 1) {
+      msgAndCount.count--;
+    } else {
+      this.store.delete(messageKeyBigInt);
+    }
+  }
+}
+
+/**
+ * Useful to keep track of the number of times a message has been seen.
+ */
+type L1ToL2MessageAndCount = {
+  /**
+   * The message.
+   */
+  message: L1ToL2Message;
+  /**
+   * The number of times the message has been seen.
+   */
+  count: number;
+};