@aztec/archiver 0.1.0-alpha11

package/src/archiver/eth_log_handlers.ts

@@ -0,0 +1,238 @@
+import { Hex, Log, PublicClient, decodeFunctionData, getAbiItem, getAddress, hexToBytes } from 'viem';
+import { InboxAbi, RollupAbi, ContractDeploymentEmitterAbi } from '@aztec/l1-artifacts';
+import { Fr } from '@aztec/foundation/fields';
+import {
+  L1ToL2Message,
+  L1Actor,
+  L2Actor,
+  L2Block,
+  ContractPublicData,
+  BufferReader,
+  ContractData,
+  EncodedContractFunction,
+} from '@aztec/types';
+import { EthAddress } from '@aztec/foundation/eth-address';
+import { AztecAddress } from '@aztec/foundation/aztec-address';
+
+/**
+ * Processes newly received MessageAdded (L1 to L2) logs.
+ * @param logs - MessageAdded logs.
+ * @returns Array of all Pending L1 to L2 messages that were processed
+ */
+export function processPendingL1ToL2MessageAddedLogs(
+  logs: Log<bigint, number, undefined, true, typeof InboxAbi, 'MessageAdded'>[],
+): L1ToL2Message[] {
+  const l1ToL2Messages: L1ToL2Message[] = [];
+  for (const log of logs) {
+    const { sender, senderChainId, recipient, recipientVersion, content, secretHash, deadline, fee, entryKey } =
+      log.args;
+    l1ToL2Messages.push(
+      new L1ToL2Message(
+        new L1Actor(EthAddress.fromString(sender), Number(senderChainId)),
+        new L2Actor(AztecAddress.fromString(recipient), Number(recipientVersion)),
+        Fr.fromString(content),
+        Fr.fromString(secretHash),
+        deadline,
+        Number(fee),
+        Fr.fromString(entryKey),
+      ),
+    );
+  }
+  return l1ToL2Messages;
+}
+
+/**
+ * Process newly received L1ToL2MessageCancelled logs.
+ * @param logs - L1ToL2MessageCancelled logs.
+ * @returns Array of message keys of the L1 to L2 messages that were cancelled
+ */
+export function processCancelledL1ToL2MessagesLogs(
+  logs: Log<bigint, number, undefined, true, typeof InboxAbi, 'L1ToL2MessageCancelled'>[],
+): Fr[] {
+  const cancelledL1ToL2Messages: Fr[] = [];
+  for (const log of logs) {
+    cancelledL1ToL2Messages.push(Fr.fromString(log.args.entryKey));
+  }
+  return cancelledL1ToL2Messages;
+}
+
+/**
+ * Processes newly received L2BlockProcessed logs.
+ * @param publicClient - The viem public client to use for transaction retrieval.
+ * @param expectedL2BlockNumber - The next expected L2 block number.
+ * @param logs - L2BlockProcessed logs.
+ */
+export async function processBlockLogs(
+  publicClient: PublicClient,
+  expectedL2BlockNumber: bigint,
+  logs: Log<bigint, number, undefined, true, typeof RollupAbi, 'L2BlockProcessed'>[],
+) {
+  const retrievedBlocks: L2Block[] = [];
+  for (const log of logs) {
+    const blockNum = log.args.blockNum;
+    if (blockNum !== expectedL2BlockNumber) {
+      throw new Error('Block number mismatch. Expected: ' + expectedL2BlockNumber + ' but got: ' + blockNum + '.');
+    }
+    // TODO: Fetch blocks from calldata in parallel
+    const newBlock = await getBlockFromCallData(publicClient, log.transactionHash!, log.args.blockNum);
+    retrievedBlocks.push(newBlock);
+    expectedL2BlockNumber++;
+  }
+  return retrievedBlocks;
+}
+
+/**
+ * Builds an L2 block out of calldata from the tx that published it.
+ * Assumes that the block was published from an EOA.
+ * TODO: Add retries and error management.
+ * @param publicClient - The viem public client to use for transaction retrieval.
+ * @param txHash - Hash of the tx that published it.
+ * @param l2BlockNum - L2 block number.
+ * @returns An L2 block deserialized from the calldata.
+ */
+async function getBlockFromCallData(
+  publicClient: PublicClient,
+  txHash: `0x${string}`,
+  l2BlockNum: bigint,
+): Promise<L2Block> {
+  const { input: data } = await publicClient.getTransaction({ hash: txHash });
+  // TODO: File a bug in viem who complains if we dont remove the ctor from the abi here
+  const { functionName, args } = decodeFunctionData({
+    abi: RollupAbi.filter(item => item.type.toString() !== 'constructor'),
+    data,
+  });
+  if (functionName !== 'process') throw new Error(`Unexpected method called ${functionName}`);
+  const [, l2BlockHex] = args! as [Hex, Hex];
+  const block = L2Block.decode(Buffer.from(hexToBytes(l2BlockHex)));
+  if (BigInt(block.number) !== l2BlockNum) {
+    throw new Error(`Block number mismatch: expected ${l2BlockNum} but got ${block.number}`);
+  }
+  return block;
+}
+
+/**
+ * Gets relevant `L2BlockProcessed` logs from chain.
+ * @param publicClient - The viem public client to use for transaction retrieval.
+ * @param rollupAddress - The address of the rollup contract.
+ * @param fromBlock - First block to get logs from (inclusive).
+ * @returns An array of `L2BlockProcessed` logs.
+ */
+export async function getL2BlockProcessedLogs(
+  publicClient: PublicClient,
+  rollupAddress: EthAddress,
+  fromBlock: bigint,
+) {
+  // Note: For some reason the return type of `getLogs` would not get correctly derived if I didn't set the abiItem
+  //       as a standalone constant.
+  const abiItem = getAbiItem({
+    abi: RollupAbi,
+    name: 'L2BlockProcessed',
+  });
+  return await publicClient.getLogs<typeof abiItem, true>({
+    address: getAddress(rollupAddress.toString()),
+    event: abiItem,
+    fromBlock,
+  });
+}
+
+/**
+ * Gets relevant `ContractDeployment` logs from chain.
+ * @param publicClient - The viem public client to use for transaction retrieval.
+ * @param contractDeploymentEmitterAddress - The address of the L2 contract deployment emitter contract.
+ * @param fromBlock - First block to get logs from (inclusive).
+ * @returns An array of `ContractDeployment` logs.
+ */
+export async function getContractDeploymentLogs(
+  publicClient: PublicClient,
+  contractDeploymentEmitterAddress: EthAddress,
+  fromBlock: bigint,
+): Promise<Log<bigint, number, undefined, true, typeof ContractDeploymentEmitterAbi, 'ContractDeployment'>[]> {
+  const abiItem = getAbiItem({
+    abi: ContractDeploymentEmitterAbi,
+    name: 'ContractDeployment',
+  });
+  return await publicClient.getLogs({
+    address: getAddress(contractDeploymentEmitterAddress.toString()),
+    event: abiItem,
+    fromBlock,
+  });
+}
+
+/**
+ * Processes newly received ContractDeployment logs.
+ * @param blockHashMapping - A mapping from block number to relevant block hash.
+ * @param logs - ContractDeployment logs.
+ * @returns The set of retrieved contract public data items.
+ */
+export function processContractDeploymentLogs(
+  blockHashMapping: { [key: number]: Buffer | undefined },
+  logs: Log<bigint, number, undefined, true, typeof ContractDeploymentEmitterAbi, 'ContractDeployment'>[],
+): [ContractPublicData[], number][] {
+  const contractPublicData: [ContractPublicData[], number][] = [];
+  for (let i = 0; i < logs.length; i++) {
+    const log = logs[i];
+    const l2BlockNum = Number(log.args.l2BlockNum);
+    const blockHash = Buffer.from(hexToBytes(log.args.l2BlockHash));
+    const expectedBlockHash = blockHashMapping[l2BlockNum];
+    if (expectedBlockHash === undefined || !blockHash.equals(expectedBlockHash)) {
+      continue;
+    }
+    const publicFnsReader = BufferReader.asReader(Buffer.from(log.args.acir.slice(2), 'hex'));
+    const contractData = new ContractPublicData(
+      new ContractData(AztecAddress.fromString(log.args.aztecAddress), EthAddress.fromString(log.args.portalAddress)),
+      publicFnsReader.readVector(EncodedContractFunction),
+    );
+    if (contractPublicData[i]) {
+      contractPublicData[i][0].push(contractData);
+    } else {
+      contractPublicData[i] = [[contractData], l2BlockNum];
+    }
+  }
+  return contractPublicData;
+}
+
+/**
+ * Get relevant `MessageAdded` logs emitted by Inbox on chain.
+ * @param publicClient - The viem public client to use for transaction retrieval.
+ * @param inboxAddress - The address of the inbox contract.
+ * @param fromBlock - First block to get logs from (inclusive).
+ * @returns An array of `MessageAdded` logs.
+ */
+export async function getPendingL1ToL2MessageLogs(
+  publicClient: PublicClient,
+  inboxAddress: EthAddress,
+  fromBlock: bigint,
+): Promise<Log<bigint, number, undefined, true, typeof InboxAbi, 'MessageAdded'>[]> {
+  const abiItem = getAbiItem({
+    abi: InboxAbi,
+    name: 'MessageAdded',
+  });
+  return await publicClient.getLogs({
+    address: getAddress(inboxAddress.toString()),
+    event: abiItem,
+    fromBlock,
+  });
+}
+
+/**
+ * Get relevant `L1ToL2MessageCancelled` logs emitted by Inbox on chain when pending messages are cancelled
+ * @param publicClient - The viem public client to use for transaction retrieval.
+ * @param inboxAddress - The address of the inbox contract.
+ * @param fromBlock - First block to get logs from (inclusive).
+ * @returns An array of `L1ToL2MessageCancelled` logs.
+ */
+export async function getL1ToL2MessageCancelledLogs(
+  publicClient: PublicClient,
+  inboxAddress: EthAddress,
+  fromBlock: bigint,
+): Promise<Log<bigint, number, undefined, true, typeof InboxAbi, 'L1ToL2MessageCancelled'>[]> {
+  const abiItem = getAbiItem({
+    abi: InboxAbi,
+    name: 'L1ToL2MessageCancelled',
+  });
+  return await publicClient.getLogs({
+    address: getAddress(inboxAddress.toString()),
+    event: abiItem,
+    fromBlock,
+  });
+}

package/src/archiver/index.ts

@@ -0,0 +1,2 @@
+export * from './archiver.js';
+export * from './config.js';

package/src/archiver/l1_to_l2_message_store.test.ts

@@ -0,0 +1,97 @@
+import { Fr } from '@aztec/foundation/fields';
+import { L1ToL2MessageStore, PendingL1ToL2MessageStore } from './l1_to_l2_message_store.js';
+import { L1Actor, L1ToL2Message, L2Actor } from '@aztec/types';
+
+describe('l1_to_l2_message_store', () => {
+  let store: L1ToL2MessageStore;
+  let entryKey: Fr;
+  let msg: L1ToL2Message;
+
+  beforeEach(() => {
+    // already adds a message to the store
+    store = new L1ToL2MessageStore();
+    entryKey = Fr.random();
+    msg = L1ToL2Message.random();
+  });
+
+  it('addMessage adds a message', () => {
+    store.addMessage(entryKey, msg);
+    expect(store.getMessage(entryKey)).toEqual(msg);
+  });
+
+  it('addMessage increments the count if the message is already in the store', () => {
+    store.addMessage(entryKey, msg);
+    store.addMessage(entryKey, msg);
+    expect(store.getMessageAndCount(entryKey)).toEqual({ message: msg, count: 2 });
+  });
+});
+
+describe('pending_l1_to_l2_message_store', () => {
+  let store: PendingL1ToL2MessageStore;
+  let entryKey: Fr;
+  let msg: L1ToL2Message;
+
+  beforeEach(() => {
+    // already adds a message to the store
+    store = new PendingL1ToL2MessageStore();
+    entryKey = Fr.random();
+    msg = L1ToL2Message.random();
+  });
+
+  it('removeMessage removes the message if the count is 1', () => {
+    store.addMessage(entryKey, msg);
+    store.removeMessage(entryKey);
+    expect(store.getMessage(entryKey)).toBeUndefined();
+  });
+
+  it("handles case when removing a message that doesn't exist", () => {
+    expect(() => store.removeMessage(new Fr(0))).not.toThrow();
+    const one = new Fr(1);
+    expect(() => store.removeMessage(one)).toThrow(`Message with key ${one.value} not found in store`);
+  });
+
+  it('removeMessage decrements the count if the message is already in the store', () => {
+    store.addMessage(entryKey, msg);
+    store.addMessage(entryKey, msg);
+    store.addMessage(entryKey, msg);
+    store.removeMessage(entryKey);
+    expect(store.getMessageAndCount(entryKey)).toEqual({ message: msg, count: 2 });
+  });
+
+  it('get messages for an empty store', () => {
+    expect(store.getMessageKeys(10)).toEqual([]);
+  });
+
+  it('getMessageKeys returns an empty array if take is 0', () => {
+    store.addMessage(entryKey, msg);
+    expect(store.getMessageKeys(0)).toEqual([]);
+  });
+
+  it('get messages for a non-empty store when take > number of messages in store', () => {
+    const entryKeys = [1, 2, 3, 4, 5].map(x => new Fr(x));
+    entryKeys.forEach(entryKey => {
+      store.addMessage(entryKey, L1ToL2Message.random());
+    });
+    expect(store.getMessageKeys(10).length).toEqual(5);
+  });
+
+  it('get messages returns messages sorted by fees and also includes multiple of the same message', () => {
+    const entryKeys = [1, 2, 3, 3, 3, 4].map(x => new Fr(x));
+    entryKeys.forEach(entryKey => {
+      // set msg.fee to entryKey to test the sort.
+      const msg = new L1ToL2Message(
+        L1Actor.random(),
+        L2Actor.random(),
+        Fr.random(),
+        Fr.random(),
+        100,
+        Number(entryKey.value),
+        entryKey,
+      );
+      store.addMessage(entryKey, msg);
+    });
+    const expectedMessgeFees = [4n, 3n, 3n, 3n]; // the top 4.
+    const receivedMessageFees = store.getMessageKeys(4).map(key => key.value);
+    expect(receivedMessageFees).toEqual(expectedMessgeFees);
+  });
+});

package/src/archiver/l1_to_l2_message_store.ts

@@ -0,0 +1,88 @@
+import { Fr } from '@aztec/foundation/fields';
+import { L1ToL2Message } from '@aztec/types';
+
+/**
+ * A simple in-memory implementation of an L1 to L2 message store
+ * that handles message duplication.
+ */
+export class L1ToL2MessageStore {
+  /**
+   * A map containing the message key to the corresponding L1 to L2
+   * messages (and the number of times the message has been seen).
+   */
+  protected store: Map<bigint, L1ToL2MessageAndCount> = new Map();
+
+  constructor() {}
+
+  addMessage(messageKey: Fr, msg: L1ToL2Message) {
+    const messageKeyBigInt = messageKey.value;
+    const msgAndCount = this.store.get(messageKeyBigInt);
+    if (msgAndCount) {
+      msgAndCount.count++;
+    } else {
+      this.store.set(messageKeyBigInt, { message: msg, count: 1 });
+    }
+  }
+
+  getMessage(messageKey: Fr): L1ToL2Message | undefined {
+    return this.store.get(messageKey.value)?.message;
+  }
+
+  getMessageAndCount(messageKey: Fr): L1ToL2MessageAndCount | undefined {
+    return this.store.get(messageKey.value);
+  }
+}
+
+/**
+ * Specifically for the store that will hold pending messages
+ * for removing messages or fetching multiple messages.
+ */
+export class PendingL1ToL2MessageStore extends L1ToL2MessageStore {
+  getMessageKeys(take: number): Fr[] {
+    if (take < 1) {
+      return [];
+    }
+    // fetch `take` number of messages from the store with the highest fee.
+    // Note the store has multiple of the same message. So if a message has count 2, include both of them in the result:
+    const messages: Fr[] = [];
+    const sortedMessages = Array.from(this.store.values()).sort((a, b) => b.message.fee - a.message.fee);
+    for (const messageAndCount of sortedMessages) {
+      for (let i = 0; i < messageAndCount.count; i++) {
+        messages.push(messageAndCount.message.entryKey!);
+        if (messages.length === take) {
+          return messages;
+        }
+      }
+    }
+    return messages;
+  }
+
+  removeMessage(messageKey: Fr) {
+    // ignore 0 - messageKey is a hash, so a 0 can probabilistically never occur. It is best to skip it.
+    if (messageKey.equals(Fr.ZERO)) return;
+    const messageKeyBigInt = messageKey.value;
+    const msgAndCount = this.store.get(messageKeyBigInt);
+    if (!msgAndCount) {
+      throw new Error(`Message with key ${messageKeyBigInt} not found in store`);
+    }
+    if (msgAndCount.count > 1) {
+      msgAndCount.count--;
+    } else {
+      this.store.delete(messageKeyBigInt);
+    }
+  }
+}
+
+/**
+ * Useful to keep track of the number of times a message has been seen.
+ */
+type L1ToL2MessageAndCount = {
+  /**
+   * The message.
+   */
+  message: L1ToL2Message;
+  /**
+   * The number of times the message has been seen.
+   */
+  count: number;
+};

package/src/index.ts

@@ -0,0 +1,51 @@
+import { fileURLToPath } from 'url';
+import { createPublicClient, http } from 'viem';
+import { localhost } from 'viem/chains';
+import { Archiver, getConfigEnvVars } from './archiver/index.js';
+import { MemoryArchiverStore } from './archiver/archiver_store.js';
+import { createLogger } from '@aztec/foundation/log';
+
+export * from './archiver/index.js';
+
+const log = createLogger('aztec:archiver_init');
+
+/**
+ * A function which instantiates and starts Archiver.
+ */
+// eslint-disable-next-line require-await
+async function main() {
+  const config = getConfigEnvVars();
+  const { rpcUrl, rollupContract, inboxContract, contractDeploymentEmitterContract, searchStartBlock } = config;
+
+  const publicClient = createPublicClient({
+    chain: localhost,
+    transport: http(rpcUrl),
+  });
+
+  const archiverStore = new MemoryArchiverStore();
+
+  const archiver = new Archiver(
+    publicClient,
+    rollupContract,
+    inboxContract,
+    contractDeploymentEmitterContract,
+    searchStartBlock,
+    archiverStore,
+  );
+
+  const shutdown = async () => {
+    await archiver.stop();
+    process.exit(0);
+  };
+  process.once('SIGINT', shutdown);
+  process.once('SIGTERM', shutdown);
+}
+
+// See https://twitter.com/Rich_Harris/status/1355289863130673153
+if (process.argv[1] === fileURLToPath(import.meta.url).replace(/\/index\.js$/, '')) {
+  // eslint-disable-next-line @typescript-eslint/no-floating-promises
+  main().catch(err => {
+    log(err);
+    process.exit(1);
+  });
+}

package/tsconfig.json

@@ -0,0 +1,23 @@
+{
+  "extends": "..",
+  "compilerOptions": {
+    "outDir": "dest",
+    "rootDir": "src",
+    "tsBuildInfoFile": ".tsbuildinfo"
+  },
+  "references": [
+    {
+      "path": "../ethereum"
+    },
+    {
+      "path": "../foundation"
+    },
+    {
+      "path": "../l1-artifacts"
+    },
+    {
+      "path": "../types"
+    }
+  ],
+  "include": ["src"]
+}