@aztec/pxe 0.23.0 → 0.26.1

package/src/note_processor/note_processor.ts

@@ -0,0 +1,275 @@
+import { AztecNode, KeyStore, L1NotePayload, L2BlockContext, L2BlockL2Logs, TaggedNote } from '@aztec/circuit-types';
+import { NoteProcessorStats } from '@aztec/circuit-types/stats';
+import { INITIAL_L2_BLOCK_NUM, MAX_NEW_NOTE_HASHES_PER_TX, PublicKey } from '@aztec/circuits.js';
+import { Grumpkin } from '@aztec/circuits.js/barretenberg';
+import { Fr } from '@aztec/foundation/fields';
+import { createDebugLogger } from '@aztec/foundation/log';
+import { Timer } from '@aztec/foundation/timer';
+import { ContractNotFoundError } from '@aztec/simulator';
+
+import { DeferredNoteDao } from '../database/deferred_note_dao.js';
+import { PxeDatabase } from '../database/index.js';
+import { NoteDao } from '../database/note_dao.js';
+import { getAcirSimulator } from '../simulator/index.js';
+import { produceNoteDao } from './produce_note_dao.js';
+
+/**
+ * Contains all the decrypted data in this array so that we can later batch insert it all into the database.
+ */
+interface ProcessedData {
+  /**
+   * Holds L2 block and a cache of already requested tx hashes.
+   */
+  blockContext: L2BlockContext;
+  /**
+   * DAOs of processed notes.
+   */
+  noteDaos: NoteDao[];
+}
+
+/**
+ * NoteProcessor is responsible for decrypting logs and converting them to notes via their originating contracts
+ * before storing them against their owner.
+ */
+export class NoteProcessor {
+  /** Keeps track of processing time since an instance is created. */
+  public readonly timer: Timer = new Timer();
+
+  /** Stats accumulated for this processor. */
+  public readonly stats: NoteProcessorStats = { seen: 0, decrypted: 0, deferred: 0, failed: 0, blocks: 0, txs: 0 };
+
+  constructor(
+    /**
+     * The public counterpart to the private key to be used in note decryption.
+     */
+    public readonly publicKey: PublicKey,
+    private keyStore: KeyStore,
+    private db: PxeDatabase,
+    private node: AztecNode,
+    private startingBlock: number = INITIAL_L2_BLOCK_NUM,
+    private simulator = getAcirSimulator(db, node, keyStore),
+    private log = createDebugLogger('aztec:note_processor'),
+  ) {}
+
+  /**
+   * Check if the NoteProcessor is synchronized with the remote block number.
+   * The function queries the remote block number from the AztecNode and compares it with the syncedToBlock value in the NoteProcessor.
+   * If the values are equal, then the NoteProcessor is considered to be synchronized, otherwise not.
+   *
+   * @returns A boolean indicating whether the NoteProcessor is synchronized with the remote block number or not.
+   */
+  public async isSynchronized() {
+    const remoteBlockNumber = await this.node.getBlockNumber();
+    return this.getSyncedToBlock() === remoteBlockNumber;
+  }
+
+  /**
+   * Returns synchronization status (ie up to which block has been synced ) for this note processor.
+   */
+  public get status() {
+    return { syncedToBlock: this.getSyncedToBlock() };
+  }
+
+  private getSyncedToBlock(): number {
+    return this.db.getSynchedBlockNumberForPublicKey(this.publicKey) ?? this.startingBlock - 1;
+  }
+
+  /**
+   * Process the given L2 block contexts and encrypted logs to update the note processor.
+   * It synchronizes the user's account by decrypting the encrypted logs and processing
+   * the transactions and auxiliary data associated with them.
+   * Throws an error if the number of block contexts and encrypted logs do not match.
+   *
+   * @param l2BlockContexts - An array of L2 block contexts to be processed.
+   * @param encryptedL2BlockLogs - An array of encrypted logs associated with the L2 block contexts.
+   * @returns A promise that resolves once the processing is completed.
+   */
+  public async process(l2BlockContexts: L2BlockContext[], encryptedL2BlockLogs: L2BlockL2Logs[]): Promise<void> {
+    if (l2BlockContexts.length !== encryptedL2BlockLogs.length) {
+      throw new Error(
+        `Number of blocks and EncryptedLogs is not equal. Received ${l2BlockContexts.length} blocks, ${encryptedL2BlockLogs.length} encrypted logs.`,
+      );
+    }
+    if (!l2BlockContexts.length) {
+      return;
+    }
+
+    const curve = new Grumpkin();
+    const blocksAndNotes: ProcessedData[] = [];
+    // Keep track of notes that we couldn't process because the contract was not found.
+    const deferredNoteDaos: DeferredNoteDao[] = [];
+
+    // Iterate over both blocks and encrypted logs.
+    for (let blockIndex = 0; blockIndex < encryptedL2BlockLogs.length; ++blockIndex) {
+      this.stats.blocks++;
+      const { txLogs } = encryptedL2BlockLogs[blockIndex];
+      const blockContext = l2BlockContexts[blockIndex];
+      const block = blockContext.block;
+      const dataEndIndexForBlock = block.header.state.partial.noteHashTree.nextAvailableLeafIndex;
+
+      // We are using set for `userPertainingTxIndices` to avoid duplicates. This would happen in case there were
+      // multiple encrypted logs in a tx pertaining to a user.
+      const noteDaos: NoteDao[] = [];
+      const privateKey = await this.keyStore.getAccountPrivateKey(this.publicKey);
+
+      // Iterate over all the encrypted logs and try decrypting them. If successful, store the note.
+      for (let indexOfTxInABlock = 0; indexOfTxInABlock < txLogs.length; ++indexOfTxInABlock) {
+        this.stats.txs++;
+        const dataStartIndexForTx =
+          dataEndIndexForBlock - (txLogs.length - indexOfTxInABlock) * MAX_NEW_NOTE_HASHES_PER_TX;
+        const newNoteHashes = block.body.txEffects[indexOfTxInABlock].noteHashes;
+        // Note: Each tx generates a `TxL2Logs` object and for this reason we can rely on its index corresponding
+        //       to the index of a tx in a block.
+        const txFunctionLogs = txLogs[indexOfTxInABlock].functionLogs;
+        const excludedIndices: Set<number> = new Set();
+        for (const functionLogs of txFunctionLogs) {
+          for (const log of functionLogs.logs) {
+            this.stats.seen++;
+            const taggedNote = TaggedNote.fromEncryptedBuffer(log, privateKey, curve);
+            if (taggedNote?.notePayload) {
+              const { notePayload: payload } = taggedNote;
+              // We have successfully decrypted the data.
+              const txHash = blockContext.getTxHash(indexOfTxInABlock);
+              try {
+                const noteDao = await produceNoteDao(
+                  this.simulator,
+                  this.publicKey,
+                  payload,
+                  txHash,
+                  newNoteHashes,
+                  dataStartIndexForTx,
+                  excludedIndices,
+                );
+                noteDaos.push(noteDao);
+                this.stats.decrypted++;
+              } catch (e) {
+                if (e instanceof ContractNotFoundError) {
+                  this.stats.deferred++;
+                  this.log.warn(e.message);
+                  const deferredNoteDao = new DeferredNoteDao(
+                    this.publicKey,
+                    payload.note,
+                    payload.contractAddress,
+                    payload.storageSlot,
+                    payload.noteTypeId,
+                    txHash,
+                    newNoteHashes,
+                    dataStartIndexForTx,
+                  );
+                  deferredNoteDaos.push(deferredNoteDao);
+                } else {
+                  this.stats.failed++;
+                  this.log.warn(`Could not process note because of "${e}". Discarding note...`);
+                }
+              }
+            }
+          }
+        }
+      }
+
+      blocksAndNotes.push({
+        blockContext: l2BlockContexts[blockIndex],
+        noteDaos,
+      });
+    }
+
+    await this.processBlocksAndNotes(blocksAndNotes);
+    await this.processDeferredNotes(deferredNoteDaos);
+
+    const syncedToBlock = l2BlockContexts[l2BlockContexts.length - 1].block.number;
+    await this.db.setSynchedBlockNumberForPublicKey(this.publicKey, syncedToBlock);
+
+    this.log(`Synched block ${syncedToBlock}`);
+  }
+
+  /**
+   * Process the given blocks and their associated transaction auxiliary data.
+   * This function updates the database with information about new transactions,
+   * user-pertaining transaction indices, and auxiliary data. It also removes nullified
+   * transaction auxiliary data from the database. This function keeps track of new nullifiers
+   * and ensures all other transactions are updated with newly settled block information.
+   *
+   * @param blocksAndNotes - Array of objects containing L2BlockContexts, user-pertaining transaction indices, and NoteDaos.
+   */
+  private async processBlocksAndNotes(blocksAndNotes: ProcessedData[]) {
+    const noteDaos = blocksAndNotes.flatMap(b => b.noteDaos);
+    if (noteDaos.length) {
+      await this.db.addNotes(noteDaos);
+      noteDaos.forEach(noteDao => {
+        this.log(
+          `Added note for contract ${noteDao.contractAddress} at slot ${
+            noteDao.storageSlot
+          } with nullifier ${noteDao.siloedNullifier.toString()}`,
+        );
+      });
+    }
+
+    const newNullifiers: Fr[] = blocksAndNotes.flatMap(b =>
+      b.blockContext.block.body.txEffects.flatMap(txEffect => txEffect.nullifiers),
+    );
+    const removedNotes = await this.db.removeNullifiedNotes(newNullifiers, this.publicKey);
+    removedNotes.forEach(noteDao => {
+      this.log(
+        `Removed note for contract ${noteDao.contractAddress} at slot ${
+          noteDao.storageSlot
+        } with nullifier ${noteDao.siloedNullifier.toString()}`,
+      );
+    });
+  }
+
+  /**
+   * Store the given deferred notes in the database for later decoding.
+   *
+   * @param deferredNoteDaos - notes that are intended for us but we couldn't process because the contract was not found.
+   */
+  private async processDeferredNotes(deferredNoteDaos: DeferredNoteDao[]) {
+    if (deferredNoteDaos.length) {
+      await this.db.addDeferredNotes(deferredNoteDaos);
+      deferredNoteDaos.forEach(noteDao => {
+        this.log(
+          `Deferred note for contract ${noteDao.contractAddress} at slot ${
+            noteDao.storageSlot
+          } in tx ${noteDao.txHash.toString()}`,
+        );
+      });
+    }
+  }
+
+  /**
+   * Retry decoding the given deferred notes because we now have the contract code.
+   *
+   * @param deferredNoteDaos - notes that we have previously deferred because the contract was not found
+   * @returns An array of NoteDaos that were successfully decoded.
+   *
+   * @remarks Caller is responsible for making sure that we have the contract for the
+   * deferred notes provided: we will not retry notes that fail again.
+   */
+  public async decodeDeferredNotes(deferredNoteDaos: DeferredNoteDao[]): Promise<NoteDao[]> {
+    const excludedIndices: Set<number> = new Set();
+    const noteDaos: NoteDao[] = [];
+    for (const deferredNote of deferredNoteDaos) {
+      const { note, contractAddress, storageSlot, noteTypeId, txHash, newNoteHashes, dataStartIndexForTx } =
+        deferredNote;
+      const payload = new L1NotePayload(note, contractAddress, storageSlot, noteTypeId);
+
+      try {
+        const noteDao = await produceNoteDao(
+          this.simulator,
+          this.publicKey,
+          payload,
+          txHash,
+          newNoteHashes,
+          dataStartIndexForTx,
+          excludedIndices,
+        );
+        noteDaos.push(noteDao);
+        this.stats.decrypted++;
+      } catch (e) {
+        this.stats.failed++;
+        this.log.warn(`Could not process deferred note because of "${e}". Discarding note...`);
+      }
+    }
+
+    return noteDaos;
+  }
+}

package/src/note_processor/produce_note_dao.ts

@@ -0,0 +1,132 @@
+import { L1NotePayload, TxHash } from '@aztec/circuit-types';
+import { Fr, PublicKey } from '@aztec/circuits.js';
+import { computeCommitmentNonce, siloNullifier } from '@aztec/circuits.js/hash';
+import { AcirSimulator } from '@aztec/simulator';
+
+import { NoteDao } from '../database/note_dao.js';
+
+/**
+ * Decodes a note from a transaction that we know was intended for us.
+ * Throws if we do not yet have the contract corresponding to the note in our database.
+ * Accepts a set of excluded indices, which are indices that have been assigned a note in the same tx.
+ * Inserts the index of the note into the excludedIndices set if the note is successfully decoded.
+ *
+ * @param publicKey - The public counterpart to the private key to be used in note decryption.
+ * @param payload - An instance of l1NotePayload.
+ * @param txHash - The hash of the transaction that created the note. Equivalent to the first nullifier of the transaction.
+ * @param newNoteHashes - New note hashes in this transaction, one of which belongs to this note.
+ * @param dataStartIndexForTx - The next available leaf index for the note hash tree for this transaction.
+ * @param excludedIndices - Indices that have been assigned a note in the same tx. Notes in a tx can have the same l1NotePayload, we need to find a different index for each replicate.
+ * @param simulator - An instance of AcirSimulator.
+ * @returns an instance of NoteDao, or throws. inserts the index of the note into the excludedIndices set.
+ */
+export async function produceNoteDao(
+  simulator: AcirSimulator,
+  publicKey: PublicKey,
+  payload: L1NotePayload,
+  txHash: TxHash,
+  newNoteHashes: Fr[],
+  dataStartIndexForTx: number,
+  excludedIndices: Set<number>,
+): Promise<NoteDao> {
+  const { commitmentIndex, nonce, innerNoteHash, siloedNullifier } = await findNoteIndexAndNullifier(
+    simulator,
+    newNoteHashes,
+    txHash,
+    payload,
+    excludedIndices,
+  );
+  const index = BigInt(dataStartIndexForTx + commitmentIndex);
+  excludedIndices?.add(commitmentIndex);
+  return new NoteDao(
+    payload.note,
+    payload.contractAddress,
+    payload.storageSlot,
+    payload.noteTypeId,
+    txHash,
+    nonce,
+    innerNoteHash,
+    siloedNullifier,
+    index,
+    publicKey,
+  );
+}
+
+/**
+ * Find the index of the note in the note hash tree by computing the note hash with different nonce and see which
+ * commitment for the current tx matches this value.
+ * Compute a nullifier for a given l1NotePayload.
+ * The nullifier is calculated using the private key of the account,
+ * contract address, and the note associated with the l1NotePayload.
+ * This method assists in identifying spent commitments in the private state.
+ * @param commitments - Commitments in the tx. One of them should be the note's commitment.
+ * @param txHash - First nullifier in the tx.
+ * @param l1NotePayload - An instance of l1NotePayload.
+ * @param excludedIndices - Indices that have been assigned a note in the same tx. Notes in a tx can have the same
+ * l1NotePayload. We need to find a different index for each replicate.
+ * @returns Information for a decrypted note, including the index of its commitment, nonce, inner note
+ * hash, and the siloed nullifier. Throw if cannot find the nonce for the note.
+ */
+async function findNoteIndexAndNullifier(
+  simulator: AcirSimulator,
+  commitments: Fr[],
+  txHash: TxHash,
+  { contractAddress, storageSlot, noteTypeId, note }: L1NotePayload,
+  excludedIndices: Set<number>,
+) {
+  let commitmentIndex = 0;
+  let nonce: Fr | undefined;
+  let innerNoteHash: Fr | undefined;
+  let siloedNoteHash: Fr | undefined;
+  let uniqueSiloedNoteHash: Fr | undefined;
+  let innerNullifier: Fr | undefined;
+  const firstNullifier = Fr.fromBuffer(txHash.toBuffer());
+
+  for (; commitmentIndex < commitments.length; ++commitmentIndex) {
+    if (excludedIndices.has(commitmentIndex)) {
+      continue;
+    }
+
+    const commitment = commitments[commitmentIndex];
+    if (commitment.equals(Fr.ZERO)) {
+      break;
+    }
+
+    const expectedNonce = computeCommitmentNonce(firstNullifier, commitmentIndex);
+    ({ innerNoteHash, siloedNoteHash, uniqueSiloedNoteHash, innerNullifier } =
+      await simulator.computeNoteHashAndNullifier(contractAddress, expectedNonce, storageSlot, noteTypeId, note));
+    if (commitment.equals(uniqueSiloedNoteHash)) {
+      nonce = expectedNonce;
+      break;
+    }
+  }
+
+  if (!nonce) {
+    let errorString;
+    if (siloedNoteHash == undefined) {
+      errorString = 'Cannot find a matching commitment for the note.';
+    } else {
+      errorString = `We decrypted a log, but couldn't find a corresponding note in the tree.
+This might be because the note was nullified in the same tx which created it.
+In that case, everything is fine. To check whether this is the case, look back through
+the logs for a notification
+'important: chopped commitment for siloed inner hash note
+${siloedNoteHash.toString()}'.
+If you can see that notification. Everything's fine.
+If that's not the case, and you can't find such a notification, something has gone wrong.
+There could be a problem with the way you've defined a custom note, or with the way you're
+serializing / deserializing / hashing / encrypting / decrypting that note.
+Please see the following github issue to track an improvement that we're working on:
+https://github.com/AztecProtocol/aztec-packages/issues/1641`;
+    }
+
+    throw new Error(errorString);
+  }
+
+  return {
+    commitmentIndex,
+    nonce,
+    innerNoteHash: innerNoteHash!,
+    siloedNullifier: siloNullifier(contractAddress, innerNullifier!),
+  };
+}

package/src/pxe_http/index.ts

@@ -0,0 +1 @@
+export * from './pxe_http_server.js';

package/src/pxe_http/pxe_http_server.ts

@@ -0,0 +1,75 @@
+import {
+  AuthWitness,
+  CompleteAddress,
+  ContractData,
+  ExtendedContractData,
+  ExtendedNote,
+  ExtendedUnencryptedL2Log,
+  L2Block,
+  L2BlockL2Logs,
+  LogId,
+  Note,
+  NullifierMembershipWitness,
+  PXE,
+  Tx,
+  TxEffect,
+  TxExecutionRequest,
+  TxHash,
+  TxReceipt,
+} from '@aztec/circuit-types';
+import { FunctionSelector } from '@aztec/circuits.js';
+import { AztecAddress } from '@aztec/foundation/aztec-address';
+import { EthAddress } from '@aztec/foundation/eth-address';
+import { Fr, GrumpkinScalar, Point } from '@aztec/foundation/fields';
+import { JsonRpcServer, createNamespacedJsonRpcServer } from '@aztec/foundation/json-rpc/server';
+
+import http from 'http';
+
+/**
+ * Wraps an instance of Private eXecution Environment (PXE) implementation to a JSON RPC HTTP interface.
+ * @returns A new instance of the HTTP server.
+ */
+export function createPXERpcServer(pxeService: PXE): JsonRpcServer {
+  return new JsonRpcServer(
+    pxeService,
+    {
+      CompleteAddress,
+      AztecAddress,
+      TxExecutionRequest,
+      ContractData,
+      ExtendedContractData,
+      ExtendedUnencryptedL2Log,
+      FunctionSelector,
+      TxHash,
+      EthAddress,
+      Point,
+      Fr,
+      GrumpkinScalar,
+      Note,
+      ExtendedNote,
+      AuthWitness,
+      L2Block,
+      TxEffect,
+      LogId,
+    },
+    { Tx, TxReceipt, L2BlockL2Logs, NullifierMembershipWitness },
+    ['start', 'stop'],
+  );
+}
+
+/**
+ * Creates an http server that forwards calls to the PXE and starts it on the given port.
+ * @param pxeService - PXE that answers queries to the created HTTP server.
+ * @param port - Port to listen in.
+ * @returns A running http server.
+ */
+export function startPXEHttpServer(pxeService: PXE, port: string | number): http.Server {
+  const pxeServer = createPXERpcServer(pxeService);
+  const rpcServer = createNamespacedJsonRpcServer([{ pxe: pxeServer }]);
+
+  const app = rpcServer.getApp();
+  const httpServer = http.createServer(app.callback());
+  httpServer.listen(port);
+
+  return httpServer;
+}

package/src/pxe_service/create_pxe_service.ts

@@ -0,0 +1,53 @@
+import { AztecNode } from '@aztec/circuit-types';
+import { Grumpkin } from '@aztec/circuits.js/barretenberg';
+import { TestKeyStore } from '@aztec/key-store';
+import { AztecLmdbStore } from '@aztec/kv-store/lmdb';
+import { initStoreForRollup } from '@aztec/kv-store/utils';
+import { getCanonicalClassRegisterer } from '@aztec/protocol-contracts/class-registerer';
+import { getCanonicalGasToken } from '@aztec/protocol-contracts/gas-token';
+import { getCanonicalInstanceDeployer } from '@aztec/protocol-contracts/instance-deployer';
+
+import { join } from 'path';
+
+import { PXEServiceConfig } from '../config/index.js';
+import { KVPxeDatabase } from '../database/kv_pxe_database.js';
+import { PXEService } from './pxe_service.js';
+
+/**
+ * Create and start an PXEService instance with the given AztecNode.
+ * If no keyStore or database is provided, it will use TestKeyStore and MemoryDB as default values.
+ * Returns a Promise that resolves to the started PXEService instance.
+ *
+ * @param aztecNode - The AztecNode instance to be used by the server.
+ * @param config - The PXE Service Config to use
+ * @param options - (Optional) Optional information for creating an PXEService.
+ * @returns A Promise that resolves to the started PXEService instance.
+ */
+export async function createPXEService(
+  aztecNode: AztecNode,
+  config: PXEServiceConfig,
+  useLogSuffix: string | boolean | undefined = undefined,
+) {
+  const logSuffix =
+    typeof useLogSuffix === 'boolean'
+      ? useLogSuffix
+        ? Math.random().toString(16).slice(2, 8)
+        : undefined
+      : useLogSuffix;
+
+  const pxeDbPath = config.dataDirectory ? join(config.dataDirectory, 'pxe_data') : undefined;
+  const keyStorePath = config.dataDirectory ? join(config.dataDirectory, 'pxe_key_store') : undefined;
+  const l1Contracts = await aztecNode.getL1ContractAddresses();
+
+  const keyStore = new TestKeyStore(
+    new Grumpkin(),
+    await initStoreForRollup(AztecLmdbStore.open(keyStorePath), l1Contracts.rollupAddress),
+  );
+  const db = new KVPxeDatabase(await initStoreForRollup(AztecLmdbStore.open(pxeDbPath), l1Contracts.rollupAddress));
+
+  const server = new PXEService(keyStore, aztecNode, db, config, logSuffix);
+  await server.addContracts([getCanonicalClassRegisterer(), getCanonicalInstanceDeployer(), getCanonicalGasToken()]);
+
+  await server.start();
+  return server;
+}

package/src/pxe_service/index.ts

@@ -0,0 +1,3 @@
+export * from './pxe_service.js';
+export * from './create_pxe_service.js';
+export { pxeTestSuite } from './test/pxe_test_suite.js';