@aztec/pxe 0.60.0 → 0.61.0

package/src/database/kv_pxe_database.ts

@@ -14,7 +14,7 @@ import {
 } from '@aztec/circuits.js';
 import { type ContractArtifact } from '@aztec/foundation/abi';
 import { toBufferBE } from '@aztec/foundation/bigint-buffer';
-import { Fr, type Point } from '@aztec/foundation/fields';
+import { Fr } from '@aztec/foundation/fields';
 import {
   type AztecArray,
   type AztecKVStore,
@@ -66,6 +66,8 @@ export class KVPxeDatabase implements PxeDatabase {
   #notesByTxHashAndScope: Map<string, AztecMultiMap<string, string>>;
   #notesByIvpkMAndScope: Map<string, AztecMultiMap<string, string>>;
 
+  #taggingSecretIndexes: AztecMap<string, number>;
+
   constructor(private db: AztecKVStore) {
     this.#db = db;
 
@@ -111,6 +113,8 @@ export class KVPxeDatabase implements PxeDatabase {
       this.#notesByTxHashAndScope.set(scope, db.openMultiMap(`${scope}:notes_by_tx_hash`));
       this.#notesByIvpkMAndScope.set(scope, db.openMultiMap(`${scope}:notes_by_ivpk_m`));
     }
+
+    this.#taggingSecretIndexes = db.openMap('tagging_secret_indices');
   }
 
   public async getContract(
@@ -215,7 +219,7 @@ export class KVPxeDatabase implements PxeDatabase {
     const newLength = await this.#deferredNotes.push(...deferredNotes.map(note => note.toBuffer()));
     for (const [index, note] of deferredNotes.entries()) {
       const noteId = newLength - deferredNotes.length + index;
-      await this.#deferredNotesByContract.set(note.contractAddress.toString(), noteId);
+      await this.#deferredNotesByContract.set(note.payload.contractAddress.toString(), noteId);
     }
   }
 
@@ -545,12 +549,12 @@ export class KVPxeDatabase implements PxeDatabase {
     return Promise.resolve(Array.from(this.#addresses).map(v => CompleteAddress.fromBuffer(v)));
   }
 
-  getSynchedBlockNumberForPublicKey(publicKey: Point): number | undefined {
-    return this.#syncedBlockPerPublicKey.get(publicKey.toString());
+  getSynchedBlockNumberForAccount(account: AztecAddress): number | undefined {
+    return this.#syncedBlockPerPublicKey.get(account.toString());
   }
 
-  setSynchedBlockNumberForPublicKey(publicKey: Point, blockNumber: number): Promise<void> {
-    return this.#syncedBlockPerPublicKey.set(publicKey.toString(), blockNumber);
+  setSynchedBlockNumberForAccount(account: AztecAddress, blockNumber: number): Promise<void> {
+    return this.#syncedBlockPerPublicKey.set(account.toString(), blockNumber);
   }
 
   async estimateSize(): Promise<number> {
@@ -572,4 +576,20 @@ export class KVPxeDatabase implements PxeDatabase {
 
     return incomingNotesSize + outgoingNotesSize + treeRootsSize + authWitsSize + addressesSize;
   }
+
+  async incrementTaggingSecretsIndexes(appTaggingSecrets: Fr[]): Promise<void> {
+    const indexes = await this.getTaggingSecretsIndexes(appTaggingSecrets);
+    await this.db.transaction(() => {
+      indexes.forEach(index => {
+        const nextIndex = index ? index + 1 : 1;
+        void this.#taggingSecretIndexes.set(appTaggingSecrets.toString(), nextIndex);
+      });
+    });
+  }
+
+  getTaggingSecretsIndexes(appTaggingSecrets: Fr[]): Promise<number[]> {
+    return this.db.transaction(() =>
+      appTaggingSecrets.map(secret => this.#taggingSecretIndexes.get(secret.toString()) ?? 0),
+    );
+  }
 }

package/src/database/outgoing_note_dao.ts

@@ -35,6 +35,7 @@ export class OutgoingNoteDao {
   ) {}
 
   static fromPayloadAndNoteInfo(
+    note: Note,
     payload: L1NotePayload,
     noteInfo: NoteInfo,
     dataStartIndexForTx: number,
@@ -42,7 +43,7 @@ export class OutgoingNoteDao {
   ) {
     const noteHashIndexInTheWholeTree = BigInt(dataStartIndexForTx + noteInfo.noteHashIndex);
     return new OutgoingNoteDao(
-      payload.note,
+      note,
       payload.contractAddress,
       payload.storageSlot,
       payload.noteTypeId,

package/src/database/pxe_database.ts

@@ -169,20 +169,24 @@ export interface PxeDatabase extends ContractArtifactDatabase, ContractInstanceD
 
   /**
    * Updates up to which block number we have processed notes for a given public key.
-   * @param publicKey - The public key to set the synched block number for.
+   * @param account - The account to set the synched block number for.
    * @param blockNumber - The block number to set.
    */
-  setSynchedBlockNumberForPublicKey(publicKey: PublicKey, blockNumber: number): Promise<void>;
+  setSynchedBlockNumberForAccount(account: AztecAddress, blockNumber: number): Promise<void>;
 
   /**
    * Get the synched block number for a given public key.
-   * @param publicKey - The public key to get the synched block number for.
+   * @param account - The account to get the synched block number for.
    */
-  getSynchedBlockNumberForPublicKey(publicKey: PublicKey): number | undefined;
+  getSynchedBlockNumberForAccount(account: AztecAddress): number | undefined;
 
   /**
    * Returns the estimated size in bytes of this db.
    * @returns The estimated size in bytes of this db.
    */
   estimateSize(): Promise<number>;
+
+  getTaggingSecretsIndexes(appTaggingSecrets: Fr[]): Promise<number[]>;
+
+  incrementTaggingSecretsIndexes(appTaggingSecrets: Fr[]): Promise<void>;
 }

package/src/note_processor/note_processor.ts

@@ -1,6 +1,11 @@
 import { type AztecNode, L1NotePayload, type L2Block } from '@aztec/circuit-types';
 import { type NoteProcessorStats } from '@aztec/circuit-types/stats';
-import { type CompleteAddress, INITIAL_L2_BLOCK_NUM, MAX_NOTE_HASHES_PER_TX, type PublicKey } from '@aztec/circuits.js';
+import {
+  type CompleteAddress,
+  INITIAL_L2_BLOCK_NUM,
+  MAX_NOTE_HASHES_PER_TX,
+  computeAddressSecret,
+} from '@aztec/circuits.js';
 import { type Fr } from '@aztec/foundation/fields';
 import { type Logger, createDebugLogger } from '@aztec/foundation/log';
 import { Timer } from '@aztec/foundation/timer';
@@ -48,10 +53,6 @@ export class NoteProcessor {
 
   private constructor(
     public readonly account: CompleteAddress,
-    /** The public counterpart to the secret key to be used in the decryption of incoming note logs. */
-    private readonly ivpkM: PublicKey,
-    /** The public counterpart to the secret key to be used in the decryption of outgoing note logs. */
-    private readonly ovpkM: PublicKey,
     private keyStore: KeyStore,
     private db: PxeDatabase,
     private node: AztecNode,
@@ -60,7 +61,7 @@ export class NoteProcessor {
     private log: Logger,
   ) {}
 
-  public static async create(
+  public static create(
     account: CompleteAddress,
     keyStore: KeyStore,
     db: PxeDatabase,
@@ -69,10 +70,7 @@ export class NoteProcessor {
     simulator = getAcirSimulator(db, node, keyStore),
     log = createDebugLogger('aztec:note_processor'),
   ) {
-    const ivpkM = await keyStore.getMasterIncomingViewingPublicKey(account.address);
-    const ovpkM = await keyStore.getMasterOutgoingViewingPublicKey(account.address);
-
-    return new NoteProcessor(account, ivpkM, ovpkM, keyStore, db, node, startingBlock, simulator, log);
+    return new NoteProcessor(account, keyStore, db, node, startingBlock, simulator, log);
   }
 
   /**
@@ -95,7 +93,7 @@ export class NoteProcessor {
   }
 
   private getSyncedToBlock(): number {
-    return this.db.getSynchedBlockNumberForPublicKey(this.ivpkM) ?? this.startingBlock - 1;
+    return this.db.getSynchedBlockNumberForAccount(this.account.address) ?? this.startingBlock - 1;
   }
 
   /**
@@ -114,13 +112,17 @@ export class NoteProcessor {
     const deferredIncomingNotes: DeferredNoteDao[] = [];
     const deferredOutgoingNotes: DeferredNoteDao[] = [];
 
-    const ivskM = await this.keyStore.getMasterSecretKey(this.ivpkM);
-    const ovskM = await this.keyStore.getMasterSecretKey(this.ovpkM);
+    const ivskM = await this.keyStore.getMasterSecretKey(this.account.publicKeys.masterIncomingViewingPublicKey);
+    const addressSecret = computeAddressSecret(this.account.getPreaddress(), ivskM);
+
+    const ovskM = await this.keyStore.getMasterSecretKey(this.account.publicKeys.masterOutgoingViewingPublicKey);
 
     // Iterate over both blocks and encrypted logs.
     for (const block of blocks) {
       this.stats.blocks++;
-      const { txLogs } = block.body.noteEncryptedLogs;
+      const { txLogs: encryptedTxLogs } = block.body.noteEncryptedLogs;
+      const { txLogs: unencryptedTxLogs } = block.body.unencryptedLogs;
+
       const dataStartIndexForBlock =
         block.header.state.partial.noteHashTree.nextAvailableLeafIndex -
         block.body.numberOfTxsIncludingPadded * MAX_NOTE_HASHES_PER_TX;
@@ -131,65 +133,72 @@ export class NoteProcessor {
       const outgoingNotes: OutgoingNoteDao[] = [];
 
       // Iterate over all the encrypted logs and try decrypting them. If successful, store the note.
-      for (let indexOfTxInABlock = 0; indexOfTxInABlock < txLogs.length; ++indexOfTxInABlock) {
+      for (let indexOfTxInABlock = 0; indexOfTxInABlock < encryptedTxLogs.length; ++indexOfTxInABlock) {
         this.stats.txs++;
         const dataStartIndexForTx = dataStartIndexForBlock + indexOfTxInABlock * MAX_NOTE_HASHES_PER_TX;
         const noteHashes = block.body.txEffects[indexOfTxInABlock].noteHashes;
         // Note: Each tx generates a `TxL2Logs` object and for this reason we can rely on its index corresponding
         //       to the index of a tx in a block.
-        const txFunctionLogs = txLogs[indexOfTxInABlock].functionLogs;
+        const encryptedTxFunctionLogs = encryptedTxLogs[indexOfTxInABlock].functionLogs;
+        const unencryptedTxFunctionLogs = unencryptedTxLogs[indexOfTxInABlock].functionLogs;
         const excludedIndices: Set<number> = new Set();
-        for (const functionLogs of txFunctionLogs) {
-          for (const log of functionLogs.logs) {
-            this.stats.seen++;
-            const incomingNotePayload = L1NotePayload.decryptAsIncoming(log, ivskM);
-            const outgoingNotePayload = L1NotePayload.decryptAsOutgoing(log, ovskM);
-
-            if (incomingNotePayload || outgoingNotePayload) {
-              if (incomingNotePayload && outgoingNotePayload && !incomingNotePayload.equals(outgoingNotePayload)) {
-                throw new Error(
-                  `Incoming and outgoing note payloads do not match. Incoming: ${JSON.stringify(
-                    incomingNotePayload,
-                  )}, Outgoing: ${JSON.stringify(outgoingNotePayload)}`,
-                );
-              }
-
-              const payload = incomingNotePayload || outgoingNotePayload;
-
-              const txEffect = block.body.txEffects[indexOfTxInABlock];
-              const { incomingNote, outgoingNote, incomingDeferredNote, outgoingDeferredNote } = await produceNoteDaos(
-                this.simulator,
-                this.db,
-                incomingNotePayload ? this.ivpkM : undefined,
-                outgoingNotePayload ? this.ovpkM : undefined,
-                payload!,
-                txEffect.txHash,
-                noteHashes,
-                dataStartIndexForTx,
-                excludedIndices,
-                this.log,
-                txEffect.unencryptedLogs,
-              );
-
-              if (incomingNote) {
-                incomingNotes.push(incomingNote);
-                this.stats.decryptedIncoming++;
-              }
-              if (outgoingNote) {
-                outgoingNotes.push(outgoingNote);
-                this.stats.decryptedOutgoing++;
-              }
-              if (incomingDeferredNote) {
-                deferredIncomingNotes.push(incomingDeferredNote);
-                this.stats.deferredIncoming++;
-              }
-              if (outgoingDeferredNote) {
-                deferredOutgoingNotes.push(outgoingDeferredNote);
-                this.stats.deferredOutgoing++;
-              }
 
-              if (incomingNote == undefined && outgoingNote == undefined && incomingDeferredNote == undefined) {
-                this.stats.failed++;
+        // We iterate over both encrypted and unencrypted logs to decrypt the notes since partial notes are passed
+        // via the unencrypted logs stream.
+        for (const txFunctionLogs of [encryptedTxFunctionLogs, unencryptedTxFunctionLogs]) {
+          for (const functionLogs of txFunctionLogs) {
+            for (const unprocessedLog of functionLogs.logs) {
+              this.stats.seen++;
+              const incomingNotePayload = L1NotePayload.decryptAsIncoming(unprocessedLog.data, addressSecret);
+              const outgoingNotePayload = L1NotePayload.decryptAsOutgoing(unprocessedLog.data, ovskM);
+
+              if (incomingNotePayload || outgoingNotePayload) {
+                if (incomingNotePayload && outgoingNotePayload && !incomingNotePayload.equals(outgoingNotePayload)) {
+                  throw new Error(
+                    `Incoming and outgoing note payloads do not match. Incoming: ${JSON.stringify(
+                      incomingNotePayload,
+                    )}, Outgoing: ${JSON.stringify(outgoingNotePayload)}`,
+                  );
+                }
+
+                const payload = incomingNotePayload || outgoingNotePayload;
+
+                const txEffect = block.body.txEffects[indexOfTxInABlock];
+                const { incomingNote, outgoingNote, incomingDeferredNote, outgoingDeferredNote } =
+                  await produceNoteDaos(
+                    this.simulator,
+                    this.db,
+                    incomingNotePayload ? this.account.publicKeys.masterIncomingViewingPublicKey : undefined,
+                    outgoingNotePayload ? this.account.publicKeys.masterOutgoingViewingPublicKey : undefined,
+                    payload!,
+                    txEffect.txHash,
+                    noteHashes,
+                    dataStartIndexForTx,
+                    excludedIndices,
+                    this.log,
+                    txEffect.unencryptedLogs,
+                  );
+
+                if (incomingNote) {
+                  incomingNotes.push(incomingNote);
+                  this.stats.decryptedIncoming++;
+                }
+                if (outgoingNote) {
+                  outgoingNotes.push(outgoingNote);
+                  this.stats.decryptedOutgoing++;
+                }
+                if (incomingDeferredNote) {
+                  deferredIncomingNotes.push(incomingDeferredNote);
+                  this.stats.deferredIncoming++;
+                }
+                if (outgoingDeferredNote) {
+                  deferredOutgoingNotes.push(outgoingDeferredNote);
+                  this.stats.deferredOutgoing++;
+                }
+
+                if (incomingNote == undefined && outgoingNote == undefined && incomingDeferredNote == undefined) {
+                  this.stats.failed++;
+                }
               }
             }
           }
@@ -207,7 +216,7 @@ export class NoteProcessor {
     await this.processDeferredNotes(deferredIncomingNotes, deferredOutgoingNotes);
 
     const syncedToBlock = blocks[blocks.length - 1].number;
-    await this.db.setSynchedBlockNumberForPublicKey(this.ivpkM, syncedToBlock);
+    await this.db.setSynchedBlockNumberForAccount(this.account.address, syncedToBlock);
 
     this.log.debug(`Synched block ${syncedToBlock}`);
   }
@@ -241,7 +250,10 @@ export class NoteProcessor {
     const nullifiers: Fr[] = blocksAndNotes.flatMap(b =>
       b.block.body.txEffects.flatMap(txEffect => txEffect.nullifiers),
     );
-    const removedNotes = await this.db.removeNullifiedNotes(nullifiers, this.ivpkM);
+    const removedNotes = await this.db.removeNullifiedNotes(
+      nullifiers,
+      this.account.publicKeys.masterIncomingViewingPublicKey,
+    );
     removedNotes.forEach(noteDao => {
       this.log.verbose(
         `Removed note for contract ${noteDao.contractAddress} at slot ${
@@ -265,15 +277,15 @@ export class NoteProcessor {
       await this.db.addDeferredNotes([...deferredIncomingNotes, ...deferredOutgoingNotes]);
       deferredIncomingNotes.forEach(noteDao => {
         this.log.verbose(
-          `Deferred incoming note for contract ${noteDao.contractAddress} at slot ${
-            noteDao.storageSlot
+          `Deferred incoming note for contract ${noteDao.payload.contractAddress} at slot ${
+            noteDao.payload.storageSlot
           } in tx ${noteDao.txHash.toString()}`,
         );
       });
       deferredOutgoingNotes.forEach(noteDao => {
         this.log.verbose(
-          `Deferred outgoing note for contract ${noteDao.contractAddress} at slot ${
-            noteDao.storageSlot
+          `Deferred outgoing note for contract ${noteDao.payload.contractAddress} at slot ${
+            noteDao.payload.storageSlot
           } in tx ${noteDao.txHash.toString()}`,
         );
       });
@@ -298,21 +310,10 @@ export class NoteProcessor {
     const outgoingNotes: OutgoingNoteDao[] = [];
 
     for (const deferredNote of deferredNoteDaos) {
-      const {
-        publicKey,
-        note,
-        contractAddress,
-        storageSlot,
-        noteTypeId,
-        txHash,
-        noteHashes,
-        dataStartIndexForTx,
-        unencryptedLogs,
-      } = deferredNote;
-      const payload = new L1NotePayload(note, contractAddress, storageSlot, noteTypeId);
+      const { publicKey, payload, txHash, noteHashes, dataStartIndexForTx, unencryptedLogs } = deferredNote;
 
-      const isIncoming = publicKey.equals(this.ivpkM);
-      const isOutgoing = publicKey.equals(this.ovpkM);
+      const isIncoming = publicKey.equals(this.account.publicKeys.masterIncomingViewingPublicKey);
+      const isOutgoing = publicKey.equals(this.account.publicKeys.masterOutgoingViewingPublicKey);
 
       if (!isIncoming && !isOutgoing) {
         // The note does not belong to this note processor
@@ -322,8 +323,8 @@ export class NoteProcessor {
       const { incomingNote, outgoingNote } = await produceNoteDaos(
         this.simulator,
         this.db,
-        isIncoming ? this.ivpkM : undefined,
-        isOutgoing ? this.ovpkM : undefined,
+        isIncoming ? this.account.publicKeys.masterIncomingViewingPublicKey : undefined,
+        isOutgoing ? this.account.publicKeys.masterOutgoingViewingPublicKey : undefined,
         payload,
         txHash,
         noteHashes,

package/src/note_processor/utils/add_public_values_to_payload.ts

@@ -0,0 +1,63 @@
+import { type L1NotePayload, Note } from '@aztec/circuit-types';
+import { ContractNotFoundError } from '@aztec/simulator';
+
+import { type PxeDatabase } from '../../database/pxe_database.js';
+
+/**
+ * Merges privately and publicly delivered note values.
+ * @param db - PXE database used to fetch contract instance and artifact.
+ * @param payload - Payload corresponding to the note.
+ * @returns Note payload with public fields added.
+ */
+export async function getOrderedNoteItems(
+  db: PxeDatabase,
+  { contractAddress, noteTypeId, privateNoteValues, publicNoteValues }: L1NotePayload,
+): Promise<Note> {
+  if (publicNoteValues.length === 0) {
+    return new Note(privateNoteValues);
+  }
+
+  const instance = await db.getContractInstance(contractAddress);
+  if (!instance) {
+    throw new ContractNotFoundError(
+      `Could not find instance for ${contractAddress.toString()}. This should never happen here as the partial notes flow should be triggered only for non-deferred notes.`,
+    );
+  }
+
+  const artifact = await db.getContractArtifact(instance.contractClassId);
+  if (!artifact) {
+    throw new Error(
+      `Could not find artifact for contract class ${instance.contractClassId.toString()}. This should never happen here as the partial notes flow should be triggered only for non-deferred notes.`,
+    );
+  }
+
+  const noteFields = Object.values(artifact.notes).find(note => note.id.equals(noteTypeId))?.fields;
+
+  if (!noteFields) {
+    throw new Error(`Could not find note fields for note type ${noteTypeId.toString()}.`);
+  }
+
+  // We sort note fields by index so that we can iterate over them in order.
+  noteFields.sort((a, b) => a.index - b.index);
+
+  // Now we insert the public fields into the note based on its indices defined in the ABI.
+  const modifiedNoteItems = privateNoteValues;
+  let indexInPublicValues = 0;
+  for (let i = 0; i < noteFields.length; i++) {
+    const noteField = noteFields[i];
+    if (noteField.nullable) {
+      if (i == noteFields.length - 1) {
+        // We are processing the last field so we simply insert the rest of the public fields at the end
+        modifiedNoteItems.push(...publicNoteValues.slice(indexInPublicValues));
+      } else {
+        const noteFieldLength = noteFields[i + 1].index - noteField.index;
+        const publicValuesToInsert = publicNoteValues.slice(indexInPublicValues, indexInPublicValues + noteFieldLength);
+        indexInPublicValues += noteFieldLength;
+        // Now we insert the public fields at the note field index
+        modifiedNoteItems.splice(noteField.index, 0, ...publicValuesToInsert);
+      }
+    }
+  }
+
+  return new Note(modifiedNoteItems);
+}

package/src/note_processor/utils/brute_force_note_info.ts

@@ -1,5 +1,7 @@
-import { type L1NotePayload, type TxHash } from '@aztec/circuit-types';
+import { type Note, type TxHash } from '@aztec/circuit-types';
+import { type AztecAddress } from '@aztec/circuits.js';
 import { computeNoteHashNonce, siloNullifier } from '@aztec/circuits.js/hash';
+import { type NoteSelector } from '@aztec/foundation/abi';
 import { Fr } from '@aztec/foundation/fields';
 import { type AcirSimulator } from '@aztec/simulator';
 
@@ -18,7 +20,10 @@ export interface NoteInfo {
  * @remarks This method assists in identifying spent notes in the note hash tree.
  * @param siloedNoteHashes - Note hashes in the tx. One of them should correspond to the note we are looking for
  * @param txHash - Hash of a tx the note was emitted in.
- * @param l1NotePayload - The note payload.
+ * @param contractAddress - Address of the contract the note was emitted in.
+ * @param storageSlot - Storage slot of the note.
+ * @param noteTypeId - Type of the note.
+ * @param note - Note items.
  * @param excludedIndices - Indices that have been assigned a note in the same tx. Notes in a tx can have the same
  * l1NotePayload. We need to find a different index for each replicate.
  * @param computeNullifier - A flag indicating whether to compute the nullifier or just return 0.
@@ -29,7 +34,10 @@ export async function bruteForceNoteInfo(
   simulator: AcirSimulator,
   siloedNoteHashes: Fr[],
   txHash: TxHash,
-  { contractAddress, storageSlot, noteTypeId, note }: L1NotePayload,
+  contractAddress: AztecAddress,
+  storageSlot: Fr,
+  noteTypeId: NoteSelector,
+  note: Note,
   excludedIndices: Set<number>,
   computeNullifier: boolean,
 ): Promise<NoteInfo> {

package/src/note_processor/utils/produce_note_daos.ts

@@ -46,8 +46,6 @@ export async function produceNoteDaos(
   incomingDeferredNote: DeferredNoteDao | undefined;
   outgoingDeferredNote: DeferredNoteDao | undefined;
 }> {
-  // WARNING: This code is full of tech debt and will be refactored once we have final design of partial notes
-  // delivery.
   if (!ivpkM && !ovpkM) {
     throw new Error('Both ivpkM and ovpkM are undefined. Cannot create note.');
   }
@@ -78,11 +76,11 @@ export async function produceNoteDaos(
       // Incoming note is defined meaning that this PXE has both the incoming and outgoing keys. We can skip computing
       // note hash and note index since we already have them in the incoming note.
       outgoingNote = new OutgoingNoteDao(
-        payload.note,
-        payload.contractAddress,
-        payload.storageSlot,
-        payload.noteTypeId,
-        txHash,
+        incomingNote.note,
+        incomingNote.contractAddress,
+        incomingNote.storageSlot,
+        incomingNote.noteTypeId,
+        incomingNote.txHash,
         incomingNote.nonce,
         incomingNote.noteHash,
         incomingNote.index,