@aztec/p2p 0.71.0 → 0.73.0

package/src/mem_pools/attestation_pool/attestation_pool_test_suite.ts

@@ -29,7 +29,7 @@ export function describeAttestationPool(getAttestationPool: () => AttestationPoo
 
   const createAttestationsForSlot = (slotNumber: number) => {
     const archive = Fr.random();
-    return signers.map(signer => mockAttestation(signer, slotNumber, archive));
+    return Promise.all(signers.map(signer => mockAttestation(signer, slotNumber, archive)));
   };
 
   // We compare buffers as the objects can have cached values attached to them which are not serialised
@@ -44,7 +44,7 @@ export function describeAttestationPool(getAttestationPool: () => AttestationPoo
   it('should add attestations to pool', async () => {
     const slotNumber = 420;
     const archive = Fr.random();
-    const attestations = signers.map(signer => mockAttestation(signer, slotNumber, archive));
+    const attestations = await Promise.all(signers.map(signer => mockAttestation(signer, slotNumber, archive)));
 
     await ap.addAttestations(attestations);
 
@@ -75,7 +75,7 @@ export function describeAttestationPool(getAttestationPool: () => AttestationPoo
     const attestations: BlockAttestation[] = [];
     const signer = signers[0];
     for (let i = 0; i < NUMBER_OF_SIGNERS_PER_TEST; i++) {
-      attestations.push(mockAttestation(signer, slotNumber, archive, txs));
+      attestations.push(await mockAttestation(signer, slotNumber, archive, txs));
     }
 
     await ap.addAttestations(attestations);
@@ -84,12 +84,12 @@ export function describeAttestationPool(getAttestationPool: () => AttestationPoo
     expect(retreivedAttestations.length).toBe(1);
     expect(retreivedAttestations[0].toBuffer()).toEqual(attestations[0].toBuffer());
     expect(retreivedAttestations[0].payload.txHashes).toEqual(txs);
-    expect(retreivedAttestations[0].getSender().toString()).toEqual(signer.address.toString());
+    expect((await retreivedAttestations[0].getSender()).toString()).toEqual(signer.address.toString());
   });
 
   it('Should store attestations by differing slot', async () => {
     const slotNumbers = [1, 2, 3, 4];
-    const attestations = signers.map((signer, i) => mockAttestation(signer, slotNumbers[i]));
+    const attestations = await Promise.all(signers.map((signer, i) => mockAttestation(signer, slotNumbers[i])));
 
     await ap.addAttestations(attestations);
 
@@ -107,7 +107,9 @@ export function describeAttestationPool(getAttestationPool: () => AttestationPoo
   it('Should store attestations by differing slot and archive', async () => {
     const slotNumbers = [1, 1, 2, 3];
     const archives = [Fr.random(), Fr.random(), Fr.random(), Fr.random()];
-    const attestations = signers.map((signer, i) => mockAttestation(signer, slotNumbers[i], archives[i]));
+    const attestations = await Promise.all(
+      signers.map((signer, i) => mockAttestation(signer, slotNumbers[i], archives[i])),
+    );
 
     await ap.addAttestations(attestations);
 
@@ -125,7 +127,7 @@ export function describeAttestationPool(getAttestationPool: () => AttestationPoo
   it('Should delete attestations', async () => {
     const slotNumber = 420;
     const archive = Fr.random();
-    const attestations = signers.map(signer => mockAttestation(signer, slotNumber, archive));
+    const attestations = await Promise.all(signers.map(signer => mockAttestation(signer, slotNumber, archive)));
     const proposalId = attestations[0].archive.toString();
 
     await ap.addAttestations(attestations);
@@ -165,12 +167,12 @@ export function describeAttestationPool(getAttestationPool: () => AttestationPoo
   it('Should blanket delete attestations per slot and proposal', async () => {
     const slotNumber = 420;
     const archive = Fr.random();
-    const attestations = signers.map(signer => mockAttestation(signer, slotNumber, archive));
+    const attestations = await Promise.all(signers.map(signer => mockAttestation(signer, slotNumber, archive)));
     const proposalId = attestations[0].archive.toString();
 
     // Add another set of attestations with a different proposalId, yet the same slot
     const archive2 = Fr.random();
-    const attestations2 = signers.map(signer => mockAttestation(signer, slotNumber, archive2));
+    const attestations2 = await Promise.all(signers.map(signer => mockAttestation(signer, slotNumber, archive2)));
     const proposalId2 = attestations2[0].archive.toString();
 
     await ap.addAttestations(attestations);
@@ -198,21 +200,11 @@ export function describeAttestationPool(getAttestationPool: () => AttestationPoo
     compareAttestations(retreivedAttestationsAfterDeleteForOtherProposal, attestations2);
   });
 
-  it('Should blanket delete attestations per slot and proposal (does not perform db ops if there are no attestations)', async () => {
-    const slotNumber = 420;
-    const proposalId = 'proposalId';
-
-    const retreivedAttestations = await ap.getAttestationsForSlot(BigInt(slotNumber), proposalId);
-    expect(retreivedAttestations.length).toBe(0);
-
-    await ap.deleteAttestationsForSlotAndProposal(BigInt(slotNumber), proposalId);
-
-    expect(metricsMock.recordRemovedObjects).toHaveBeenCalledTimes(0);
-  });
-
   it('Should delete attestations older than a given slot', async () => {
     const slotNumbers = [1, 2, 3, 69, 72, 74, 88, 420];
-    const attestations = slotNumbers.map(slotNumber => createAttestationsForSlot(slotNumber)).flat();
+    const attestations = (
+      await Promise.all(slotNumbers.map(slotNumber => createAttestationsForSlot(slotNumber)))
+    ).flat();
     const proposalId = attestations[0].archive.toString();
 
     await ap.addAttestations(attestations);

package/src/mem_pools/attestation_pool/kv_attestation_pool.ts

@@ -1,7 +1,8 @@
 import { BlockAttestation } from '@aztec/circuit-types';
 import { Fr } from '@aztec/foundation/fields';
+import { toArray } from '@aztec/foundation/iterable';
 import { createLogger } from '@aztec/foundation/log';
-import { type AztecKVStore, type AztecMapWithSize, type AztecMultiMap } from '@aztec/kv-store';
+import { type AztecAsyncKVStore, type AztecAsyncMap, type AztecAsyncMultiMap } from '@aztec/kv-store';
 import { type TelemetryClient, getTelemetryClient } from '@aztec/telemetry-client';
 
 import { PoolInstrumentation, PoolName } from '../instrumentation.js';
@@ -10,144 +11,149 @@ import { type AttestationPool } from './attestation_pool.js';
 export class KvAttestationPool implements AttestationPool {
   private metrics: PoolInstrumentation<BlockAttestation>;
 
-  // Index of all proposal ids in a slot
-  private attestations: AztecMultiMap<string, string>;
+  private attestations: AztecAsyncMap<string, Buffer>;
+  private proposalsForSlot: AztecAsyncMultiMap<string, string>;
+  private attestationsForProposal: AztecAsyncMultiMap<string, string>;
 
   constructor(
-    private store: AztecKVStore,
+    private store: AztecAsyncKVStore,
     telemetry: TelemetryClient = getTelemetryClient(),
     private log = createLogger('aztec:attestation_pool'),
   ) {
-    this.attestations = store.openMultiMap('attestations');
+    this.attestations = store.openMap('attestations');
+    this.proposalsForSlot = store.openMultiMap('proposals_for_slot');
+    this.attestationsForProposal = store.openMultiMap('attestations_for_proposal');
+
     this.metrics = new PoolInstrumentation(telemetry, PoolName.ATTESTATION_POOL);
   }
 
-  private getProposalMapKey(slot: string, proposalId: string): string {
-    return `proposal-${slot}-${proposalId}`;
+  private getProposalKey(slot: number | bigint | Fr | string, proposalId: Fr | string | Buffer): string {
+    const slotStr = typeof slot === 'string' ? slot : new Fr(slot).toString();
+    const proposalIdStr =
+      typeof proposalId === 'string'
+        ? proposalId
+        : Buffer.isBuffer(proposalId)
+        ? Fr.fromBuffer(proposalId).toString()
+        : proposalId.toString();
+
+    return `${slotStr}-${proposalIdStr}`;
   }
 
-  /**
-   * Get the proposal map for a given slot and proposalId
-   *
-   * Essentially a nested mapping of address -> attestation
-   *
-   * @param slot - The slot to get the proposal map for
-   * @param proposalId - The proposalId to get the map for
-   * @returns The proposal map
-   */
-  private getProposalMap(slot: string, proposalId: string): AztecMapWithSize<string, Buffer> {
-    const mapKey = this.getProposalMapKey(slot, proposalId);
-    return this.store.openMapWithSize(mapKey);
+  private getAttestationKey(slot: number | bigint | Fr | string, proposalId: Fr | string, address: string): string {
+    return `${this.getProposalKey(slot, proposalId)}-${address}`;
   }
 
   public async addAttestations(attestations: BlockAttestation[]): Promise<void> {
-    for (const attestation of attestations) {
-      const slotNumber = attestation.payload.header.globalVariables.slotNumber.toString();
-      const proposalId = attestation.archive.toString();
-      const address = attestation.getSender().toString();
+    await this.store.transactionAsync(async () => {
+      for (const attestation of attestations) {
+        const slotNumber = attestation.payload.header.globalVariables.slotNumber;
+        const proposalId = attestation.archive;
+        const address = (await attestation.getSender()).toString();
 
-      // Index the proposalId in the slot map
-      await this.attestations.set(slotNumber, proposalId);
+        await this.attestations.set(this.getAttestationKey(slotNumber, proposalId, address), attestation.toBuffer());
 
-      // Store the actual attestation in the proposal map
-      const proposalMap = this.getProposalMap(slotNumber, proposalId);
-      await proposalMap.set(address, attestation.toBuffer());
+        await this.proposalsForSlot.set(slotNumber.toString(), proposalId.toString());
+        await this.attestationsForProposal.set(
+          this.getProposalKey(slotNumber, proposalId),
+          this.getAttestationKey(slotNumber, proposalId, address),
+        );
 
-      this.log.verbose(`Added attestation for slot ${slotNumber} from ${address}`);
-    }
+        this.log.verbose(`Added attestation for slot ${slotNumber} from ${address}`);
+      }
+    });
 
     this.metrics.recordAddedObjects(attestations.length);
   }
 
-  public getAttestationsForSlot(slot: bigint, proposalId: string): Promise<BlockAttestation[]> {
-    const slotNumber = new Fr(slot).toString();
-    const proposalMap = this.getProposalMap(slotNumber, proposalId);
-    const attestations = proposalMap.values();
-    const attestationsArray = Array.from(attestations).map(attestation => BlockAttestation.fromBuffer(attestation));
-    return Promise.resolve(attestationsArray);
-  }
+  public async getAttestationsForSlot(slot: bigint, proposalId: string): Promise<BlockAttestation[]> {
+    const attestationIds = await toArray(
+      this.attestationsForProposal.getValuesAsync(this.getProposalKey(slot, proposalId)),
+    );
+    const attestations: BlockAttestation[] = [];
 
-  public async deleteAttestationsOlderThan(oldestSlot: bigint): Promise<void> {
-    const olderThan = [];
+    // alternatively iterate this.attestaions starting from slot-proposal-EthAddress.zero
+    for (const id of attestationIds) {
+      const buf = await this.attestations.getAsync(id);
 
-    const slots = this.attestations.keys();
-    for (const slot of slots) {
-      if (BigInt(slot) < oldestSlot) {
-        olderThan.push(slot);
+      if (!buf) {
+        // this should not happen unless we lost writes
+        throw new Error('Attestation not found ' + id);
       }
+
+      const attestation = BlockAttestation.fromBuffer(buf);
+      attestations.push(attestation);
     }
 
-    await Promise.all(olderThan.map(oldSlot => this.deleteAttestationsForSlot(BigInt(oldSlot))));
-    return Promise.resolve();
+    return attestations;
   }
 
-  public async deleteAttestationsForSlot(slot: bigint): Promise<void> {
-    const deletionPromises = [];
+  public async deleteAttestationsOlderThan(oldestSlot: bigint): Promise<void> {
+    const olderThan = await toArray(this.proposalsForSlot.keysAsync({ end: new Fr(oldestSlot).toString() }));
+    for (const oldSlot of olderThan) {
+      await this.deleteAttestationsForSlot(BigInt(oldSlot));
+    }
+  }
 
-    const slotString = new Fr(slot).toString();
+  public async deleteAttestationsForSlot(slot: bigint): Promise<void> {
+    const slotFr = new Fr(slot);
     let numberOfAttestations = 0;
-    const proposalIds = this.attestations.getValues(slotString);
-
-    if (proposalIds) {
+    await this.store.transactionAsync(async () => {
+      const proposalIds = await toArray(this.proposalsForSlot.getValuesAsync(slotFr.toString()));
       for (const proposalId of proposalIds) {
-        const proposalMap = this.getProposalMap(slotString, proposalId);
-        numberOfAttestations += proposalMap.size();
-        deletionPromises.push(proposalMap.clear());
-      }
-    }
+        const attestations = await toArray(
+          this.attestationsForProposal.getValuesAsync(this.getProposalKey(slotFr, proposalId)),
+        );
 
-    await Promise.all(deletionPromises);
+        numberOfAttestations += attestations.length;
+        for (const attestation of attestations) {
+          await this.attestations.delete(attestation);
+        }
+
+        await this.attestationsForProposal.delete(this.getProposalKey(slotFr, proposalId));
+      }
+    });
 
     this.log.verbose(`Removed ${numberOfAttestations} attestations for slot ${slot}`);
     this.metrics.recordRemovedObjects(numberOfAttestations);
-    return Promise.resolve();
   }
 
   public async deleteAttestationsForSlotAndProposal(slot: bigint, proposalId: string): Promise<void> {
-    const deletionPromises = [];
-
-    const slotString = new Fr(slot).toString();
-    const exists = this.attestations.get(slotString);
-
-    if (exists) {
-      // Remove the proposalId from the slot index
-      deletionPromises.push(this.attestations.deleteValue(slotString, proposalId));
-
-      // Delete all attestations for the proposalId
-      const proposalMap = this.getProposalMap(slotString, proposalId);
-      const numberOfAttestations = proposalMap.size();
-      deletionPromises.push(proposalMap.clear());
+    let numberOfAttestations = 0;
+    await this.store.transactionAsync(async () => {
+      const slotString = new Fr(slot).toString();
+      const attestations = await toArray(
+        this.attestationsForProposal.getValuesAsync(this.getProposalKey(slot, proposalId)),
+      );
+
+      numberOfAttestations += attestations.length;
+      for (const attestation of attestations) {
+        await this.attestations.delete(attestation);
+      }
 
-      this.log.verbose(`Removed ${numberOfAttestations} attestations for slot ${slot} and proposal ${proposalId}`);
-      this.metrics.recordRemovedObjects(numberOfAttestations);
-    }
+      await this.proposalsForSlot.deleteValue(slotString, proposalId);
+      await this.attestationsForProposal.delete(this.getProposalKey(slotString, proposalId));
+    });
 
-    await Promise.all(deletionPromises);
-    return Promise.resolve();
+    this.log.verbose(`Removed ${numberOfAttestations} attestations for slot ${slot} and proposal ${proposalId}`);
+    this.metrics.recordRemovedObjects(numberOfAttestations);
   }
 
   public async deleteAttestations(attestations: BlockAttestation[]): Promise<void> {
-    const deletionPromises = [];
-
-    for (const attestation of attestations) {
-      const slotNumber = attestation.payload.header.globalVariables.slotNumber.toString();
-      const proposalId = attestation.archive.toString();
-      const proposalMap = this.getProposalMap(slotNumber, proposalId);
+    await this.store.transactionAsync(async () => {
+      for (const attestation of attestations) {
+        const slotNumber = attestation.payload.header.globalVariables.slotNumber;
+        const proposalId = attestation.archive;
+        const address = (await attestation.getSender()).toString();
+
+        await this.attestations.delete(this.getAttestationKey(slotNumber, proposalId, address));
+        await this.attestationsForProposal.deleteValue(
+          this.getProposalKey(slotNumber, proposalId),
+          this.getAttestationKey(slotNumber, proposalId, address),
+        );
 
-      if (proposalMap) {
-        const address = attestation.getSender().toString();
-        deletionPromises.push(proposalMap.delete(address));
         this.log.debug(`Deleted attestation for slot ${slotNumber} from ${address}`);
       }
-
-      if (proposalMap.size() === 0) {
-        deletionPromises.push(this.attestations.deleteValue(slotNumber, proposalId));
-      }
-    }
-
-    await Promise.all(deletionPromises);
-
+    });
     this.metrics.recordRemovedObjects(attestations.length);
-    return Promise.resolve();
   }
 }

package/src/mem_pools/attestation_pool/memory_attestation_pool.ts

@@ -26,13 +26,13 @@ export class InMemoryAttestationPool implements AttestationPool {
     return Promise.resolve([]);
   }
 
-  public addAttestations(attestations: BlockAttestation[]): Promise<void> {
+  public async addAttestations(attestations: BlockAttestation[]): Promise<void> {
     for (const attestation of attestations) {
       // Perf: order and group by slot before insertion
       const slotNumber = attestation.payload.header.globalVariables.slotNumber;
 
       const proposalId = attestation.archive.toString();
-      const address = attestation.getSender();
+      const address = await attestation.getSender();
 
       const slotAttestationMap = getSlotOrDefault(this.attestations, slotNumber.toBigInt());
       const proposalAttestationMap = getProposalOrDefault(slotAttestationMap, proposalId);
@@ -105,7 +105,7 @@ export class InMemoryAttestationPool implements AttestationPool {
     return Promise.resolve();
   }
 
-  public deleteAttestations(attestations: BlockAttestation[]): Promise<void> {
+  public async deleteAttestations(attestations: BlockAttestation[]): Promise<void> {
     for (const attestation of attestations) {
       const slotNumber = attestation.payload.header.globalVariables.slotNumber;
       const slotAttestationMap = this.attestations.get(slotNumber.toBigInt());
@@ -113,7 +113,7 @@ export class InMemoryAttestationPool implements AttestationPool {
         const proposalId = attestation.archive.toString();
         const proposalAttestationMap = getProposalOrDefault(slotAttestationMap, proposalId);
         if (proposalAttestationMap) {
-          const address = attestation.getSender();
+          const address = await attestation.getSender();
           proposalAttestationMap.delete(address.toString());
           this.log.debug(`Deleted attestation for slot ${slotNumber} from ${address}`);
         }

package/src/mem_pools/attestation_pool/mocks.ts

@@ -9,14 +9,14 @@ import { makeHeader } from '@aztec/circuits.js/testing';
 import { type Secp256k1Signer } from '@aztec/foundation/crypto';
 import { Fr } from '@aztec/foundation/fields';
 
-import { generatePrivateKey, privateKeyToAccount } from 'viem/accounts';
+import { type LocalAccount, generatePrivateKey, privateKeyToAccount } from 'viem/accounts';
 
 /** Generate Account
  *
  * Create a random signer
  * @returns A random viem signer
  */
-export const generateAccount = () => {
+export const generateAccount = (): LocalAccount => {
   const privateKey = generatePrivateKey();
   return privateKeyToAccount(privateKey);
 };
@@ -27,17 +27,17 @@ export const generateAccount = () => {
  * @param slot The slot number the attestation is for
  * @returns A Block Attestation
  */
-export const mockAttestation = (
+export const mockAttestation = async (
   signer: Secp256k1Signer,
   slot: number = 0,
   archive: Fr = Fr.random(),
   txs: TxHash[] = [0, 1, 2, 3, 4, 5].map(() => TxHash.random()),
-): BlockAttestation => {
+): Promise<BlockAttestation> => {
   // Use arbitrary numbers for all other than slot
   const header = makeHeader(1, 2, slot);
   const payload = new ConsensusPayload(header, archive, txs);
 
-  const hash = getHashedSignaturePayloadEthSignedMessage(payload, SignatureDomainSeparator.blockAttestation);
+  const hash = await getHashedSignaturePayloadEthSignedMessage(payload, SignatureDomainSeparator.blockAttestation);
   const signature = signer.sign(hash);
 
   return new BlockAttestation(payload, signature);