@aztec/p2p 0.0.0-test.0 → 0.0.1-commit.24de95ac

package/src/mem_pools/attestation_pool/kv_attestation_pool.ts

@@ -2,16 +2,20 @@ import { Fr } from '@aztec/foundation/fields';
 import { toArray } from '@aztec/foundation/iterable';
 import { createLogger } from '@aztec/foundation/log';
 import type { AztecAsyncKVStore, AztecAsyncMap, AztecAsyncMultiMap } from '@aztec/kv-store';
-import { BlockAttestation } from '@aztec/stdlib/p2p';
+import { BlockAttestation, BlockProposal } from '@aztec/stdlib/p2p';
 import { type TelemetryClient, getTelemetryClient } from '@aztec/telemetry-client';
 
-import { PoolInstrumentation, PoolName } from '../instrumentation.js';
+import { PoolInstrumentation, PoolName, type PoolStatsCallback } from '../instrumentation.js';
 import type { AttestationPool } from './attestation_pool.js';
 
 export class KvAttestationPool implements AttestationPool {
   private metrics: PoolInstrumentation<BlockAttestation>;
 
   private attestations: AztecAsyncMap<string, Buffer>;
+  private proposals: AztecAsyncMap<
+    /*  proposal.payload.archive  */ string,
+    /* buffer representation of proposal */ Buffer
+  >;
   private proposalsForSlot: AztecAsyncMultiMap<string, string>;
   private attestationsForProposal: AztecAsyncMultiMap<string, string>;
 
@@ -21,10 +25,24 @@ export class KvAttestationPool implements AttestationPool {
     private log = createLogger('aztec:attestation_pool'),
   ) {
     this.attestations = store.openMap('attestations');
+    this.proposals = store.openMap('proposals');
     this.proposalsForSlot = store.openMultiMap('proposals_for_slot');
     this.attestationsForProposal = store.openMultiMap('attestations_for_proposal');
 
-    this.metrics = new PoolInstrumentation(telemetry, PoolName.ATTESTATION_POOL);
+    this.metrics = new PoolInstrumentation(telemetry, PoolName.ATTESTATION_POOL, this.poolStats);
+  }
+
+  private poolStats: PoolStatsCallback = async () => {
+    return {
+      itemCount: await this.attestations.sizeAsync(),
+    };
+  };
+
+  public async isEmpty(): Promise<boolean> {
+    for await (const _ of this.attestations.entriesAsync()) {
+      return false;
+    }
+    return true;
   }
 
   private getProposalKey(slot: number | bigint | Fr | string, proposalId: Fr | string | Buffer): string {
@@ -33,8 +51,8 @@ export class KvAttestationPool implements AttestationPool {
       typeof proposalId === 'string'
         ? proposalId
         : Buffer.isBuffer(proposalId)
-        ? Fr.fromBuffer(proposalId).toString()
-        : proposalId.toString();
+          ? Fr.fromBuffer(proposalId).toString()
+          : proposalId.toString();
 
     return `${slotStr}-${proposalIdStr}`;
   }
@@ -46,9 +64,21 @@ export class KvAttestationPool implements AttestationPool {
   public async addAttestations(attestations: BlockAttestation[]): Promise<void> {
     await this.store.transactionAsync(async () => {
       for (const attestation of attestations) {
-        const slotNumber = attestation.payload.header.globalVariables.slotNumber;
+        const slotNumber = attestation.payload.header.slotNumber;
         const proposalId = attestation.archive;
-        const address = (await attestation.getSender()).toString();
+        const sender = attestation.getSender();
+
+        // Skip attestations with invalid signatures
+        if (!sender) {
+          this.log.warn(`Skipping attestation with invalid signature for slot ${slotNumber.toBigInt()}`, {
+            signature: attestation.signature.toString(),
+            slotNumber,
+            proposalId,
+          });
+          continue;
+        }
+
+        const address = sender.toString();
 
         await this.attestations.set(this.getAttestationKey(slotNumber, proposalId, address), attestation.toBuffer());
 
@@ -58,14 +88,29 @@ export class KvAttestationPool implements AttestationPool {
           this.getAttestationKey(slotNumber, proposalId, address),
         );
 
-        this.log.verbose(`Added attestation for slot ${slotNumber} from ${address}`);
+        this.log.verbose(`Added attestation for slot ${slotNumber.toBigInt()} from ${address}`, {
+          signature: attestation.signature.toString(),
+          slotNumber,
+          address,
+          proposalId,
+        });
       }
     });
+  }
+
+  public async getAttestationsForSlot(slot: bigint): Promise<BlockAttestation[]> {
+    const slotFr = new Fr(slot);
+    const proposalIds = await toArray(this.proposalsForSlot.getValuesAsync(slotFr.toString()));
+    const attestations: BlockAttestation[] = [];
+
+    for (const proposalId of proposalIds) {
+      attestations.push(...(await this.getAttestationsForSlotAndProposal(slot, proposalId)));
+    }
 
-    this.metrics.recordAddedObjects(attestations.length);
+    return attestations;
   }
 
-  public async getAttestationsForSlot(slot: bigint, proposalId: string): Promise<BlockAttestation[]> {
+  public async getAttestationsForSlotAndProposal(slot: bigint, proposalId: string): Promise<BlockAttestation[]> {
     const attestationIds = await toArray(
       this.attestationsForProposal.getValuesAsync(this.getProposalKey(slot, proposalId)),
     );
@@ -109,12 +154,12 @@ export class KvAttestationPool implements AttestationPool {
           await this.attestations.delete(attestation);
         }
 
+        await this.proposals.delete(proposalId);
         await this.attestationsForProposal.delete(this.getProposalKey(slotFr, proposalId));
       }
-    });
 
-    this.log.verbose(`Removed ${numberOfAttestations} attestations for slot ${slot}`);
-    this.metrics.recordRemovedObjects(numberOfAttestations);
+      this.log.verbose(`Removed ${numberOfAttestations} attestations for slot ${slot}`);
+    });
   }
 
   public async deleteAttestationsForSlotAndProposal(slot: bigint, proposalId: string): Promise<void> {
@@ -130,22 +175,34 @@ export class KvAttestationPool implements AttestationPool {
         await this.attestations.delete(attestation);
       }
 
+      await this.proposals.delete(proposalId);
       await this.proposalsForSlot.deleteValue(slotString, proposalId);
       await this.attestationsForProposal.delete(this.getProposalKey(slotString, proposalId));
-    });
 
-    this.log.verbose(`Removed ${numberOfAttestations} attestations for slot ${slot} and proposal ${proposalId}`);
-    this.metrics.recordRemovedObjects(numberOfAttestations);
+      this.log.verbose(`Removed ${numberOfAttestations} attestations for slot ${slot} and proposal ${proposalId}`);
+    });
   }
 
   public async deleteAttestations(attestations: BlockAttestation[]): Promise<void> {
     await this.store.transactionAsync(async () => {
       for (const attestation of attestations) {
-        const slotNumber = attestation.payload.header.globalVariables.slotNumber;
+        const slotNumber = attestation.payload.header.slotNumber;
         const proposalId = attestation.archive;
-        const address = (await attestation.getSender()).toString();
+        const sender = attestation.getSender();
+
+        // Skip attestations with invalid signatures
+        if (!sender) {
+          this.log.warn(`Skipping deletion of attestation with invalid signature for slot ${slotNumber.toBigInt()}`);
+          continue;
+        }
+
+        const address = sender.toString();
+        const key = this.getAttestationKey(slotNumber, proposalId, address);
+
+        if (await this.attestations.hasAsync(key)) {
+          await this.attestations.delete(key);
+        }
 
-        await this.attestations.delete(this.getAttestationKey(slotNumber, proposalId, address));
         await this.attestationsForProposal.deleteValue(
           this.getProposalKey(slotNumber, proposalId),
           this.getAttestationKey(slotNumber, proposalId, address),
@@ -154,6 +211,46 @@ export class KvAttestationPool implements AttestationPool {
         this.log.debug(`Deleted attestation for slot ${slotNumber} from ${address}`);
       }
     });
-    this.metrics.recordRemovedObjects(attestations.length);
+  }
+
+  public async hasAttestation(attestation: BlockAttestation): Promise<boolean> {
+    const slotNumber = attestation.payload.header.slotNumber;
+    const proposalId = attestation.archive;
+    const sender = attestation.getSender();
+
+    // Attestations with invalid signatures are never in the pool
+    if (!sender) {
+      return false;
+    }
+
+    const address = sender.toString();
+    const key = this.getAttestationKey(slotNumber, proposalId, address);
+
+    return await this.attestations.hasAsync(key);
+  }
+
+  public async getBlockProposal(id: string): Promise<BlockProposal | undefined> {
+    const buffer = await this.proposals.getAsync(id);
+    try {
+      if (buffer && buffer.length > 0) {
+        return BlockProposal.fromBuffer(buffer);
+      }
+    } catch {
+      return Promise.resolve(undefined);
+    }
+
+    return Promise.resolve(undefined);
+  }
+
+  public async hasBlockProposal(idOrProposal: string | BlockProposal): Promise<boolean> {
+    const id = typeof idOrProposal === 'string' ? idOrProposal : idOrProposal.payload.archive.toString();
+    return await this.proposals.hasAsync(id);
+  }
+
+  public async addBlockProposal(blockProposal: BlockProposal): Promise<void> {
+    await this.store.transactionAsync(async () => {
+      await this.proposalsForSlot.set(blockProposal.slotNumber.toString(), blockProposal.archive.toString());
+      await this.proposals.set(blockProposal.payload.archive.toString(), blockProposal.toBuffer());
+    });
   }
 }

package/src/mem_pools/attestation_pool/memory_attestation_pool.ts

@@ -1,21 +1,44 @@
 import { createLogger } from '@aztec/foundation/log';
-import type { BlockAttestation } from '@aztec/stdlib/p2p';
+import type { BlockAttestation, BlockProposal } from '@aztec/stdlib/p2p';
 import { type TelemetryClient, getTelemetryClient } from '@aztec/telemetry-client';
 
-import { PoolInstrumentation, PoolName } from '../instrumentation.js';
+import { PoolInstrumentation, PoolName, type PoolStatsCallback } from '../instrumentation.js';
 import type { AttestationPool } from './attestation_pool.js';
 
 export class InMemoryAttestationPool implements AttestationPool {
   private metrics: PoolInstrumentation<BlockAttestation>;
 
   private attestations: Map</*slot=*/ bigint, Map</*proposalId*/ string, Map</*address=*/ string, BlockAttestation>>>;
+  private proposals: Map<string, BlockProposal>;
 
-  constructor(telemetry: TelemetryClient = getTelemetryClient(), private log = createLogger('p2p:attestation_pool')) {
+  constructor(
+    telemetry: TelemetryClient = getTelemetryClient(),
+    private log = createLogger('p2p:attestation_pool'),
+  ) {
     this.attestations = new Map();
-    this.metrics = new PoolInstrumentation(telemetry, PoolName.ATTESTATION_POOL);
+    this.proposals = new Map();
+    this.metrics = new PoolInstrumentation(telemetry, PoolName.ATTESTATION_POOL, this.poolStats);
   }
 
-  public getAttestationsForSlot(slot: bigint, proposalId: string): Promise<BlockAttestation[]> {
+  private poolStats: PoolStatsCallback = () => {
+    return Promise.resolve({
+      itemCount: this.attestations.size,
+    });
+  };
+
+  public isEmpty(): Promise<boolean> {
+    return Promise.resolve(this.attestations.size === 0);
+  }
+
+  public getAttestationsForSlot(slot: bigint): Promise<BlockAttestation[]> {
+    return Promise.resolve(
+      Array.from(this.attestations.get(slot)?.values() ?? []).flatMap(proposalAttestationMap =>
+        Array.from(proposalAttestationMap.values()),
+      ),
+    );
+  }
+
+  public getAttestationsForSlotAndProposal(slot: bigint, proposalId: string): Promise<BlockAttestation[]> {
     const slotAttestationMap = this.attestations.get(slot);
     if (slotAttestationMap) {
       const proposalAttestationMap = slotAttestationMap.get(proposalId);
@@ -26,23 +49,36 @@ export class InMemoryAttestationPool implements AttestationPool {
     return Promise.resolve([]);
   }
 
-  public async addAttestations(attestations: BlockAttestation[]): Promise<void> {
+  public addAttestations(attestations: BlockAttestation[]): Promise<void> {
     for (const attestation of attestations) {
       // Perf: order and group by slot before insertion
-      const slotNumber = attestation.payload.header.globalVariables.slotNumber;
+      const slotNumber = attestation.payload.header.slotNumber;
 
       const proposalId = attestation.archive.toString();
-      const address = await attestation.getSender();
+      const sender = attestation.getSender();
+
+      // Skip attestations with invalid signatures
+      if (!sender) {
+        this.log.warn(`Skipping attestation with invalid signature for slot ${slotNumber.toBigInt()}`, {
+          signature: attestation.signature.toString(),
+          slotNumber,
+          proposalId,
+        });
+        continue;
+      }
 
       const slotAttestationMap = getSlotOrDefault(this.attestations, slotNumber.toBigInt());
       const proposalAttestationMap = getProposalOrDefault(slotAttestationMap, proposalId);
-      proposalAttestationMap.set(address.toString(), attestation);
-
-      this.log.verbose(`Added attestation for slot ${slotNumber} from ${address}`);
+      proposalAttestationMap.set(sender.toString(), attestation);
+
+      this.log.verbose(`Added attestation for slot ${slotNumber.toBigInt()} from ${sender}`, {
+        signature: attestation.signature.toString(),
+        slotNumber,
+        address: sender,
+        proposalId,
+      });
     }
 
-    // TODO: set these to pending or something ????
-    this.metrics.recordAddedObjects(attestations.length);
     return Promise.resolve();
   }
 
@@ -82,11 +118,18 @@ export class InMemoryAttestationPool implements AttestationPool {
   public deleteAttestationsForSlot(slot: bigint): Promise<void> {
     // We count the number of attestations we are removing
     const numberOfAttestations = this.#getNumberOfAttestationsInSlot(slot);
+    const proposalIdsToDelete = this.attestations.get(slot)?.keys();
+    let proposalIdsToDeleteCount = 0;
+    proposalIdsToDelete?.forEach(proposalId => {
+      this.proposals.delete(proposalId);
+      proposalIdsToDeleteCount++;
+    });
 
     this.attestations.delete(slot);
-    this.log.verbose(`Removed ${numberOfAttestations} attestations for slot ${slot}`);
+    this.log.verbose(
+      `Removed ${numberOfAttestations} attestations and ${proposalIdsToDeleteCount} proposals for slot ${slot}`,
+    );
 
-    this.metrics.recordRemovedObjects(numberOfAttestations);
     return Promise.resolve();
   }
 
@@ -99,29 +142,78 @@ export class InMemoryAttestationPool implements AttestationPool {
         slotAttestationMap.delete(proposalId);
 
         this.log.verbose(`Removed ${numberOfAttestations} attestations for slot ${slot} and proposal ${proposalId}`);
-        this.metrics.recordRemovedObjects(numberOfAttestations);
       }
     }
+
+    this.proposals.delete(proposalId);
     return Promise.resolve();
   }
 
-  public async deleteAttestations(attestations: BlockAttestation[]): Promise<void> {
+  public deleteAttestations(attestations: BlockAttestation[]): Promise<void> {
     for (const attestation of attestations) {
-      const slotNumber = attestation.payload.header.globalVariables.slotNumber;
+      const slotNumber = attestation.payload.header.slotNumber;
       const slotAttestationMap = this.attestations.get(slotNumber.toBigInt());
       if (slotAttestationMap) {
         const proposalId = attestation.archive.toString();
         const proposalAttestationMap = getProposalOrDefault(slotAttestationMap, proposalId);
         if (proposalAttestationMap) {
-          const address = await attestation.getSender();
-          proposalAttestationMap.delete(address.toString());
-          this.log.debug(`Deleted attestation for slot ${slotNumber} from ${address}`);
+          const sender = attestation.getSender();
+
+          // Skip attestations with invalid signatures
+          if (!sender) {
+            this.log.warn(`Skipping deletion of attestation with invalid signature for slot ${slotNumber.toBigInt()}`);
+            continue;
+          }
+
+          proposalAttestationMap.delete(sender.toString());
+          this.log.debug(`Deleted attestation for slot ${slotNumber} from ${sender}`);
         }
       }
     }
-    this.metrics.recordRemovedObjects(attestations.length);
     return Promise.resolve();
   }
+
+  public hasAttestation(attestation: BlockAttestation): Promise<boolean> {
+    const slotNumber = attestation.payload.header.slotNumber;
+    const proposalId = attestation.archive.toString();
+    const sender = attestation.getSender();
+
+    // Attestations with invalid signatures are never in the pool
+    if (!sender) {
+      return Promise.resolve(false);
+    }
+
+    const slotAttestationMap = this.attestations.get(slotNumber.toBigInt());
+    if (!slotAttestationMap) {
+      return Promise.resolve(false);
+    }
+
+    const proposalAttestationMap = slotAttestationMap.get(proposalId);
+    if (!proposalAttestationMap) {
+      return Promise.resolve(false);
+    }
+
+    return Promise.resolve(proposalAttestationMap.has(sender.toString()));
+  }
+
+  public addBlockProposal(blockProposal: BlockProposal): Promise<void> {
+    // We initialize slot-proposal mapping if it does not exist
+    // This is important to ensure we can delete this proposal if there were not attestations for it
+    const slotProposalMapping = getSlotOrDefault(this.attestations, blockProposal.slotNumber.toBigInt());
+    slotProposalMapping.set(blockProposal.payload.archive.toString(), new Map<string, BlockAttestation>());
+
+    this.proposals.set(blockProposal.payload.archive.toString(), blockProposal);
+    return Promise.resolve();
+  }
+
+  public getBlockProposal(id: string): Promise<BlockProposal | undefined> {
+    return Promise.resolve(this.proposals.get(id));
+  }
+
+  public hasBlockProposal(idOrProposal: string | BlockProposal): Promise<boolean> {
+    const id = typeof idOrProposal === 'string' ? idOrProposal : idOrProposal.payload.archive.toString();
+    return Promise.resolve(this.proposals.has(id));
+  }
 }
 
 /**

package/src/mem_pools/attestation_pool/mocks.ts

@@ -6,8 +6,7 @@ import {
   SignatureDomainSeparator,
   getHashedSignaturePayloadEthSignedMessage,
 } from '@aztec/stdlib/p2p';
-import { makeHeader } from '@aztec/stdlib/testing';
-import { TxHash } from '@aztec/stdlib/tx';
+import { makeL2BlockHeader } from '@aztec/stdlib/testing';
 
 import { type LocalAccount, generatePrivateKey, privateKeyToAccount } from 'viem/accounts';
 
@@ -27,18 +26,20 @@ export const generateAccount = (): LocalAccount => {
  * @param slot The slot number the attestation is for
  * @returns A Block Attestation
  */
-export const mockAttestation = async (
+export const mockAttestation = (
   signer: Secp256k1Signer,
   slot: number = 0,
   archive: Fr = Fr.random(),
-  txs: TxHash[] = [0, 1, 2, 3, 4, 5].map(() => TxHash.random()),
-): Promise<BlockAttestation> => {
+): BlockAttestation => {
   // Use arbitrary numbers for all other than slot
-  const header = makeHeader(1, 2, slot);
-  const payload = new ConsensusPayload(header, archive, txs);
+  const header = makeL2BlockHeader(1, 2, slot);
+  const payload = new ConsensusPayload(header.toCheckpointHeader(), archive, header.state);
 
-  const hash = await getHashedSignaturePayloadEthSignedMessage(payload, SignatureDomainSeparator.blockAttestation);
-  const signature = signer.sign(hash);
+  const attestationHash = getHashedSignaturePayloadEthSignedMessage(payload, SignatureDomainSeparator.blockAttestation);
+  const attestationSignature = signer.sign(attestationHash);
 
-  return new BlockAttestation(payload, signature);
+  const proposalHash = getHashedSignaturePayloadEthSignedMessage(payload, SignatureDomainSeparator.blockProposal);
+  const proposerSignature = signer.sign(proposalHash);
+
+  return new BlockAttestation(payload, attestationSignature, proposerSignature);
 };

package/src/mem_pools/instrumentation.ts

@@ -1,13 +1,15 @@
 import type { Gossipable } from '@aztec/stdlib/p2p';
 import {
   Attributes,
+  type BatchObservableResult,
   type Histogram,
   LmdbMetrics,
   type LmdbStatsCallback,
+  type Meter,
   Metrics,
   type MetricsType,
+  type ObservableGauge,
   type TelemetryClient,
-  type UpDownCounter,
 } from '@aztec/telemetry-client';
 
 export enum PoolName {
@@ -41,86 +43,70 @@ function getMetricsLabels(name: PoolName): MetricsLabels {
   throw new Error('Invalid pool type');
 }
 
+export type PoolStatsCallback = () => Promise<{
+  itemCount: number | Record<string, number>;
+}>;
+
 /**
  * Instrumentation class for the Pools (TxPool, AttestationPool, etc).
  */
 export class PoolInstrumentation<PoolObject extends Gossipable> {
   /** The number of txs in the mempool */
-  private objectsInMempool: UpDownCounter;
+  private objectsInMempool: ObservableGauge;
   /** Tracks tx size */
   private objectSize: Histogram;
 
   private dbMetrics: LmdbMetrics;
 
   private defaultAttributes;
-
-  constructor(telemetry: TelemetryClient, name: PoolName, dbStats?: LmdbStatsCallback) {
-    const meter = telemetry.getMeter(name);
+  private meter: Meter;
+
+  constructor(
+    telemetry: TelemetryClient,
+    name: PoolName,
+    private poolStats: PoolStatsCallback,
+    dbStats?: LmdbStatsCallback,
+  ) {
+    this.meter = telemetry.getMeter(name);
     this.defaultAttributes = { [Attributes.POOL_NAME]: name };
 
     const metricsLabels = getMetricsLabels(name);
 
-    this.objectsInMempool = meter.createUpDownCounter(metricsLabels.objectInMempool, {
+    this.objectsInMempool = this.meter.createObservableGauge(metricsLabels.objectInMempool, {
       description: 'The current number of transactions in the mempool',
     });
 
-    this.objectSize = meter.createHistogram(metricsLabels.objectSize, {
+    this.objectSize = this.meter.createHistogram(metricsLabels.objectSize, {
       unit: 'By',
       description: 'The size of transactions in the mempool',
     });
 
     this.dbMetrics = new LmdbMetrics(
-      meter,
+      this.meter,
       {
         [Attributes.DB_DATA_TYPE]: 'tx-pool',
       },
       dbStats,
     );
+
+    this.meter.addBatchObservableCallback(this.observeStats, [this.objectsInMempool]);
   }
 
   public recordSize(poolObject: PoolObject) {
     this.objectSize.record(poolObject.getSize());
   }
 
-  /**
-   * Updates the metrics with the new objects.
-   * @param txs - The objects to record
-   */
-  public recordAddedObjects(count = 1, status?: string) {
-    if (count < 0) {
-      throw new Error('Count must be positive');
-    }
-    if (count === 0) {
-      return;
-    }
-    const attributes = status
-      ? {
+  private observeStats = async (observer: BatchObservableResult) => {
+    const { itemCount } = await this.poolStats();
+    if (typeof itemCount === 'number') {
+      observer.observe(this.objectsInMempool, itemCount, this.defaultAttributes);
+    } else {
+      for (const [status, count] of Object.entries(itemCount)) {
+        observer.observe(this.objectsInMempool, count, {
           ...this.defaultAttributes,
           [Attributes.STATUS]: status,
-        }
-      : this.defaultAttributes;
-
-    this.objectsInMempool.add(count, attributes);
-  }
-
-  /**
-   * Updates the metrics by removing objects from the mempool.
-   * @param count - The number of objects to remove from the mempool
-   */
-  public recordRemovedObjects(count = 1, status?: string) {
-    if (count < 0) {
-      throw new Error('Count must be positive');
-    }
-    if (count === 0) {
-      return;
+        });
+      }
     }
-
-    const attributes = status
-      ? {
-          ...this.defaultAttributes,
-          [Attributes.STATUS]: status,
-        }
-      : this.defaultAttributes;
-    this.objectsInMempool.add(-1 * count, attributes);
-  }
+  };
 }