@aztec/p2p 0.0.1-commit.96bb3f7 → 0.0.1-commit.993d240

package/dest/services/tx_collection/slow_tx_collection.js

@@ -1,177 +0,0 @@
-import { EpochNumber } from '@aztec/foundation/branded-types';
-import { chunk } from '@aztec/foundation/collection';
-import { createLogger } from '@aztec/foundation/log';
-import { boundInclusive } from '@aztec/foundation/number';
-import { RunningPromise } from '@aztec/foundation/promise';
-import { DateProvider } from '@aztec/foundation/timer';
-import { getEpochAtSlot, getTimestampRangeForEpoch } from '@aztec/stdlib/epoch-helpers';
-import { TxHash } from '@aztec/stdlib/tx';
-import { ReqRespSubProtocol } from '../reqresp/interface.js';
-import { chunkTxHashesRequest } from '../reqresp/protocols/tx.js';
-export class SlowTxCollection {
-    reqResp;
-    nodes;
-    txCollectionSink;
-    fastCollection;
-    constants;
-    config;
-    dateProvider;
-    log;
-    /** Map from tx hash to missing tx info to collect via slow loop */ missingTxs;
-    /** One slow collection loop for each node tx source we have */ nodesSlowCollectionLoops;
-    /** Slow collection loop for reqresp which collects txs from peers */ reqrespSlowCollectionLoop;
-    constructor(reqResp, nodes, txCollectionSink, fastCollection, constants, config, dateProvider = new DateProvider(), log = createLogger('p2p:tx_collection_service')){
-        this.reqResp = reqResp;
-        this.nodes = nodes;
-        this.txCollectionSink = txCollectionSink;
-        this.fastCollection = fastCollection;
-        this.constants = constants;
-        this.config = config;
-        this.dateProvider = dateProvider;
-        this.log = log;
-        this.missingTxs = new Map();
-        this.nodesSlowCollectionLoops = this.nodes.map((node)=>new RunningPromise(()=>this.collectMissingTxsFromNode(node), this.log, this.config.txCollectionSlowNodesIntervalMs));
-        this.reqrespSlowCollectionLoop = new RunningPromise(()=>this.collectMissingTxsViaReqResp(), this.log, this.config.txCollectionSlowReqRespIntervalMs);
-    }
-    getMissingTxHashes() {
-        return Array.from(this.missingTxs.keys()).map(TxHash.fromString);
-    }
-    start() {
-        this.nodesSlowCollectionLoops.forEach((loop)=>loop.start());
-        this.reqrespSlowCollectionLoop.start();
-    }
-    async stop() {
-        await Promise.all([
-            this.reqrespSlowCollectionLoop.stop(),
-            ...this.nodesSlowCollectionLoops.map((loop)=>loop.stop())
-        ]);
-    }
-    async trigger() {
-        await Promise.all([
-            this.reqrespSlowCollectionLoop.trigger(),
-            ...this.nodesSlowCollectionLoops.map((loop)=>loop.trigger())
-        ]);
-    }
-    /** Starts collecting the given tx hashes for the given L2Block in the slow loop */ startCollecting(block, txHashes) {
-        const slot = block.header.getSlot();
-        const deadline = this.getDeadlineForSlot(slot);
-        if (+deadline < this.dateProvider.now()) {
-            this.log.debug(`Skipping collection of txs for block ${block.number} at slot ${slot} as it is already expired`, {
-                blockNumber: block.number,
-                slot: slot.toString(),
-                txHashes: txHashes.map((txHash)=>txHash.toString()),
-                deadline: +deadline,
-                now: this.dateProvider.now()
-            });
-        }
-        for (const txHash of txHashes){
-            this.missingTxs.set(txHash.toString(), {
-                blockNumber: block.number,
-                deadline: this.getDeadlineForSlot(block.header.getSlot()),
-                readyForReqResp: this.nodes.length === 0
-            });
-        }
-    }
-    /** Entrypoint for the node slow collection loop */ async collectMissingTxsFromNode(node) {
-        // If we have any fast requests, we skip the slow collection for this node if configured to do so
-        const requests = this.fastCollection.getFastCollectionRequests();
-        if (this.config.txCollectionDisableSlowDuringFastRequests && requests.size > 0) {
-            this.log.trace(`Skipping node slow collection due to active fast requests`);
-            return;
-        }
-        // Gather all missing txs that are not in fast collection and request them from the node
-        const missingTxs = this.getMissingTxsForSlowCollection();
-        const missingTxHashes = missingTxs.map(([txHash])=>txHash).map(TxHash.fromString);
-        if (missingTxHashes.length === 0) {
-            return;
-        }
-        // Request in chunks to avoid hitting RPC limits
-        for (const batch of chunk(missingTxHashes, this.config.txCollectionNodeRpcMaxBatchSize)){
-            await this.txCollectionSink.collect((txHashes)=>node.getTxsByHash(txHashes), batch, {
-                description: `node ${node.getInfo()}`,
-                node: node.getInfo(),
-                method: 'slow-node-rpc'
-            });
-        }
-        // Mark every tx that is still missing as ready for reqresp.
-        // Note that we can just mark all requested txs as ready for reqresp, without filtering out the ones
-        // we retrieved, since the ones already found will have been removed from `missingTxs`.
-        missingTxs.forEach(([_txHash, info])=>{
-            info.readyForReqResp = true;
-        });
-    }
-    /** Entrypoint for the reqresp slow collection loop */ async collectMissingTxsViaReqResp() {
-        // If we have any fast requests, we skip the slow collection if configured to do so
-        const requests = this.fastCollection.getFastCollectionRequests();
-        if (this.config.txCollectionDisableSlowDuringFastRequests && requests.size > 0) {
-            this.log.trace(`Skipping reqresp slow collection due to active fast requests`);
-            return;
-        }
-        // Gather all missing txs that are not in fast collection and are ready for reqresp
-        // A tx is flagged as ready for reqresp if it has been requested from a node at least once
-        const missingTxs = this.getMissingTxsForSlowCollection({
-            onlyReqRespReady: true
-        });
-        if (missingTxs.length === 0) {
-            return;
-        }
-        const pinnedPeer = undefined;
-        const timeoutMs = this.config.txCollectionSlowReqRespTimeoutMs;
-        const maxPeers = boundInclusive(Math.ceil(missingTxs.length / 3), 4, 16);
-        const maxRetryAttempts = 3;
-        // Send a batch request via reqresp for the missing txs
-        await this.txCollectionSink.collect(async (txHashes)=>{
-            const txs = await this.reqResp.sendBatchRequest(ReqRespSubProtocol.TX, chunkTxHashesRequest(txHashes), pinnedPeer, timeoutMs, maxPeers, maxRetryAttempts);
-            return txs.flat();
-        }, missingTxs.map(([txHash])=>TxHash.fromString(txHash)), {
-            description: 'slow reqresp',
-            timeoutMs,
-            method: 'slow-req-resp'
-        });
-    }
-    /** Retrieves all missing txs for the slow collection process. This is, all missing txs that are not part of a fast request. */ getMissingTxsForSlowCollection(opts = {}) {
-        // Remove expired txs from missingTxs
-        const now = this.dateProvider.now();
-        const expiredTxs = Array.from(this.missingTxs.entries()).filter(([_, value])=>+value.deadline < now);
-        expiredTxs.forEach(([txHash])=>this.missingTxs.delete(txHash));
-        // Gather all txs that are marked for fast collection, we do not want to collect them via slow collection.
-        // There are some situations where a tx is in both slow and fast collection, for example when a prover node
-        // is fast-collecting missing txs for proving an epoch, and still has the tx in the slow collection loops
-        // from mined unproven blocks it has seen in the past.
-        const fastRequests = this.fastCollection.getFastCollectionRequests();
-        const fastCollectionTxs = new Set(...Array.from(fastRequests.values()).flatMap((r)=>r.missingTxHashes));
-        // Return all missing txs that are not in fastCollectionTxs and are ready for reqresp if requested
-        return Array.from(this.missingTxs.entries()).filter(([txHash, _])=>!fastCollectionTxs.has(txHash)).filter(([_, value])=>!opts.onlyReqRespReady || value.readyForReqResp);
-    }
-    /** Stop collecting the given txs since we have found them. Called whenever tx pool emits a tx-added event. */ foundTxs(txs) {
-        for (const txHash of txs.map((tx)=>tx.txHash)){
-            this.missingTxs.delete(txHash.toString());
-        }
-    }
-    /**
-   * Stop collecting all txs for blocks less than or requal to the block number specified.
-   * To be called when we no longer care about gathering txs up to a certain block, eg when they become proven or finalized.
-   */ stopCollectingForBlocksUpTo(blockNumber) {
-        for (const [txHash, info] of this.missingTxs.entries()){
-            if (info.blockNumber <= blockNumber) {
-                this.missingTxs.delete(txHash);
-            }
-        }
-    }
-    /**
-   * Stop collecting all txs for blocks greater than the block number specified.
-   * To be called when there is a chain prune and previously mined txs are no longer relevant.
-   */ stopCollectingForBlocksAfter(blockNumber) {
-        for (const [txHash, info] of this.missingTxs.entries()){
-            if (info.blockNumber > blockNumber) {
-                this.missingTxs.delete(txHash);
-            }
-        }
-    }
-    /** Computes the proof submission deadline for a given slot, a tx mined in this slot is no longer interesting after this deadline */ getDeadlineForSlot(slotNumber) {
-        const epoch = getEpochAtSlot(slotNumber, this.constants);
-        const submissionEndEpoch = EpochNumber(epoch + this.constants.proofSubmissionEpochs);
-        const submissionEndTimestamp = getTimestampRangeForEpoch(submissionEndEpoch, this.constants)[1];
-        return new Date(Number(submissionEndTimestamp) * 1000);
-    }
-}

package/src/mem_pools/attestation_pool/kv_attestation_pool.ts

@@ -1,320 +0,0 @@
-import { SlotNumber } from '@aztec/foundation/branded-types';
-import { Fr } from '@aztec/foundation/curves/bn254';
-import { toArray } from '@aztec/foundation/iterable';
-import { createLogger } from '@aztec/foundation/log';
-import type { AztecAsyncKVStore, AztecAsyncMap, AztecAsyncMultiMap } from '@aztec/kv-store';
-import {
-  BlockProposal,
-  CheckpointAttestation,
-  CheckpointProposal,
-  type CheckpointProposalCore,
-} from '@aztec/stdlib/p2p';
-import { type TelemetryClient, getTelemetryClient } from '@aztec/telemetry-client';
-
-import { ProposalSlotCapExceededError } from '../../errors/attestation-pool.error.js';
-import { PoolInstrumentation, PoolName, type PoolStatsCallback } from '../instrumentation.js';
-import type { AttestationPool } from './attestation_pool.js';
-
-export const MAX_PROPOSALS_PER_SLOT = 5;
-export const ATTESTATION_CAP_BUFFER = 10;
-
-export class KvAttestationPool implements AttestationPool {
-  private metrics: PoolInstrumentation<CheckpointAttestation>;
-
-  private proposals: AztecAsyncMap<
-    /*  proposal.payload.archive  */ string,
-    /* buffer representation of proposal */ Buffer
-  >;
-
-  // Checkpoint attestation storage
-  private checkpointAttestations: AztecAsyncMap<string, Buffer>;
-  private checkpointProposals: AztecAsyncMap<string, Buffer>;
-  private checkpointProposalsForSlot: AztecAsyncMultiMap<number, string>;
-  private checkpointAttestationsForProposal: AztecAsyncMultiMap<string, string>;
-
-  constructor(
-    private store: AztecAsyncKVStore,
-    telemetry: TelemetryClient = getTelemetryClient(),
-    private log = createLogger('aztec:attestation_pool'),
-  ) {
-    this.proposals = store.openMap('proposals');
-
-    // Initialize checkpoint attestation storage
-    this.checkpointAttestations = store.openMap('checkpoint_attestations');
-    this.checkpointProposals = store.openMap('checkpoint_proposals');
-    this.checkpointProposalsForSlot = store.openMultiMap('checkpoint_proposals_for_slot');
-    this.checkpointAttestationsForProposal = store.openMultiMap('checkpoint_attestations_for_proposal');
-
-    this.metrics = new PoolInstrumentation(telemetry, PoolName.ATTESTATION_POOL, this.poolStats);
-  }
-
-  private poolStats: PoolStatsCallback = async () => {
-    return {
-      itemCount: await this.checkpointAttestations.sizeAsync(),
-    };
-  };
-
-  public async isEmpty(): Promise<boolean> {
-    for await (const _ of this.checkpointAttestations.entriesAsync()) {
-      return false;
-    }
-    for await (const _ of this.proposals.entriesAsync()) {
-      return false;
-    }
-    return true;
-  }
-
-  private getProposalKey(slot: number | bigint | Fr | string, proposalId: Fr | string | Buffer): string {
-    const slotStr = typeof slot === 'string' ? slot : new Fr(slot).toString();
-    const proposalIdStr =
-      typeof proposalId === 'string'
-        ? proposalId
-        : Buffer.isBuffer(proposalId)
-          ? Fr.fromBuffer(proposalId).toString()
-          : proposalId.toString();
-
-    return `${slotStr}-${proposalIdStr}`;
-  }
-
-  private getAttestationKey(slot: number | bigint | Fr | string, proposalId: Fr | string, address: string): string {
-    return `${this.getProposalKey(slot, proposalId)}-${address}`;
-  }
-
-  public async addBlockProposal(blockProposal: BlockProposal): Promise<void> {
-    await this.store.transactionAsync(async () => {
-      const proposalId = blockProposal.archive.toString();
-      // Strip signedTxs before storing to avoid persisting full tx data
-      await this.proposals.set(proposalId, blockProposal.withoutSignedTxs().toBuffer());
-    });
-  }
-
-  public async getBlockProposal(id: string): Promise<BlockProposal | undefined> {
-    const buffer = await this.proposals.getAsync(id);
-    try {
-      if (buffer && buffer.length > 0) {
-        return BlockProposal.fromBuffer(buffer);
-      }
-    } catch {
-      return Promise.resolve(undefined);
-    }
-
-    return Promise.resolve(undefined);
-  }
-
-  public async hasBlockProposal(idOrProposal: string | BlockProposal): Promise<boolean> {
-    const id = typeof idOrProposal === 'string' ? idOrProposal : idOrProposal.archive.toString();
-    return await this.proposals.hasAsync(id);
-  }
-
-  public async addCheckpointProposal(proposal: CheckpointProposal): Promise<void> {
-    if (!(await this.canAddCheckpointProposal(proposal))) {
-      throw new ProposalSlotCapExceededError(
-        `Maximum checkpoint proposals per slot reached: slot=${proposal.slotNumber} cap=${MAX_PROPOSALS_PER_SLOT} proposal=${proposal.archive.toString()}`,
-      );
-    }
-
-    // Extract and validate the block proposal if present
-    const blockProposal = proposal.getBlockProposal();
-    if (blockProposal && !(await this.canAddProposal(blockProposal))) {
-      throw new ProposalSlotCapExceededError(
-        `Maximum block proposals per slot reached when extracting from checkpoint: slot=${proposal.slotNumber} proposal=${blockProposal.archive.toString()}`,
-      );
-    }
-
-    await this.store.transactionAsync(async () => {
-      const slotKey = proposal.slotNumber;
-      const proposalId = proposal.archive.toString();
-
-      await this.checkpointProposalsForSlot.set(slotKey, proposalId);
-      // Store the checkpoint proposal as core (without lastBlock) to avoid duplication
-      await this.checkpointProposals.set(proposalId, proposal.toCore().toBuffer());
-
-      // Store the extracted block proposal separately
-      if (blockProposal) {
-        await this.proposals.set(blockProposal.archive.toString(), blockProposal.withoutSignedTxs().toBuffer());
-      }
-    });
-  }
-
-  public async getCheckpointProposal(id: string): Promise<CheckpointProposalCore | undefined> {
-    const buffer = await this.checkpointProposals.getAsync(id);
-    try {
-      if (buffer && buffer.length > 0) {
-        return CheckpointProposal.fromBuffer(buffer);
-      }
-    } catch {
-      return Promise.resolve(undefined);
-    }
-
-    return Promise.resolve(undefined);
-  }
-
-  public async hasCheckpointProposal(idOrProposal: string | CheckpointProposal): Promise<boolean> {
-    const id = typeof idOrProposal === 'string' ? idOrProposal : idOrProposal.archive.toString();
-    return await this.checkpointProposals.hasAsync(id);
-  }
-
-  public async addCheckpointAttestations(attestations: CheckpointAttestation[]): Promise<void> {
-    await this.store.transactionAsync(async () => {
-      for (const attestation of attestations) {
-        const slotNumber = attestation.payload.header.slotNumber;
-        const proposalId = attestation.archive;
-        const sender = attestation.getSender();
-
-        // Skip attestations with invalid signatures
-        if (!sender) {
-          this.log.warn(`Skipping checkpoint attestation with invalid signature for slot ${slotNumber}`, {
-            signature: attestation.signature.toString(),
-            slotNumber,
-            proposalId,
-          });
-          continue;
-        }
-
-        const address = sender.toString();
-
-        await this.checkpointAttestations.set(
-          this.getAttestationKey(slotNumber, proposalId, address),
-          attestation.toBuffer(),
-        );
-
-        await this.checkpointProposalsForSlot.set(slotNumber, proposalId.toString());
-        await this.checkpointAttestationsForProposal.set(
-          this.getProposalKey(slotNumber, proposalId),
-          this.getAttestationKey(slotNumber, proposalId, address),
-        );
-
-        this.log.verbose(`Added checkpoint attestation for slot ${slotNumber} from ${address}`, {
-          signature: attestation.signature.toString(),
-          slotNumber,
-          address,
-          proposalId,
-        });
-      }
-    });
-  }
-
-  public async getCheckpointAttestationsForSlot(slot: SlotNumber): Promise<CheckpointAttestation[]> {
-    const proposalIds = await toArray(this.checkpointProposalsForSlot.getValuesAsync(slot));
-    const attestations: CheckpointAttestation[] = [];
-
-    for (const proposalId of proposalIds) {
-      attestations.push(...(await this.getCheckpointAttestationsForSlotAndProposal(slot, proposalId)));
-    }
-
-    return attestations;
-  }
-
-  public async getCheckpointAttestationsForSlotAndProposal(
-    slot: SlotNumber,
-    proposalId: string,
-  ): Promise<CheckpointAttestation[]> {
-    const attestationIds = await toArray(
-      this.checkpointAttestationsForProposal.getValuesAsync(this.getProposalKey(slot, proposalId)),
-    );
-    const attestations: CheckpointAttestation[] = [];
-
-    for (const id of attestationIds) {
-      const buf = await this.checkpointAttestations.getAsync(id);
-
-      if (!buf) {
-        throw new Error('Checkpoint attestation not found ' + id);
-      }
-
-      const attestation = CheckpointAttestation.fromBuffer(buf);
-      attestations.push(attestation);
-    }
-
-    return attestations;
-  }
-
-  public async deleteCheckpointAttestationsOlderThan(oldestSlot: SlotNumber): Promise<void> {
-    const olderThan = await toArray(this.checkpointProposalsForSlot.keysAsync({ end: oldestSlot }));
-    for (const oldSlot of olderThan) {
-      await this.deleteCheckpointAttestationsForSlot(SlotNumber(oldSlot));
-    }
-  }
-
-  private async deleteCheckpointAttestationsForSlot(slot: SlotNumber): Promise<void> {
-    let numberOfAttestations = 0;
-    await this.store.transactionAsync(async () => {
-      const proposalIds = await toArray(this.checkpointProposalsForSlot.getValuesAsync(slot));
-      for (const proposalId of proposalIds) {
-        const attestations = await toArray(
-          this.checkpointAttestationsForProposal.getValuesAsync(this.getProposalKey(slot, proposalId)),
-        );
-
-        numberOfAttestations += attestations.length;
-        for (const attestation of attestations) {
-          await this.checkpointAttestations.delete(attestation);
-        }
-
-        await this.checkpointProposals.delete(proposalId);
-        await this.checkpointAttestationsForProposal.delete(this.getProposalKey(slot, proposalId));
-      }
-
-      await this.checkpointProposalsForSlot.delete(slot);
-
-      this.log.verbose(`Removed ${numberOfAttestations} checkpoint attestations for slot ${slot}`);
-    });
-  }
-
-  public async hasCheckpointAttestation(attestation: CheckpointAttestation): Promise<boolean> {
-    const slotNumber = attestation.payload.header.slotNumber;
-    const proposalId = attestation.archive;
-    const sender = attestation.getSender();
-
-    // Attestations with invalid signatures are never in the pool
-    if (!sender) {
-      return false;
-    }
-
-    const address = sender.toString();
-    const key = this.getAttestationKey(slotNumber, proposalId, address);
-
-    return await this.checkpointAttestations.hasAsync(key);
-  }
-
-  public canAddProposal(_block: BlockProposal): Promise<boolean> {
-    // TODO(palla/mbps): implement proposal cap logic
-    return Promise.resolve(true);
-  }
-
-  public async canAddCheckpointProposal(proposal: CheckpointProposal): Promise<boolean> {
-    // TODO(palla/mbps): Adjust checkpoint proposal limit to 1. Also connect to slashing condition in the caller.
-    return (
-      (await this.checkpointProposals.hasAsync(proposal.archive.toString())) ||
-      !(await this.hasReachedCheckpointProposalCap(proposal.slotNumber))
-    );
-  }
-
-  public async canAddCheckpointAttestation(
-    attestation: CheckpointAttestation,
-    committeeSize: number,
-  ): Promise<boolean> {
-    return (
-      (await this.hasCheckpointAttestation(attestation)) ||
-      !(await this.hasReachedCheckpointAttestationCap(
-        attestation.payload.header.slotNumber,
-        attestation.archive.toString(),
-        committeeSize,
-      ))
-    );
-  }
-
-  public async hasReachedCheckpointProposalCap(slot: SlotNumber): Promise<boolean> {
-    const uniqueProposalCount = await this.checkpointProposalsForSlot.getValueCountAsync(slot);
-    return uniqueProposalCount >= MAX_PROPOSALS_PER_SLOT;
-  }
-
-  public async hasReachedCheckpointAttestationCap(
-    slot: SlotNumber,
-    proposalId: string,
-    committeeSize: number,
-  ): Promise<boolean> {
-    const limit = committeeSize + ATTESTATION_CAP_BUFFER;
-    return (
-      (await this.checkpointAttestationsForProposal.getValueCountAsync(this.getProposalKey(slot, proposalId))) >= limit
-    );
-  }
-}