@aztec/p2p 0.0.1-commit.f295ac2 → 0.0.1-commit.f8ca9b2f3

package/dest/client/p2p_client.js

@@ -374,6 +374,7 @@ var _dec, _dec1, _initProto;
 import { GENESIS_BLOCK_HEADER_HASH } from '@aztec/constants';
 import { BlockNumber, SlotNumber } from '@aztec/foundation/branded-types';
 import { createLogger } from '@aztec/foundation/log';
+import { RunningPromise } from '@aztec/foundation/promise';
 import { DateProvider } from '@aztec/foundation/timer';
 import { L2TipsKVStore } from '@aztec/kv-store/stores';
 import { L2BlockStream } from '@aztec/stdlib/block';
@@ -401,6 +402,8 @@ _dec = trackSpan('p2pClient.broadcastProposal', async (proposal)=>({
     l2BlockSource;
     p2pService;
     txCollection;
+    txFileStore;
+    epochCache;
     _dateProvider;
     telemetry;
     log;
@@ -433,15 +436,10 @@ _dec = trackSpan('p2pClient.broadcastProposal', async (proposal)=>({
     blockStream;
     txProvider;
     validatorAddresses;
-    /**
-   * In-memory P2P client constructor.
-   * @param store - The client's instance of the KV store.
-   * @param l2BlockSource - P2P client's source for fetching existing blocks.
-   * @param txPool - The client's instance of a transaction pool. Defaults to in-memory implementation.
-   * @param p2pService - The concrete instance of p2p networking to use.
-   * @param log - A logger.
-   */ constructor(_clientType, store, l2BlockSource, mempools, p2pService, txCollection, config = {}, _dateProvider = new DateProvider(), telemetry = getTelemetryClient(), log = createLogger('p2p')){
-        super(telemetry, 'P2PClient'), this.store = store, this.l2BlockSource = l2BlockSource, this.p2pService = p2pService, this.txCollection = txCollection, this._dateProvider = _dateProvider, this.telemetry = telemetry, this.log = log, this.currentState = (_initProto(this), P2PClientState.IDLE), this.syncPromise = Promise.resolve(), this.syncResolve = undefined, this.latestBlockNumberAtStart = -1, this.provenBlockNumberAtStart = -1, this.finalizedBlockNumberAtStart = -1, this.validatorAddresses = [];
+    /** Tracks the last slot for which we called prepareForSlot */ lastSlotProcessed;
+    /** Polls for slot changes and calls prepareForSlot on the tx pool */ slotMonitor;
+    constructor(_clientType, store, l2BlockSource, mempools, p2pService, txCollection, txFileStore, epochCache, config = {}, _dateProvider = new DateProvider(), telemetry = getTelemetryClient(), log = createLogger('p2p')){
+        super(telemetry, 'P2PClient'), this.store = store, this.l2BlockSource = l2BlockSource, this.p2pService = p2pService, this.txCollection = txCollection, this.txFileStore = txFileStore, this.epochCache = epochCache, this._dateProvider = _dateProvider, this.telemetry = telemetry, this.log = log, this.currentState = (_initProto(this), P2PClientState.IDLE), this.syncPromise = Promise.resolve(), this.syncResolve = undefined, this.latestBlockNumberAtStart = -1, this.provenBlockNumberAtStart = -1, this.finalizedBlockNumberAtStart = -1, this.validatorAddresses = [], this.lastSlotProcessed = SlotNumber.ZERO;
         this.config = {
             ...getP2PDefaultConfig(),
             ...config
@@ -496,10 +494,9 @@ _dec = trackSpan('p2pClient.broadcastProposal', async (proposal)=>({
     getL2BlockHash(number) {
         return this.l2Tips.getL2BlockHash(number);
     }
-    updateP2PConfig(config) {
-        this.txPool.updateConfig(config);
+    async updateP2PConfig(config) {
+        await this.txPool.updateConfig(config);
         this.p2pService.updateConfig(config);
-        return Promise.resolve();
     }
     getL2Tips() {
         return this.l2Tips.getL2Tips();
@@ -526,7 +523,7 @@ _dec = trackSpan('p2pClient.broadcastProposal', async (proposal)=>({
                 break;
             case 'chain-pruned':
                 this.txCollection.stopCollectingForBlocksAfter(event.block.number);
-                await this.handlePruneL2Blocks(event.block.number);
+                await this.handlePruneL2Blocks(event.block);
                 break;
             case 'chain-checkpointed':
                 break;
@@ -541,7 +538,6 @@ _dec = trackSpan('p2pClient.broadcastProposal', async (proposal)=>({
         await this.startServiceIfSynched();
     }
     #assertIsReady() {
-        // this.log.info('Checking if p2p client is ready, current state: ', this.currentState);
         if (!this.isReady()) {
             throw new Error('P2P client not ready');
         }
@@ -556,6 +552,8 @@ _dec = trackSpan('p2pClient.broadcastProposal', async (proposal)=>({
         if (this.currentState !== P2PClientState.IDLE) {
             return this.syncPromise;
         }
+        // Start the tx pool first, as it needs to hydrate state from persistence
+        await this.txPool.start();
         // get the current latest block numbers
         const latestBlockNumbers = await this.l2BlockSource.getL2Tips();
         this.latestBlockNumberAtStart = latestBlockNumbers.proposed.number;
@@ -598,6 +596,10 @@ _dec = trackSpan('p2pClient.broadcastProposal', async (proposal)=>({
         }
         this.blockStream.start();
         await this.txCollection.start();
+        this.txFileStore?.start();
+        // Start slot monitor to call prepareForSlot when the slot changes
+        this.slotMonitor = new RunningPromise(()=>this.maybeCallPrepareForSlot(), this.log, this.config.slotCheckIntervalMS);
+        this.slotMonitor.start();
         return this.syncPromise;
     }
     addReqRespSubProtocol(subProtocol, handler, validator) {
@@ -618,12 +620,18 @@ _dec = trackSpan('p2pClient.broadcastProposal', async (proposal)=>({
    * 'ready' will now return 'false' and the running promise that keeps the client synced is interrupted.
    */ async stop() {
         this.log.debug('Stopping p2p client...');
+        await this.slotMonitor?.stop();
+        this.log.debug('Stopped slot monitor');
         await tryStop(this.txCollection);
         this.log.debug('Stopped tx collection service');
+        await this.txFileStore?.stop();
+        this.log.debug('Stopped tx file store');
         await this.p2pService.stop();
         this.log.debug('Stopped p2p service');
         await this.blockStream?.stop();
         this.log.debug('Stopped block downloader');
+        await this.txPool.stop();
+        this.log.debug('Stopped tx pool');
         await this.runningPromise;
         this.setCurrentState(P2PClientState.STOPPED);
         this.log.info('P2P client stopped');
@@ -632,12 +640,30 @@ _dec = trackSpan('p2pClient.broadcastProposal', async (proposal)=>({
         this.initBlockStream();
         await this.blockStream.sync();
     }
-    broadcastProposal(proposal) {
+    async broadcastProposal(proposal) {
         this.log.verbose(`Broadcasting proposal for slot ${proposal.slotNumber} to peers`);
+        // Store our own proposal so we can respond to req/resp requests for it
+        const { count } = await this.attestationPool.tryAddBlockProposal(proposal);
+        if (count > 1) {
+            if (this.config.broadcastEquivocatedProposals) {
+                this.log.warn(`Broadcasting equivocated block proposal for slot ${proposal.slotNumber}`, {
+                    slot: proposal.slotNumber,
+                    archive: proposal.archive.toString(),
+                    count
+                });
+            } else {
+                throw new Error(`Attempted to broadcast a duplicate block proposal for slot ${proposal.slotNumber}`);
+            }
+        }
         return this.p2pService.propagate(proposal);
     }
-    broadcastCheckpointProposal(proposal) {
+    async broadcastCheckpointProposal(proposal) {
         this.log.verbose(`Broadcasting checkpoint proposal for slot ${proposal.slotNumber} to peers`);
+        const blockProposal = proposal.getBlockProposal();
+        if (blockProposal) {
+            // Store our own last-block proposal so we can respond to req/resp requests for it.
+            await this.attestationPool.tryAddBlockProposal(blockProposal);
+        }
         return this.p2pService.propagate(proposal);
     }
     async broadcastCheckpointAttestations(attestations) {
@@ -647,8 +673,8 @@ _dec = trackSpan('p2pClient.broadcastProposal', async (proposal)=>({
     async getCheckpointAttestationsForSlot(slot, proposalId) {
         return await (proposalId ? this.attestationPool.getCheckpointAttestationsForSlotAndProposal(slot, proposalId) : this.attestationPool.getCheckpointAttestationsForSlot(slot));
     }
-    addCheckpointAttestations(attestations) {
-        return this.attestationPool.addCheckpointAttestations(attestations);
+    addOwnCheckpointAttestations(attestations) {
+        return this.attestationPool.addOwnCheckpointAttestations(attestations);
     }
     // REVIEW: https://github.com/AztecProtocol/aztec-packages/issues/7963
     // ^ This pattern is not my favorite (md)
@@ -658,6 +684,12 @@ _dec = trackSpan('p2pClient.broadcastProposal', async (proposal)=>({
     registerCheckpointProposalHandler(handler) {
         this.p2pService.registerCheckpointReceivedCallback(handler);
     }
+    registerDuplicateProposalCallback(callback) {
+        this.p2pService.registerDuplicateProposalCallback(callback);
+    }
+    registerDuplicateAttestationCallback(callback) {
+        this.p2pService.registerDuplicateAttestationCallback(callback);
+    }
     /**
    * Uses the batched Request Response protocol to request a set of transactions from the network.
    */ async requestTxsByHash(txHashes, pinnedPeerId) {
@@ -668,73 +700,41 @@ _dec = trackSpan('p2pClient.broadcastProposal', async (proposal)=>({
         const txBatches = await this.p2pService.sendBatchRequest(ReqRespSubProtocol.TX, requests, pinnedPeerId, timeoutMs, maxPeers, maxRetryAttempts);
         const txs = txBatches.flat();
         if (txs.length > 0) {
-            await this.txPool.addTxs(txs);
+            await this.txPool.addPendingTxs(txs);
         }
         const txHashesStr = txHashes.map((tx)=>tx.toString()).join(', ');
         this.log.debug(`Requested txs ${txHashesStr} (${txs.length} / ${txHashes.length}) from peers`);
         // We return all transactions, even the not found ones to the caller, such they can handle missing items themselves.
         return txs;
     }
-    getPendingTxs(limit, after) {
-        return this.getTxs('pending', limit, after);
-    }
-    getPendingTxCount() {
-        return this.txPool.getPendingTxCount();
-    }
-    async *iteratePendingTxs() {
-        for (const txHash of (await this.txPool.getPendingTxHashes())){
-            const tx = await this.txPool.getTxByHash(txHash);
-            if (tx) {
-                yield tx;
-            }
-        }
-    }
-    /**
-   * Returns all transactions in the transaction pool.
-   * @param filter - The type of txs to return
-   * @param limit - How many txs to return
-   * @param after - If paginating, the last known tx hash. Will return txs after this hash
-   * @returns An array of Txs.
-   */ async getTxs(filter, limit, after) {
+    async getPendingTxs(limit, after) {
         if (limit !== undefined && limit <= 0) {
             throw new TypeError('limit must be greater than 0');
         }
-        let txs = undefined;
-        let txHashes;
-        if (filter === 'all') {
-            txs = await this.txPool.getAllTxs();
-            txHashes = await Promise.all(txs.map((tx)=>tx.getTxHash()));
-        } else if (filter === 'mined') {
-            const minedTxHashes = await this.txPool.getMinedTxHashes();
-            txHashes = minedTxHashes.map(([txHash])=>txHash);
-        } else if (filter === 'pending') {
-            txHashes = await this.txPool.getPendingTxHashes();
-        } else {
-            const _ = filter;
-            throw new Error(`Unknown filter ${filter}`);
-        }
+        let txHashes = await this.txPool.getPendingTxHashes();
         let startIndex = 0;
-        let endIndex = undefined;
         if (after) {
             startIndex = txHashes.findIndex((txHash)=>after.equals(txHash));
-            // if we can't find the last tx in our set then return an empty array as pagination is no longer valid.
             if (startIndex === -1) {
                 return [];
             }
-            // increment by one because we don't want to return the same tx again
             startIndex++;
         }
-        if (limit !== undefined) {
-            endIndex = startIndex + limit;
-        }
+        const endIndex = limit !== undefined ? startIndex + limit : undefined;
         txHashes = txHashes.slice(startIndex, endIndex);
-        if (txs) {
-            txs = txs.slice(startIndex, endIndex);
-        } else {
-            const maybeTxs = await Promise.all(txHashes.map((txHash)=>this.txPool.getTxByHash(txHash)));
-            txs = maybeTxs.filter((tx)=>!!tx);
+        const maybeTxs = await Promise.all(txHashes.map((txHash)=>this.txPool.getTxByHash(txHash)));
+        return maybeTxs.filter((tx)=>!!tx);
+    }
+    getPendingTxCount() {
+        return this.txPool.getPendingTxCount();
+    }
+    async *iteratePendingTxs() {
+        for (const txHash of (await this.txPool.getPendingTxHashes())){
+            const tx = await this.txPool.getTxByHash(txHash);
+            if (tx) {
+                yield tx;
+            }
         }
-        return txs;
     }
     /**
    * Returns a transaction in the transaction pool by its hash.
@@ -797,16 +797,18 @@ _dec = trackSpan('p2pClient.broadcastProposal', async (proposal)=>({
         return this.txPool.getArchivedTxByHash(txHash);
     }
     /**
-   * Verifies the 'tx' and, if valid, adds it to local tx pool and forwards it to other peers.
-   * @param tx - The tx to verify.
+   * Accepts a transaction, adds it to local tx pool and forwards it to other peers.
+   * @param tx - The tx to send.
    * @returns Empty promise.
    **/ async sendTx(tx) {
-        const addedCount = await this.addTxsToPool([
+        this.#assertIsReady();
+        const result = await this.txPool.addPendingTxs([
             tx
         ]);
-        const txAddedSuccessfully = addedCount === 1;
-        if (txAddedSuccessfully) {
+        if (result.accepted.length === 1) {
             await this.p2pService.propagate(tx);
+        } else {
+            this.log.warn(`Tx ${tx.getTxHash()} not propagated: accepted=${result.accepted.length} ignored=${result.ignored.length} rejected=${result.rejected.length}`);
         }
     }
     /**
@@ -814,14 +816,15 @@ _dec = trackSpan('p2pClient.broadcastProposal', async (proposal)=>({
    * @param txs - The transactions.
    **/ async addTxsToPool(txs) {
         this.#assertIsReady();
-        return await this.txPool.addTxs(txs);
+        return (await this.txPool.addPendingTxs(txs)).accepted.length;
     }
     /**
    * Returns whether the given tx hash is flagged as pending or mined.
    * @param txHash - Hash of the tx to query.
    * @returns Pending or mined depending on its status, or undefined if not found.
-   */ getTxStatus(txHash) {
-        return this.txPool.getTxStatus(txHash);
+   */ async getTxStatus(txHash) {
+        const status = await this.txPool.getTxStatus(txHash);
+        return status === 'protected' ? 'pending' : status;
     }
     getEnr() {
         return this.p2pService.getEnr();
@@ -830,13 +833,11 @@ _dec = trackSpan('p2pClient.broadcastProposal', async (proposal)=>({
         return Promise.resolve(this.p2pService.getEnr()?.encodeTxt());
     }
     /**
-   * Deletes the 'txs' from the pool.
-   * NOT used if we use sendTx as reconcileTxPool will handle this.
-   * @param txHashes - Hashes of the transactions to delete.
-   * @returns Empty promise.
-   **/ async deleteTxs(txHashes) {
+   * Handles failed transaction execution by removing txs from the pool.
+   * @param txHashes - Hashes of the transactions that failed execution.
+   **/ async handleFailedExecution(txHashes) {
         this.#assertIsReady();
-        await this.txPool.deleteTxs(txHashes);
+        await this.txPool.handleFailedExecution(txHashes);
     }
     /**
    * Public function to check if the p2p client is fully synced and ready to receive txs.
@@ -880,13 +881,12 @@ _dec = trackSpan('p2pClient.broadcastProposal', async (proposal)=>({
         };
     }
     /**
-   * Mark all txs from these blocks as mined.
+   * Handles mined blocks by marking the txs in them as mined.
    * @param blocks - A list of existing blocks with txs that the P2P client needs to ensure the tx pool is reconciled with.
    * @returns Empty promise.
-   */ async markTxsAsMinedFromBlocks(blocks) {
+   */ async handleMinedBlocks(blocks) {
         for (const block of blocks){
-            const txHashes = block.body.txEffects.map((txEffect)=>txEffect.txHash);
-            await this.txPool.markAsMined(txHashes, block.header);
+            await this.txPool.handleMinedBlock(block);
         }
     }
     /**
@@ -895,14 +895,13 @@ _dec = trackSpan('p2pClient.broadcastProposal', async (proposal)=>({
    * @returns Empty promise.
    */ async handleLatestL2Blocks(blocks) {
         if (!blocks.length) {
-            return Promise.resolve();
+            return;
         }
-        await this.markTxsAsMinedFromBlocks(blocks);
-        await this.txPool.clearNonEvictableTxs();
+        await this.handleMinedBlocks(blocks);
+        await this.maybeCallPrepareForSlot();
         await this.startCollectingMissingTxs(blocks);
         const lastBlock = blocks.at(-1);
         await this.synchedLatestSlot.set(BigInt(lastBlock.header.getSlot()));
-        this.log.verbose(`Synched to latest block ${lastBlock.number}`);
     }
     /** Request txs for unproven blocks so the prover node has more chances to get them. */ async startCollectingMissingTxs(blocks) {
         try {
@@ -936,59 +935,26 @@ _dec = trackSpan('p2pClient.broadcastProposal', async (proposal)=>({
    * @returns Empty promise.
    */ async handleFinalizedL2Blocks(blocks) {
         if (!blocks.length) {
-            return Promise.resolve();
+            return;
         }
-        this.log.debug(`Handling finalized blocks ${blocks.length} up to ${blocks.at(-1)?.number}`);
-        const lastBlockNum = blocks[blocks.length - 1].number;
-        const lastBlockSlot = blocks[blocks.length - 1].header.getSlot();
-        const txHashes = blocks.flatMap((block)=>block.body.txEffects.map((txEffect)=>txEffect.txHash));
-        this.log.debug(`Deleting ${txHashes.length} txs from pool from finalized blocks up to ${lastBlockNum}`);
-        await this.txPool.deleteTxs(txHashes, {
-            permanently: true
-        });
-        await this.txPool.cleanupDeletedMinedTxs(lastBlockNum);
-        await this.attestationPool.deleteCheckpointAttestationsOlderThan(lastBlockSlot);
-        this.log.debug(`Synched to finalized block ${lastBlockNum} at slot ${lastBlockSlot}`);
+        // Finalization is monotonic, so we only need to call with the last block
+        const lastBlock = blocks.at(-1);
+        await this.txPool.handleFinalizedBlock(lastBlock.header);
+        await this.attestationPool.deleteOlderThan(lastBlock.header.getSlot());
     }
     /**
    * Updates the tx pool after a chain prune.
-   * @param latestBlock - The block number the chain was pruned to.
+   * @param latestBlock - The block ID the chain was pruned to.
    */ async handlePruneL2Blocks(latestBlock) {
-        const txsToDelete = new Map();
-        const minedTxs = await this.txPool.getMinedTxHashes();
-        // Find transactions that reference pruned blocks in their historical header
-        for (const tx of (await this.txPool.getAllTxs())){
-            // every tx that's been generated against a block that has now been pruned is no longer valid
-            if (tx.data.constants.anchorBlockHeader.globalVariables.blockNumber > latestBlock) {
-                const txHash = tx.getTxHash();
-                txsToDelete.set(txHash.toString(), txHash);
-            }
-        }
-        this.log.info(`Detected chain prune. Removing ${txsToDelete.size} txs built against pruned blocks.`, {
-            newLatestBlock: latestBlock,
-            previousLatestBlock: await this.getSyncedLatestBlockNum(),
-            txsToDelete: Array.from(txsToDelete.keys())
-        });
-        // delete invalid txs (both pending and mined)
-        await this.txPool.deleteTxs(Array.from(txsToDelete.values()));
-        // everything left in the mined set was built against a block on the proven chain so its still valid
-        // move back to pending the txs that were reorged out of the chain, unless txPoolDeleteTxsAfterReorg is set,
-        // in which case we clean them up to avoid potential reorg loops
-        // NOTE: we can't move _all_ txs back to pending because the tx pool could keep hold of mined txs for longer
-        // (see this.keepProvenTxsFor)
-        const minedTxsFromReorg = [];
-        for (const [txHash, blockNumber] of minedTxs){
-            // We keep the txsToDelete out of this list as they have already been deleted above
-            if (blockNumber > latestBlock && !txsToDelete.has(txHash.toString())) {
-                minedTxsFromReorg.push(txHash);
-            }
-        }
-        if (this.config.txPoolDeleteTxsAfterReorg) {
-            this.log.info(`Deleting ${minedTxsFromReorg.length} mined txs from reorg`);
-            await this.txPool.deleteTxs(minedTxsFromReorg);
-        } else {
-            await this.txPool.markMinedAsPending(minedTxsFromReorg, latestBlock);
+        await this.txPool.handlePrunedBlocks(latestBlock);
+    }
+    /** Checks if the slot has changed and calls prepareForSlot if so. */ async maybeCallPrepareForSlot() {
+        const { currentSlot } = this.epochCache.getCurrentAndNextSlot();
+        if (currentSlot <= this.lastSlotProcessed) {
+            return;
         }
+        this.lastSlotProcessed = currentSlot;
+        await this.txPool.prepareForSlot(currentSlot);
     }
     async startServiceIfSynched() {
         if (this.currentState !== P2PClientState.SYNCHING) {
@@ -1023,10 +989,20 @@ _dec = trackSpan('p2pClient.broadcastProposal', async (proposal)=>({
         return this.p2pService.validate(txs);
     }
     /**
-   * Marks transactions as non-evictable in the pool.
-   * @param txHashes - Hashes of the transactions to mark as non-evictable.
-   */ markTxsAsNonEvictable(txHashes) {
-        return this.txPool.markTxsAsNonEvictable(txHashes);
+   * Protects existing transactions by hash for a given slot.
+   * Returns hashes of transactions that weren't found in the pool.
+   * @param txHashes - Hashes of the transactions to protect.
+   * @param blockHeader - The block header providing slot context.
+   * @returns Hashes of transactions not found in the pool.
+   */ protectTxs(txHashes, blockHeader) {
+        return this.txPool.protectTxs(txHashes, blockHeader);
+    }
+    /**
+   * Prepares the pool for a new slot.
+   * Unprotects transactions from earlier slots and validates them.
+   * @param slotNumber - The slot number to prepare for
+   */ async prepareForSlot(slotNumber) {
+        await this.txPool.prepareForSlot(slotNumber);
     }
     handleAuthRequestFromPeer(authRequest, peerId) {
         return this.p2pService.handleAuthRequestFromPeer(authRequest, peerId);

package/dest/client/test/tx_proposal_collector/proposal_tx_collector_worker.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoicHJvcG9zYWxfdHhfY29sbGVjdG9yX3dvcmtlci5kLnRzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vLi4vLi4vLi4vc3JjL2NsaWVudC90ZXN0L3R4X3Byb3Bvc2FsX2NvbGxlY3Rvci9wcm9wb3NhbF90eF9jb2xsZWN0b3Jfd29ya2VyLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiIifQ==

package/dest/client/test/tx_proposal_collector/proposal_tx_collector_worker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"proposal_tx_collector_worker.d.ts","sourceRoot":"","sources":["../../../../src/client/test/tx_proposal_collector/proposal_tx_collector_worker.ts"],"names":[],"mappings":""}