@aztec/p2p 5.0.0-rc.2 → 5.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. package/README.md +3 -5
  2. package/dest/client/factory.d.ts +1 -1
  3. package/dest/client/factory.d.ts.map +1 -1
  4. package/dest/client/factory.js +2 -1
  5. package/dest/client/interface.d.ts +6 -1
  6. package/dest/client/interface.d.ts.map +1 -1
  7. package/dest/client/p2p_client.d.ts +2 -1
  8. package/dest/client/p2p_client.d.ts.map +1 -1
  9. package/dest/client/p2p_client.js +5 -2
  10. package/dest/config.d.ts +7 -1
  11. package/dest/config.d.ts.map +1 -1
  12. package/dest/config.js +9 -4
  13. package/dest/errors/reqresp.error.d.ts +13 -1
  14. package/dest/errors/reqresp.error.d.ts.map +1 -1
  15. package/dest/errors/reqresp.error.js +14 -0
  16. package/dest/mem_pools/attestation_pool/attestation_pool.d.ts +1 -1
  17. package/dest/mem_pools/attestation_pool/attestation_pool.d.ts.map +1 -1
  18. package/dest/mem_pools/attestation_pool/attestation_pool.js +6 -2
  19. package/dest/mem_pools/attestation_pool/attestation_pool_test_suite.d.ts +1 -1
  20. package/dest/mem_pools/attestation_pool/attestation_pool_test_suite.d.ts.map +1 -1
  21. package/dest/mem_pools/attestation_pool/attestation_pool_test_suite.js +43 -0
  22. package/dest/mem_pools/instrumentation.d.ts +9 -4
  23. package/dest/mem_pools/instrumentation.d.ts.map +1 -1
  24. package/dest/mem_pools/instrumentation.js +11 -8
  25. package/dest/mem_pools/tx_pool_v2/interfaces.d.ts +12 -1
  26. package/dest/mem_pools/tx_pool_v2/interfaces.d.ts.map +1 -1
  27. package/dest/mem_pools/tx_pool_v2/interfaces.js +2 -1
  28. package/dest/mem_pools/tx_pool_v2/tx_pool_indices.d.ts +9 -1
  29. package/dest/mem_pools/tx_pool_v2/tx_pool_indices.d.ts.map +1 -1
  30. package/dest/mem_pools/tx_pool_v2/tx_pool_indices.js +21 -0
  31. package/dest/mem_pools/tx_pool_v2/tx_pool_v2.d.ts +2 -1
  32. package/dest/mem_pools/tx_pool_v2/tx_pool_v2.d.ts.map +1 -1
  33. package/dest/mem_pools/tx_pool_v2/tx_pool_v2.js +12 -4
  34. package/dest/mem_pools/tx_pool_v2/tx_pool_v2_impl.d.ts +10 -2
  35. package/dest/mem_pools/tx_pool_v2/tx_pool_v2_impl.d.ts.map +1 -1
  36. package/dest/mem_pools/tx_pool_v2/tx_pool_v2_impl.js +55 -7
  37. package/dest/msg_validators/proposal_validator/proposal_validator.d.ts +1 -1
  38. package/dest/msg_validators/proposal_validator/proposal_validator.d.ts.map +1 -1
  39. package/dest/msg_validators/proposal_validator/proposal_validator.js +8 -1
  40. package/dest/services/encoding.d.ts +14 -1
  41. package/dest/services/encoding.d.ts.map +1 -1
  42. package/dest/services/encoding.js +15 -1
  43. package/dest/services/libp2p/libp2p_service.d.ts +2 -2
  44. package/dest/services/libp2p/libp2p_service.d.ts.map +1 -1
  45. package/dest/services/libp2p/libp2p_service.js +5 -1
  46. package/dest/services/reqresp/config.d.ts +13 -1
  47. package/dest/services/reqresp/config.d.ts.map +1 -1
  48. package/dest/services/reqresp/config.js +11 -0
  49. package/dest/services/reqresp/rate-limiter/rate_limits.js +3 -3
  50. package/dest/services/reqresp/reqresp.d.ts +1 -1
  51. package/dest/services/reqresp/reqresp.d.ts.map +1 -1
  52. package/dest/services/reqresp/reqresp.js +34 -3
  53. package/dest/test-helpers/testbench-utils.d.ts +2 -1
  54. package/dest/test-helpers/testbench-utils.d.ts.map +1 -1
  55. package/dest/test-helpers/testbench-utils.js +3 -0
  56. package/package.json +15 -15
  57. package/src/client/factory.ts +1 -0
  58. package/src/client/interface.ts +6 -0
  59. package/src/client/p2p_client.ts +7 -3
  60. package/src/config.ts +16 -3
  61. package/src/errors/reqresp.error.ts +19 -0
  62. package/src/mem_pools/attestation_pool/attestation_pool.ts +6 -2
  63. package/src/mem_pools/attestation_pool/attestation_pool_test_suite.ts +40 -1
  64. package/src/mem_pools/instrumentation.ts +13 -11
  65. package/src/mem_pools/tx_pool_v2/interfaces.ts +13 -0
  66. package/src/mem_pools/tx_pool_v2/tx_pool_indices.ts +23 -0
  67. package/src/mem_pools/tx_pool_v2/tx_pool_v2.ts +14 -4
  68. package/src/mem_pools/tx_pool_v2/tx_pool_v2_impl.ts +70 -6
  69. package/src/msg_validators/proposal_validator/proposal_validator.ts +5 -1
  70. package/src/services/encoding.ts +19 -1
  71. package/src/services/libp2p/libp2p_service.ts +3 -2
  72. package/src/services/reqresp/README.md +2 -23
  73. package/src/services/reqresp/config.ts +17 -0
  74. package/src/services/reqresp/rate-limiter/rate_limits.ts +3 -3
  75. package/src/services/reqresp/reqresp.ts +42 -2
  76. package/src/test-helpers/testbench-utils.ts +4 -0
@@ -1,7 +1,9 @@
1
1
  import { IndexWithinCheckpoint, SlotNumber } from '@aztec/foundation/branded-types';
2
2
  import { Secp256k1Signer } from '@aztec/foundation/crypto/secp256k1-signer';
3
3
  import { Fr } from '@aztec/foundation/curves/bn254';
4
- import type { BlockProposal, CheckpointAttestation, CheckpointProposalCore } from '@aztec/stdlib/p2p';
4
+ import { Signature } from '@aztec/foundation/eth-signature';
5
+ import type { BlockProposal, CheckpointProposalCore } from '@aztec/stdlib/p2p';
6
+ import { CheckpointAttestation } from '@aztec/stdlib/p2p';
5
7
  import { CheckpointHeader } from '@aztec/stdlib/rollup';
6
8
  import {
7
9
  makeBlockHeader,
@@ -62,7 +64,44 @@ export function describeAttestationPool(getAttestationPool: () => AttestationPoo
62
64
  expect(a1Buffer).toEqual(expect.arrayContaining(a2Buffer));
63
65
  };
64
66
 
67
+ /**
68
+ * Rewrites the attester signature into yParity form (27 -> 0, 28 -> 1), keeping (r, s). The recovery
69
+ * bit is unchanged so it still recovers to the same signer, mimicking a peer that gossips (or mutates
70
+ * in flight) a signature with a non-canonical recovery byte.
71
+ */
72
+ const toYParityForm = (attestation: CheckpointAttestation): CheckpointAttestation => {
73
+ const sig = attestation.signature;
74
+ const yParityV = sig.v === 27 ? 0 : sig.v === 28 ? 1 : sig.v;
75
+ return new CheckpointAttestation(
76
+ attestation.payload,
77
+ new Signature(sig.r, sig.s, yParityV),
78
+ attestation.proposerSignature,
79
+ );
80
+ };
81
+
65
82
  describe('CheckpointAttestation', () => {
83
+ it('normalizes yParity (v=0/v=1) attester signatures on ingress from both gossip and own keys', async () => {
84
+ const slotNumber = 421;
85
+ const canonical = createCheckpointAttestationsForSlot(slotNumber);
86
+ const payloadHash = canonical[0].getPayloadHash();
87
+ const expectedSenders = new Set([canonical[0].getSender()!.toString(), canonical[1].getSender()!.toString()]);
88
+
89
+ const gossiped = toYParityForm(canonical[0]);
90
+ expect([0, 1]).toContain(gossiped.signature.v);
91
+ await ap.tryAddCheckpointAttestation(gossiped);
92
+
93
+ const own = toYParityForm(canonical[1]);
94
+ expect([0, 1]).toContain(own.signature.v);
95
+ await ap.addOwnCheckpointAttestations([own]);
96
+
97
+ const retrieved = await ap.getCheckpointAttestationsForSlotAndProposal(SlotNumber(slotNumber), payloadHash);
98
+ expect(retrieved.length).toBe(2);
99
+ for (const attestation of retrieved) {
100
+ expect([27, 28]).toContain(attestation.signature.v);
101
+ expect(expectedSenders.has(attestation.getSender()!.toString())).toBe(true);
102
+ }
103
+ });
104
+
66
105
  it('should add attestations to pool', async () => {
67
106
  const slotNumber = 420;
68
107
  const archive = Fr.random();
@@ -1,5 +1,4 @@
1
1
  import type { Gossipable } from '@aztec/stdlib/p2p';
2
- import type { Tx } from '@aztec/stdlib/tx';
3
2
  import {
4
3
  Attributes,
5
4
  type BatchObservableResult,
@@ -117,16 +116,6 @@ export class PoolInstrumentation<PoolObject extends Gossipable> {
117
116
  this.addObjectCounter.add(count);
118
117
  }
119
118
 
120
- public transactionsAdded(transactions: Tx[]) {
121
- transactions.forEach(tx => this.trackMempoolItemAdded(tx.txHash.toBigInt()));
122
- }
123
-
124
- public transactionsRemoved(hashes: Iterable<bigint> | Iterable<string>) {
125
- for (const hash of hashes) {
126
- this.trackMempoolItemRemoved(BigInt(hash));
127
- }
128
- }
129
-
130
119
  public trackMempoolItemAdded(key: bigint | string): void {
131
120
  this.mempoolItemAddedTimestamp.set(key, Date.now());
132
121
  }
@@ -142,6 +131,19 @@ export class PoolInstrumentation<PoolObject extends Gossipable> {
142
131
  }
143
132
  }
144
133
 
134
+ /**
135
+ * Records a pre-computed pending-to-mined delay directly, bypassing the
136
+ * `mempoolItemAddedTimestamp` map. Used by the tx pool, which derives the
137
+ * delay from the persisted `receivedAt` on tx metadata at the mined
138
+ * transition — more accurate than the add/remove map (which also fires on
139
+ * eviction) and resilient to the caller not having tracked the add.
140
+ */
141
+ public recordMinedDelay(delayMs: number): void {
142
+ if (delayMs > 0) {
143
+ this.minedDelay.record(delayMs);
144
+ }
145
+ }
146
+
145
147
  private observeStats = async (observer: BatchObservableResult) => {
146
148
  const { itemCount } = await this.poolStats();
147
149
  if (typeof itemCount === 'number') {
@@ -49,6 +49,11 @@ export type TxPoolV2Config = {
49
49
  dropTransactionsProbability: number;
50
50
  /** Minimum percentage fee increase required to replace an existing tx via RPC (0 = no bump). */
51
51
  priceBumpPercentage: bigint;
52
+ /**
53
+ * Number of slots behind the finalized tip to keep finalized txs for before deleting them. 0 deletes
54
+ * at the finalized tip (default).
55
+ */
56
+ keepFinalizedTxsForSlots: number;
52
57
  };
53
58
 
54
59
  /**
@@ -61,6 +66,7 @@ export const DEFAULT_TX_POOL_V2_CONFIG: TxPoolV2Config = {
61
66
  evictedTxCacheSize: 10_000,
62
67
  dropTransactionsProbability: 0,
63
68
  priceBumpPercentage: 10n,
69
+ keepFinalizedTxsForSlots: 0,
64
70
  };
65
71
 
66
72
  /**
@@ -224,6 +230,13 @@ export interface TxPoolV2 extends TypedEventEmitter<TxPoolV2Events> {
224
230
  /** Gets the count of pending transactions */
225
231
  getPendingTxCount(): Promise<number>;
226
232
 
233
+ /**
234
+ * Returns whether at least `minCount` pending transactions are old enough per minTxPoolAgeMs to be eligible
235
+ * for block building. Stops scanning once the threshold is reached, so it is cheaper than counting all
236
+ * eligible txs when only a few are needed.
237
+ */
238
+ hasEligiblePendingTxs(minCount: number): Promise<boolean>;
239
+
227
240
  /** Gets mined transaction hashes with their block IDs */
228
241
  getMinedTxHashes(): Promise<[TxHash, L2BlockId][]>;
229
242
 
@@ -228,6 +228,29 @@ export class TxPoolIndices {
228
228
  return this.#pendingByPriority.length;
229
229
  }
230
230
 
231
+ /**
232
+ * Returns whether at least `minCount` pending transactions were received at or before maxReceivedAt,
233
+ * i.e. are old enough to be eligible for block building. Stops as soon as the threshold is met, so it
234
+ * does not scan the whole pool when only a handful of eligible txs are needed. The total pending count
235
+ * is an upper bound on the eligible count, so a pool with fewer than `minCount` pending can never
236
+ * satisfy the threshold and short-circuits without scanning.
237
+ */
238
+ hasEligiblePendingTxs(maxReceivedAt: number, minCount: number): boolean {
239
+ if (minCount <= 0) {
240
+ return true;
241
+ }
242
+ if (this.#pendingByPriority.length < minCount) {
243
+ return false;
244
+ }
245
+ let count = 0;
246
+ for (const _ of this.iterateEligiblePendingByPriority('desc', maxReceivedAt)) {
247
+ if (++count >= minCount) {
248
+ return true;
249
+ }
250
+ }
251
+ return false;
252
+ }
253
+
231
254
  /** Gets the lowest priority pending transaction hashes (up to limit) */
232
255
  getLowestPriorityPending(limit: number): string[] {
233
256
  if (limit <= 0) {
@@ -20,6 +20,7 @@ import type {
20
20
  TxPoolV2Events,
21
21
  } from './interfaces.js';
22
22
  import type { TxState } from './tx_metadata.js';
23
+ import type { MinedTxInfo } from './tx_pool_v2_impl.js';
23
24
  import { TxPoolV2Impl } from './tx_pool_v2_impl.js';
24
25
 
25
26
  /**
@@ -56,17 +57,22 @@ export class AztecKVTxPoolV2 extends (EventEmitter as new () => TypedEventEmitte
56
57
  // Create callbacks that the impl uses to notify us about events and metrics
57
58
  const callbacks = {
58
59
  onTxsAdded: (txs: Tx[], opts: { source?: string }) => {
59
- this.#metrics?.transactionsAdded(txs);
60
60
  this.emit('txs-added', { txs, ...opts });
61
61
  },
62
62
  onTxsRemoved: (txHashes: string[] | bigint[]) => {
63
- this.#metrics?.transactionsRemoved(txHashes);
64
63
  // Convert to TxHash objects for the event
65
64
  const hashes = txHashes.map(h => (typeof h === 'string' ? TxHash.fromString(h) : TxHash.fromBigInt(h)));
66
65
  this.emit('txs-removed', { txHashes: hashes });
67
66
  },
68
- onTxsMined: (txHashes: string[]) => {
69
- this.#metrics?.transactionsRemoved(txHashes);
67
+ onTxsMined: (minedTxs: MinedTxInfo[]) => {
68
+ // Pending-to-mined delay is derived from the tx's persisted receivedAt at the mined
69
+ // transition (see TxPoolV2Impl.handleMinedBlock), not the add/remove timestamp map —
70
+ // so eviction no longer pollutes MEMPOOL_TX_MINED_DELAY.
71
+ for (const { minedDelayMs } of minedTxs) {
72
+ if (minedDelayMs !== undefined) {
73
+ this.#metrics?.recordMinedDelay(minedDelayMs);
74
+ }
75
+ }
70
76
  },
71
77
  };
72
78
 
@@ -156,6 +162,10 @@ export class AztecKVTxPoolV2 extends (EventEmitter as new () => TypedEventEmitte
156
162
  return this.#queue.put(() => Promise.resolve(this.#impl.getPendingTxCount()));
157
163
  }
158
164
 
165
+ hasEligiblePendingTxs(minCount: number): Promise<boolean> {
166
+ return this.#queue.put(() => Promise.resolve(this.#impl.hasEligiblePendingTxs(minCount)));
167
+ }
168
+
159
169
  getMinedTxHashes(): Promise<[TxHash, L2BlockId][]> {
160
170
  return this.#queue.put(() => Promise.resolve(this.#impl.getMinedTxHashes()));
161
171
  }
@@ -51,10 +51,18 @@ const FINALIZE_BLOCK_CHUNK_SIZE = 100;
51
51
  /**
52
52
  * Callbacks for the implementation to notify the outer class about events and metrics.
53
53
  */
54
+ /** A tx that has just transitioned to mined, with the time it spent in the pool. */
55
+ export interface MinedTxInfo {
56
+ txHash: string;
57
+ /** Wall-clock ms from receipt into the pool to being marked mined. Undefined when the
58
+ * receive time is unknown (e.g. a tx hydrated from the DB on restart, receivedAt === 0). */
59
+ minedDelayMs?: number;
60
+ }
61
+
54
62
  export interface TxPoolV2Callbacks {
55
63
  onTxsAdded: (txs: Tx[], opts: { source?: string }) => void;
56
64
  onTxsRemoved: (txHashes: string[] | bigint[]) => void;
57
- onTxsMined: (txHashes: string[]) => void;
65
+ onTxsMined: (minedTxs: MinedTxInfo[]) => void;
58
66
  }
59
67
 
60
68
  /**
@@ -560,7 +568,15 @@ export class TxPoolV2Impl {
560
568
  });
561
569
 
562
570
  if (found.length > 0) {
563
- this.#callbacks.onTxsMined(found.map(m => m.txHash));
571
+ // receivedAt is 0 for txs hydrated from the DB on restart (true receive time lost) — leave
572
+ // their delay undefined so the metric isn't polluted by epoch-sized values.
573
+ const now = this.#dateProvider.now();
574
+ this.#callbacks.onTxsMined(
575
+ found.map(m => ({
576
+ txHash: m.txHash,
577
+ minedDelayMs: m.receivedAt > 0 ? now - m.receivedAt : undefined,
578
+ })),
579
+ );
564
580
  }
565
581
 
566
582
  this.#log.info(`Marked ${found.length} txs as mined in block ${blockId.number}`);
@@ -712,8 +728,12 @@ export class TxPoolV2Impl {
712
728
  async handleFinalizedBlock(block: BlockHeader): Promise<void> {
713
729
  const blockNumber = block.globalVariables.blockNumber;
714
730
 
715
- // Step 1: Find mined txs at or before finalized block
716
- const minedTxsToFinalize = this.#indices.findTxsMinedAtOrBefore(blockNumber);
731
+ // Hold finalized txs for a configurable margin behind the finalized tip so a prover still
732
+ // proving an epoch with already-finalized blocks isn't starved of its txs. 0 deletes at the finalized tip.
733
+ const cutoffBlock = await this.#finalizationCutoffBlock(block);
734
+
735
+ // Step 1: Find mined txs at or before the cutoff block
736
+ const minedTxsToFinalize = this.#indices.findTxsMinedAtOrBefore(cutoffBlock);
717
737
 
718
738
  // Step 2: Archive in chunks if archiving is enabled. Hydrating an entire epoch's worth of
719
739
  // mined txs at once would OOM under load. When archiving is disabled there is no need to hydrate the txs at all.
@@ -737,16 +757,52 @@ export class TxPoolV2Impl {
737
757
  // transaction. Only tx hashes are touched here, so memory is bounded and atomicity is preserved.
738
758
  await this.#store.transactionAsync(async () => {
739
759
  await this.#deleteTxsBatch(minedTxsToFinalize);
740
- await this.#deletedPool.finalizeBlock(blockNumber);
760
+ await this.#deletedPool.finalizeBlock(cutoffBlock);
741
761
  });
742
762
 
743
763
  if (minedTxsToFinalize.length > 0) {
744
- this.#log.info(`Finalized ${minedTxsToFinalize.length} mined txs from blocks up to ${blockNumber}`, {
764
+ this.#log.info(`Finalized ${minedTxsToFinalize.length} mined txs from blocks up to ${cutoffBlock}`, {
745
765
  txHashes: minedTxsToFinalize,
766
+ finalizedBlockNumber: blockNumber,
767
+ cutoffBlock,
746
768
  });
747
769
  }
748
770
  }
749
771
 
772
+ /**
773
+ * Resolves the block up to which finalized txs may be deleted, given `keepFinalizedTxsForSlots`.
774
+ * With a margin of 0 this is just the finalized block. Otherwise we keep txs whose block is within
775
+ * the margin (in slots) of the finalized tip: take the target slot `finalizedSlot - margin`, find the
776
+ * latest checkpoint at or before it with a single reverse range query (`limit: 1`), and use that
777
+ * checkpoint's last block as the cutoff. This rounds to a checkpoint boundary, so it can retain slightly
778
+ * more than the configured margin but never less, and never deletes past the finalized block.
779
+ */
780
+ async #finalizationCutoffBlock(block: BlockHeader): Promise<BlockNumber> {
781
+ const keepSlots = this.#config.keepFinalizedTxsForSlots;
782
+ if (keepSlots <= 0) {
783
+ return block.globalVariables.blockNumber;
784
+ }
785
+
786
+ // The goal here really is to ensure we keep transactions long enough for them to be proven.
787
+ // So we could be smart and calculate the number of slots until the end of the proof submission window.
788
+ // Our approach here is a lot simpler however and we just keep transactions for the configured number
789
+ // of slots past L1 finalisation. This means we may keep transactions in the mined pool for longer
790
+ // than strictly necessary.
791
+ const targetSlot = block.getSlot() - keepSlots;
792
+ if (targetSlot < 0) {
793
+ // The margin reaches past genesis: keep everything.
794
+ return BlockNumber(0);
795
+ }
796
+
797
+ const [checkpoint] = await this.#l2BlockSource.getCheckpointsData({
798
+ fromSlot: SlotNumber(targetSlot),
799
+ limit: 1,
800
+ reverse: true,
801
+ });
802
+ // No checkpoint at or before the target slot (e.g. early chain): keep everything.
803
+ return checkpoint ? BlockNumber(checkpoint.startBlock + checkpoint.blockCount - 1) : BlockNumber(0);
804
+ }
805
+
750
806
  // === Query Methods ===
751
807
 
752
808
  async getTxByHash(txHash: TxHash, opts: { includeProof?: boolean } = {}): Promise<Tx | undefined> {
@@ -814,6 +870,11 @@ export class TxPoolV2Impl {
814
870
  return this.#indices.getPendingTxCount();
815
871
  }
816
872
 
873
+ hasEligiblePendingTxs(minCount: number): boolean {
874
+ const maxReceivedAt = this.#dateProvider.now() - this.#config.minTxPoolAgeMs;
875
+ return this.#indices.hasEligiblePendingTxs(maxReceivedAt, minCount);
876
+ }
877
+
817
878
  getMinedTxHashes(): [TxHash, L2BlockId][] {
818
879
  return this.#indices.getMinedTxs().map(([hash, blockId]) => [TxHash.fromString(hash), blockId]);
819
880
  }
@@ -857,6 +918,9 @@ export class TxPoolV2Impl {
857
918
  if (config.minTxPoolAgeMs !== undefined) {
858
919
  this.#config.minTxPoolAgeMs = config.minTxPoolAgeMs;
859
920
  }
921
+ if (config.keepFinalizedTxsForSlots !== undefined) {
922
+ this.#config.keepFinalizedTxsForSlots = config.keepFinalizedTxsForSlots;
923
+ }
860
924
  // Update eviction rules with new config
861
925
  this.#evictionManager.updateConfig(config);
862
926
  }
@@ -97,7 +97,11 @@ export class ProposalValidator {
97
97
 
98
98
  // Proposer check
99
99
  const expectedProposer = await this.epochCache.getProposerAttesterAddressInSlot(slotNumber);
100
- if (expectedProposer !== undefined && !proposer.equals(expectedProposer)) {
100
+ if (expectedProposer === undefined) {
101
+ this.logger.warn(`Penalizing peer for proposal with no expected proposer for current slot ${slotNumber}`);
102
+ return { result: 'reject', severity: PeerErrorSeverity.HighToleranceError };
103
+ }
104
+ if (!proposer.equals(expectedProposer)) {
101
105
  this.logger.warn(`Penalizing peer for invalid proposer for current slot ${slotNumber}`, {
102
106
  expectedProposer,
103
107
  proposer: proposer.toString(),
@@ -7,6 +7,12 @@ import type { Message } from '@libp2p/interface';
7
7
  import { webcrypto } from 'node:crypto';
8
8
  import { compressSync, uncompressSync } from 'snappy';
9
9
 
10
+ /**
11
+ * Fallback maximum response size (in KB) applied when no explicit per-request bound is supplied.
12
+ * Used both as the `SnappyTransform` default and as the reception-side fallback in `readMessage`.
13
+ */
14
+ export const DEFAULT_MAX_RESPONSE_SIZE_KB = 10 * 1024;
15
+
10
16
  /** Thrown when a Snappy-compressed response exceeds the allowed decompressed size. */
11
17
  export class OversizedSnappyResponseError extends Error {
12
18
  constructor(decompressedSize: number, maxSizeKb: number) {
@@ -15,6 +21,18 @@ export class OversizedSnappyResponseError extends Error {
15
21
  }
16
22
  }
17
23
 
24
+ /**
25
+ * Thrown when the compressed bytes accumulated while reading a response stream exceed the reception
26
+ * bound, before the full payload is buffered. Distinct from {@link OversizedSnappyResponseError},
27
+ * which guards the decompressed size after the stream has already been fully received.
28
+ */
29
+ export class ResponseSizeLimitExceededError extends Error {
30
+ constructor(accumulatedBytes: number, limitBytes: number) {
31
+ super(`Accumulated response size ${accumulatedBytes} bytes exceeds reception limit of ${limitBytes} bytes`);
32
+ this.name = 'ResponseSizeLimitExceededError';
33
+ }
34
+ }
35
+
18
36
  // Shared buffer to convert msgId to string
19
37
  const sharedMsgIdBuf = Buffer.alloc(20);
20
38
 
@@ -66,7 +84,7 @@ const DefaultMaxSizesKb: Record<TopicType, number> = {
66
84
  export class SnappyTransform implements DataTransform {
67
85
  constructor(
68
86
  private maxSizesKb: Record<TopicType, number> = DefaultMaxSizesKb,
69
- private defaultMaxSizeKb: number = 10 * 1024,
87
+ private defaultMaxSizeKb: number = DEFAULT_MAX_RESPONSE_SIZE_KB,
70
88
  private logger = createLogger('p2p:snappy-transform'),
71
89
  ) {}
72
90
 
@@ -90,7 +90,7 @@ import type { PeerManagerInterface } from '../peer-manager/interface.js';
90
90
  import { PeerManager } from '../peer-manager/peer_manager.js';
91
91
  import { PeerScoring } from '../peer-manager/peer_scoring.js';
92
92
  import type { BatchTxRequesterLibP2PService } from '../reqresp/batch-tx-requester/interface.js';
93
- import type { P2PReqRespConfig } from '../reqresp/config.js';
93
+ import { type P2PReqRespConfig, YAMUX_MAX_MESSAGE_SIZE_BYTES } from '../reqresp/config.js';
94
94
  import {
95
95
  AuthRequest,
96
96
  BlockTxsRequest,
@@ -457,7 +457,8 @@ export class LibP2PService extends WithTracer implements P2PService {
457
457
  ],
458
458
  datastore,
459
459
  peerDiscovery,
460
- streamMuxers: [yamux(), mplex()],
460
+ // Pin the yamux frame size: MAX_REQRESP_REQUEST_SIZE_BYTES relies on a reqresp request fitting in one frame.
461
+ streamMuxers: [yamux({ maxMessageSize: YAMUX_MAX_MESSAGE_SIZE_BYTES }), mplex()],
461
462
  connectionEncryption: [noise()],
462
463
  connectionManager: {
463
464
  minConnections: 0, // Disable libp2p peer dialing, we do it manually
@@ -14,9 +14,8 @@ Applied before the protocol handler runs.
14
14
  | STATUS | 5/s | 10/s | same |
15
15
  | AUTH | 5/s | 10/s | same |
16
16
  | GOODBYE | 5/s | 10/s | same |
17
- | BLOCK | 2/s | 5/s | same |
18
17
  | BLOCK_TXS | 10/s | 200/s | same |
19
- | TX | (see rate limits file) | (see rate limits file) | same |
18
+ | TX | 10/s | 200/s | same |
20
19
 
21
20
  - Per-peer limit exceeded: `HighToleranceError` penalty + `RATE_LIMIT_EXCEEDED` status. Penalty fires inside `RequestResponseRateLimiter.allow()`, not the stream handler.
22
21
  - Global limit exceeded: `RATE_LIMIT_EXCEEDED` status only (no peer penalty).
@@ -149,26 +148,6 @@ Protected peers (private/trusted/preferred) are always considered "authenticated
149
148
 
150
149
  ## Block Data Protocols
151
150
 
152
- ### BLOCK Protocol (`/aztec/req/block/1.0.0`)
153
-
154
- **Server side**:
155
-
156
- | Rule | Consequence | File |
157
- |------|-------------|------|
158
- | Request must parse as `Fr` | `BADLY_FORMED_REQUEST` + LowToleranceError | `protocols/block.ts` |
159
- | Block lookup throws | `INTERNAL_ERROR` status | same |
160
- | Block not found | SUCCESS + empty buffer (design choice; no `NOT_FOUND` status used) | same |
161
-
162
- **Requester side** (Snappy limit: 3 MB):
163
-
164
- | Rule | Consequence | File |
165
- |------|-------------|------|
166
- | Response block number must match requested | LowToleranceError; rejected | `libp2p_service.ts` (`validateRequestedBlock`) |
167
- | Local block must exist for hash verification | Rejected (no penalty) | same |
168
- | Response block hash must equal local block hash | MidToleranceError; rejected | same |
169
-
170
- **Limitation**: the local-block requirement means BLOCK req/resp is unusable for initial P2P-only sync (before L1 sync provides local copies for verification). A TODO in the code acknowledges this.
171
-
172
151
  ### BLOCK_TXS Protocol (`/aztec/req/block_txs/1.0.0`)
173
152
 
174
153
  **Server side**:
@@ -178,7 +157,7 @@ Protected peers (private/trusted/preferred) are always considered "authenticated
178
157
  | Request must parse as `BlockTxsRequest` (Fr + TxHashArray + BitVector) | `BADLY_FORMED_REQUEST` + LowToleranceError | `protocols/block_txs/block_txs_handler.ts` |
179
158
  | BitVector length: non-negative and <= `MAX_TXS_PER_BLOCK` (65536) | Deserialization throws -> `BADLY_FORMED_REQUEST` | `protocols/block_txs/bitvector.ts` |
180
159
  | Archive root not found and no explicit txHashes | `NOT_FOUND` status | handler |
181
- | Internal error during lookup | Unhandled exception -> stream abort (no `INTERNAL_ERROR` status, unlike BLOCK) | handler |
160
+ | Internal error during lookup | Unhandled exception -> stream abort (no `INTERNAL_ERROR` status) | handler |
182
161
 
183
162
  Conditional registration: BLOCK_TXS handler only registered when `config.disableTransactions` is false. Otherwise peers get `ERR_UNSUPPORTED_PROTOCOL`.
184
163
 
@@ -5,6 +5,23 @@ export const DEFAULT_OVERALL_REQUEST_TIMEOUT_MS = 10_000; // Not currently used
5
5
  export const DEFAULT_REQRESP_DIAL_TIMEOUT_MS = 5_000;
6
6
  export const DEFAULT_OPTIMISTIC_NEGOTIATION = false;
7
7
 
8
+ /**
9
+ * Max size of a yamux frame, header included. This is the library default, pinned explicitly when configuring the
10
+ * muxer in the libp2p service so a dependency upgrade cannot silently change it.
11
+ */
12
+ export const YAMUX_MAX_MESSAGE_SIZE_BYTES = 64 * 1024;
13
+
14
+ /** Size of a yamux data frame header. */
15
+ const YAMUX_HEADER_LENGTH_BYTES = 12;
16
+
17
+ /**
18
+ * Max size of a reqresp request payload. A request must fit in a single muxer frame: the responder never reassembles
19
+ * a request from multiple chunks (see `ReqResp.processStream`), and yamux splits writes larger than one frame into
20
+ * multiple frames, each of which arrives as a separate chunk. mplex only splits at 1 MiB, so yamux is the binding
21
+ * constraint.
22
+ */
23
+ export const MAX_REQRESP_REQUEST_SIZE_BYTES = YAMUX_MAX_MESSAGE_SIZE_BYTES - YAMUX_HEADER_LENGTH_BYTES;
24
+
8
25
  // For use in tests.
9
26
  export const DEFAULT_P2P_REQRESP_CONFIG: P2PReqRespConfig = {
10
27
  overallRequestTimeoutMs: DEFAULT_OVERALL_REQUEST_TIMEOUT_MS,
@@ -9,7 +9,7 @@ export const DEFAULT_RATE_LIMITS: ReqRespSubProtocolRateLimits = {
9
9
  },
10
10
  globalLimit: {
11
11
  quotaTimeMs: 1000,
12
- quotaCount: 10,
12
+ quotaCount: 50,
13
13
  },
14
14
  },
15
15
  [ReqRespSubProtocol.STATUS]: {
@@ -19,7 +19,7 @@ export const DEFAULT_RATE_LIMITS: ReqRespSubProtocolRateLimits = {
19
19
  },
20
20
  globalLimit: {
21
21
  quotaTimeMs: 1000,
22
- quotaCount: 10,
22
+ quotaCount: 50,
23
23
  },
24
24
  },
25
25
  [ReqRespSubProtocol.AUTH]: {
@@ -49,7 +49,7 @@ export const DEFAULT_RATE_LIMITS: ReqRespSubProtocolRateLimits = {
49
49
  },
50
50
  globalLimit: {
51
51
  quotaTimeMs: 1000,
52
- quotaCount: 10,
52
+ quotaCount: 50,
53
53
  },
54
54
  },
55
55
  [ReqRespSubProtocol.BLOCK_TXS]: {
@@ -10,12 +10,18 @@ import type { Libp2p } from 'libp2p';
10
10
  import { pipeline } from 'node:stream/promises';
11
11
  import type { Uint8ArrayList } from 'uint8arraylist';
12
12
 
13
- import { IndividualReqRespTimeoutError } from '../../errors/reqresp.error.js';
14
- import { OversizedSnappyResponseError, SnappyTransform } from '../encoding.js';
13
+ import { IndividualReqRespTimeoutError, OversizedReqRespRequestError } from '../../errors/reqresp.error.js';
14
+ import {
15
+ DEFAULT_MAX_RESPONSE_SIZE_KB,
16
+ OversizedSnappyResponseError,
17
+ ResponseSizeLimitExceededError,
18
+ SnappyTransform,
19
+ } from '../encoding.js';
15
20
  import type { PeerScoring } from '../peer-manager/peer_scoring.js';
16
21
  import {
17
22
  DEFAULT_INDIVIDUAL_REQUEST_TIMEOUT_MS,
18
23
  DEFAULT_REQRESP_DIAL_TIMEOUT_MS,
24
+ MAX_REQRESP_REQUEST_SIZE_BYTES,
19
25
  type P2PReqRespConfig,
20
26
  } from './config.js';
21
27
  import { ConnectionSampler, RandomSampler } from './connection-sampler/connection_sampler.js';
@@ -207,6 +213,11 @@ export class ReqResp implements ReqRespInterface {
207
213
  payload: Buffer,
208
214
  dialTimeout: number = this.dialTimeoutMs,
209
215
  ): Promise<ReqRespResponse> {
216
+ // Thrown before the try block so the generic error handling below does not penalize the peer for our own bug.
217
+ if (payload.length > MAX_REQRESP_REQUEST_SIZE_BYTES) {
218
+ throw new OversizedReqRespRequestError(subProtocol, payload.length, MAX_REQRESP_REQUEST_SIZE_BYTES);
219
+ }
220
+
210
221
  let stream: Stream | undefined;
211
222
  try {
212
223
  this.metrics.recordRequestSent(subProtocol);
@@ -291,6 +302,14 @@ export class ReqResp implements ReqRespInterface {
291
302
  * - The first chunk should contain a control byte, indicating the status of the response see `ReqRespStatus`
292
303
  * - The second chunk should contain the response data
293
304
  *
305
+ * To bound memory usage we abort reception as soon as the accumulated compressed bytes exceed a
306
+ * bound derived from `maxSizeKb`, before the full stream has been buffered. Without this a peer we
307
+ * dialed could stream data up to the request timeout, forcing us to buffer arbitrarily much (the
308
+ * `maxSizeKb` guard in `inboundTransformData` only runs after the whole compressed payload is
309
+ * concatenated, so it does nothing to cap reception memory). Snappy can expand incompressible input
310
+ * to ~1.17x its size, so we allow 2x `maxSizeKb`: comfortably above the worst-case expansion of a
311
+ * legitimate max-size response, while still capping buffered memory at twice the permitted size.
312
+ *
294
313
  * @param source - The async iterable source of data chunks
295
314
  * @param maxSizeKb - Optional maximum expected size in KB for the decompressed response
296
315
  */
@@ -298,10 +317,20 @@ export class ReqResp implements ReqRespInterface {
298
317
  let status: ReqRespStatus | undefined;
299
318
  const chunks: Uint8Array[] = [];
300
319
 
320
+ const maxAccumulatedBytes = (maxSizeKb ?? DEFAULT_MAX_RESPONSE_SIZE_KB) * 1024 * 2;
321
+ let accumulatedBytes = 0;
322
+
301
323
  try {
302
324
  for await (const chunk of source) {
303
325
  const statusParsed = status !== undefined;
304
326
  if (statusParsed) {
327
+ // Use `byteLength` (the logical length) rather than `chunk.subarray().length`: calling
328
+ // `subarray()` on a `Uint8ArrayList` consolidates its backing buffers into a single copy,
329
+ // which would materialize the very chunk we are trying to reject.
330
+ accumulatedBytes += chunk.byteLength;
331
+ if (accumulatedBytes > maxAccumulatedBytes) {
332
+ throw new ResponseSizeLimitExceededError(accumulatedBytes, maxAccumulatedBytes);
333
+ }
305
334
  chunks.push(chunk.subarray());
306
335
  continue;
307
336
  }
@@ -440,6 +469,9 @@ export class ReqResp implements ReqRespInterface {
440
469
  await pipeline(
441
470
  stream.source,
442
471
  async function* (source: any) {
472
+ // Req/resp is one-request-one-response: an honest sender writes a single payload and half-closes.
473
+ // Process only the first chunk and return, so extra frames a peer queues on the same stream cannot
474
+ // drive further handler invocations that bypass the per-stream rate-limit check in streamHandler.
443
475
  for await (const chunk of source) {
444
476
  const response = await handler(connection.remotePeer, chunk.subarray());
445
477
 
@@ -454,6 +486,7 @@ export class ReqResp implements ReqRespInterface {
454
486
 
455
487
  yield SUCCESS;
456
488
  yield snappy.outboundTransformData(response);
489
+ return;
457
490
  }
458
491
  },
459
492
  stream.sink,
@@ -564,6 +597,13 @@ export class ReqResp implements ReqRespInterface {
564
597
  return PeerErrorSeverity.LowToleranceError;
565
598
  }
566
599
 
600
+ // Response exceeded the reception buffer bound: the peer streamed more compressed data than the
601
+ // per-request size limit allows. Same protocol violation as above, penalize harshly.
602
+ if (e instanceof ResponseSizeLimitExceededError) {
603
+ this.logger.warn(`Oversized response from peer ${peerId.toString()} in ${subProtocol}: ${e.message}`, logTags);
604
+ return PeerErrorSeverity.LowToleranceError;
605
+ }
606
+
567
607
  return this.categorizeConnectionErrors(e, peerId, subProtocol);
568
608
  }
569
609
 
@@ -171,6 +171,10 @@ export class InMemoryTxPool extends EventEmitter implements TxPoolV2 {
171
171
  return Promise.resolve(this.txsByHash.size);
172
172
  }
173
173
 
174
+ async hasEligiblePendingTxs(minCount: number): Promise<boolean> {
175
+ return (await this.getPendingTxCount()) >= minCount;
176
+ }
177
+
174
178
  getMinedTxHashes(): Promise<[TxHash, L2BlockId][]> {
175
179
  return Promise.resolve([]);
176
180
  }