@aztec/p2p 5.0.0-rc.2 → 5.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -5
- package/dest/client/factory.d.ts +1 -1
- package/dest/client/factory.d.ts.map +1 -1
- package/dest/client/factory.js +2 -1
- package/dest/client/interface.d.ts +6 -1
- package/dest/client/interface.d.ts.map +1 -1
- package/dest/client/p2p_client.d.ts +2 -1
- package/dest/client/p2p_client.d.ts.map +1 -1
- package/dest/client/p2p_client.js +5 -2
- package/dest/config.d.ts +7 -1
- package/dest/config.d.ts.map +1 -1
- package/dest/config.js +9 -4
- package/dest/errors/reqresp.error.d.ts +13 -1
- package/dest/errors/reqresp.error.d.ts.map +1 -1
- package/dest/errors/reqresp.error.js +14 -0
- package/dest/mem_pools/attestation_pool/attestation_pool.d.ts +1 -1
- package/dest/mem_pools/attestation_pool/attestation_pool.d.ts.map +1 -1
- package/dest/mem_pools/attestation_pool/attestation_pool.js +6 -2
- package/dest/mem_pools/attestation_pool/attestation_pool_test_suite.d.ts +1 -1
- package/dest/mem_pools/attestation_pool/attestation_pool_test_suite.d.ts.map +1 -1
- package/dest/mem_pools/attestation_pool/attestation_pool_test_suite.js +43 -0
- package/dest/mem_pools/instrumentation.d.ts +9 -4
- package/dest/mem_pools/instrumentation.d.ts.map +1 -1
- package/dest/mem_pools/instrumentation.js +11 -8
- package/dest/mem_pools/tx_pool_v2/interfaces.d.ts +12 -1
- package/dest/mem_pools/tx_pool_v2/interfaces.d.ts.map +1 -1
- package/dest/mem_pools/tx_pool_v2/interfaces.js +2 -1
- package/dest/mem_pools/tx_pool_v2/tx_pool_indices.d.ts +9 -1
- package/dest/mem_pools/tx_pool_v2/tx_pool_indices.d.ts.map +1 -1
- package/dest/mem_pools/tx_pool_v2/tx_pool_indices.js +21 -0
- package/dest/mem_pools/tx_pool_v2/tx_pool_v2.d.ts +2 -1
- package/dest/mem_pools/tx_pool_v2/tx_pool_v2.d.ts.map +1 -1
- package/dest/mem_pools/tx_pool_v2/tx_pool_v2.js +12 -4
- package/dest/mem_pools/tx_pool_v2/tx_pool_v2_impl.d.ts +10 -2
- package/dest/mem_pools/tx_pool_v2/tx_pool_v2_impl.d.ts.map +1 -1
- package/dest/mem_pools/tx_pool_v2/tx_pool_v2_impl.js +55 -7
- package/dest/msg_validators/proposal_validator/proposal_validator.d.ts +1 -1
- package/dest/msg_validators/proposal_validator/proposal_validator.d.ts.map +1 -1
- package/dest/msg_validators/proposal_validator/proposal_validator.js +8 -1
- package/dest/services/encoding.d.ts +14 -1
- package/dest/services/encoding.d.ts.map +1 -1
- package/dest/services/encoding.js +15 -1
- package/dest/services/libp2p/libp2p_service.d.ts +2 -2
- package/dest/services/libp2p/libp2p_service.d.ts.map +1 -1
- package/dest/services/libp2p/libp2p_service.js +5 -1
- package/dest/services/reqresp/config.d.ts +13 -1
- package/dest/services/reqresp/config.d.ts.map +1 -1
- package/dest/services/reqresp/config.js +11 -0
- package/dest/services/reqresp/rate-limiter/rate_limits.js +3 -3
- package/dest/services/reqresp/reqresp.d.ts +1 -1
- package/dest/services/reqresp/reqresp.d.ts.map +1 -1
- package/dest/services/reqresp/reqresp.js +34 -3
- package/dest/test-helpers/testbench-utils.d.ts +2 -1
- package/dest/test-helpers/testbench-utils.d.ts.map +1 -1
- package/dest/test-helpers/testbench-utils.js +3 -0
- package/package.json +15 -15
- package/src/client/factory.ts +1 -0
- package/src/client/interface.ts +6 -0
- package/src/client/p2p_client.ts +7 -3
- package/src/config.ts +16 -3
- package/src/errors/reqresp.error.ts +19 -0
- package/src/mem_pools/attestation_pool/attestation_pool.ts +6 -2
- package/src/mem_pools/attestation_pool/attestation_pool_test_suite.ts +40 -1
- package/src/mem_pools/instrumentation.ts +13 -11
- package/src/mem_pools/tx_pool_v2/interfaces.ts +13 -0
- package/src/mem_pools/tx_pool_v2/tx_pool_indices.ts +23 -0
- package/src/mem_pools/tx_pool_v2/tx_pool_v2.ts +14 -4
- package/src/mem_pools/tx_pool_v2/tx_pool_v2_impl.ts +70 -6
- package/src/msg_validators/proposal_validator/proposal_validator.ts +5 -1
- package/src/services/encoding.ts +19 -1
- package/src/services/libp2p/libp2p_service.ts +3 -2
- package/src/services/reqresp/README.md +2 -23
- package/src/services/reqresp/config.ts +17 -0
- package/src/services/reqresp/rate-limiter/rate_limits.ts +3 -3
- package/src/services/reqresp/reqresp.ts +42 -2
- package/src/test-helpers/testbench-utils.ts +4 -0
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
import { IndexWithinCheckpoint, SlotNumber } from '@aztec/foundation/branded-types';
|
|
2
2
|
import { Secp256k1Signer } from '@aztec/foundation/crypto/secp256k1-signer';
|
|
3
3
|
import { Fr } from '@aztec/foundation/curves/bn254';
|
|
4
|
-
import
|
|
4
|
+
import { Signature } from '@aztec/foundation/eth-signature';
|
|
5
|
+
import type { BlockProposal, CheckpointProposalCore } from '@aztec/stdlib/p2p';
|
|
6
|
+
import { CheckpointAttestation } from '@aztec/stdlib/p2p';
|
|
5
7
|
import { CheckpointHeader } from '@aztec/stdlib/rollup';
|
|
6
8
|
import {
|
|
7
9
|
makeBlockHeader,
|
|
@@ -62,7 +64,44 @@ export function describeAttestationPool(getAttestationPool: () => AttestationPoo
|
|
|
62
64
|
expect(a1Buffer).toEqual(expect.arrayContaining(a2Buffer));
|
|
63
65
|
};
|
|
64
66
|
|
|
67
|
+
/**
|
|
68
|
+
* Rewrites the attester signature into yParity form (27 -> 0, 28 -> 1), keeping (r, s). The recovery
|
|
69
|
+
* bit is unchanged so it still recovers to the same signer, mimicking a peer that gossips (or mutates
|
|
70
|
+
* in flight) a signature with a non-canonical recovery byte.
|
|
71
|
+
*/
|
|
72
|
+
const toYParityForm = (attestation: CheckpointAttestation): CheckpointAttestation => {
|
|
73
|
+
const sig = attestation.signature;
|
|
74
|
+
const yParityV = sig.v === 27 ? 0 : sig.v === 28 ? 1 : sig.v;
|
|
75
|
+
return new CheckpointAttestation(
|
|
76
|
+
attestation.payload,
|
|
77
|
+
new Signature(sig.r, sig.s, yParityV),
|
|
78
|
+
attestation.proposerSignature,
|
|
79
|
+
);
|
|
80
|
+
};
|
|
81
|
+
|
|
65
82
|
describe('CheckpointAttestation', () => {
|
|
83
|
+
it('normalizes yParity (v=0/v=1) attester signatures on ingress from both gossip and own keys', async () => {
|
|
84
|
+
const slotNumber = 421;
|
|
85
|
+
const canonical = createCheckpointAttestationsForSlot(slotNumber);
|
|
86
|
+
const payloadHash = canonical[0].getPayloadHash();
|
|
87
|
+
const expectedSenders = new Set([canonical[0].getSender()!.toString(), canonical[1].getSender()!.toString()]);
|
|
88
|
+
|
|
89
|
+
const gossiped = toYParityForm(canonical[0]);
|
|
90
|
+
expect([0, 1]).toContain(gossiped.signature.v);
|
|
91
|
+
await ap.tryAddCheckpointAttestation(gossiped);
|
|
92
|
+
|
|
93
|
+
const own = toYParityForm(canonical[1]);
|
|
94
|
+
expect([0, 1]).toContain(own.signature.v);
|
|
95
|
+
await ap.addOwnCheckpointAttestations([own]);
|
|
96
|
+
|
|
97
|
+
const retrieved = await ap.getCheckpointAttestationsForSlotAndProposal(SlotNumber(slotNumber), payloadHash);
|
|
98
|
+
expect(retrieved.length).toBe(2);
|
|
99
|
+
for (const attestation of retrieved) {
|
|
100
|
+
expect([27, 28]).toContain(attestation.signature.v);
|
|
101
|
+
expect(expectedSenders.has(attestation.getSender()!.toString())).toBe(true);
|
|
102
|
+
}
|
|
103
|
+
});
|
|
104
|
+
|
|
66
105
|
it('should add attestations to pool', async () => {
|
|
67
106
|
const slotNumber = 420;
|
|
68
107
|
const archive = Fr.random();
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import type { Gossipable } from '@aztec/stdlib/p2p';
|
|
2
|
-
import type { Tx } from '@aztec/stdlib/tx';
|
|
3
2
|
import {
|
|
4
3
|
Attributes,
|
|
5
4
|
type BatchObservableResult,
|
|
@@ -117,16 +116,6 @@ export class PoolInstrumentation<PoolObject extends Gossipable> {
|
|
|
117
116
|
this.addObjectCounter.add(count);
|
|
118
117
|
}
|
|
119
118
|
|
|
120
|
-
public transactionsAdded(transactions: Tx[]) {
|
|
121
|
-
transactions.forEach(tx => this.trackMempoolItemAdded(tx.txHash.toBigInt()));
|
|
122
|
-
}
|
|
123
|
-
|
|
124
|
-
public transactionsRemoved(hashes: Iterable<bigint> | Iterable<string>) {
|
|
125
|
-
for (const hash of hashes) {
|
|
126
|
-
this.trackMempoolItemRemoved(BigInt(hash));
|
|
127
|
-
}
|
|
128
|
-
}
|
|
129
|
-
|
|
130
119
|
public trackMempoolItemAdded(key: bigint | string): void {
|
|
131
120
|
this.mempoolItemAddedTimestamp.set(key, Date.now());
|
|
132
121
|
}
|
|
@@ -142,6 +131,19 @@ export class PoolInstrumentation<PoolObject extends Gossipable> {
|
|
|
142
131
|
}
|
|
143
132
|
}
|
|
144
133
|
|
|
134
|
+
/**
|
|
135
|
+
* Records a pre-computed pending-to-mined delay directly, bypassing the
|
|
136
|
+
* `mempoolItemAddedTimestamp` map. Used by the tx pool, which derives the
|
|
137
|
+
* delay from the persisted `receivedAt` on tx metadata at the mined
|
|
138
|
+
* transition — more accurate than the add/remove map (which also fires on
|
|
139
|
+
* eviction) and resilient to the caller not having tracked the add.
|
|
140
|
+
*/
|
|
141
|
+
public recordMinedDelay(delayMs: number): void {
|
|
142
|
+
if (delayMs > 0) {
|
|
143
|
+
this.minedDelay.record(delayMs);
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
|
|
145
147
|
private observeStats = async (observer: BatchObservableResult) => {
|
|
146
148
|
const { itemCount } = await this.poolStats();
|
|
147
149
|
if (typeof itemCount === 'number') {
|
|
@@ -49,6 +49,11 @@ export type TxPoolV2Config = {
|
|
|
49
49
|
dropTransactionsProbability: number;
|
|
50
50
|
/** Minimum percentage fee increase required to replace an existing tx via RPC (0 = no bump). */
|
|
51
51
|
priceBumpPercentage: bigint;
|
|
52
|
+
/**
|
|
53
|
+
* Number of slots behind the finalized tip to keep finalized txs for before deleting them. 0 deletes
|
|
54
|
+
* at the finalized tip (default).
|
|
55
|
+
*/
|
|
56
|
+
keepFinalizedTxsForSlots: number;
|
|
52
57
|
};
|
|
53
58
|
|
|
54
59
|
/**
|
|
@@ -61,6 +66,7 @@ export const DEFAULT_TX_POOL_V2_CONFIG: TxPoolV2Config = {
|
|
|
61
66
|
evictedTxCacheSize: 10_000,
|
|
62
67
|
dropTransactionsProbability: 0,
|
|
63
68
|
priceBumpPercentage: 10n,
|
|
69
|
+
keepFinalizedTxsForSlots: 0,
|
|
64
70
|
};
|
|
65
71
|
|
|
66
72
|
/**
|
|
@@ -224,6 +230,13 @@ export interface TxPoolV2 extends TypedEventEmitter<TxPoolV2Events> {
|
|
|
224
230
|
/** Gets the count of pending transactions */
|
|
225
231
|
getPendingTxCount(): Promise<number>;
|
|
226
232
|
|
|
233
|
+
/**
|
|
234
|
+
* Returns whether at least `minCount` pending transactions are old enough per minTxPoolAgeMs to be eligible
|
|
235
|
+
* for block building. Stops scanning once the threshold is reached, so it is cheaper than counting all
|
|
236
|
+
* eligible txs when only a few are needed.
|
|
237
|
+
*/
|
|
238
|
+
hasEligiblePendingTxs(minCount: number): Promise<boolean>;
|
|
239
|
+
|
|
227
240
|
/** Gets mined transaction hashes with their block IDs */
|
|
228
241
|
getMinedTxHashes(): Promise<[TxHash, L2BlockId][]>;
|
|
229
242
|
|
|
@@ -228,6 +228,29 @@ export class TxPoolIndices {
|
|
|
228
228
|
return this.#pendingByPriority.length;
|
|
229
229
|
}
|
|
230
230
|
|
|
231
|
+
/**
|
|
232
|
+
* Returns whether at least `minCount` pending transactions were received at or before maxReceivedAt,
|
|
233
|
+
* i.e. are old enough to be eligible for block building. Stops as soon as the threshold is met, so it
|
|
234
|
+
* does not scan the whole pool when only a handful of eligible txs are needed. The total pending count
|
|
235
|
+
* is an upper bound on the eligible count, so a pool with fewer than `minCount` pending can never
|
|
236
|
+
* satisfy the threshold and short-circuits without scanning.
|
|
237
|
+
*/
|
|
238
|
+
hasEligiblePendingTxs(maxReceivedAt: number, minCount: number): boolean {
|
|
239
|
+
if (minCount <= 0) {
|
|
240
|
+
return true;
|
|
241
|
+
}
|
|
242
|
+
if (this.#pendingByPriority.length < minCount) {
|
|
243
|
+
return false;
|
|
244
|
+
}
|
|
245
|
+
let count = 0;
|
|
246
|
+
for (const _ of this.iterateEligiblePendingByPriority('desc', maxReceivedAt)) {
|
|
247
|
+
if (++count >= minCount) {
|
|
248
|
+
return true;
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
return false;
|
|
252
|
+
}
|
|
253
|
+
|
|
231
254
|
/** Gets the lowest priority pending transaction hashes (up to limit) */
|
|
232
255
|
getLowestPriorityPending(limit: number): string[] {
|
|
233
256
|
if (limit <= 0) {
|
|
@@ -20,6 +20,7 @@ import type {
|
|
|
20
20
|
TxPoolV2Events,
|
|
21
21
|
} from './interfaces.js';
|
|
22
22
|
import type { TxState } from './tx_metadata.js';
|
|
23
|
+
import type { MinedTxInfo } from './tx_pool_v2_impl.js';
|
|
23
24
|
import { TxPoolV2Impl } from './tx_pool_v2_impl.js';
|
|
24
25
|
|
|
25
26
|
/**
|
|
@@ -56,17 +57,22 @@ export class AztecKVTxPoolV2 extends (EventEmitter as new () => TypedEventEmitte
|
|
|
56
57
|
// Create callbacks that the impl uses to notify us about events and metrics
|
|
57
58
|
const callbacks = {
|
|
58
59
|
onTxsAdded: (txs: Tx[], opts: { source?: string }) => {
|
|
59
|
-
this.#metrics?.transactionsAdded(txs);
|
|
60
60
|
this.emit('txs-added', { txs, ...opts });
|
|
61
61
|
},
|
|
62
62
|
onTxsRemoved: (txHashes: string[] | bigint[]) => {
|
|
63
|
-
this.#metrics?.transactionsRemoved(txHashes);
|
|
64
63
|
// Convert to TxHash objects for the event
|
|
65
64
|
const hashes = txHashes.map(h => (typeof h === 'string' ? TxHash.fromString(h) : TxHash.fromBigInt(h)));
|
|
66
65
|
this.emit('txs-removed', { txHashes: hashes });
|
|
67
66
|
},
|
|
68
|
-
onTxsMined: (
|
|
69
|
-
|
|
67
|
+
onTxsMined: (minedTxs: MinedTxInfo[]) => {
|
|
68
|
+
// Pending-to-mined delay is derived from the tx's persisted receivedAt at the mined
|
|
69
|
+
// transition (see TxPoolV2Impl.handleMinedBlock), not the add/remove timestamp map —
|
|
70
|
+
// so eviction no longer pollutes MEMPOOL_TX_MINED_DELAY.
|
|
71
|
+
for (const { minedDelayMs } of minedTxs) {
|
|
72
|
+
if (minedDelayMs !== undefined) {
|
|
73
|
+
this.#metrics?.recordMinedDelay(minedDelayMs);
|
|
74
|
+
}
|
|
75
|
+
}
|
|
70
76
|
},
|
|
71
77
|
};
|
|
72
78
|
|
|
@@ -156,6 +162,10 @@ export class AztecKVTxPoolV2 extends (EventEmitter as new () => TypedEventEmitte
|
|
|
156
162
|
return this.#queue.put(() => Promise.resolve(this.#impl.getPendingTxCount()));
|
|
157
163
|
}
|
|
158
164
|
|
|
165
|
+
hasEligiblePendingTxs(minCount: number): Promise<boolean> {
|
|
166
|
+
return this.#queue.put(() => Promise.resolve(this.#impl.hasEligiblePendingTxs(minCount)));
|
|
167
|
+
}
|
|
168
|
+
|
|
159
169
|
getMinedTxHashes(): Promise<[TxHash, L2BlockId][]> {
|
|
160
170
|
return this.#queue.put(() => Promise.resolve(this.#impl.getMinedTxHashes()));
|
|
161
171
|
}
|
|
@@ -51,10 +51,18 @@ const FINALIZE_BLOCK_CHUNK_SIZE = 100;
|
|
|
51
51
|
/**
|
|
52
52
|
* Callbacks for the implementation to notify the outer class about events and metrics.
|
|
53
53
|
*/
|
|
54
|
+
/** A tx that has just transitioned to mined, with the time it spent in the pool. */
|
|
55
|
+
export interface MinedTxInfo {
|
|
56
|
+
txHash: string;
|
|
57
|
+
/** Wall-clock ms from receipt into the pool to being marked mined. Undefined when the
|
|
58
|
+
* receive time is unknown (e.g. a tx hydrated from the DB on restart, receivedAt === 0). */
|
|
59
|
+
minedDelayMs?: number;
|
|
60
|
+
}
|
|
61
|
+
|
|
54
62
|
export interface TxPoolV2Callbacks {
|
|
55
63
|
onTxsAdded: (txs: Tx[], opts: { source?: string }) => void;
|
|
56
64
|
onTxsRemoved: (txHashes: string[] | bigint[]) => void;
|
|
57
|
-
onTxsMined: (
|
|
65
|
+
onTxsMined: (minedTxs: MinedTxInfo[]) => void;
|
|
58
66
|
}
|
|
59
67
|
|
|
60
68
|
/**
|
|
@@ -560,7 +568,15 @@ export class TxPoolV2Impl {
|
|
|
560
568
|
});
|
|
561
569
|
|
|
562
570
|
if (found.length > 0) {
|
|
563
|
-
|
|
571
|
+
// receivedAt is 0 for txs hydrated from the DB on restart (true receive time lost) — leave
|
|
572
|
+
// their delay undefined so the metric isn't polluted by epoch-sized values.
|
|
573
|
+
const now = this.#dateProvider.now();
|
|
574
|
+
this.#callbacks.onTxsMined(
|
|
575
|
+
found.map(m => ({
|
|
576
|
+
txHash: m.txHash,
|
|
577
|
+
minedDelayMs: m.receivedAt > 0 ? now - m.receivedAt : undefined,
|
|
578
|
+
})),
|
|
579
|
+
);
|
|
564
580
|
}
|
|
565
581
|
|
|
566
582
|
this.#log.info(`Marked ${found.length} txs as mined in block ${blockId.number}`);
|
|
@@ -712,8 +728,12 @@ export class TxPoolV2Impl {
|
|
|
712
728
|
async handleFinalizedBlock(block: BlockHeader): Promise<void> {
|
|
713
729
|
const blockNumber = block.globalVariables.blockNumber;
|
|
714
730
|
|
|
715
|
-
//
|
|
716
|
-
|
|
731
|
+
// Hold finalized txs for a configurable margin behind the finalized tip so a prover still
|
|
732
|
+
// proving an epoch with already-finalized blocks isn't starved of its txs. 0 deletes at the finalized tip.
|
|
733
|
+
const cutoffBlock = await this.#finalizationCutoffBlock(block);
|
|
734
|
+
|
|
735
|
+
// Step 1: Find mined txs at or before the cutoff block
|
|
736
|
+
const minedTxsToFinalize = this.#indices.findTxsMinedAtOrBefore(cutoffBlock);
|
|
717
737
|
|
|
718
738
|
// Step 2: Archive in chunks if archiving is enabled. Hydrating an entire epoch's worth of
|
|
719
739
|
// mined txs at once would OOM under load. When archiving is disabled there is no need to hydrate the txs at all.
|
|
@@ -737,16 +757,52 @@ export class TxPoolV2Impl {
|
|
|
737
757
|
// transaction. Only tx hashes are touched here, so memory is bounded and atomicity is preserved.
|
|
738
758
|
await this.#store.transactionAsync(async () => {
|
|
739
759
|
await this.#deleteTxsBatch(minedTxsToFinalize);
|
|
740
|
-
await this.#deletedPool.finalizeBlock(
|
|
760
|
+
await this.#deletedPool.finalizeBlock(cutoffBlock);
|
|
741
761
|
});
|
|
742
762
|
|
|
743
763
|
if (minedTxsToFinalize.length > 0) {
|
|
744
|
-
this.#log.info(`Finalized ${minedTxsToFinalize.length} mined txs from blocks up to ${
|
|
764
|
+
this.#log.info(`Finalized ${minedTxsToFinalize.length} mined txs from blocks up to ${cutoffBlock}`, {
|
|
745
765
|
txHashes: minedTxsToFinalize,
|
|
766
|
+
finalizedBlockNumber: blockNumber,
|
|
767
|
+
cutoffBlock,
|
|
746
768
|
});
|
|
747
769
|
}
|
|
748
770
|
}
|
|
749
771
|
|
|
772
|
+
/**
|
|
773
|
+
* Resolves the block up to which finalized txs may be deleted, given `keepFinalizedTxsForSlots`.
|
|
774
|
+
* With a margin of 0 this is just the finalized block. Otherwise we keep txs whose block is within
|
|
775
|
+
* the margin (in slots) of the finalized tip: take the target slot `finalizedSlot - margin`, find the
|
|
776
|
+
* latest checkpoint at or before it with a single reverse range query (`limit: 1`), and use that
|
|
777
|
+
* checkpoint's last block as the cutoff. This rounds to a checkpoint boundary, so it can retain slightly
|
|
778
|
+
* more than the configured margin but never less, and never deletes past the finalized block.
|
|
779
|
+
*/
|
|
780
|
+
async #finalizationCutoffBlock(block: BlockHeader): Promise<BlockNumber> {
|
|
781
|
+
const keepSlots = this.#config.keepFinalizedTxsForSlots;
|
|
782
|
+
if (keepSlots <= 0) {
|
|
783
|
+
return block.globalVariables.blockNumber;
|
|
784
|
+
}
|
|
785
|
+
|
|
786
|
+
// The goal here really is to ensure we keep transactions long enough for them to be proven.
|
|
787
|
+
// So we could be smart and calculate the number of slots until the end of the proof submission window.
|
|
788
|
+
// Our approach here is a lot simpler however and we just keep transactions for the configured number
|
|
789
|
+
// of slots past L1 finalisation. This means we may keep transactions in the mined pool for longer
|
|
790
|
+
// than strictly necessary.
|
|
791
|
+
const targetSlot = block.getSlot() - keepSlots;
|
|
792
|
+
if (targetSlot < 0) {
|
|
793
|
+
// The margin reaches past genesis: keep everything.
|
|
794
|
+
return BlockNumber(0);
|
|
795
|
+
}
|
|
796
|
+
|
|
797
|
+
const [checkpoint] = await this.#l2BlockSource.getCheckpointsData({
|
|
798
|
+
fromSlot: SlotNumber(targetSlot),
|
|
799
|
+
limit: 1,
|
|
800
|
+
reverse: true,
|
|
801
|
+
});
|
|
802
|
+
// No checkpoint at or before the target slot (e.g. early chain): keep everything.
|
|
803
|
+
return checkpoint ? BlockNumber(checkpoint.startBlock + checkpoint.blockCount - 1) : BlockNumber(0);
|
|
804
|
+
}
|
|
805
|
+
|
|
750
806
|
// === Query Methods ===
|
|
751
807
|
|
|
752
808
|
async getTxByHash(txHash: TxHash, opts: { includeProof?: boolean } = {}): Promise<Tx | undefined> {
|
|
@@ -814,6 +870,11 @@ export class TxPoolV2Impl {
|
|
|
814
870
|
return this.#indices.getPendingTxCount();
|
|
815
871
|
}
|
|
816
872
|
|
|
873
|
+
hasEligiblePendingTxs(minCount: number): boolean {
|
|
874
|
+
const maxReceivedAt = this.#dateProvider.now() - this.#config.minTxPoolAgeMs;
|
|
875
|
+
return this.#indices.hasEligiblePendingTxs(maxReceivedAt, minCount);
|
|
876
|
+
}
|
|
877
|
+
|
|
817
878
|
getMinedTxHashes(): [TxHash, L2BlockId][] {
|
|
818
879
|
return this.#indices.getMinedTxs().map(([hash, blockId]) => [TxHash.fromString(hash), blockId]);
|
|
819
880
|
}
|
|
@@ -857,6 +918,9 @@ export class TxPoolV2Impl {
|
|
|
857
918
|
if (config.minTxPoolAgeMs !== undefined) {
|
|
858
919
|
this.#config.minTxPoolAgeMs = config.minTxPoolAgeMs;
|
|
859
920
|
}
|
|
921
|
+
if (config.keepFinalizedTxsForSlots !== undefined) {
|
|
922
|
+
this.#config.keepFinalizedTxsForSlots = config.keepFinalizedTxsForSlots;
|
|
923
|
+
}
|
|
860
924
|
// Update eviction rules with new config
|
|
861
925
|
this.#evictionManager.updateConfig(config);
|
|
862
926
|
}
|
|
@@ -97,7 +97,11 @@ export class ProposalValidator {
|
|
|
97
97
|
|
|
98
98
|
// Proposer check
|
|
99
99
|
const expectedProposer = await this.epochCache.getProposerAttesterAddressInSlot(slotNumber);
|
|
100
|
-
if (expectedProposer
|
|
100
|
+
if (expectedProposer === undefined) {
|
|
101
|
+
this.logger.warn(`Penalizing peer for proposal with no expected proposer for current slot ${slotNumber}`);
|
|
102
|
+
return { result: 'reject', severity: PeerErrorSeverity.HighToleranceError };
|
|
103
|
+
}
|
|
104
|
+
if (!proposer.equals(expectedProposer)) {
|
|
101
105
|
this.logger.warn(`Penalizing peer for invalid proposer for current slot ${slotNumber}`, {
|
|
102
106
|
expectedProposer,
|
|
103
107
|
proposer: proposer.toString(),
|
package/src/services/encoding.ts
CHANGED
|
@@ -7,6 +7,12 @@ import type { Message } from '@libp2p/interface';
|
|
|
7
7
|
import { webcrypto } from 'node:crypto';
|
|
8
8
|
import { compressSync, uncompressSync } from 'snappy';
|
|
9
9
|
|
|
10
|
+
/**
|
|
11
|
+
* Fallback maximum response size (in KB) applied when no explicit per-request bound is supplied.
|
|
12
|
+
* Used both as the `SnappyTransform` default and as the reception-side fallback in `readMessage`.
|
|
13
|
+
*/
|
|
14
|
+
export const DEFAULT_MAX_RESPONSE_SIZE_KB = 10 * 1024;
|
|
15
|
+
|
|
10
16
|
/** Thrown when a Snappy-compressed response exceeds the allowed decompressed size. */
|
|
11
17
|
export class OversizedSnappyResponseError extends Error {
|
|
12
18
|
constructor(decompressedSize: number, maxSizeKb: number) {
|
|
@@ -15,6 +21,18 @@ export class OversizedSnappyResponseError extends Error {
|
|
|
15
21
|
}
|
|
16
22
|
}
|
|
17
23
|
|
|
24
|
+
/**
|
|
25
|
+
* Thrown when the compressed bytes accumulated while reading a response stream exceed the reception
|
|
26
|
+
* bound, before the full payload is buffered. Distinct from {@link OversizedSnappyResponseError},
|
|
27
|
+
* which guards the decompressed size after the stream has already been fully received.
|
|
28
|
+
*/
|
|
29
|
+
export class ResponseSizeLimitExceededError extends Error {
|
|
30
|
+
constructor(accumulatedBytes: number, limitBytes: number) {
|
|
31
|
+
super(`Accumulated response size ${accumulatedBytes} bytes exceeds reception limit of ${limitBytes} bytes`);
|
|
32
|
+
this.name = 'ResponseSizeLimitExceededError';
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
|
|
18
36
|
// Shared buffer to convert msgId to string
|
|
19
37
|
const sharedMsgIdBuf = Buffer.alloc(20);
|
|
20
38
|
|
|
@@ -66,7 +84,7 @@ const DefaultMaxSizesKb: Record<TopicType, number> = {
|
|
|
66
84
|
export class SnappyTransform implements DataTransform {
|
|
67
85
|
constructor(
|
|
68
86
|
private maxSizesKb: Record<TopicType, number> = DefaultMaxSizesKb,
|
|
69
|
-
private defaultMaxSizeKb: number =
|
|
87
|
+
private defaultMaxSizeKb: number = DEFAULT_MAX_RESPONSE_SIZE_KB,
|
|
70
88
|
private logger = createLogger('p2p:snappy-transform'),
|
|
71
89
|
) {}
|
|
72
90
|
|
|
@@ -90,7 +90,7 @@ import type { PeerManagerInterface } from '../peer-manager/interface.js';
|
|
|
90
90
|
import { PeerManager } from '../peer-manager/peer_manager.js';
|
|
91
91
|
import { PeerScoring } from '../peer-manager/peer_scoring.js';
|
|
92
92
|
import type { BatchTxRequesterLibP2PService } from '../reqresp/batch-tx-requester/interface.js';
|
|
93
|
-
import type
|
|
93
|
+
import { type P2PReqRespConfig, YAMUX_MAX_MESSAGE_SIZE_BYTES } from '../reqresp/config.js';
|
|
94
94
|
import {
|
|
95
95
|
AuthRequest,
|
|
96
96
|
BlockTxsRequest,
|
|
@@ -457,7 +457,8 @@ export class LibP2PService extends WithTracer implements P2PService {
|
|
|
457
457
|
],
|
|
458
458
|
datastore,
|
|
459
459
|
peerDiscovery,
|
|
460
|
-
|
|
460
|
+
// Pin the yamux frame size: MAX_REQRESP_REQUEST_SIZE_BYTES relies on a reqresp request fitting in one frame.
|
|
461
|
+
streamMuxers: [yamux({ maxMessageSize: YAMUX_MAX_MESSAGE_SIZE_BYTES }), mplex()],
|
|
461
462
|
connectionEncryption: [noise()],
|
|
462
463
|
connectionManager: {
|
|
463
464
|
minConnections: 0, // Disable libp2p peer dialing, we do it manually
|
|
@@ -14,9 +14,8 @@ Applied before the protocol handler runs.
|
|
|
14
14
|
| STATUS | 5/s | 10/s | same |
|
|
15
15
|
| AUTH | 5/s | 10/s | same |
|
|
16
16
|
| GOODBYE | 5/s | 10/s | same |
|
|
17
|
-
| BLOCK | 2/s | 5/s | same |
|
|
18
17
|
| BLOCK_TXS | 10/s | 200/s | same |
|
|
19
|
-
| TX |
|
|
18
|
+
| TX | 10/s | 200/s | same |
|
|
20
19
|
|
|
21
20
|
- Per-peer limit exceeded: `HighToleranceError` penalty + `RATE_LIMIT_EXCEEDED` status. Penalty fires inside `RequestResponseRateLimiter.allow()`, not the stream handler.
|
|
22
21
|
- Global limit exceeded: `RATE_LIMIT_EXCEEDED` status only (no peer penalty).
|
|
@@ -149,26 +148,6 @@ Protected peers (private/trusted/preferred) are always considered "authenticated
|
|
|
149
148
|
|
|
150
149
|
## Block Data Protocols
|
|
151
150
|
|
|
152
|
-
### BLOCK Protocol (`/aztec/req/block/1.0.0`)
|
|
153
|
-
|
|
154
|
-
**Server side**:
|
|
155
|
-
|
|
156
|
-
| Rule | Consequence | File |
|
|
157
|
-
|------|-------------|------|
|
|
158
|
-
| Request must parse as `Fr` | `BADLY_FORMED_REQUEST` + LowToleranceError | `protocols/block.ts` |
|
|
159
|
-
| Block lookup throws | `INTERNAL_ERROR` status | same |
|
|
160
|
-
| Block not found | SUCCESS + empty buffer (design choice; no `NOT_FOUND` status used) | same |
|
|
161
|
-
|
|
162
|
-
**Requester side** (Snappy limit: 3 MB):
|
|
163
|
-
|
|
164
|
-
| Rule | Consequence | File |
|
|
165
|
-
|------|-------------|------|
|
|
166
|
-
| Response block number must match requested | LowToleranceError; rejected | `libp2p_service.ts` (`validateRequestedBlock`) |
|
|
167
|
-
| Local block must exist for hash verification | Rejected (no penalty) | same |
|
|
168
|
-
| Response block hash must equal local block hash | MidToleranceError; rejected | same |
|
|
169
|
-
|
|
170
|
-
**Limitation**: the local-block requirement means BLOCK req/resp is unusable for initial P2P-only sync (before L1 sync provides local copies for verification). A TODO in the code acknowledges this.
|
|
171
|
-
|
|
172
151
|
### BLOCK_TXS Protocol (`/aztec/req/block_txs/1.0.0`)
|
|
173
152
|
|
|
174
153
|
**Server side**:
|
|
@@ -178,7 +157,7 @@ Protected peers (private/trusted/preferred) are always considered "authenticated
|
|
|
178
157
|
| Request must parse as `BlockTxsRequest` (Fr + TxHashArray + BitVector) | `BADLY_FORMED_REQUEST` + LowToleranceError | `protocols/block_txs/block_txs_handler.ts` |
|
|
179
158
|
| BitVector length: non-negative and <= `MAX_TXS_PER_BLOCK` (65536) | Deserialization throws -> `BADLY_FORMED_REQUEST` | `protocols/block_txs/bitvector.ts` |
|
|
180
159
|
| Archive root not found and no explicit txHashes | `NOT_FOUND` status | handler |
|
|
181
|
-
| Internal error during lookup | Unhandled exception -> stream abort (no `INTERNAL_ERROR` status
|
|
160
|
+
| Internal error during lookup | Unhandled exception -> stream abort (no `INTERNAL_ERROR` status) | handler |
|
|
182
161
|
|
|
183
162
|
Conditional registration: BLOCK_TXS handler only registered when `config.disableTransactions` is false. Otherwise peers get `ERR_UNSUPPORTED_PROTOCOL`.
|
|
184
163
|
|
|
@@ -5,6 +5,23 @@ export const DEFAULT_OVERALL_REQUEST_TIMEOUT_MS = 10_000; // Not currently used
|
|
|
5
5
|
export const DEFAULT_REQRESP_DIAL_TIMEOUT_MS = 5_000;
|
|
6
6
|
export const DEFAULT_OPTIMISTIC_NEGOTIATION = false;
|
|
7
7
|
|
|
8
|
+
/**
|
|
9
|
+
* Max size of a yamux frame, header included. This is the library default, pinned explicitly when configuring the
|
|
10
|
+
* muxer in the libp2p service so a dependency upgrade cannot silently change it.
|
|
11
|
+
*/
|
|
12
|
+
export const YAMUX_MAX_MESSAGE_SIZE_BYTES = 64 * 1024;
|
|
13
|
+
|
|
14
|
+
/** Size of a yamux data frame header. */
|
|
15
|
+
const YAMUX_HEADER_LENGTH_BYTES = 12;
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* Max size of a reqresp request payload. A request must fit in a single muxer frame: the responder never reassembles
|
|
19
|
+
* a request from multiple chunks (see `ReqResp.processStream`), and yamux splits writes larger than one frame into
|
|
20
|
+
* multiple frames, each of which arrives as a separate chunk. mplex only splits at 1 MiB, so yamux is the binding
|
|
21
|
+
* constraint.
|
|
22
|
+
*/
|
|
23
|
+
export const MAX_REQRESP_REQUEST_SIZE_BYTES = YAMUX_MAX_MESSAGE_SIZE_BYTES - YAMUX_HEADER_LENGTH_BYTES;
|
|
24
|
+
|
|
8
25
|
// For use in tests.
|
|
9
26
|
export const DEFAULT_P2P_REQRESP_CONFIG: P2PReqRespConfig = {
|
|
10
27
|
overallRequestTimeoutMs: DEFAULT_OVERALL_REQUEST_TIMEOUT_MS,
|
|
@@ -9,7 +9,7 @@ export const DEFAULT_RATE_LIMITS: ReqRespSubProtocolRateLimits = {
|
|
|
9
9
|
},
|
|
10
10
|
globalLimit: {
|
|
11
11
|
quotaTimeMs: 1000,
|
|
12
|
-
quotaCount:
|
|
12
|
+
quotaCount: 50,
|
|
13
13
|
},
|
|
14
14
|
},
|
|
15
15
|
[ReqRespSubProtocol.STATUS]: {
|
|
@@ -19,7 +19,7 @@ export const DEFAULT_RATE_LIMITS: ReqRespSubProtocolRateLimits = {
|
|
|
19
19
|
},
|
|
20
20
|
globalLimit: {
|
|
21
21
|
quotaTimeMs: 1000,
|
|
22
|
-
quotaCount:
|
|
22
|
+
quotaCount: 50,
|
|
23
23
|
},
|
|
24
24
|
},
|
|
25
25
|
[ReqRespSubProtocol.AUTH]: {
|
|
@@ -49,7 +49,7 @@ export const DEFAULT_RATE_LIMITS: ReqRespSubProtocolRateLimits = {
|
|
|
49
49
|
},
|
|
50
50
|
globalLimit: {
|
|
51
51
|
quotaTimeMs: 1000,
|
|
52
|
-
quotaCount:
|
|
52
|
+
quotaCount: 50,
|
|
53
53
|
},
|
|
54
54
|
},
|
|
55
55
|
[ReqRespSubProtocol.BLOCK_TXS]: {
|
|
@@ -10,12 +10,18 @@ import type { Libp2p } from 'libp2p';
|
|
|
10
10
|
import { pipeline } from 'node:stream/promises';
|
|
11
11
|
import type { Uint8ArrayList } from 'uint8arraylist';
|
|
12
12
|
|
|
13
|
-
import { IndividualReqRespTimeoutError } from '../../errors/reqresp.error.js';
|
|
14
|
-
import {
|
|
13
|
+
import { IndividualReqRespTimeoutError, OversizedReqRespRequestError } from '../../errors/reqresp.error.js';
|
|
14
|
+
import {
|
|
15
|
+
DEFAULT_MAX_RESPONSE_SIZE_KB,
|
|
16
|
+
OversizedSnappyResponseError,
|
|
17
|
+
ResponseSizeLimitExceededError,
|
|
18
|
+
SnappyTransform,
|
|
19
|
+
} from '../encoding.js';
|
|
15
20
|
import type { PeerScoring } from '../peer-manager/peer_scoring.js';
|
|
16
21
|
import {
|
|
17
22
|
DEFAULT_INDIVIDUAL_REQUEST_TIMEOUT_MS,
|
|
18
23
|
DEFAULT_REQRESP_DIAL_TIMEOUT_MS,
|
|
24
|
+
MAX_REQRESP_REQUEST_SIZE_BYTES,
|
|
19
25
|
type P2PReqRespConfig,
|
|
20
26
|
} from './config.js';
|
|
21
27
|
import { ConnectionSampler, RandomSampler } from './connection-sampler/connection_sampler.js';
|
|
@@ -207,6 +213,11 @@ export class ReqResp implements ReqRespInterface {
|
|
|
207
213
|
payload: Buffer,
|
|
208
214
|
dialTimeout: number = this.dialTimeoutMs,
|
|
209
215
|
): Promise<ReqRespResponse> {
|
|
216
|
+
// Thrown before the try block so the generic error handling below does not penalize the peer for our own bug.
|
|
217
|
+
if (payload.length > MAX_REQRESP_REQUEST_SIZE_BYTES) {
|
|
218
|
+
throw new OversizedReqRespRequestError(subProtocol, payload.length, MAX_REQRESP_REQUEST_SIZE_BYTES);
|
|
219
|
+
}
|
|
220
|
+
|
|
210
221
|
let stream: Stream | undefined;
|
|
211
222
|
try {
|
|
212
223
|
this.metrics.recordRequestSent(subProtocol);
|
|
@@ -291,6 +302,14 @@ export class ReqResp implements ReqRespInterface {
|
|
|
291
302
|
* - The first chunk should contain a control byte, indicating the status of the response see `ReqRespStatus`
|
|
292
303
|
* - The second chunk should contain the response data
|
|
293
304
|
*
|
|
305
|
+
* To bound memory usage we abort reception as soon as the accumulated compressed bytes exceed a
|
|
306
|
+
* bound derived from `maxSizeKb`, before the full stream has been buffered. Without this a peer we
|
|
307
|
+
* dialed could stream data up to the request timeout, forcing us to buffer arbitrarily much (the
|
|
308
|
+
* `maxSizeKb` guard in `inboundTransformData` only runs after the whole compressed payload is
|
|
309
|
+
* concatenated, so it does nothing to cap reception memory). Snappy can expand incompressible input
|
|
310
|
+
* to ~1.17x its size, so we allow 2x `maxSizeKb`: comfortably above the worst-case expansion of a
|
|
311
|
+
* legitimate max-size response, while still capping buffered memory at twice the permitted size.
|
|
312
|
+
*
|
|
294
313
|
* @param source - The async iterable source of data chunks
|
|
295
314
|
* @param maxSizeKb - Optional maximum expected size in KB for the decompressed response
|
|
296
315
|
*/
|
|
@@ -298,10 +317,20 @@ export class ReqResp implements ReqRespInterface {
|
|
|
298
317
|
let status: ReqRespStatus | undefined;
|
|
299
318
|
const chunks: Uint8Array[] = [];
|
|
300
319
|
|
|
320
|
+
const maxAccumulatedBytes = (maxSizeKb ?? DEFAULT_MAX_RESPONSE_SIZE_KB) * 1024 * 2;
|
|
321
|
+
let accumulatedBytes = 0;
|
|
322
|
+
|
|
301
323
|
try {
|
|
302
324
|
for await (const chunk of source) {
|
|
303
325
|
const statusParsed = status !== undefined;
|
|
304
326
|
if (statusParsed) {
|
|
327
|
+
// Use `byteLength` (the logical length) rather than `chunk.subarray().length`: calling
|
|
328
|
+
// `subarray()` on a `Uint8ArrayList` consolidates its backing buffers into a single copy,
|
|
329
|
+
// which would materialize the very chunk we are trying to reject.
|
|
330
|
+
accumulatedBytes += chunk.byteLength;
|
|
331
|
+
if (accumulatedBytes > maxAccumulatedBytes) {
|
|
332
|
+
throw new ResponseSizeLimitExceededError(accumulatedBytes, maxAccumulatedBytes);
|
|
333
|
+
}
|
|
305
334
|
chunks.push(chunk.subarray());
|
|
306
335
|
continue;
|
|
307
336
|
}
|
|
@@ -440,6 +469,9 @@ export class ReqResp implements ReqRespInterface {
|
|
|
440
469
|
await pipeline(
|
|
441
470
|
stream.source,
|
|
442
471
|
async function* (source: any) {
|
|
472
|
+
// Req/resp is one-request-one-response: an honest sender writes a single payload and half-closes.
|
|
473
|
+
// Process only the first chunk and return, so extra frames a peer queues on the same stream cannot
|
|
474
|
+
// drive further handler invocations that bypass the per-stream rate-limit check in streamHandler.
|
|
443
475
|
for await (const chunk of source) {
|
|
444
476
|
const response = await handler(connection.remotePeer, chunk.subarray());
|
|
445
477
|
|
|
@@ -454,6 +486,7 @@ export class ReqResp implements ReqRespInterface {
|
|
|
454
486
|
|
|
455
487
|
yield SUCCESS;
|
|
456
488
|
yield snappy.outboundTransformData(response);
|
|
489
|
+
return;
|
|
457
490
|
}
|
|
458
491
|
},
|
|
459
492
|
stream.sink,
|
|
@@ -564,6 +597,13 @@ export class ReqResp implements ReqRespInterface {
|
|
|
564
597
|
return PeerErrorSeverity.LowToleranceError;
|
|
565
598
|
}
|
|
566
599
|
|
|
600
|
+
// Response exceeded the reception buffer bound: the peer streamed more compressed data than the
|
|
601
|
+
// per-request size limit allows. Same protocol violation as above, penalize harshly.
|
|
602
|
+
if (e instanceof ResponseSizeLimitExceededError) {
|
|
603
|
+
this.logger.warn(`Oversized response from peer ${peerId.toString()} in ${subProtocol}: ${e.message}`, logTags);
|
|
604
|
+
return PeerErrorSeverity.LowToleranceError;
|
|
605
|
+
}
|
|
606
|
+
|
|
567
607
|
return this.categorizeConnectionErrors(e, peerId, subProtocol);
|
|
568
608
|
}
|
|
569
609
|
|
|
@@ -171,6 +171,10 @@ export class InMemoryTxPool extends EventEmitter implements TxPoolV2 {
|
|
|
171
171
|
return Promise.resolve(this.txsByHash.size);
|
|
172
172
|
}
|
|
173
173
|
|
|
174
|
+
async hasEligiblePendingTxs(minCount: number): Promise<boolean> {
|
|
175
|
+
return (await this.getPendingTxCount()) >= minCount;
|
|
176
|
+
}
|
|
177
|
+
|
|
174
178
|
getMinedTxHashes(): Promise<[TxHash, L2BlockId][]> {
|
|
175
179
|
return Promise.resolve([]);
|
|
176
180
|
}
|