@aztec/p2p 0.0.1-commit.4ad48494d → 0.0.1-commit.4eabbdb
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dest/client/factory.d.ts +3 -2
- package/dest/client/factory.d.ts.map +1 -1
- package/dest/client/factory.js +3 -2
- package/dest/client/interface.d.ts +7 -9
- package/dest/client/interface.d.ts.map +1 -1
- package/dest/client/p2p_client.d.ts +4 -7
- package/dest/client/p2p_client.d.ts.map +1 -1
- package/dest/client/p2p_client.js +38 -14
- package/dest/client/test/tx_proposal_collector/proposal_tx_collector_worker.js +5 -5
- package/dest/config.d.ts +3 -2
- package/dest/config.d.ts.map +1 -1
- package/dest/errors/tx-pool.error.d.ts +8 -0
- package/dest/errors/tx-pool.error.d.ts.map +1 -0
- package/dest/errors/tx-pool.error.js +9 -0
- package/dest/mem_pools/attestation_pool/mocks.d.ts +2 -2
- package/dest/mem_pools/attestation_pool/mocks.d.ts.map +1 -1
- package/dest/mem_pools/attestation_pool/mocks.js +2 -2
- package/dest/mem_pools/tx_pool/eviction/invalid_txs_after_mining_rule.js +3 -3
- package/dest/mem_pools/tx_pool_v2/deleted_pool.d.ts +3 -1
- package/dest/mem_pools/tx_pool_v2/deleted_pool.d.ts.map +1 -1
- package/dest/mem_pools/tx_pool_v2/deleted_pool.js +9 -0
- package/dest/mem_pools/tx_pool_v2/eviction/eviction_manager.d.ts +3 -3
- package/dest/mem_pools/tx_pool_v2/eviction/eviction_manager.d.ts.map +1 -1
- package/dest/mem_pools/tx_pool_v2/eviction/eviction_manager.js +18 -9
- package/dest/mem_pools/tx_pool_v2/eviction/fee_payer_balance_eviction_rule.js +2 -2
- package/dest/mem_pools/tx_pool_v2/eviction/fee_payer_balance_pre_add_rule.d.ts +3 -3
- package/dest/mem_pools/tx_pool_v2/eviction/fee_payer_balance_pre_add_rule.d.ts.map +1 -1
- package/dest/mem_pools/tx_pool_v2/eviction/fee_payer_balance_pre_add_rule.js +10 -4
- package/dest/mem_pools/tx_pool_v2/eviction/index.d.ts +2 -2
- package/dest/mem_pools/tx_pool_v2/eviction/index.d.ts.map +1 -1
- package/dest/mem_pools/tx_pool_v2/eviction/index.js +1 -1
- package/dest/mem_pools/tx_pool_v2/eviction/interfaces.d.ts +48 -5
- package/dest/mem_pools/tx_pool_v2/eviction/interfaces.d.ts.map +1 -1
- package/dest/mem_pools/tx_pool_v2/eviction/interfaces.js +8 -0
- package/dest/mem_pools/tx_pool_v2/eviction/invalid_txs_after_mining_rule.js +4 -4
- package/dest/mem_pools/tx_pool_v2/eviction/invalid_txs_after_reorg_rule.js +3 -3
- package/dest/mem_pools/tx_pool_v2/eviction/low_priority_eviction_rule.d.ts +1 -1
- package/dest/mem_pools/tx_pool_v2/eviction/low_priority_eviction_rule.d.ts.map +1 -1
- package/dest/mem_pools/tx_pool_v2/eviction/low_priority_eviction_rule.js +6 -4
- package/dest/mem_pools/tx_pool_v2/eviction/low_priority_pre_add_rule.d.ts +4 -4
- package/dest/mem_pools/tx_pool_v2/eviction/low_priority_pre_add_rule.d.ts.map +1 -1
- package/dest/mem_pools/tx_pool_v2/eviction/low_priority_pre_add_rule.js +14 -4
- package/dest/mem_pools/tx_pool_v2/eviction/nullifier_conflict_rule.d.ts +3 -3
- package/dest/mem_pools/tx_pool_v2/eviction/nullifier_conflict_rule.d.ts.map +1 -1
- package/dest/mem_pools/tx_pool_v2/eviction/nullifier_conflict_rule.js +2 -2
- package/dest/mem_pools/tx_pool_v2/instrumentation.d.ts +15 -0
- package/dest/mem_pools/tx_pool_v2/instrumentation.d.ts.map +1 -0
- package/dest/mem_pools/tx_pool_v2/instrumentation.js +43 -0
- package/dest/mem_pools/tx_pool_v2/interfaces.d.ts +10 -2
- package/dest/mem_pools/tx_pool_v2/interfaces.d.ts.map +1 -1
- package/dest/mem_pools/tx_pool_v2/interfaces.js +2 -1
- package/dest/mem_pools/tx_pool_v2/tx_metadata.d.ts +7 -5
- package/dest/mem_pools/tx_pool_v2/tx_metadata.d.ts.map +1 -1
- package/dest/mem_pools/tx_pool_v2/tx_metadata.js +29 -5
- package/dest/mem_pools/tx_pool_v2/tx_pool_indices.d.ts +5 -2
- package/dest/mem_pools/tx_pool_v2/tx_pool_indices.d.ts.map +1 -1
- package/dest/mem_pools/tx_pool_v2/tx_pool_indices.js +12 -2
- package/dest/mem_pools/tx_pool_v2/tx_pool_v2.d.ts +5 -2
- package/dest/mem_pools/tx_pool_v2/tx_pool_v2.d.ts.map +1 -1
- package/dest/mem_pools/tx_pool_v2/tx_pool_v2.js +6 -5
- package/dest/mem_pools/tx_pool_v2/tx_pool_v2_impl.d.ts +10 -4
- package/dest/mem_pools/tx_pool_v2/tx_pool_v2_impl.d.ts.map +1 -1
- package/dest/mem_pools/tx_pool_v2/tx_pool_v2_impl.js +135 -37
- package/dest/msg_validators/tx_validator/timestamp_validator.d.ts +2 -2
- package/dest/msg_validators/tx_validator/timestamp_validator.d.ts.map +1 -1
- package/dest/msg_validators/tx_validator/timestamp_validator.js +6 -6
- package/dest/services/dummy_service.d.ts +3 -2
- package/dest/services/dummy_service.d.ts.map +1 -1
- package/dest/services/dummy_service.js +3 -0
- package/dest/services/encoding.d.ts +1 -1
- package/dest/services/encoding.d.ts.map +1 -1
- package/dest/services/encoding.js +2 -1
- package/dest/services/gossipsub/topic_score_params.d.ts +18 -6
- package/dest/services/gossipsub/topic_score_params.d.ts.map +1 -1
- package/dest/services/gossipsub/topic_score_params.js +32 -10
- package/dest/services/libp2p/libp2p_service.d.ts +2 -1
- package/dest/services/libp2p/libp2p_service.d.ts.map +1 -1
- package/dest/services/libp2p/libp2p_service.js +7 -3
- package/dest/services/reqresp/batch-tx-requester/batch_tx_requester.d.ts +4 -3
- package/dest/services/reqresp/batch-tx-requester/batch_tx_requester.d.ts.map +1 -1
- package/dest/services/reqresp/batch-tx-requester/batch_tx_requester.js +5 -9
- package/dest/services/reqresp/batch-tx-requester/interface.d.ts +2 -6
- package/dest/services/reqresp/batch-tx-requester/interface.d.ts.map +1 -1
- package/dest/services/reqresp/batch-tx-requester/missing_txs.d.ts +10 -13
- package/dest/services/reqresp/batch-tx-requester/missing_txs.d.ts.map +1 -1
- package/dest/services/reqresp/batch-tx-requester/missing_txs.js +25 -46
- package/dest/services/service.d.ts +4 -2
- package/dest/services/service.d.ts.map +1 -1
- package/dest/services/tx_collection/fast_tx_collection.d.ts +1 -1
- package/dest/services/tx_collection/fast_tx_collection.d.ts.map +1 -1
- package/dest/services/tx_collection/fast_tx_collection.js +39 -33
- package/dest/services/tx_collection/file_store_tx_collection.d.ts +1 -1
- package/dest/services/tx_collection/file_store_tx_collection.d.ts.map +1 -1
- package/dest/services/tx_collection/file_store_tx_collection.js +4 -2
- package/dest/services/tx_collection/file_store_tx_source.d.ts +15 -6
- package/dest/services/tx_collection/file_store_tx_source.d.ts.map +1 -1
- package/dest/services/tx_collection/file_store_tx_source.js +47 -16
- package/dest/services/tx_collection/instrumentation.d.ts +1 -1
- package/dest/services/tx_collection/instrumentation.d.ts.map +1 -1
- package/dest/services/tx_collection/instrumentation.js +2 -1
- package/dest/services/tx_collection/missing_txs_tracker.d.ts +32 -0
- package/dest/services/tx_collection/missing_txs_tracker.d.ts.map +1 -0
- package/dest/services/tx_collection/missing_txs_tracker.js +27 -0
- package/dest/services/tx_collection/proposal_tx_collector.d.ts +7 -6
- package/dest/services/tx_collection/proposal_tx_collector.d.ts.map +1 -1
- package/dest/services/tx_collection/proposal_tx_collector.js +5 -4
- package/dest/services/tx_collection/slow_tx_collection.d.ts +2 -2
- package/dest/services/tx_collection/slow_tx_collection.d.ts.map +1 -1
- package/dest/services/tx_collection/slow_tx_collection.js +10 -8
- package/dest/services/tx_collection/tx_collection.d.ts +5 -4
- package/dest/services/tx_collection/tx_collection.d.ts.map +1 -1
- package/dest/services/tx_collection/tx_collection_sink.d.ts +6 -5
- package/dest/services/tx_collection/tx_collection_sink.d.ts.map +1 -1
- package/dest/services/tx_collection/tx_collection_sink.js +13 -22
- package/dest/services/tx_collection/tx_source.d.ts +8 -3
- package/dest/services/tx_collection/tx_source.d.ts.map +1 -1
- package/dest/services/tx_collection/tx_source.js +19 -2
- package/dest/services/tx_file_store/tx_file_store.js +1 -1
- package/dest/test-helpers/mock-pubsub.d.ts +3 -2
- package/dest/test-helpers/mock-pubsub.d.ts.map +1 -1
- package/dest/test-helpers/mock-pubsub.js +6 -0
- package/dest/test-helpers/testbench-utils.d.ts +5 -2
- package/dest/test-helpers/testbench-utils.d.ts.map +1 -1
- package/dest/test-helpers/testbench-utils.js +1 -1
- package/dest/testbench/p2p_client_testbench_worker.d.ts +2 -2
- package/dest/testbench/p2p_client_testbench_worker.d.ts.map +1 -1
- package/dest/testbench/p2p_client_testbench_worker.js +8 -8
- package/dest/util.d.ts +2 -2
- package/dest/util.d.ts.map +1 -1
- package/package.json +14 -14
- package/src/client/factory.ts +5 -2
- package/src/client/interface.ts +14 -9
- package/src/client/p2p_client.ts +44 -17
- package/src/client/test/tx_proposal_collector/proposal_tx_collector_worker.ts +18 -8
- package/src/config.ts +1 -1
- package/src/errors/tx-pool.error.ts +12 -0
- package/src/mem_pools/attestation_pool/mocks.ts +2 -1
- package/src/mem_pools/tx_pool/README.md +1 -1
- package/src/mem_pools/tx_pool/eviction/invalid_txs_after_mining_rule.ts +3 -3
- package/src/mem_pools/tx_pool_v2/README.md +1 -1
- package/src/mem_pools/tx_pool_v2/deleted_pool.ts +11 -0
- package/src/mem_pools/tx_pool_v2/eviction/eviction_manager.ts +21 -8
- package/src/mem_pools/tx_pool_v2/eviction/fee_payer_balance_eviction_rule.ts +2 -2
- package/src/mem_pools/tx_pool_v2/eviction/fee_payer_balance_pre_add_rule.ts +15 -4
- package/src/mem_pools/tx_pool_v2/eviction/index.ts +4 -0
- package/src/mem_pools/tx_pool_v2/eviction/interfaces.ts +49 -4
- package/src/mem_pools/tx_pool_v2/eviction/invalid_txs_after_mining_rule.ts +4 -4
- package/src/mem_pools/tx_pool_v2/eviction/invalid_txs_after_reorg_rule.ts +3 -3
- package/src/mem_pools/tx_pool_v2/eviction/low_priority_eviction_rule.ts +6 -7
- package/src/mem_pools/tx_pool_v2/eviction/low_priority_pre_add_rule.ts +24 -6
- package/src/mem_pools/tx_pool_v2/eviction/nullifier_conflict_rule.ts +3 -3
- package/src/mem_pools/tx_pool_v2/instrumentation.ts +69 -0
- package/src/mem_pools/tx_pool_v2/interfaces.ts +8 -2
- package/src/mem_pools/tx_pool_v2/tx_metadata.ts +40 -9
- package/src/mem_pools/tx_pool_v2/tx_pool_indices.ts +14 -3
- package/src/mem_pools/tx_pool_v2/tx_pool_v2.ts +11 -6
- package/src/mem_pools/tx_pool_v2/tx_pool_v2_impl.ts +157 -29
- package/src/msg_validators/tx_validator/timestamp_validator.ts +7 -7
- package/src/services/dummy_service.ts +5 -1
- package/src/services/encoding.ts +2 -1
- package/src/services/gossipsub/README.md +29 -14
- package/src/services/gossipsub/topic_score_params.ts +49 -13
- package/src/services/libp2p/libp2p_service.ts +7 -2
- package/src/services/reqresp/batch-tx-requester/batch_tx_requester.ts +6 -6
- package/src/services/reqresp/batch-tx-requester/interface.ts +1 -5
- package/src/services/reqresp/batch-tx-requester/missing_txs.ts +23 -71
- package/src/services/service.ts +10 -1
- package/src/services/tx_collection/fast_tx_collection.ts +51 -30
- package/src/services/tx_collection/file_store_tx_collection.ts +7 -3
- package/src/services/tx_collection/file_store_tx_source.ts +61 -17
- package/src/services/tx_collection/instrumentation.ts +7 -1
- package/src/services/tx_collection/missing_txs_tracker.ts +52 -0
- package/src/services/tx_collection/proposal_tx_collector.ts +8 -7
- package/src/services/tx_collection/slow_tx_collection.ts +8 -9
- package/src/services/tx_collection/tx_collection.ts +4 -3
- package/src/services/tx_collection/tx_collection_sink.ts +15 -29
- package/src/services/tx_collection/tx_source.ts +22 -3
- package/src/services/tx_file_store/tx_file_store.ts +1 -1
- package/src/test-helpers/mock-pubsub.ts +10 -0
- package/src/test-helpers/testbench-utils.ts +2 -2
- package/src/testbench/p2p_client_testbench_worker.ts +20 -13
- package/src/util.ts +7 -1
|
@@ -341,6 +341,7 @@ export class LibP2PService<T extends P2PClientType = P2PClientType.Full> extends
|
|
|
341
341
|
heartbeatIntervalMs: config.gossipsubInterval,
|
|
342
342
|
targetCommitteeSize: l1Constants.targetCommitteeSize,
|
|
343
343
|
blockDurationMs: config.blockDurationMs,
|
|
344
|
+
expectedBlockProposalsPerSlot: config.expectedBlockProposalsPerSlot,
|
|
344
345
|
});
|
|
345
346
|
|
|
346
347
|
const node = await createLibp2p({
|
|
@@ -614,6 +615,10 @@ export class LibP2PService<T extends P2PClientType = P2PClientType.Full> extends
|
|
|
614
615
|
return this.peerManager.getPeers(includePending);
|
|
615
616
|
}
|
|
616
617
|
|
|
618
|
+
public getGossipMeshPeerCount(topicType: TopicType): number {
|
|
619
|
+
return this.node.services.pubsub.getMeshPeers(this.topicStrings[topicType]).length;
|
|
620
|
+
}
|
|
621
|
+
|
|
617
622
|
private handleGossipSubEvent(e: CustomEvent<GossipsubMessage>) {
|
|
618
623
|
this.logger.trace(`Received PUBSUB message.`);
|
|
619
624
|
|
|
@@ -1544,7 +1549,7 @@ export class LibP2PService<T extends P2PClientType = P2PClientType.Full> extends
|
|
|
1544
1549
|
protected async validatePropagatedTx(tx: Tx, peerId: PeerId): Promise<boolean> {
|
|
1545
1550
|
const currentBlockNumber = await this.archiver.getBlockNumber();
|
|
1546
1551
|
|
|
1547
|
-
// We accept transactions if they are not expired by the next slot (checked based on the
|
|
1552
|
+
// We accept transactions if they are not expired by the next slot (checked based on the ExpirationTimestamp field)
|
|
1548
1553
|
const { ts: nextSlotTimestamp } = this.epochCache.getEpochAndSlotInNextL1Slot();
|
|
1549
1554
|
const messageValidators = await this.createMessageValidators(currentBlockNumber, nextSlotTimestamp);
|
|
1550
1555
|
|
|
@@ -1599,7 +1604,7 @@ export class LibP2PService<T extends P2PClientType = P2PClientType.Full> extends
|
|
|
1599
1604
|
public async validate(txs: Tx[]): Promise<void> {
|
|
1600
1605
|
const currentBlockNumber = await this.archiver.getBlockNumber();
|
|
1601
1606
|
|
|
1602
|
-
// We accept transactions if they are not expired by the next slot (checked based on the
|
|
1607
|
+
// We accept transactions if they are not expired by the next slot (checked based on the ExpirationTimestamp field)
|
|
1603
1608
|
const { ts: nextSlotTimestamp } = this.epochCache.getEpochAndSlotInNextL1Slot();
|
|
1604
1609
|
const messageValidators = await this.createMessageValidators(currentBlockNumber, nextSlotTimestamp);
|
|
1605
1610
|
|
|
@@ -10,6 +10,7 @@ import { Tx, TxArray, TxHash } from '@aztec/stdlib/tx';
|
|
|
10
10
|
import type { PeerId } from '@libp2p/interface';
|
|
11
11
|
import { peerIdFromString } from '@libp2p/peer-id';
|
|
12
12
|
|
|
13
|
+
import type { IMissingTxsTracker } from '../../tx_collection/missing_txs_tracker.js';
|
|
13
14
|
import { ReqRespSubProtocol } from '.././interface.js';
|
|
14
15
|
import { BlockTxsRequest, BlockTxsResponse, type BlockTxsSource } from '.././protocols/index.js';
|
|
15
16
|
import { ReqRespStatus } from '.././status.js';
|
|
@@ -20,7 +21,7 @@ import {
|
|
|
20
21
|
DEFAULT_BATCH_TX_REQUESTER_TX_BATCH_SIZE,
|
|
21
22
|
} from './config.js';
|
|
22
23
|
import type { BatchTxRequesterLibP2PService, BatchTxRequesterOptions, ITxMetadataCollection } from './interface.js';
|
|
23
|
-
import {
|
|
24
|
+
import { MissingTxMetadataCollection } from './missing_txs.js';
|
|
24
25
|
import { type IPeerCollection, PeerCollection } from './peer_collection.js';
|
|
25
26
|
import { BatchRequestTxValidator, type IBatchRequestTxValidator } from './tx_validator.js';
|
|
26
27
|
|
|
@@ -60,7 +61,7 @@ export class BatchTxRequester {
|
|
|
60
61
|
private readonly txBatchSize: number;
|
|
61
62
|
|
|
62
63
|
constructor(
|
|
63
|
-
|
|
64
|
+
missingTxsTracker: IMissingTxsTracker,
|
|
64
65
|
blockTxsSource: BlockTxsSource,
|
|
65
66
|
pinnedPeer: PeerId | undefined,
|
|
66
67
|
timeoutMs: number,
|
|
@@ -99,8 +100,7 @@ export class BatchTxRequester {
|
|
|
99
100
|
this.p2pService.peerScoring,
|
|
100
101
|
);
|
|
101
102
|
}
|
|
102
|
-
|
|
103
|
-
this.txsMetadata = new MissingTxMetadataCollection(entries, this.txBatchSize);
|
|
103
|
+
this.txsMetadata = new MissingTxMetadataCollection(missingTxsTracker, this.txBatchSize);
|
|
104
104
|
this.smartRequesterSemaphore = this.opts.semaphore ?? new Semaphore(0);
|
|
105
105
|
}
|
|
106
106
|
|
|
@@ -661,7 +661,7 @@ export class BatchTxRequester {
|
|
|
661
661
|
/*
|
|
662
662
|
* @returns true if all missing txs have been fetched */
|
|
663
663
|
private fetchedAllTxs() {
|
|
664
|
-
return
|
|
664
|
+
return this.txsMetadata.getMissingTxHashes().size == 0;
|
|
665
665
|
}
|
|
666
666
|
|
|
667
667
|
/*
|
|
@@ -679,7 +679,7 @@ export class BatchTxRequester {
|
|
|
679
679
|
this.unlockSmartRequesterSemaphores();
|
|
680
680
|
}
|
|
681
681
|
|
|
682
|
-
return aborted || this.
|
|
682
|
+
return aborted || this.fetchedAllTxs() || this.dateProvider.now() > this.deadline;
|
|
683
683
|
}
|
|
684
684
|
|
|
685
685
|
/*
|
|
@@ -6,7 +6,6 @@ import type { PeerId } from '@libp2p/interface';
|
|
|
6
6
|
|
|
7
7
|
import type { ConnectionSampler } from '../connection-sampler/connection_sampler.js';
|
|
8
8
|
import type { ReqRespInterface } from '../interface.js';
|
|
9
|
-
import type { MissingTxMetadata } from './missing_txs.js';
|
|
10
9
|
import type { IPeerCollection } from './peer_collection.js';
|
|
11
10
|
import type { BatchRequestTxValidatorConfig, IBatchRequestTxValidator } from './tx_validator.js';
|
|
12
11
|
|
|
@@ -15,18 +14,15 @@ export interface IPeerPenalizer {
|
|
|
15
14
|
}
|
|
16
15
|
|
|
17
16
|
export interface ITxMetadataCollection {
|
|
18
|
-
size: number;
|
|
19
|
-
values(): IterableIterator<MissingTxMetadata>;
|
|
20
17
|
getMissingTxHashes(): Set<string>;
|
|
18
|
+
markFetched(peerId: PeerId, tx: Tx): boolean;
|
|
21
19
|
getTxsToRequestFromThePeer(peer: PeerId): TxHash[];
|
|
22
20
|
markRequested(txHash: TxHash): void;
|
|
23
21
|
markInFlightBySmartPeer(txHash: TxHash): void;
|
|
24
22
|
markNotInFlightBySmartPeer(txHash: TxHash): void;
|
|
25
23
|
alreadyFetched(txHash: TxHash): boolean;
|
|
26
24
|
// Returns true if tx was marked as fetched, false if it was already marked as fetched
|
|
27
|
-
markFetched(peerId: PeerId, tx: Tx): boolean;
|
|
28
25
|
markPeerHas(peerId: PeerId, txHashes: TxHash[]): void;
|
|
29
|
-
getFetchedTxs(): Tx[];
|
|
30
26
|
}
|
|
31
27
|
|
|
32
28
|
/**
|
|
@@ -2,13 +2,13 @@ import { type Tx, TxHash } from '@aztec/stdlib/tx';
|
|
|
2
2
|
|
|
3
3
|
import type { PeerId } from '@libp2p/interface';
|
|
4
4
|
|
|
5
|
+
import type { IMissingTxsTracker } from '../../tx_collection/missing_txs_tracker.js';
|
|
5
6
|
import { DEFAULT_BATCH_TX_REQUESTER_TX_BATCH_SIZE } from './config.js';
|
|
6
7
|
import type { ITxMetadataCollection } from './interface.js';
|
|
7
8
|
|
|
8
|
-
|
|
9
|
+
class MissingTxMetadata {
|
|
9
10
|
constructor(
|
|
10
|
-
public readonly txHash:
|
|
11
|
-
public fetched = false,
|
|
11
|
+
public readonly txHash: string,
|
|
12
12
|
public requestedCount = 0,
|
|
13
13
|
public inFlightCount = 0,
|
|
14
14
|
public tx: Tx | undefined = undefined,
|
|
@@ -30,24 +30,6 @@ export class MissingTxMetadata {
|
|
|
30
30
|
public isInFlight(): boolean {
|
|
31
31
|
return this.inFlightCount > 0;
|
|
32
32
|
}
|
|
33
|
-
|
|
34
|
-
//Returns true if this is the first time we mark it as fetched
|
|
35
|
-
public markAsFetched(peerId: PeerId, tx: Tx): boolean {
|
|
36
|
-
if (this.fetched) {
|
|
37
|
-
return false;
|
|
38
|
-
}
|
|
39
|
-
|
|
40
|
-
this.fetched = true;
|
|
41
|
-
this.tx = tx;
|
|
42
|
-
|
|
43
|
-
this.peers.add(peerId.toString());
|
|
44
|
-
|
|
45
|
-
return true;
|
|
46
|
-
}
|
|
47
|
-
|
|
48
|
-
public toString() {
|
|
49
|
-
return this.txHash.toString();
|
|
50
|
-
}
|
|
51
33
|
}
|
|
52
34
|
|
|
53
35
|
/*
|
|
@@ -55,21 +37,18 @@ export class MissingTxMetadata {
|
|
|
55
37
|
* This could be better optimized but given expected count of missing txs (N < 100)
|
|
56
38
|
* At the moment there is no need for it. And benefit is that we have everything in single store
|
|
57
39
|
* */
|
|
58
|
-
export class MissingTxMetadataCollection
|
|
40
|
+
export class MissingTxMetadataCollection implements ITxMetadataCollection {
|
|
41
|
+
private txMetadata = new Map<string, MissingTxMetadata>();
|
|
42
|
+
|
|
59
43
|
constructor(
|
|
60
|
-
|
|
44
|
+
private missingTxsTracker: IMissingTxsTracker,
|
|
61
45
|
private readonly txBatchSize: number = DEFAULT_BATCH_TX_REQUESTER_TX_BATCH_SIZE,
|
|
62
46
|
) {
|
|
63
|
-
|
|
64
|
-
}
|
|
65
|
-
public getSortedByRequestedCountAsc(txs: string[]): MissingTxMetadata[] {
|
|
66
|
-
return Array.from(this.values().filter(txMeta => txs.includes(txMeta.txHash.toString()))).sort(
|
|
67
|
-
(a, b) => a.requestedCount - b.requestedCount,
|
|
68
|
-
);
|
|
47
|
+
missingTxsTracker.missingTxHashes.forEach(hash => this.txMetadata.set(hash, new MissingTxMetadata(hash)));
|
|
69
48
|
}
|
|
70
49
|
|
|
71
50
|
public getPrioritizingNotInFlightAndLowerRequestCount(txs: string[]): MissingTxMetadata[] {
|
|
72
|
-
const filtered = Array.from(this.values()).filter(txMeta => txs.includes(txMeta.txHash.toString()));
|
|
51
|
+
const filtered = Array.from(this.txMetadata.values()).filter(txMeta => txs.includes(txMeta.txHash.toString()));
|
|
73
52
|
|
|
74
53
|
const [notInFlight, inFlight] = filtered.reduce<[MissingTxMetadata[], MissingTxMetadata[]]>(
|
|
75
54
|
(buckets, tx) => {
|
|
@@ -85,43 +64,15 @@ export class MissingTxMetadataCollection extends Map<string, MissingTxMetadata>
|
|
|
85
64
|
return [...notInFlight, ...inFlight];
|
|
86
65
|
}
|
|
87
66
|
|
|
88
|
-
public getFetchedTxHashes(): Set<string> {
|
|
89
|
-
return new Set(
|
|
90
|
-
this.values()
|
|
91
|
-
.filter(t => t.fetched)
|
|
92
|
-
.map(t => t.txHash.toString()),
|
|
93
|
-
);
|
|
94
|
-
}
|
|
95
|
-
|
|
96
67
|
public getMissingTxHashes(): Set<string> {
|
|
97
|
-
return
|
|
98
|
-
this.values()
|
|
99
|
-
.filter(t => !t.fetched)
|
|
100
|
-
.map(t => t.txHash.toString()),
|
|
101
|
-
);
|
|
102
|
-
}
|
|
103
|
-
|
|
104
|
-
public getInFlightTxHashes(): Set<string> {
|
|
105
|
-
return new Set(
|
|
106
|
-
this.values()
|
|
107
|
-
.filter(t => t.isInFlight())
|
|
108
|
-
.map(t => t.txHash.toString()),
|
|
109
|
-
);
|
|
110
|
-
}
|
|
111
|
-
|
|
112
|
-
public getFetchedTxs(): Tx[] {
|
|
113
|
-
return Array.from(
|
|
114
|
-
this.values()
|
|
115
|
-
.map(t => t.tx)
|
|
116
|
-
.filter(t => !!t),
|
|
117
|
-
);
|
|
68
|
+
return this.missingTxsTracker.missingTxHashes;
|
|
118
69
|
}
|
|
119
70
|
|
|
120
71
|
public getTxsPeerHas(peer: PeerId): Set<string> {
|
|
121
72
|
const peerIdStr = peer.toString();
|
|
122
73
|
const txsPeerHas = new Set<string>();
|
|
123
74
|
|
|
124
|
-
this.values().forEach(txMeta => {
|
|
75
|
+
this.txMetadata.values().forEach(txMeta => {
|
|
125
76
|
if (txMeta.peers.has(peerIdStr)) {
|
|
126
77
|
txsPeerHas.add(txMeta.txHash.toString());
|
|
127
78
|
}
|
|
@@ -132,13 +83,13 @@ export class MissingTxMetadataCollection extends Map<string, MissingTxMetadata>
|
|
|
132
83
|
|
|
133
84
|
public getTxsToRequestFromThePeer(peer: PeerId): TxHash[] {
|
|
134
85
|
const txsPeerHas = this.getTxsPeerHas(peer);
|
|
135
|
-
const
|
|
86
|
+
const missingTxHashes = this.getMissingTxHashes();
|
|
136
87
|
|
|
137
|
-
const txsToRequest = txsPeerHas.
|
|
88
|
+
const txsToRequest = txsPeerHas.intersection(missingTxHashes);
|
|
138
89
|
|
|
139
90
|
if (txsToRequest.size >= this.txBatchSize) {
|
|
140
91
|
return this.getPrioritizingNotInFlightAndLowerRequestCount(Array.from(txsToRequest))
|
|
141
|
-
.map(t => t.txHash)
|
|
92
|
+
.map(t => TxHash.fromString(t.txHash))
|
|
142
93
|
.slice(0, this.txBatchSize);
|
|
143
94
|
}
|
|
144
95
|
|
|
@@ -150,13 +101,13 @@ export class MissingTxMetadataCollection extends Map<string, MissingTxMetadata>
|
|
|
150
101
|
Array.from(this.getMissingTxHashes().difference(txsToRequest)),
|
|
151
102
|
)
|
|
152
103
|
.slice(0, countToFill)
|
|
153
|
-
.map(t => t.txHash);
|
|
104
|
+
.map(t => TxHash.fromString(t.txHash));
|
|
154
105
|
|
|
155
106
|
return [...Array.from(txsToRequest).map(t => TxHash.fromString(t)), ...txsToFill];
|
|
156
107
|
}
|
|
157
108
|
|
|
158
109
|
public markRequested(txHash: TxHash) {
|
|
159
|
-
this.get(txHash.toString())?.markAsRequested();
|
|
110
|
+
this.txMetadata.get(txHash.toString())?.markAsRequested();
|
|
160
111
|
}
|
|
161
112
|
|
|
162
113
|
/*
|
|
@@ -165,7 +116,7 @@ export class MissingTxMetadataCollection extends Map<string, MissingTxMetadata>
|
|
|
165
116
|
* "dumb" peer might return it, or might not - we don't know
|
|
166
117
|
* */
|
|
167
118
|
public markInFlightBySmartPeer(txHash: TxHash) {
|
|
168
|
-
this.get(txHash.toString())?.markInFlight();
|
|
119
|
+
this.txMetadata.get(txHash.toString())?.markInFlight();
|
|
169
120
|
}
|
|
170
121
|
|
|
171
122
|
/*
|
|
@@ -173,16 +124,16 @@ export class MissingTxMetadataCollection extends Map<string, MissingTxMetadata>
|
|
|
173
124
|
* Because the smart peer should return this tx, whereas
|
|
174
125
|
* "dumb" peer might return it, or might not - we don't know*/
|
|
175
126
|
public markNotInFlightBySmartPeer(txHash: TxHash) {
|
|
176
|
-
this.get(txHash.toString())?.markNotInFlight();
|
|
127
|
+
this.txMetadata.get(txHash.toString())?.markNotInFlight();
|
|
177
128
|
}
|
|
178
129
|
|
|
179
130
|
public alreadyFetched(txHash: TxHash): boolean {
|
|
180
|
-
return this.
|
|
131
|
+
return !this.missingTxsTracker.isMissing(txHash.toString());
|
|
181
132
|
}
|
|
182
133
|
|
|
183
134
|
public markFetched(peerId: PeerId, tx: Tx): boolean {
|
|
184
135
|
const txHashStr = tx.txHash.toString();
|
|
185
|
-
const txMeta = this.get(txHashStr);
|
|
136
|
+
const txMeta = this.txMetadata.get(txHashStr);
|
|
186
137
|
if (!txMeta) {
|
|
187
138
|
//TODO: what to do about peer which sent txs we didn't request?
|
|
188
139
|
// 1. don't request from it in the scope of this batch request
|
|
@@ -192,7 +143,8 @@ export class MissingTxMetadataCollection extends Map<string, MissingTxMetadata>
|
|
|
192
143
|
return false;
|
|
193
144
|
}
|
|
194
145
|
|
|
195
|
-
|
|
146
|
+
txMeta.peers.add(peerId.toString());
|
|
147
|
+
return this.missingTxsTracker.markFetched(tx);
|
|
196
148
|
}
|
|
197
149
|
|
|
198
150
|
public markPeerHas(peerId: PeerId, txHash: TxHash[]) {
|
|
@@ -200,7 +152,7 @@ export class MissingTxMetadataCollection extends Map<string, MissingTxMetadata>
|
|
|
200
152
|
txHash
|
|
201
153
|
.map(t => t.toString())
|
|
202
154
|
.forEach(txh => {
|
|
203
|
-
const txMeta = this.get(txh);
|
|
155
|
+
const txMeta = this.txMetadata.get(txh);
|
|
204
156
|
if (txMeta) {
|
|
205
157
|
txMeta.peers.add(peerIdStr);
|
|
206
158
|
}
|
package/src/services/service.ts
CHANGED
|
@@ -1,7 +1,13 @@
|
|
|
1
1
|
import type { SlotNumber } from '@aztec/foundation/branded-types';
|
|
2
2
|
import type { EthAddress } from '@aztec/foundation/eth-address';
|
|
3
3
|
import type { PeerInfo } from '@aztec/stdlib/interfaces/server';
|
|
4
|
-
import type {
|
|
4
|
+
import type {
|
|
5
|
+
BlockProposal,
|
|
6
|
+
CheckpointAttestation,
|
|
7
|
+
CheckpointProposalCore,
|
|
8
|
+
Gossipable,
|
|
9
|
+
TopicType,
|
|
10
|
+
} from '@aztec/stdlib/p2p';
|
|
5
11
|
import type { Tx } from '@aztec/stdlib/tx';
|
|
6
12
|
|
|
7
13
|
import type { PeerId } from '@libp2p/interface';
|
|
@@ -130,6 +136,9 @@ export interface P2PService {
|
|
|
130
136
|
|
|
131
137
|
getPeers(includePending?: boolean): PeerInfo[];
|
|
132
138
|
|
|
139
|
+
/** Returns the number of peers in the GossipSub mesh for a given topic type. */
|
|
140
|
+
getGossipMeshPeerCount(topicType: TopicType): number;
|
|
141
|
+
|
|
133
142
|
validate(txs: Tx[]): Promise<void>;
|
|
134
143
|
|
|
135
144
|
addReqRespSubProtocol(
|
|
@@ -14,6 +14,7 @@ import type { PeerId } from '@libp2p/interface';
|
|
|
14
14
|
import type { BatchTxRequesterConfig } from '../reqresp/batch-tx-requester/config.js';
|
|
15
15
|
import type { BatchTxRequesterLibP2PService } from '../reqresp/batch-tx-requester/interface.js';
|
|
16
16
|
import type { TxCollectionConfig } from './config.js';
|
|
17
|
+
import { MissingTxsTracker } from './missing_txs_tracker.js';
|
|
17
18
|
import {
|
|
18
19
|
BatchTxRequesterCollector,
|
|
19
20
|
type MissingTxsCollector,
|
|
@@ -83,8 +84,7 @@ export class FastTxCollection {
|
|
|
83
84
|
...input,
|
|
84
85
|
blockInfo,
|
|
85
86
|
promise,
|
|
86
|
-
|
|
87
|
-
missingTxHashes: new Set(txHashes.map(t => t.toString())),
|
|
87
|
+
missingTxTracker: MissingTxsTracker.fromArray(txHashes),
|
|
88
88
|
deadline: opts.deadline,
|
|
89
89
|
};
|
|
90
90
|
|
|
@@ -92,15 +92,15 @@ export class FastTxCollection {
|
|
|
92
92
|
clearTimeout(timeoutTimer);
|
|
93
93
|
|
|
94
94
|
this.log.verbose(
|
|
95
|
-
`Collected ${request.
|
|
95
|
+
`Collected ${request.missingTxTracker.collectedTxs.length} txs out of ${txHashes.length} for ${input.type} at slot ${blockInfo.slotNumber}`,
|
|
96
96
|
{
|
|
97
97
|
...blockInfo,
|
|
98
98
|
duration,
|
|
99
99
|
requestType: input.type,
|
|
100
|
-
missingTxs: [...request.missingTxHashes],
|
|
100
|
+
missingTxs: [...request.missingTxTracker.missingTxHashes],
|
|
101
101
|
},
|
|
102
102
|
);
|
|
103
|
-
return
|
|
103
|
+
return request.missingTxTracker.collectedTxs;
|
|
104
104
|
}
|
|
105
105
|
|
|
106
106
|
protected async collectFast(
|
|
@@ -111,7 +111,7 @@ export class FastTxCollection {
|
|
|
111
111
|
const { blockInfo } = request;
|
|
112
112
|
|
|
113
113
|
this.log.debug(
|
|
114
|
-
`Starting fast collection of ${request.
|
|
114
|
+
`Starting fast collection of ${request.missingTxTracker.numberOfMissingTxs} txs for ${request.type} at slot ${blockInfo.slotNumber}`,
|
|
115
115
|
{ ...blockInfo, requestType: request.type, deadline: opts.deadline },
|
|
116
116
|
);
|
|
117
117
|
|
|
@@ -124,7 +124,7 @@ export class FastTxCollection {
|
|
|
124
124
|
await Promise.race([request.promise.promise, waitBeforeReqResp]);
|
|
125
125
|
|
|
126
126
|
// If we have collected all txs, we can stop here
|
|
127
|
-
if (request.
|
|
127
|
+
if (request.missingTxTracker.allFetched()) {
|
|
128
128
|
this.log.debug(`All txs collected for slot ${blockInfo.slotNumber} without reqresp`, blockInfo);
|
|
129
129
|
return;
|
|
130
130
|
}
|
|
@@ -138,7 +138,7 @@ export class FastTxCollection {
|
|
|
138
138
|
const logCtx = {
|
|
139
139
|
...blockInfo,
|
|
140
140
|
errorMessage: err instanceof Error ? err.message : undefined,
|
|
141
|
-
missingTxs:
|
|
141
|
+
missingTxs: request.missingTxTracker.missingTxHashes.values().map(txHash => txHash.toString()),
|
|
142
142
|
};
|
|
143
143
|
if (err instanceof Error && err.name === 'TimeoutError') {
|
|
144
144
|
this.log.warn(`Timed out collecting txs for ${request.type} at slot ${blockInfo.slotNumber}`, logCtx);
|
|
@@ -166,7 +166,11 @@ export class FastTxCollection {
|
|
|
166
166
|
}
|
|
167
167
|
|
|
168
168
|
// Keep a shared priority queue of all txs pending to be requested, sorted by the number of attempts made to collect them.
|
|
169
|
-
const attemptsPerTx = [...request.missingTxHashes].map(txHash => ({
|
|
169
|
+
const attemptsPerTx = [...request.missingTxTracker.missingTxHashes].map(txHash => ({
|
|
170
|
+
txHash,
|
|
171
|
+
attempts: 0,
|
|
172
|
+
found: false,
|
|
173
|
+
}));
|
|
170
174
|
|
|
171
175
|
// Returns once we have finished all node loops. Each loop finishes when the deadline is hit, or all txs have been collected.
|
|
172
176
|
await Promise.allSettled(this.nodes.map(node => this.collectFastFromNode(request, node, attemptsPerTx, opts)));
|
|
@@ -179,7 +183,7 @@ export class FastTxCollection {
|
|
|
179
183
|
opts: { deadline: Date },
|
|
180
184
|
) {
|
|
181
185
|
const notFinished = () =>
|
|
182
|
-
this.dateProvider.now() <= +opts.deadline && request.
|
|
186
|
+
this.dateProvider.now() <= +opts.deadline && !request.missingTxTracker.allFetched() && this.requests.has(request);
|
|
183
187
|
|
|
184
188
|
const maxParallelRequests = this.config.txCollectionFastMaxParallelRequestsPerNode;
|
|
185
189
|
const maxBatchSize = this.config.txCollectionNodeRpcMaxBatchSize;
|
|
@@ -196,7 +200,7 @@ export class FastTxCollection {
|
|
|
196
200
|
if (!txToRequest) {
|
|
197
201
|
// No more txs to process
|
|
198
202
|
break;
|
|
199
|
-
} else if (!request.
|
|
203
|
+
} else if (!request.missingTxTracker.isMissing(txToRequest.txHash)) {
|
|
200
204
|
// Mark as found if it was found somewhere else, we'll then remove it from the array.
|
|
201
205
|
// We don't delete it now since 'array.splice' is pretty expensive, so we do it after sorting.
|
|
202
206
|
txToRequest.found = true;
|
|
@@ -225,10 +229,17 @@ export class FastTxCollection {
|
|
|
225
229
|
return;
|
|
226
230
|
}
|
|
227
231
|
|
|
232
|
+
const txHashes = batch.map(({ txHash }) => txHash);
|
|
228
233
|
// Collect this batch from the node
|
|
229
234
|
await this.txCollectionSink.collect(
|
|
230
|
-
|
|
231
|
-
|
|
235
|
+
async () => {
|
|
236
|
+
const result = await node.getTxsByHash(txHashes.map(TxHash.fromString));
|
|
237
|
+
for (const tx of result.validTxs) {
|
|
238
|
+
request.missingTxTracker.markFetched(tx);
|
|
239
|
+
}
|
|
240
|
+
return result;
|
|
241
|
+
},
|
|
242
|
+
txHashes,
|
|
232
243
|
{
|
|
233
244
|
description: `fast ${node.getInfo()}`,
|
|
234
245
|
node: node.getInfo(),
|
|
@@ -268,32 +279,44 @@ export class FastTxCollection {
|
|
|
268
279
|
}
|
|
269
280
|
|
|
270
281
|
this.log.debug(
|
|
271
|
-
`Starting fast reqresp for ${request.
|
|
282
|
+
`Starting fast reqresp for ${request.missingTxTracker.numberOfMissingTxs} txs for ${request.type} at slot ${blockInfo.slotNumber}`,
|
|
272
283
|
{ ...blockInfo, timeoutMs, pinnedPeer },
|
|
273
284
|
);
|
|
274
285
|
|
|
275
286
|
try {
|
|
276
287
|
await this.txCollectionSink.collect(
|
|
277
|
-
async
|
|
288
|
+
async () => {
|
|
289
|
+
let result: Tx[];
|
|
278
290
|
if (request.type === 'proposal') {
|
|
279
|
-
|
|
291
|
+
result = await this.missingTxsCollector.collectTxs(
|
|
292
|
+
request.missingTxTracker,
|
|
293
|
+
request.blockProposal,
|
|
294
|
+
pinnedPeer,
|
|
295
|
+
timeoutMs,
|
|
296
|
+
);
|
|
280
297
|
} else if (request.type === 'block') {
|
|
281
298
|
const blockTxsSource = {
|
|
282
299
|
txHashes: request.block.body.txEffects.map(e => e.txHash),
|
|
283
300
|
archive: request.block.archive.root,
|
|
284
301
|
};
|
|
285
|
-
|
|
302
|
+
result = await this.missingTxsCollector.collectTxs(
|
|
303
|
+
request.missingTxTracker,
|
|
304
|
+
blockTxsSource,
|
|
305
|
+
pinnedPeer,
|
|
306
|
+
timeoutMs,
|
|
307
|
+
);
|
|
286
308
|
} else {
|
|
287
309
|
throw new Error(`Unknown request type: ${(request as any).type}`);
|
|
288
310
|
}
|
|
311
|
+
return { validTxs: result, invalidTxHashes: [] };
|
|
289
312
|
},
|
|
290
|
-
Array.from(request.missingTxHashes)
|
|
313
|
+
Array.from(request.missingTxTracker.missingTxHashes),
|
|
291
314
|
{ description: `reqresp for slot ${slotNumber}`, method: 'fast-req-resp', ...opts, ...request.blockInfo },
|
|
292
315
|
this.getAddContext(request),
|
|
293
316
|
);
|
|
294
317
|
} catch (err) {
|
|
295
318
|
this.log.error(`Error sending fast reqresp request for txs`, err, {
|
|
296
|
-
txs: [...request.missingTxHashes],
|
|
319
|
+
txs: [...request.missingTxTracker.missingTxHashes],
|
|
297
320
|
...blockInfo,
|
|
298
321
|
});
|
|
299
322
|
}
|
|
@@ -317,22 +340,20 @@ export class FastTxCollection {
|
|
|
317
340
|
for (const tx of txs) {
|
|
318
341
|
const txHash = tx.txHash.toString();
|
|
319
342
|
// Remove the tx hash from the missing set, and add it to the found set.
|
|
320
|
-
if (request.
|
|
321
|
-
request.missingTxHashes.delete(txHash);
|
|
322
|
-
request.foundTxs.set(txHash, tx);
|
|
343
|
+
if (request.missingTxTracker.markFetched(tx)) {
|
|
323
344
|
this.log.trace(`Found tx ${txHash} for fast collection request`, {
|
|
324
345
|
...request.blockInfo,
|
|
325
346
|
txHash: tx.txHash.toString(),
|
|
326
347
|
type: request.type,
|
|
327
348
|
});
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
349
|
+
}
|
|
350
|
+
// If we found all txs for this request, we resolve the promise
|
|
351
|
+
if (request.missingTxTracker.allFetched()) {
|
|
352
|
+
this.log.trace(`All txs found for fast collection request`, {
|
|
353
|
+
...request.blockInfo,
|
|
354
|
+
type: request.type,
|
|
355
|
+
});
|
|
356
|
+
request.promise.resolve();
|
|
336
357
|
}
|
|
337
358
|
}
|
|
338
359
|
}
|
|
@@ -126,9 +126,13 @@ export class FileStoreTxCollection {
|
|
|
126
126
|
|
|
127
127
|
try {
|
|
128
128
|
const result = await this.txCollectionSink.collect(
|
|
129
|
-
|
|
130
|
-
[
|
|
131
|
-
{
|
|
129
|
+
() => source.getTxsByHash([TxHash.fromString(entry.txHash)]),
|
|
130
|
+
[entry.txHash],
|
|
131
|
+
{
|
|
132
|
+
description: `file-store ${source.getInfo()}`,
|
|
133
|
+
method: 'file-store',
|
|
134
|
+
fileStore: source.getInfo(),
|
|
135
|
+
},
|
|
132
136
|
entry.context,
|
|
133
137
|
);
|
|
134
138
|
if (result.txs.length > 0) {
|