@aztec/p2p 0.0.0-test.0

package/dest/services/reqresp/rate-limiter/rate_limiter.js

@@ -0,0 +1,184 @@
+/**
+ * @attribution Rate limiter approach implemented in the lodestar ethereum 2 client.
+ * Rationale is that if it was good enough for them, then it should be good enough for us.
+ * https://github.com/ChainSafe/lodestar
+ */ import { PeerErrorSeverity } from '@aztec/stdlib/p2p';
+import { DEFAULT_RATE_LIMITS } from './rate_limits.js';
+// Check for disconnected peers every 10 minutes
+const CHECK_DISCONNECTED_PEERS_INTERVAL_MS = 10 * 60 * 1000;
+/**
+ * GCRARateLimiter: A Generic Cell Rate Algorithm (GCRA) based rate limiter.
+ *
+ * How it works:
+ * 1. The rate limiter allows a certain number of operations (quotaCount) within a specified
+ *    time interval (quotaTimeMs).
+ * 2. It uses a "virtual scheduling time" (VST) to determine when the next operation should be allowed.
+ * 3. When an operation is requested, the limiter checks if enough time has passed since the last
+ *    allowed operation.
+ * 4. If sufficient time has passed, the operation is allowed, and the VST is updated.
+ * 5. If not enough time has passed, the operation is denied.
+ *
+ * The limiter also allows for short bursts of activity, as long as the overall rate doesn't exceed
+ * the specified quota over time.
+ *
+ * Usage example:
+ * ```
+ * const limiter = new GCRARateLimiter(100, 60000); // 100 operations per minute
+ * ```
+ */ export class GCRARateLimiter {
+    // Virtual scheduling time: i.e. the time at which we should allow the next request
+    vst;
+    // The interval at which we emit a new token
+    emissionInterval;
+    // The interval over which we limit the number of requests
+    limitInterval;
+    /**
+   * @param quotaCount - The number of requests to allow over the limit interval
+   * @param quotaTimeMs - The time interval over which the quotaCount applies
+   */ constructor(quotaCount, quotaTimeMs){
+        this.emissionInterval = quotaTimeMs / quotaCount;
+        this.limitInterval = quotaTimeMs;
+        this.vst = Date.now();
+    }
+    allow() {
+        const now = Date.now();
+        const newVst = Math.max(this.vst, now) + this.emissionInterval;
+        if (newVst - now <= this.limitInterval) {
+            this.vst = newVst;
+            return true;
+        }
+        return false;
+    }
+}
+export var RateLimitStatus = /*#__PURE__*/ function(RateLimitStatus) {
+    RateLimitStatus[RateLimitStatus["DeniedGlobal"] = 0] = "DeniedGlobal";
+    RateLimitStatus[RateLimitStatus["DeniedPeer"] = 1] = "DeniedPeer";
+    RateLimitStatus[RateLimitStatus["Allowed"] = 2] = "Allowed";
+    return RateLimitStatus;
+}({});
+export function prettyPrintRateLimitStatus(status) {
+    switch(status){
+        case 0:
+            return 'DeniedGlobal';
+        case 1:
+            return 'DeniedPeer';
+        case 2:
+            return 'Allowed';
+    }
+}
+/**
+ * SubProtocolRateLimiter: A rate limiter for managing request rates on a per-peer and global basis for a specific subprotocol.
+ *
+ * This class provides a two-tier rate limiting system:
+ * 1. A global rate limit for all requests across all peers for this subprotocol.
+ * 2. Individual rate limits for each peer.
+ *
+ * How it works:
+ * - When a request comes in, it first checks against the global rate limit.
+ * - If the global limit allows, it then checks against the specific peer's rate limit.
+ * - The request is only allowed if both the global and peer-specific limits allow it.
+ * - It automatically creates and manages rate limiters for new peers as they make requests.
+ * - It periodically cleans up rate limiters for inactive peers to conserve memory.
+ *
+ * Note: Remember to call `start()` to begin the cleanup process and `stop()` when shutting down to clear the cleanup interval.
+ */ export class SubProtocolRateLimiter {
+    peerLimiters = new Map();
+    globalLimiter;
+    peerQuotaCount;
+    peerQuotaTimeMs;
+    constructor(peerQuotaCount, peerQuotaTimeMs, globalQuotaCount, globalQuotaTimeMs){
+        this.peerLimiters = new Map();
+        this.globalLimiter = new GCRARateLimiter(globalQuotaCount, globalQuotaTimeMs);
+        this.peerQuotaCount = peerQuotaCount;
+        this.peerQuotaTimeMs = peerQuotaTimeMs;
+    }
+    allow(peerId) {
+        if (!this.globalLimiter.allow()) {
+            return 0;
+        }
+        const peerIdStr = peerId.toString();
+        let peerLimiter = this.peerLimiters.get(peerIdStr);
+        if (!peerLimiter) {
+            // Create a limiter for this peer
+            peerLimiter = {
+                limiter: new GCRARateLimiter(this.peerQuotaCount, this.peerQuotaTimeMs),
+                lastAccess: Date.now()
+            };
+            this.peerLimiters.set(peerIdStr, peerLimiter);
+        } else {
+            peerLimiter.lastAccess = Date.now();
+        }
+        const peerLimitAllowed = peerLimiter.limiter.allow();
+        if (!peerLimitAllowed) {
+            return 1;
+        }
+        return 2;
+    }
+    cleanupInactivePeers() {
+        const now = Date.now();
+        this.peerLimiters.forEach((peerLimiter, peerId)=>{
+            if (now - peerLimiter.lastAccess > CHECK_DISCONNECTED_PEERS_INTERVAL_MS) {
+                this.peerLimiters.delete(peerId);
+            }
+        });
+    }
+}
+/**
+ * RequestResponseRateLimiter.
+ *
+ * A rate limiter that is protocol aware, then peer aware.
+ * SubProtocols can have their own global / peer level rate limits.
+ *
+ * How it works:
+ * - Initializes with a set of rate limit configurations for different subprotocols.
+ * - Creates a separate SubProtocolRateLimiter for each configured subprotocol.
+ * - When a request comes in, it routes the rate limiting decision to the appropriate subprotocol limiter.
+ * - Peers who exceed their peer rate limits will be penalised by the peer manager.
+ *
+ * Usage:
+ * ```
+ * const peerManager = new PeerManager(...);
+ * const rateLimits = {
+ *   subprotocol1: { peerLimit: { quotaCount: 10, quotaTimeMs: 1000 }, globalLimit: { quotaCount: 100, quotaTimeMs: 1000 } },
+ *   subprotocol2: { peerLimit: { quotaCount: 5, quotaTimeMs: 1000 }, globalLimit: { quotaCount: 50, quotaTimeMs: 1000 } }
+ * };
+ * const limiter = new RequestResponseRateLimiter(peerManager, rateLimits);
+ *
+ * Note: Ensure to call `stop()` when shutting down to properly clean up all subprotocol limiters.
+ */ export class RequestResponseRateLimiter {
+    peerScoring;
+    subProtocolRateLimiters;
+    cleanupInterval;
+    constructor(peerScoring, rateLimits = DEFAULT_RATE_LIMITS){
+        this.peerScoring = peerScoring;
+        this.cleanupInterval = undefined;
+        this.subProtocolRateLimiters = new Map();
+        for (const [subProtocol, protocolLimits] of Object.entries(rateLimits)){
+            this.subProtocolRateLimiters.set(subProtocol, new SubProtocolRateLimiter(protocolLimits.peerLimit.quotaCount, protocolLimits.peerLimit.quotaTimeMs, protocolLimits.globalLimit.quotaCount, protocolLimits.globalLimit.quotaTimeMs));
+        }
+    }
+    start() {
+        this.cleanupInterval = setInterval(()=>{
+            this.cleanupInactivePeers();
+        }, CHECK_DISCONNECTED_PEERS_INTERVAL_MS);
+    }
+    allow(subProtocol, peerId) {
+        const limiter = this.subProtocolRateLimiters.get(subProtocol);
+        if (!limiter) {
+            return 2;
+        }
+        const rateLimitStatus = limiter.allow(peerId);
+        if (rateLimitStatus === 1) {
+            this.peerScoring.penalizePeer(peerId, PeerErrorSeverity.HighToleranceError);
+        }
+        return rateLimitStatus;
+    }
+    cleanupInactivePeers() {
+        this.subProtocolRateLimiters.forEach((limiter)=>limiter.cleanupInactivePeers());
+    }
+    /**
+   * Make sure to call destroy on each of the sub protocol rate limiters when cleaning up
+   */ stop() {
+        clearInterval(this.cleanupInterval);
+    }
+}

package/dest/services/reqresp/rate-limiter/rate_limits.d.ts

@@ -0,0 +1,3 @@
+import { type ReqRespSubProtocolRateLimits } from '../interface.js';
+export declare const DEFAULT_RATE_LIMITS: ReqRespSubProtocolRateLimits;
+//# sourceMappingURL=rate_limits.d.ts.map

package/dest/services/reqresp/rate-limiter/rate_limits.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"rate_limits.d.ts","sourceRoot":"","sources":["../../../../src/services/reqresp/rate-limiter/rate_limits.ts"],"names":[],"mappings":"AAAA,OAAO,EAAsB,KAAK,4BAA4B,EAAE,MAAM,iBAAiB,CAAC;AAGxF,eAAO,MAAM,mBAAmB,EAAE,4BAmDjC,CAAC"}

package/dest/services/reqresp/rate-limiter/rate_limits.js

@@ -0,0 +1,54 @@
+import { ReqRespSubProtocol } from '../interface.js';
+// TODO(md): these defaults need to be tuned
+export const DEFAULT_RATE_LIMITS = {
+    [ReqRespSubProtocol.PING]: {
+        peerLimit: {
+            quotaTimeMs: 1000,
+            quotaCount: 5
+        },
+        globalLimit: {
+            quotaTimeMs: 1000,
+            quotaCount: 10
+        }
+    },
+    [ReqRespSubProtocol.STATUS]: {
+        peerLimit: {
+            quotaTimeMs: 1000,
+            quotaCount: 5
+        },
+        globalLimit: {
+            quotaTimeMs: 1000,
+            quotaCount: 10
+        }
+    },
+    [ReqRespSubProtocol.TX]: {
+        peerLimit: {
+            quotaTimeMs: 1000,
+            quotaCount: 10
+        },
+        globalLimit: {
+            quotaTimeMs: 1000,
+            quotaCount: 20
+        }
+    },
+    [ReqRespSubProtocol.BLOCK]: {
+        peerLimit: {
+            quotaTimeMs: 1000,
+            quotaCount: 2
+        },
+        globalLimit: {
+            quotaTimeMs: 1000,
+            quotaCount: 5
+        }
+    },
+    [ReqRespSubProtocol.GOODBYE]: {
+        peerLimit: {
+            quotaTimeMs: 1000,
+            quotaCount: 5
+        },
+        globalLimit: {
+            quotaTimeMs: 1000,
+            quotaCount: 10
+        }
+    }
+};

package/dest/services/reqresp/reqresp.d.ts

@@ -0,0 +1,166 @@
+/// <reference types="node" resolution-mode="require"/>
+/// <reference types="node" resolution-mode="require"/>
+import { type Logger } from '@aztec/foundation/log';
+import { type TelemetryClient } from '@aztec/telemetry-client';
+import type { PeerId } from '@libp2p/interface';
+import type { Libp2p } from 'libp2p';
+import type { PeerScoring } from '../peer-manager/peer_scoring.js';
+import type { P2PReqRespConfig } from './config.js';
+import { type ReqRespResponse, ReqRespSubProtocol, type ReqRespSubProtocolHandlers, type ReqRespSubProtocolValidators, type SubProtocolMap } from './interface.js';
+/**
+ * The Request Response Service
+ *
+ * It allows nodes to request specific information from their peers, its use case covers recovering
+ * information that was missed during a syncronisation or a gossip event.
+ *
+ * This service implements the request response sub protocol, it is heavily inspired from
+ * ethereum implementations of the same name.
+ *
+ * Note, responses get compressed in streamHandler
+ *       so they get decompressed in readMessage
+ *
+ * see: https://github.com/ethereum/consensus-specs/blob/dev/specs/phase0/p2p-interface.md#the-reqresp-domain
+ */
+export declare class ReqResp {
+    private libp2p;
+    private peerScoring;
+    protected readonly logger: Logger;
+    private overallRequestTimeoutMs;
+    private individualRequestTimeoutMs;
+    private subProtocolHandlers;
+    private subProtocolValidators;
+    private connectionSampler;
+    private rateLimiter;
+    private snappyTransform;
+    private metrics;
+    constructor(config: P2PReqRespConfig, libp2p: Libp2p, peerScoring: PeerScoring, telemetryClient?: TelemetryClient);
+    get tracer(): import("@aztec/telemetry-client").Tracer;
+    /**
+     * Start the reqresp service
+     */
+    start(subProtocolHandlers: ReqRespSubProtocolHandlers, subProtocolValidators: ReqRespSubProtocolValidators): Promise<void>;
+    /**
+     * Stop the reqresp service
+     */
+    stop(): Promise<void>;
+    /**
+     * Send a request to peers, returns the first response
+     *
+     * @param subProtocol - The protocol being requested
+     * @param request - The request to send
+     * @returns - The response from the peer, otherwise undefined
+     *
+     * @description
+     * This method attempts to send a request to all active peers using the specified sub-protocol.
+     * It opens a stream with each peer, sends the request, and awaits a response.
+     * If a valid response is received, it returns the response; otherwise, it continues to the next peer.
+     * If no response is received from any peer, it returns undefined.
+     *
+     * The method performs the following steps:
+     * - Sample a peer to send the request to.
+     * - Opens a stream with the peer using the specified sub-protocol.
+     *
+     * When a response is received, it is validated using the given sub protocols response validator.
+     * To see the interface for the response validator - see `interface.ts`
+     *
+     * Failing a response validation requests in a severe peer penalty, and will
+     * prompt the node to continue to search to the next peer.
+     * For example, a transaction request validator will check that the payload returned does in fact
+     * match the txHash that was requested. A peer that fails this check an only be an extremely naughty peer.
+     *
+     * This entire operation is wrapped in an overall timeout, that is independent of the
+     * peer it is requesting data from.
+     *
+     */
+    sendRequest<SubProtocol extends ReqRespSubProtocol>(subProtocol: SubProtocol, request: InstanceType<SubProtocolMap[SubProtocol]['request']>): Promise<InstanceType<SubProtocolMap[SubProtocol]['response']> | undefined>;
+    /**
+     * Request multiple messages over the same sub protocol, balancing the requests across peers.
+     *
+     * @devnote
+     * - The function prioritizes sending requests to free peers using a batch sampling strategy.
+     * - If a peer fails to respond or returns an invalid response, it is removed from the sampling pool and replaced.
+     * - The function stops retrying once all requests are processed, no active peers remain, or the maximum retry attempts are reached.
+     * - Responses are validated using a custom validator for the sub-protocol.*
+     *
+     * Requests are sent in parallel to each peer, but multiple requests are sent to the same peer in series
+     * - If a peer fails to respond or returns an invalid response, it is removed from the sampling pool and replaced.
+     * - The function stops retrying once all requests are processed, no active peers remain, or the maximum retry attempts are reached.
+     * - Responses are validated using a custom validator for the sub-protocol.*
+     *
+     * @param subProtocol
+     * @param requests
+     * @param timeoutMs
+     * @param maxPeers
+     * @returns
+     *
+     * @throws {CollectiveReqRespTimeoutError} - If the request batch exceeds the specified timeout (`timeoutMs`).
+     */
+    sendBatchRequest<SubProtocol extends ReqRespSubProtocol>(subProtocol: SubProtocol, requests: InstanceType<SubProtocolMap[SubProtocol]['request']>[], timeoutMs?: number, maxPeers?: number, maxRetryAttempts?: number): Promise<InstanceType<SubProtocolMap[SubProtocol]['response']>[]>;
+    /**
+     * Sends a request to a specific peer
+     *
+     * We first dial a particular protocol for the peer, this ensures that the peer knows
+     * what to respond with
+     *
+     *
+     * @param peerId - The peer to send the request to
+     * @param subProtocol - The protocol to use to request
+     * @param payload - The payload to send
+     * @returns If the request is successful, the response is returned, otherwise undefined
+     *
+     * @description
+     * This method attempts to open a stream with the specified peer, send the payload,
+     * and await a response.
+     * If an error occurs, it penalizes the peer and returns undefined.
+     *
+     * The method performs the following steps:
+     * - Opens a stream with the peer using the specified sub-protocol.
+     * - Sends the payload and awaits a response with a timeout.
+     *
+     * If the stream is not closed by the dialled peer, and a timeout occurs, then
+     * the stream is closed on the requester's end and sender (us) updates its peer score
+     */
+    sendRequestToPeer(peerId: PeerId, subProtocol: ReqRespSubProtocol, payload: Buffer): Promise<ReqRespResponse | undefined>;
+    /**
+     * Handle a response error
+     *
+     * ReqResp errors are punished differently depending on the severity of the offense
+     *
+     * @param e - The error
+     * @param peerId - The peer id
+     * @param subProtocol - The sub protocol
+     * @returns If the error is non pubishable, then undefined is returned, otherwise the peer is penalized
+     */
+    private handleResponseError;
+    /**
+     * Categorize the error and log it.
+     */
+    private categorizeError;
+    /**
+     * Read a message returned from a stream into a single buffer
+     *
+     * The message is split into two components
+     * - The first chunk should contain a control byte, indicating the status of the response see `ReqRespStatus`
+     * - The second chunk should contain the response data
+     */
+    private readMessage;
+    /**
+     * Stream Handler
+     * Reads the incoming stream, determines the protocol, then triggers the appropriate handler
+     *
+     * @param param0 - The incoming stream data
+     *
+     * @description
+     * An individual stream handler will be bound to each sub protocol, and handles returning data back
+     * to the requesting peer.
+     *
+     * The sub protocol handler interface is defined within `interface.ts` and will be assigned to the
+     * req resp service on start up.
+     *
+     * We check rate limits for each peer, note the peer will be penalised within the rate limiter implementation
+     * if they exceed their peer specific limits.
+     */
+    private streamHandler;
+    private sendErrorChunk;
+}
+//# sourceMappingURL=reqresp.d.ts.map

package/dest/services/reqresp/reqresp.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"reqresp.d.ts","sourceRoot":"","sources":["../../../src/services/reqresp/reqresp.ts"],"names":[],"mappings":";;AACA,OAAO,EAAE,KAAK,MAAM,EAAgB,MAAM,uBAAuB,CAAC;AAGlE,OAAO,EAAc,KAAK,eAAe,EAAiC,MAAM,yBAAyB,CAAC;AAE1G,OAAO,KAAK,EAAsB,MAAM,EAAU,MAAM,mBAAmB,CAAC;AAE5E,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AASrC,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,iCAAiC,CAAC;AACnE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAGpD,OAAO,EAGL,KAAK,eAAe,EACpB,kBAAkB,EAClB,KAAK,0BAA0B,EAC/B,KAAK,4BAA4B,EACjC,KAAK,cAAc,EAEpB,MAAM,gBAAgB,CAAC;AASxB;;;;;;;;;;;;;GAaG;AACH,qBAAa,OAAO;IAmBhB,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,WAAW;IAnBrB,SAAS,CAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IAElC,OAAO,CAAC,uBAAuB,CAAS;IACxC,OAAO,CAAC,0BAA0B,CAAS;IAG3C,OAAO,CAAC,mBAAmB,CAA6D;IACxF,OAAO,CAAC,qBAAqB,CAAiE;IAE9F,OAAO,CAAC,iBAAiB,CAAoB;IAC7C,OAAO,CAAC,WAAW,CAA6B;IAEhD,OAAO,CAAC,eAAe,CAAkB;IAEzC,OAAO,CAAC,OAAO,CAAiB;gBAG9B,MAAM,EAAE,gBAAgB,EAChB,MAAM,EAAE,MAAM,EACd,WAAW,EAAE,WAAW,EAChC,eAAe,GAAE,eAAsC;IAgBzD,IAAI,MAAM,6CAET;IAED;;OAEG;IACG,KAAK,CAAC,mBAAmB,EAAE,0BAA0B,EAAE,qBAAqB,EAAE,4BAA4B;IAiBhH;;OAEG;IACG,IAAI;IAoBV;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA4BG;IACG,WAAW,CAAC,WAAW,SAAS,kBAAkB,EACtD,WAAW,EAAE,WAAW,EACxB,OAAO,EAAE,YAAY,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC,SAAS,CAAC,CAAC,GAC5D,OAAO,CAAC,YAAY,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC,UAAU,CAAC,CAAC,GAAG,SAAS,CAAC;IA8D7E;;;;;;;;;;;;;;;;;;;;;OAqBG;IAQG,gBAAgB,CAAC,WAAW,SAAS,kBAAkB,EAC3D,WAAW,EAAE,WAAW,EACxB,QAAQ,EAAE,YAAY,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC,SAAS,CAAC,CAAC,EAAE,EAChE,SAAS,SAAQ,EACjB,QAAQ,SAAgC,EACxC,gBAAgB,SAAI,GACnB,OAAO,CAAC,YAAY,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC;IA6HnE;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IAKU,iBAAiB,CAC5B,MAAM,EAAE,MAAM,EACd,WAAW,EAAE,kBAAkB,EAC/B,OAAO,EAAE,MAAM,GACd,OAAO,CAAC,eAAe,GAAG,SAAS,CAAC;IAgCvC;;;;;;;;;OASG;IACH,OAAO,CAAC,mBAAmB;IAO3B;;OAEG;IACH,OAAO,CAAC,eAAe;IAoDvB;;;;;;OAMG;YACW,WAAW;IAoCzB;;;;;;;;;;;;;;;OAeG;YAKW,aAAa;YAkEZ,cAAc;CAI9B"}