@aztec/p2p 0.0.0-test.0 → 0.0.1-commit.023c3e5

package/src/services/reqresp/reqresp.ts

@@ -1,12 +1,14 @@
 // @attribution: lodestar impl for inspiration
-import { type Logger, createLogger } from '@aztec/foundation/log';
+import { compactArray } from '@aztec/foundation/collection';
+import { AbortError, TimeoutError } from '@aztec/foundation/error';
+import { createLogger } from '@aztec/foundation/log';
 import { executeTimeout } from '@aztec/foundation/timer';
 import { PeerErrorSeverity } from '@aztec/stdlib/p2p';
 import { Attributes, type TelemetryClient, getTelemetryClient, trackSpan } from '@aztec/telemetry-client';
 
 import type { IncomingStreamData, PeerId, Stream } from '@libp2p/interface';
-import { pipe } from 'it-pipe';
 import type { Libp2p } from 'libp2p';
+import { pipeline } from 'node:stream/promises';
 import type { Uint8ArrayList } from 'uint8arraylist';
 
 import {
@@ -16,18 +18,24 @@ import {
 } from '../../errors/reqresp.error.js';
 import { SnappyTransform } from '../encoding.js';
 import type { PeerScoring } from '../peer-manager/peer_scoring.js';
-import type { P2PReqRespConfig } from './config.js';
+import {
+  DEFAULT_INDIVIDUAL_REQUEST_TIMEOUT_MS,
+  DEFAULT_REQRESP_DIAL_TIMEOUT_MS,
+  type P2PReqRespConfig,
+} from './config.js';
 import { BatchConnectionSampler } from './connection-sampler/batch_connection_sampler.js';
-import { ConnectionSampler } from './connection-sampler/connection_sampler.js';
+import { ConnectionSampler, RandomSampler } from './connection-sampler/connection_sampler.js';
 import {
-  DEFAULT_SUB_PROTOCOL_HANDLERS,
   DEFAULT_SUB_PROTOCOL_VALIDATORS,
+  type ReqRespInterface,
   type ReqRespResponse,
   ReqRespSubProtocol,
+  type ReqRespSubProtocolHandler,
   type ReqRespSubProtocolHandlers,
+  type ReqRespSubProtocolRateLimits,
   type ReqRespSubProtocolValidators,
   type SubProtocolMap,
-  subProtocolMap,
+  responseFromBuffer,
 } from './interface.js';
 import { ReqRespMetrics } from './metrics.js';
 import {
@@ -51,15 +59,12 @@ import { ReqRespStatus, ReqRespStatusError, parseStatusChunk, prettyPrintReqResp
  *
  * see: https://github.com/ethereum/consensus-specs/blob/dev/specs/phase0/p2p-interface.md#the-reqresp-domain
  */
-export class ReqResp {
-  protected readonly logger: Logger;
-
-  private overallRequestTimeoutMs: number;
-  private individualRequestTimeoutMs: number;
+export class ReqResp implements ReqRespInterface {
+  private individualRequestTimeoutMs: number = DEFAULT_INDIVIDUAL_REQUEST_TIMEOUT_MS;
+  private dialTimeoutMs: number = DEFAULT_REQRESP_DIAL_TIMEOUT_MS;
 
-  // Warning, if the `start` function is not called as the parent class constructor, then the default sub protocol handlers will be used ( not good )
-  private subProtocolHandlers: ReqRespSubProtocolHandlers = DEFAULT_SUB_PROTOCOL_HANDLERS;
-  private subProtocolValidators: ReqRespSubProtocolValidators = DEFAULT_SUB_PROTOCOL_VALIDATORS;
+  private subProtocolHandlers: Partial<ReqRespSubProtocolHandlers> = {};
+  private subProtocolValidators: Partial<ReqRespSubProtocolValidators> = {};
 
   private connectionSampler: ConnectionSampler;
   private rateLimiter: RequestResponseRateLimiter;
@@ -72,35 +77,57 @@ export class ReqResp {
     config: P2PReqRespConfig,
     private libp2p: Libp2p,
     private peerScoring: PeerScoring,
+    private logger = createLogger('p2p:reqresp'),
+    rateLimits: Partial<ReqRespSubProtocolRateLimits> = {},
     telemetryClient: TelemetryClient = getTelemetryClient(),
   ) {
-    this.logger = createLogger('p2p:reqresp');
+    this.updateConfig(config);
 
-    this.overallRequestTimeoutMs = config.overallRequestTimeoutMs;
-    this.individualRequestTimeoutMs = config.individualRequestTimeoutMs;
-
-    this.rateLimiter = new RequestResponseRateLimiter(peerScoring);
+    this.rateLimiter = new RequestResponseRateLimiter(peerScoring, rateLimits);
 
     // Connection sampler is used to sample our connected peers
-    this.connectionSampler = new ConnectionSampler(libp2p);
+    this.connectionSampler = new ConnectionSampler(
+      libp2p,
+      new RandomSampler(),
+      createLogger(`${logger.module}:connection-sampler`),
+      config,
+    );
 
     this.snappyTransform = new SnappyTransform();
     this.metrics = new ReqRespMetrics(telemetryClient);
   }
 
+  public updateConfig(config: Partial<P2PReqRespConfig>): void {
+    if (typeof config.individualRequestTimeoutMs === 'number') {
+      this.individualRequestTimeoutMs = config.individualRequestTimeoutMs;
+    }
+
+    if (typeof config.dialTimeoutMs === 'number') {
+      this.dialTimeoutMs = config.dialTimeoutMs;
+    }
+  }
+
   get tracer() {
     return this.metrics.tracer;
   }
 
+  /**
+   * Get the connection sampler instance
+   */
+  getConnectionSampler(): Pick<ConnectionSampler, 'getPeerListSortedByConnectionCountAsc'> {
+    return this.connectionSampler;
+  }
+
   /**
    * Start the reqresp service
    */
   async start(subProtocolHandlers: ReqRespSubProtocolHandlers, subProtocolValidators: ReqRespSubProtocolValidators) {
-    this.subProtocolHandlers = subProtocolHandlers;
-    this.subProtocolValidators = subProtocolValidators;
+    Object.assign(this.subProtocolHandlers, subProtocolHandlers);
+    Object.assign(this.subProtocolValidators, subProtocolValidators);
 
     // Register all protocol handlers
-    for (const subProtocol of Object.keys(this.subProtocolHandlers)) {
+    for (const subProtocol of Object.keys(subProtocolHandlers)) {
+      this.logger.debug(`Registering handler for sub protocol ${subProtocol}`);
       await this.libp2p.handle(
         subProtocol,
         (data: IncomingStreamData) =>
@@ -112,6 +139,23 @@ export class ReqResp {
     this.rateLimiter.start();
   }
 
+  async addSubProtocol(
+    subProtocol: ReqRespSubProtocol,
+    handler: ReqRespSubProtocolHandler,
+    validator: ReqRespSubProtocolValidators[ReqRespSubProtocol] = DEFAULT_SUB_PROTOCOL_VALIDATORS[subProtocol],
+  ): Promise<void> {
+    this.subProtocolHandlers[subProtocol] = handler;
+    this.subProtocolValidators[subProtocol] = validator;
+    this.logger.debug(`Registering handler for sub protocol ${subProtocol}`);
+    await this.libp2p.handle(
+      subProtocol,
+      (data: IncomingStreamData) =>
+        void this.streamHandler(subProtocol as ReqRespSubProtocol, data).catch(err =>
+          this.logger.error(`Error on libp2p subprotocol ${subProtocol} handler`, err),
+        ),
+    );
+  }
+
   /**
    * Stop the reqresp service
    */
@@ -135,100 +179,6 @@ export class ReqResp {
     // NOTE: We assume libp2p instance is managed by the caller
   }
 
-  /**
-   * Send a request to peers, returns the first response
-   *
-   * @param subProtocol - The protocol being requested
-   * @param request - The request to send
-   * @returns - The response from the peer, otherwise undefined
-   *
-   * @description
-   * This method attempts to send a request to all active peers using the specified sub-protocol.
-   * It opens a stream with each peer, sends the request, and awaits a response.
-   * If a valid response is received, it returns the response; otherwise, it continues to the next peer.
-   * If no response is received from any peer, it returns undefined.
-   *
-   * The method performs the following steps:
-   * - Sample a peer to send the request to.
-   * - Opens a stream with the peer using the specified sub-protocol.
-   *
-   * When a response is received, it is validated using the given sub protocols response validator.
-   * To see the interface for the response validator - see `interface.ts`
-   *
-   * Failing a response validation requests in a severe peer penalty, and will
-   * prompt the node to continue to search to the next peer.
-   * For example, a transaction request validator will check that the payload returned does in fact
-   * match the txHash that was requested. A peer that fails this check an only be an extremely naughty peer.
-   *
-   * This entire operation is wrapped in an overall timeout, that is independent of the
-   * peer it is requesting data from.
-   *
-   */
-  async sendRequest<SubProtocol extends ReqRespSubProtocol>(
-    subProtocol: SubProtocol,
-    request: InstanceType<SubProtocolMap[SubProtocol]['request']>,
-  ): Promise<InstanceType<SubProtocolMap[SubProtocol]['response']> | undefined> {
-    const responseValidator = this.subProtocolValidators[subProtocol];
-    const requestBuffer = request.toBuffer();
-
-    const requestFunction = async () => {
-      // Attempt to ask all of our peers, but sampled in a random order
-      // This function is wrapped in a timeout, so we will exit the loop if we have not received a response
-      const numberOfPeers = this.libp2p.getPeers().length;
-
-      if (numberOfPeers === 0) {
-        this.logger.debug('No active peers to send requests to');
-        return undefined;
-      }
-
-      const attemptedPeers: Map<string, boolean> = new Map();
-      for (let i = 0; i < numberOfPeers; i++) {
-        // Sample a peer to make a request to
-        const peer = this.connectionSampler.getPeer(attemptedPeers);
-        this.logger.trace(`Attempting to send request to peer: ${peer?.toString()}`);
-        if (!peer) {
-          this.logger.debug('No peers available to send requests to');
-          return undefined;
-        }
-
-        attemptedPeers.set(peer.toString(), true);
-
-        this.logger.trace(`Sending request to peer: ${peer.toString()}`);
-        const response = await this.sendRequestToPeer(peer, subProtocol, requestBuffer);
-
-        if (response && response.status !== ReqRespStatus.SUCCESS) {
-          this.logger.debug(
-            `Request to peer ${peer.toString()} failed with status ${prettyPrintReqRespStatus(response.status)}`,
-          );
-          continue;
-        }
-
-        // If we get a response, return it, otherwise we iterate onto the next peer
-        // We do not consider it a success if we have an empty buffer
-        if (response && response.data.length > 0) {
-          const object = subProtocolMap[subProtocol].response.fromBuffer(response.data);
-          // The response validator handles peer punishment within
-          const isValid = await responseValidator(request, object, peer);
-          if (!isValid) {
-            throw new InvalidResponseError();
-          }
-          return object;
-        }
-      }
-    };
-
-    try {
-      return await executeTimeout<InstanceType<SubProtocolMap[SubProtocol]['response']> | undefined>(
-        requestFunction,
-        this.overallRequestTimeoutMs,
-        () => new CollectiveReqRespTimeoutError(),
-      );
-    } catch (e: any) {
-      this.logger.debug(`${e.message} | subProtocol: ${subProtocol}`);
-      return undefined;
-    }
-  }
-
   /**
    * Request multiple messages over the same sub protocol, balancing the requests across peers.
    *
@@ -261,23 +211,38 @@ export class ReqResp {
   async sendBatchRequest<SubProtocol extends ReqRespSubProtocol>(
     subProtocol: SubProtocol,
     requests: InstanceType<SubProtocolMap[SubProtocol]['request']>[],
+    pinnedPeer: PeerId | undefined,
     timeoutMs = 10000,
-    maxPeers = Math.min(10, requests.length),
+    maxPeers = Math.max(10, Math.ceil(requests.length / 3)),
     maxRetryAttempts = 3,
   ): Promise<InstanceType<SubProtocolMap[SubProtocol]['response']>[]> {
-    const responseValidator = this.subProtocolValidators[subProtocol];
+    const responseValidator = this.subProtocolValidators[subProtocol] ?? DEFAULT_SUB_PROTOCOL_VALIDATORS[subProtocol];
     const responses: InstanceType<SubProtocolMap[SubProtocol]['response']>[] = new Array(requests.length);
     const requestBuffers = requests.map(req => req.toBuffer());
+    const isEmptyResponse = (value: unknown): boolean => {
+      // Some responses serialize to a non-empty buffer even when they contain no items (e.g., empty TxArray).
+      if (!value || typeof value !== 'object') {
+        return false;
+      }
+      const length = (value as { length?: number }).length;
+      return typeof length === 'number' && length === 0;
+    };
 
-    const requestFunction = async () => {
+    const requestFunction = async (signal: AbortSignal) => {
       // Track which requests still need to be processed
       const pendingRequestIndices = new Set(requestBuffers.map((_, i) => i));
 
       // Create batch sampler with the total number of requests and max peers
-      const batchSampler = new BatchConnectionSampler(this.connectionSampler, requests.length, maxPeers);
+      const batchSampler = new BatchConnectionSampler(
+        this.connectionSampler,
+        requests.length,
+        maxPeers,
+        compactArray([pinnedPeer]), // Exclude pinned peer from sampling, we will forcefully send all requests to it
+        createLogger(`${this.logger.module}:batch-connection-sampler`),
+      );
 
-      if (batchSampler.activePeerCount === 0) {
-        this.logger.debug('No active peers to send requests to');
+      if (batchSampler.activePeerCount === 0 && !pinnedPeer) {
+        this.logger.warn('No active peers to send requests to');
         return [];
       }
 
@@ -291,20 +256,42 @@ export class ReqResp {
 
       let retryAttempts = 0;
       while (pendingRequestIndices.size > 0 && batchSampler.activePeerCount > 0 && retryAttempts < maxRetryAttempts) {
+        if (signal.aborted) {
+          throw new AbortError('Batch request aborted');
+        }
         // Process requests in parallel for each available peer
-        const requestBatches = new Map<PeerId, number[]>();
+        type BatchEntry = { peerId: PeerId; indices: number[] };
+        const requestBatches = new Map<string, BatchEntry>();
 
         // Group requests by peer
         for (const requestIndex of pendingRequestIndices) {
           const peer = batchSampler.getPeerForRequest(requestIndex);
           if (!peer) {
-            break;
+            // No peer available for this specific index (all peers exhausted for it)
+            // Skip this index for now - it stays in pendingRequestIndices for retry
+            continue;
           }
-
-          if (!requestBatches.has(peer)) {
-            requestBatches.set(peer, []);
+          const peerAsString = peer.toString();
+          if (!requestBatches.has(peerAsString)) {
+            requestBatches.set(peerAsString, { peerId: peer, indices: [] });
           }
-          requestBatches.get(peer)!.push(requestIndex);
+          requestBatches.get(peerAsString)!.indices.push(requestIndex);
+        }
+
+        // If there is a pinned peer, we will always send every request to that peer
+        // We use the default limits for the subprotocol to avoid hitting the rate limiter
+        if (pinnedPeer) {
+          const limit = this.rateLimiter.getRateLimits(subProtocol).peerLimit.quotaCount;
+          requestBatches.set(pinnedPeer.toString(), {
+            peerId: pinnedPeer,
+            indices: Array.from(pendingRequestIndices.values()).slice(0, limit),
+          });
+        }
+
+        // If no requests could be assigned (all peers exhausted for all indices), exit early
+        if (requestBatches.size === 0) {
+          this.logger.warn('No peers available for any pending request indices, stopping batch request');
+          break;
         }
 
         // Make parallel requests for each peer's batch
@@ -316,41 +303,63 @@ export class ReqResp {
         // while simultaneously Peer Id 1 will send requests 4, 5, 6, 7 in serial
 
         const batchResults = await Promise.all(
-          Array.from(requestBatches.entries()).map(async ([peer, indices]) => {
+          Array.from(requestBatches.entries()).map(async ([peerAsString, { peerId: peer, indices }]) => {
             try {
+              const markIndexFailed = (index: number) => batchSampler.markPeerFailedForIndex(peer, index);
               // Requests all going to the same peer are sent synchronously
               const peerResults: { index: number; response: InstanceType<SubProtocolMap[SubProtocol]['response']> }[] =
                 [];
+              let shouldReplacePeer = false;
+              const handleFailure = (status: ReqRespStatus, index: number) => {
+                this.logger.warn(
+                  `Request to peer ${peerAsString} failed with status ${prettyPrintReqRespStatus(status)}`,
+                );
+                markIndexFailed(index);
+                return status === ReqRespStatus.RATE_LIMIT_EXCEEDED;
+              };
+
               for (const index of indices) {
+                this.logger.info(`Sending request ${index} to peer ${peerAsString}`);
                 const response = await this.sendRequestToPeer(peer, subProtocol, requestBuffers[index]);
 
                 // Check the status of the response buffer
-                if (response && response.status !== ReqRespStatus.SUCCESS) {
-                  this.logger.debug(
-                    `Request to peer ${peer.toString()} failed with status ${prettyPrintReqRespStatus(
-                      response.status,
-                    )}`,
-                  );
-
-                  // If we hit a rate limit or some failure, we remove the peer and return the results,
-                  // they will be split among remaining peers and the new sampled peer
-                  batchSampler.removePeerAndReplace(peer);
-                  return { peer, results: peerResults };
+                if (response.status !== ReqRespStatus.SUCCESS) {
+                  shouldReplacePeer = handleFailure(response.status, index);
+                  if (shouldReplacePeer) {
+                    break;
+                  }
+                  continue;
                 }
 
-                if (response && response.data.length > 0) {
-                  const object = subProtocolMap[subProtocol].response.fromBuffer(response.data);
-                  const isValid = await responseValidator(requests[index], object, peer);
+                if (response.data.length === 0) {
+                  markIndexFailed(index);
+                  continue;
+                }
 
-                  if (isValid) {
-                    peerResults.push({ index, response: object });
-                  }
+                const object = responseFromBuffer(subProtocol, response.data);
+                if (isEmptyResponse(object)) {
+                  markIndexFailed(index);
+                  continue;
                 }
+
+                const isValid = await responseValidator(requests[index], object, peer);
+                if (!isValid) {
+                  markIndexFailed(index);
+                  continue;
+                }
+
+                peerResults.push({ index, response: object });
+              }
+
+              // If peer had a hard failure (rate limit), replace it for future iterations
+              if (shouldReplacePeer) {
+                this.logger.warn(`Peer ${peerAsString} hit a hard failure, removing from sampler`);
+                batchSampler.removePeerAndReplace(peer);
               }
 
               return { peer, results: peerResults };
             } catch (error) {
-              this.logger.debug(`Failed batch request to peer ${peer.toString()}:`, error);
+              this.logger.warn(`Failed batch request to peer ${peerAsString}:`, error);
               batchSampler.removePeerAndReplace(peer);
               return { peer, results: [] };
             }
@@ -371,7 +380,7 @@ export class ReqResp {
       }
 
       if (retryAttempts >= maxRetryAttempts) {
-        this.logger.debug(`Max retry attempts ${maxRetryAttempts} reached for batch request`);
+        this.logger.warn(`Max retry attempts ${maxRetryAttempts} reached for batch request`);
       }
 
       return responses;
@@ -384,7 +393,7 @@ export class ReqResp {
         () => new CollectiveReqRespTimeoutError(),
       );
     } catch (e: any) {
-      this.logger.debug(`${e.message} | subProtocol: ${subProtocol}`);
+      this.logger.warn(`${e.message} | subProtocol: ${subProtocol}`);
       return [];
     }
   }
@@ -399,6 +408,7 @@ export class ReqResp {
    * @param peerId - The peer to send the request to
    * @param subProtocol - The protocol to use to request
    * @param payload - The payload to send
+   * @param dialTimeout - If establishing a stream takes longer than this an error will be thrown
    * @returns If the request is successful, the response is returned, otherwise undefined
    *
    * @description
@@ -421,29 +431,53 @@ export class ReqResp {
     peerId: PeerId,
     subProtocol: ReqRespSubProtocol,
     payload: Buffer,
-  ): Promise<ReqRespResponse | undefined> {
+    dialTimeout: number = this.dialTimeoutMs,
+  ): Promise<ReqRespResponse> {
     let stream: Stream | undefined;
     try {
       this.metrics.recordRequestSent(subProtocol);
 
-      stream = await this.connectionSampler.dialProtocol(peerId, subProtocol);
+      this.logger.trace(`Sending request to peer ${peerId.toString()} on sub protocol ${subProtocol}`);
+      stream = await this.connectionSampler.dialProtocol(peerId, subProtocol, dialTimeout);
+      this.logger.trace(
+        `Opened stream ${stream.id} for sending request to peer ${peerId.toString()} on sub protocol ${subProtocol}`,
+      );
 
-      // Open the stream with a timeout
-      const result = await executeTimeout<ReqRespResponse>(
-        (): Promise<ReqRespResponse> => pipe([payload], stream!, this.readMessage.bind(this)),
+      const timeoutErr = new IndividualReqRespTimeoutError();
+      const [_, resp] = await executeTimeout(
+        signal =>
+          Promise.all([
+            pipeline([payload], stream!.sink, { signal }),
+            pipeline(stream!.source, this.readMessage.bind(this), { signal }),
+          ]),
         this.individualRequestTimeoutMs,
-        () => new IndividualReqRespTimeoutError(),
+        () => timeoutErr,
       );
-
-      return result;
+      return resp;
     } catch (e: any) {
+      this.logger.warn(`SUBPROTOCOL: ${subProtocol}\n`, e);
+      // On error we immediately abort the stream, this is preferred way,
+      // because it signals to the sender that error happened, whereas
+      // closing the stream only closes our side and is much slower
+      if (stream) {
+        stream!.abort(e);
+      }
+
       this.metrics.recordRequestError(subProtocol);
       this.handleResponseError(e, peerId, subProtocol);
+
+      // If there is an exception, we return an unknown response
+      this.logger.debug(`Error sending request to peer ${peerId.toString()} on sub protocol ${subProtocol}: ${e}`);
+      return { status: ReqRespStatus.FAILURE };
     } finally {
       // Only close the stream if we created it
+      // Note even if we aborted the stream, calling close on it is ok, it's just a no-op
       if (stream) {
         try {
-          await this.connectionSampler.close(stream.id);
+          this.logger.trace(
+            `Closing stream ${stream.id} for request to peer ${peerId.toString()} on sub protocol ${subProtocol}`,
+          );
+          await this.connectionSampler.close(stream);
         } catch (closeError) {
           this.logger.error(
             `Error closing stream: ${closeError instanceof Error ? closeError.message : 'Unknown error'}`,
@@ -464,67 +498,12 @@ export class ReqResp {
    * @returns If the error is non pubishable, then undefined is returned, otherwise the peer is penalized
    */
   private handleResponseError(e: any, peerId: PeerId, subProtocol: ReqRespSubProtocol): void {
-    const severity = this.categorizeError(e, peerId, subProtocol);
+    const severity = this.categorizeResponseError(e, peerId, subProtocol);
     if (severity) {
       this.peerScoring.penalizePeer(peerId, severity);
     }
   }
 
-  /**
-   * Categorize the error and log it.
-   */
-  private categorizeError(e: any, peerId: PeerId, subProtocol: ReqRespSubProtocol): PeerErrorSeverity | undefined {
-    // Non punishable errors - we do not expect a response for goodbye messages
-    if (subProtocol === ReqRespSubProtocol.GOODBYE) {
-      this.logger.debug('Error encountered on goodbye sub protocol, no penalty', {
-        peerId: peerId.toString(),
-        subProtocol,
-      });
-      return undefined;
-    }
-
-    // We do not punish a collective timeout, as the node triggers this interupt, independent of the peer's behaviour
-    const logTags = {
-      peerId: peerId.toString(),
-      subProtocol,
-    };
-    if (e instanceof CollectiveReqRespTimeoutError || e instanceof InvalidResponseError) {
-      this.logger.debug(
-        `Non-punishable error: ${e.message} | peerId: ${peerId.toString()} | subProtocol: ${subProtocol}`,
-        logTags,
-      );
-      return undefined;
-    }
-
-    // Pubishable errors
-    // Connection reset errors in the networking stack are punished with high severity
-    // it just signals an unreliable peer
-    // We assume that the requesting node has a functioning networking stack.
-    if (e?.code === 'ECONNRESET' || e?.code === 'EPIPE') {
-      this.logger.debug(`Connection reset: ${peerId.toString()}`, logTags);
-      return PeerErrorSeverity.HighToleranceError;
-    }
-
-    if (e?.code === 'ECONNREFUSED') {
-      this.logger.debug(`Connection refused: ${peerId.toString()}`, logTags);
-      return PeerErrorSeverity.HighToleranceError;
-    }
-
-    // Timeout errors are punished with high tolerance, they can be due to a geogrpahically far away peer or an
-    // overloaded peer
-    if (e instanceof IndividualReqRespTimeoutError) {
-      this.logger.debug(
-        `Timeout error: ${e.message} | peerId: ${peerId.toString()} | subProtocol: ${subProtocol}`,
-        logTags,
-      );
-      return PeerErrorSeverity.HighToleranceError;
-    }
-
-    // Catch all error
-    this.logger.error(`Unexpected error sending request to peer`, e, logTags);
-    return PeerErrorSeverity.HighToleranceError;
-  }
-
   /**
    * Read a message returned from a stream into a single buffer
    *
@@ -533,24 +512,34 @@ export class ReqResp {
    * - The second chunk should contain the response data
    */
   private async readMessage(source: AsyncIterable<Uint8ArrayList>): Promise<ReqRespResponse> {
-    let statusBuffer: ReqRespStatus | undefined;
+    let status: ReqRespStatus | undefined;
     const chunks: Uint8Array[] = [];
 
     try {
       for await (const chunk of source) {
-        if (statusBuffer === undefined) {
-          const firstChunkBuffer = chunk.subarray();
-          statusBuffer = parseStatusChunk(firstChunkBuffer);
-        } else {
+        const statusParsed = status !== undefined;
+        if (statusParsed) {
           chunks.push(chunk.subarray());
+          continue;
+        }
+
+        const firstChunkBuffer = chunk.subarray();
+        status = parseStatusChunk(firstChunkBuffer);
+
+        // In case status is not SUCCESS, we do not expect any data in the response
+        // we can return early
+        if (status !== ReqRespStatus.SUCCESS) {
+          return {
+            status,
+          };
         }
       }
 
       const messageData = Buffer.concat(chunks);
-      const message: Buffer = this.snappyTransform.inboundTransformNoTopic(messageData);
+      const message: Buffer = this.snappyTransform.inboundTransformData(messageData);
 
       return {
-        status: statusBuffer ?? ReqRespStatus.UNKNOWN,
+        status: status ?? ReqRespStatus.UNKNOWN,
         data: message,
       };
     } catch (e: any) {
@@ -563,7 +552,6 @@ export class ReqResp {
 
       return {
         status,
-        data: Buffer.from([]),
       };
     }
   }
@@ -572,7 +560,8 @@ export class ReqResp {
    * Stream Handler
    * Reads the incoming stream, determines the protocol, then triggers the appropriate handler
    *
-   * @param param0 - The incoming stream data
+   * @param protocol - The sub protocol to handle
+   * @param incomingStream - The incoming stream data containing the stream and connection
    *
    * @description
    * An individual stream handler will be bound to each sub protocol, and handles returning data back
@@ -588,14 +577,13 @@ export class ReqResp {
     [Attributes.P2P_REQ_RESP_PROTOCOL]: protocol,
     [Attributes.P2P_ID]: connection.remotePeer.toString(),
   }))
-  private async streamHandler(protocol: ReqRespSubProtocol, { stream, connection }: IncomingStreamData) {
-    this.metrics.recordRequestReceived(protocol);
-
+  private async streamHandler(protocol: ReqRespSubProtocol, incomingStream: IncomingStreamData) {
+    const { stream, connection } = incomingStream;
     try {
-      // Store a reference to from this for the async generator
+      this.metrics.recordRequestReceived(protocol);
       const rateLimitStatus = this.rateLimiter.allow(protocol, connection.remotePeer);
-      if (rateLimitStatus != RateLimitStatus.Allowed) {
-        this.logger.warn(
+      if (rateLimitStatus !== RateLimitStatus.Allowed) {
+        this.logger.verbose(
           `Rate limit exceeded ${prettyPrintRateLimitStatus(rateLimitStatus)} for ${protocol} from ${
             connection.remotePeer
           }`,
@@ -604,58 +592,248 @@ export class ReqResp {
         throw new ReqRespStatusError(ReqRespStatus.RATE_LIMIT_EXCEEDED);
       }
 
-      const handler = this.subProtocolHandlers[protocol];
-      const transform = this.snappyTransform;
+      await this.processStream(protocol, incomingStream);
+    } catch (err: any) {
+      this.metrics.recordResponseError(protocol);
+      this.handleRequestError(err, connection.remotePeer, protocol);
+
+      if (err instanceof ReqRespStatusError) {
+        const errorSent = await this.trySendError(stream, connection.remotePeer, protocol, err.status);
+        const logMessage = errorSent
+          ? 'Protocol error sent successfully.'
+          : 'Stream already closed or poisoned, not sending error response.';
+
+        const isRateLimit = err.status === ReqRespStatus.RATE_LIMIT_EXCEEDED;
+
+        const level = isRateLimit ? 'debug' : 'warn';
+        this.logger[level](logMessage + ` Status: ${ReqRespStatus[err.status]}`, {
+          protocol,
+          err,
+          errorStatus: err.status,
+          cause: err.cause ?? 'Cause unknown',
+        });
+      } else {
+        // In erroneous case we abort the stream, this will signal the peer that something went wrong
+        // and that this stream should be dropped
+        const isMessageToNotWarn =
+          err instanceof Error &&
+          ['stream reset', 'Cannot push value onto an ended pushable'].some(msg => err.message.includes(msg));
+        const level = isMessageToNotWarn ? 'debug' : 'warn';
+        this.logger[level]('Unknown stream error while handling the stream, aborting', {
+          protocol,
+          err,
+        });
+
+        stream.abort(err);
+      }
+    } finally {
+      //NOTE: This is idempotent action, so it's ok to call it even if stream was aborted
+      await stream.close();
+    }
+  }
 
-      await pipe(
-        stream,
-        async function* (source: any) {
-          for await (const chunkList of source) {
-            const msg = Buffer.from(chunkList.subarray());
-            const response = await handler(connection.remotePeer, msg);
+  /**
+   * Reads incoming data from the stream, processes it according to the sub protocol,
+   * and puts response back into the stream.
+   *
+   * @param protocol - The sub protocol to use for processing the stream
+   * @param incomingStream - The incoming stream data containing the stream and connection
+   *
+   * */
+  private async processStream(protocol: ReqRespSubProtocol, { stream, connection }: IncomingStreamData): Promise<void> {
+    const handler = this.subProtocolHandlers[protocol];
+    if (!handler) {
+      throw new Error(`No handler defined for reqresp subprotocol ${protocol}`);
+    }
 
-            if (protocol === ReqRespSubProtocol.GOODBYE) {
-              // Don't respond
-              await stream.close();
-              return;
-            }
+    const snappy = this.snappyTransform;
+    const SUCCESS = Uint8Array.of(ReqRespStatus.SUCCESS);
 
-            // Send success code first, then the response
-            const successChunk = Buffer.from([ReqRespStatus.SUCCESS]);
-            yield new Uint8Array(successChunk);
+    await pipeline(
+      stream.source,
+      async function* (source: any) {
+        for await (const chunk of source) {
+          const response = await handler(connection.remotePeer, chunk.subarray());
 
-            yield new Uint8Array(transform.outboundTransformNoTopic(response));
+          if (protocol === ReqRespSubProtocol.GOODBYE) {
+            // NOTE: The stream was already closed by Goodbye handler
+            // peerManager.goodbyeReceived(peerId, reason); will call libp2p.hangUp closing all active streams and connections
+            // Don't try to respond
+            return;
           }
-        },
-        stream,
-      );
+
+          stream.metadata.written = true; // Mark the stream as written to;
+
+          yield SUCCESS;
+          yield snappy.outboundTransformData(response);
+        }
+      },
+      stream.sink,
+    );
+  }
+
+  /**
+   * Try to send error status to the peer. We say try, because the stream,
+   * might already be closed
+   * @param stream - The stream opened between us and the peer
+   * @param status - The error status to send back to the peer
+   * @returns true if error was sent successfully, otherwise false
+   *
+   */
+  private async trySendError(
+    stream: Stream,
+    peerId: PeerId,
+    protocol: ReqRespSubProtocol,
+    status: ReqRespStatus,
+  ): Promise<boolean> {
+    const canWriteToStream =
+      // 'writing' is a bit weird naming, but it actually means that the stream is ready to write
+      // 'ready' means that stream ready to be opened for writing
+      stream.status === 'open' && (stream.writeStatus === 'writing' || stream.writeStatus === 'ready');
+
+    // Stream was already written to, we consider it poisoned, in a sense,
+    // that even if we write an error response, it will not be interpreted correctly by the peer
+    const streamPoisoned = stream.metadata.written === true;
+    const shouldWriteToStream = canWriteToStream && !streamPoisoned;
+
+    if (!shouldWriteToStream) {
+      return false;
+    }
+
+    try {
+      await pipeline(function* () {
+        yield Uint8Array.of(status);
+      }, stream.sink);
+
+      return true;
     } catch (e: any) {
-      this.logger.warn('Reqresp Response error: ', e);
-      this.metrics.recordResponseError(protocol);
+      this.logger.warn('Error while sending error response', e);
+      stream.abort(e);
 
-      // If we receive a known error, we use the error status in the response chunk, otherwise we categorize as unknown
-      let errorStatus = ReqRespStatus.UNKNOWN;
-      if (e instanceof ReqRespStatusError) {
-        errorStatus = e.status;
-      }
+      this.handleRequestError(e, peerId, protocol);
+      return false;
+    }
+  }
 
-      const sendErrorChunk = this.sendErrorChunk(errorStatus);
+  private handleRequestError(e: any, peerId: PeerId, subProtocol: ReqRespSubProtocol): void {
+    const severity = this.categorizeRequestError(e, peerId, subProtocol);
+    if (severity) {
+      this.peerScoring.penalizePeer(peerId, severity);
+    }
+  }
 
-      // Return and yield the response chunk
-      await pipe(
-        stream,
-        async function* (_source: any) {
-          yield* sendErrorChunk;
-        },
-        stream,
-      );
-    } finally {
-      await stream.close();
+  /**
+   * Categorize the request error and log it.
+   *
+   * @returns Severity of the error, or undefined if the error is not punishable.
+   */
+  private categorizeRequestError(
+    e: any,
+    peerId: PeerId,
+    subProtocol: ReqRespSubProtocol,
+  ): PeerErrorSeverity | undefined {
+    const logTags = { peerId: peerId.toString(), subProtocol };
+
+    //Punishable error - peer should never send badly formed request
+    if (e instanceof ReqRespStatusError && e.status === ReqRespStatus.BADLY_FORMED_REQUEST) {
+      this.logger.debug(`Punishable error in ${subProtocol}: ${e.cause}`, logTags);
+      return PeerErrorSeverity.LowToleranceError;
     }
+
+    //TODO: (mralj): think if we should penalize peer here based on connection errors
+    return undefined;
+  }
+
+  /**
+   * Categorize the response error and log it.
+   *
+   * @returns Severity of the error, or undefined if the error is not punishable.
+   */
+  private categorizeResponseError(
+    e: any,
+    peerId: PeerId,
+    subProtocol: ReqRespSubProtocol,
+  ): PeerErrorSeverity | undefined {
+    const logTags = { peerId: peerId.toString(), subProtocol };
+
+    // Non punishable errors - we do not expect a response for goodbye messages
+    if (subProtocol === ReqRespSubProtocol.GOODBYE) {
+      this.logger.debug('Error encountered on goodbye sub protocol, no penalty', logTags);
+      return undefined;
+    }
+
+    // We do not punish a collective timeout, as the node triggers this interupt, independent of the peer's behaviour
+    if (e instanceof CollectiveReqRespTimeoutError || e instanceof InvalidResponseError) {
+      this.logger.debug(`Non-punishable error in ${subProtocol}: ${e.message}`, logTags);
+      return undefined;
+    }
+
+    return this.categorizeConnectionErrors(e, peerId, subProtocol);
   }
 
-  private async *sendErrorChunk(error: ReqRespStatus): AsyncIterable<Uint8Array> {
-    const errorChunk = Buffer.from([error]);
-    yield new Uint8Array(errorChunk);
+  /*
+   * Errors specific to connection  handling
+   * These can happen  both when sending request and response*/
+  private categorizeConnectionErrors(
+    e: any,
+    peerId: PeerId,
+    subProtocol: ReqRespSubProtocol,
+  ): PeerErrorSeverity | undefined {
+    const logTags = { peerId: peerId.toString(), subProtocol };
+    // Do not punish if we are stopping the service
+    if (e instanceof AbortError || e?.code == 'ABORT_ERR') {
+      this.logger.debug(`Request aborted: ${e.message}`, logTags);
+      return undefined;
+    }
+
+    // Do not punish if we are the ones closing the connection
+    if (
+      e?.code === 'ERR_CONNECTION_BEING_CLOSED' ||
+      e?.code === 'ERR_CONNECTION_CLOSED' ||
+      e?.code === 'ERR_TRANSIENT_CONNECTION' ||
+      e?.message?.includes('Muxer already closed') ||
+      e?.message?.includes('muxer closed') ||
+      e?.message?.includes('ended pushable')
+    ) {
+      this.logger.debug(
+        `Connection closed to peer from our side: ${peerId.toString()} (${e?.message ?? 'missing error message'})`,
+        logTags,
+      );
+      return undefined;
+    }
+
+    // Pubishable errors
+    // Connection reset errors in the networking stack are punished with high severity
+    // it just signals an unreliable peer
+    // We assume that the requesting node has a functioning networking stack.
+    if (e?.code === 'ECONNRESET' || e?.code === 'EPIPE') {
+      this.logger.debug(`Connection reset: ${peerId.toString()}`, logTags);
+      return PeerErrorSeverity.HighToleranceError;
+    }
+
+    if (e?.code === 'ECONNREFUSED') {
+      this.logger.debug(`Connection refused: ${peerId.toString()}`, logTags);
+      return PeerErrorSeverity.HighToleranceError;
+    }
+
+    if (e?.code === 'ERR_UNEXPECTED_EOF') {
+      this.logger.debug(`Connection unexpected EOF: ${peerId.toString()}`, logTags);
+      return PeerErrorSeverity.HighToleranceError;
+    }
+
+    if (e?.code === 'ERR_UNSUPPORTED_PROTOCOL') {
+      this.logger.debug(`Sub protocol not supported by peer: ${peerId.toString()}`, logTags);
+      return PeerErrorSeverity.HighToleranceError;
+    }
+
+    // Timeout errors are punished with high tolerance, they can be due to a geographically far away or overloaded peer
+    if (e instanceof IndividualReqRespTimeoutError || e instanceof TimeoutError) {
+      this.logger.debug(`Timeout error in ${subProtocol}: ${e.message}`, logTags);
+      return PeerErrorSeverity.HighToleranceError;
+    }
+
+    // Catch all error
+    this.logger.error(`Unexpected error in ReqResp protocol`, e, logTags);
+    return PeerErrorSeverity.HighToleranceError;
   }
 }