@waku/core 0.0.22 → 0.0.24

package/src/lib/base_protocol.ts

@@ -1,13 +1,12 @@
-import type { Stream } from "@libp2p/interface-connection";
-import type { Libp2p } from "@libp2p/interface-libp2p";
-import type { PeerId } from "@libp2p/interface-peer-id";
-import { Peer, PeerStore } from "@libp2p/interface-peer-store";
+import type { Libp2p } from "@libp2p/interface";
+import type { Stream } from "@libp2p/interface/connection";
+import type { PeerId } from "@libp2p/interface/peer-id";
+import { Peer, PeerStore } from "@libp2p/interface/peer-store";
 import type { IBaseProtocol, Libp2pComponents } from "@waku/interfaces";
-import {
-  getPeersForProtocol,
-  selectConnection,
-  selectPeerForProtocol,
-} from "@waku/utils/libp2p";
+import { getPeersForProtocol, selectPeerForProtocol } from "@waku/utils/libp2p";
+
+import { filterPeers } from "./filterPeers.js";
+import { StreamManager } from "./stream_manager.js";
 
 /**
  * A class with predefined helpers, to be used as a base to implement Waku
@@ -16,14 +15,29 @@ import {
 export class BaseProtocol implements IBaseProtocol {
   public readonly addLibp2pEventListener: Libp2p["addEventListener"];
   public readonly removeLibp2pEventListener: Libp2p["removeEventListener"];
+  protected streamManager: StreamManager;
 
-  constructor(public multicodec: string, private components: Libp2pComponents) {
+  constructor(
+    public multicodec: string,
+    private components: Libp2pComponents
+  ) {
     this.addLibp2pEventListener = components.events.addEventListener.bind(
       components.events
     );
     this.removeLibp2pEventListener = components.events.removeEventListener.bind(
       components.events
     );
+
+    this.streamManager = new StreamManager(
+      multicodec,
+      components.connectionManager.getConnections.bind(
+        components.connectionManager
+      ),
+      this.addLibp2pEventListener
+    );
+  }
+  protected async getStream(peer: Peer): Promise<Stream> {
+    return this.streamManager.getStream(peer);
   }
 
   public get peerStore(): PeerStore {
@@ -47,15 +61,32 @@ export class BaseProtocol implements IBaseProtocol {
     );
     return peer;
   }
-  protected async newStream(peer: Peer): Promise<Stream> {
-    const connections = this.components.connectionManager.getConnections(
-      peer.id
-    );
-    const connection = selectConnection(connections);
-    if (!connection) {
-      throw new Error("Failed to get a connection to the peer");
+
+  /**
+   * Retrieves a list of peers based on the specified criteria.
+   *
+   * @param numPeers - The total number of peers to retrieve. If 0, all peers are returned.
+   * @param maxBootstrapPeers - The maximum number of bootstrap peers to retrieve.
+   * @returns A Promise that resolves to an array of peers based on the specified criteria.
+   */
+  protected async getPeers(
+    {
+      numPeers,
+      maxBootstrapPeers
+    }: {
+      numPeers: number;
+      maxBootstrapPeers: number;
+    } = {
+      maxBootstrapPeers: 1,
+      numPeers: 0
     }
+  ): Promise<Peer[]> {
+    // Retrieve all peers that support the protocol
+    const allPeersForProtocol = await getPeersForProtocol(this.peerStore, [
+      this.multicodec
+    ]);
 
-    return connection.newStream(this.multicodec);
+    // Filter the peers based on the specified criteria
+    return filterPeers(allPeersForProtocol, numPeers, maxBootstrapPeers);
   }
 }

package/src/lib/connection_manager.ts

@@ -1,7 +1,9 @@
-import type { PeerId } from "@libp2p/interface-peer-id";
-import type { PeerInfo } from "@libp2p/interface-peer-info";
-import type { Peer } from "@libp2p/interface-peer-store";
+import type { PeerId } from "@libp2p/interface/peer-id";
+import type { PeerInfo } from "@libp2p/interface/peer-info";
+import type { Peer } from "@libp2p/interface/peer-store";
+import type { PeerStore } from "@libp2p/interface/peer-store";
 import { CustomEvent, EventEmitter } from "@libp2p/interfaces/events";
+import { decodeRelayShard } from "@waku/enr";
 import {
   ConnectionManagerOptions,
   EPeersByDiscoveryEvents,
@@ -10,8 +12,11 @@ import {
   IRelay,
   KeepAliveOptions,
   PeersByDiscoveryResult,
+  PubSubTopic,
+  ShardInfo
 } from "@waku/interfaces";
 import { Libp2p, Tags } from "@waku/interfaces";
+import { shardInfoToPubSubTopics } from "@waku/utils";
 import debug from "debug";
 
 import { KeepAliveManager } from "./keep_alive_manager.js";
@@ -40,6 +45,7 @@ export class ConnectionManager
     peerId: string,
     libp2p: Libp2p,
     keepAliveOptions: KeepAliveOptions,
+    pubsubTopics: PubSubTopic[],
     relay?: IRelay,
     options?: ConnectionManagerOptions
   ): ConnectionManager {
@@ -48,6 +54,7 @@ export class ConnectionManager
       instance = new ConnectionManager(
         libp2p,
         keepAliveOptions,
+        pubsubTopics,
         relay,
         options
       );
@@ -92,28 +99,30 @@ export class ConnectionManager
     return {
       DISCOVERED: {
         [Tags.BOOTSTRAP]: peersDiscoveredByBootstrap,
-        [Tags.PEER_EXCHANGE]: peersDiscoveredByPeerExchange,
+        [Tags.PEER_EXCHANGE]: peersDiscoveredByPeerExchange
       },
       CONNECTED: {
         [Tags.BOOTSTRAP]: peersConnectedByBootstrap,
-        [Tags.PEER_EXCHANGE]: peersConnectedByPeerExchange,
-      },
+        [Tags.PEER_EXCHANGE]: peersConnectedByPeerExchange
+      }
     };
   }
 
   private constructor(
     libp2p: Libp2p,
     keepAliveOptions: KeepAliveOptions,
+    private configuredPubSubTopics: PubSubTopic[],
     relay?: IRelay,
     options?: Partial<ConnectionManagerOptions>
   ) {
     super();
     this.libp2p = libp2p;
+    this.configuredPubSubTopics = configuredPubSubTopics;
     this.options = {
       maxDialAttemptsForPeer: DEFAULT_MAX_DIAL_ATTEMPTS_FOR_PEER,
       maxBootstrapPeersAllowed: DEFAULT_MAX_BOOTSTRAP_PEERS_ALLOWED,
       maxParallelDials: DEFAULT_MAX_PARALLEL_DIALS,
-      ...options,
+      ...options
     };
 
     this.keepAliveManager = new KeepAliveManager(keepAliveOptions, relay);
@@ -217,16 +226,24 @@ export class ConnectionManager
       try {
         const error = this.dialErrorsForPeer.get(peerId.toString());
 
-        let errorMessage;
-        if (error instanceof AggregateError) {
-          errorMessage = JSON.stringify(error.errors[0]);
-        } else {
-          errorMessage = error.message;
-        }
+        if (error) {
+          let errorMessage;
+          if (error instanceof AggregateError) {
+            if (!error.errors) {
+              log(`No errors array found for AggregateError`);
+            } else if (error.errors.length === 0) {
+              log(`Errors array is empty for AggregateError`);
+            } else {
+              errorMessage = JSON.stringify(error.errors[0]);
+            }
+          } else {
+            errorMessage = error.message;
+          }
 
-        log(
-          `Deleting undialable peer ${peerId.toString()} from peer store. Error: ${errorMessage}`
-        );
+          log(
+            `Deleting undialable peer ${peerId.toString()} from peer store. Error: ${errorMessage}`
+          );
+        }
 
         this.dialErrorsForPeer.delete(peerId.toString());
         await this.libp2p.peerStore.delete(peerId);
@@ -297,15 +314,15 @@ export class ConnectionManager
   }
 
   private async attemptDial(peerId: PeerId): Promise<void> {
+    if (!(await this.shouldDialPeer(peerId))) return;
+
     if (this.currentActiveDialCount >= this.options.maxParallelDials) {
       this.pendingPeerDialQueue.push(peerId);
       return;
     }
 
-    if (!(await this.shouldDialPeer(peerId))) return;
-
     this.dialPeer(peerId).catch((err) => {
-      throw `Error dialing peer ${peerId.toString()} : ${err}`;
+      log(`Error dialing peer ${peerId.toString()} : ${err}`);
     });
   }
 
@@ -314,20 +331,7 @@ export class ConnectionManager
       void (async () => {
         const { id: peerId } = evt.detail;
 
-        const isBootstrap = (await this.getTagNamesForPeer(peerId)).includes(
-          Tags.BOOTSTRAP
-        );
-
-        this.dispatchEvent(
-          new CustomEvent<PeerId>(
-            isBootstrap
-              ? EPeersByDiscoveryEvents.PEER_DISCOVERY_BOOTSTRAP
-              : EPeersByDiscoveryEvents.PEER_DISCOVERY_PEER_EXCHANGE,
-            {
-              detail: peerId,
-            }
-          )
-        );
+        await this.dispatchDiscoveryEvent(peerId);
 
         try {
           await this.attemptDial(peerId);
@@ -340,7 +344,11 @@ export class ConnectionManager
       void (async () => {
         const peerId = evt.detail;
 
-        this.keepAliveManager.start(peerId, this.libp2p.services.ping);
+        this.keepAliveManager.start(
+          peerId,
+          this.libp2p.services.ping,
+          this.libp2p.peerStore
+        );
 
         const isBootstrap = (await this.getTagNamesForPeer(peerId)).includes(
           Tags.BOOTSTRAP
@@ -361,7 +369,7 @@ export class ConnectionManager
               new CustomEvent<PeerId>(
                 EPeersByDiscoveryEvents.PEER_CONNECT_BOOTSTRAP,
                 {
-                  detail: peerId,
+                  detail: peerId
                 }
               )
             );
@@ -371,7 +379,7 @@ export class ConnectionManager
             new CustomEvent<PeerId>(
               EPeersByDiscoveryEvents.PEER_CONNECT_PEER_EXCHANGE,
               {
-                detail: peerId,
+                detail: peerId
               }
             )
           );
@@ -382,19 +390,58 @@ export class ConnectionManager
       return (evt: CustomEvent<PeerId>): void => {
         this.keepAliveManager.stop(evt.detail);
       };
-    },
+    }
   };
 
   /**
-   * Checks if the peer is dialable based on the following conditions:
-   * 1. If the peer is a bootstrap peer, it is only dialable if the number of current bootstrap connections is less than the max allowed.
-   * 2. If the peer is not a bootstrap peer
+   * Checks if the peer should be dialed based on the following conditions:
+   * 1. If the peer is already connected, don't dial
+   * 2. If the peer is not part of any of the configured pubsub topics, don't dial
+   * 3. If the peer is not dialable based on bootstrap status, don't dial
+   * @returns true if the peer should be dialed, false otherwise
    */
   private async shouldDialPeer(peerId: PeerId): Promise<boolean> {
+    // if we're already connected to the peer, don't dial
     const isConnected = this.libp2p.getConnections(peerId).length > 0;
+    if (isConnected) {
+      log(`Already connected to peer ${peerId.toString()}. Not dialing.`);
+      return false;
+    }
+
+    // if the peer is not part of any of the configured pubsub topics, don't dial
+    if (!(await this.isPeerTopicConfigured(peerId))) {
+      const shardInfo = await this.getPeerShardInfo(
+        peerId,
+        this.libp2p.peerStore
+      );
+      log(
+        `Discovered peer ${peerId.toString()} with ShardInfo ${shardInfo} is not part of any of the configured pubsub topics (${
+          this.configuredPubSubTopics
+        }). 
+            Not dialing.`
+      );
+      return false;
+    }
 
-    if (isConnected) return false;
+    // if the peer is not dialable based on bootstrap status, don't dial
+    if (!(await this.isPeerDialableBasedOnBootstrapStatus(peerId))) {
+      log(
+        `Peer ${peerId.toString()} is not dialable based on bootstrap status. Not dialing.`
+      );
+      return false;
+    }
 
+    return true;
+  }
+
+  /**
+   * Checks if the peer is dialable based on the following conditions:
+   * 1. If the peer is a bootstrap peer, it is only dialable if the number of current bootstrap connections is less than the max allowed.
+   * 2. If the peer is not a bootstrap peer
+   */
+  private async isPeerDialableBasedOnBootstrapStatus(
+    peerId: PeerId
+  ): Promise<boolean> {
     const tagNames = await this.getTagNamesForPeer(peerId);
 
     const isBootstrap = tagNames.some((tagName) => tagName === Tags.BOOTSTRAP);
@@ -414,6 +461,23 @@ export class ConnectionManager
     return false;
   }
 
+  private async dispatchDiscoveryEvent(peerId: PeerId): Promise<void> {
+    const isBootstrap = (await this.getTagNamesForPeer(peerId)).includes(
+      Tags.BOOTSTRAP
+    );
+
+    this.dispatchEvent(
+      new CustomEvent<PeerId>(
+        isBootstrap
+          ? EPeersByDiscoveryEvents.PEER_DISCOVERY_BOOTSTRAP
+          : EPeersByDiscoveryEvents.PEER_DISCOVERY_PEER_EXCHANGE,
+        {
+          detail: peerId
+        }
+      )
+    );
+  }
+
   /**
    * Fetches the tag names for a given peer
    */
@@ -426,4 +490,31 @@ export class ConnectionManager
       return [];
     }
   }
+
+  private async isPeerTopicConfigured(peerId: PeerId): Promise<boolean> {
+    const shardInfo = await this.getPeerShardInfo(
+      peerId,
+      this.libp2p.peerStore
+    );
+
+    // If there's no shard information, simply return true
+    if (!shardInfo) return true;
+
+    const pubsubTopics = shardInfoToPubSubTopics(shardInfo);
+
+    const isTopicConfigured = pubsubTopics.some((topic) =>
+      this.configuredPubSubTopics.includes(topic)
+    );
+    return isTopicConfigured;
+  }
+
+  private async getPeerShardInfo(
+    peerId: PeerId,
+    peerStore: PeerStore
+  ): Promise<ShardInfo | undefined> {
+    const peer = await peerStore.get(peerId);
+    const shardInfoBytes = peer.metadata.get("shardInfo");
+    if (!shardInfoBytes) return undefined;
+    return decodeRelayShard(shardInfoBytes);
+  }
 }

package/src/lib/filter/filter_rpc.ts

@@ -42,7 +42,7 @@ export class FilterSubscribeRpc {
       filterSubscribeType:
         proto.FilterSubscribeRequest.FilterSubscribeType.SUBSCRIBE,
       pubsubTopic,
-      contentTopics,
+      contentTopics
     });
   }
 
@@ -55,7 +55,7 @@ export class FilterSubscribeRpc {
       filterSubscribeType:
         proto.FilterSubscribeRequest.FilterSubscribeType.UNSUBSCRIBE,
       pubsubTopic,
-      contentTopics,
+      contentTopics
     });
   }
 
@@ -65,7 +65,7 @@ export class FilterSubscribeRpc {
       filterSubscribeType:
         proto.FilterSubscribeRequest.FilterSubscribeType.UNSUBSCRIBE_ALL,
       pubsubTopic,
-      contentTopics: [],
+      contentTopics: []
     });
   }
 
@@ -75,7 +75,7 @@ export class FilterSubscribeRpc {
       filterSubscribeType:
         proto.FilterSubscribeRequest.FilterSubscribeType.SUBSCRIBER_PING,
       pubsubTopic: "",
-      contentTopics: [],
+      contentTopics: []
     });
   }
 

package/src/lib/filter/index.ts

@@ -1,7 +1,6 @@
-import { Stream } from "@libp2p/interface-connection";
-import type { PeerId } from "@libp2p/interface-peer-id";
-import type { Peer } from "@libp2p/interface-peer-store";
-import type { IncomingStreamData } from "@libp2p/interface-registrar";
+import { Stream } from "@libp2p/interface/connection";
+import type { Peer } from "@libp2p/interface/peer-store";
+import type { IncomingStreamData } from "@libp2p/interface-internal/registrar";
 import type {
   Callback,
   ContentTopic,
@@ -14,12 +13,15 @@ import type {
   Libp2p,
   PeerIdStr,
   ProtocolCreateOptions,
-  ProtocolOptions,
   PubSubTopic,
-  Unsubscribe,
+  Unsubscribe
 } from "@waku/interfaces";
 import { WakuMessage } from "@waku/proto";
-import { groupByContentTopic, toAsyncIterator } from "@waku/utils";
+import {
+  ensurePubsubTopicIsConfigured,
+  groupByContentTopic,
+  toAsyncIterator
+} from "@waku/utils";
 import debug from "debug";
 import all from "it-all";
 import * as lp from "it-length-prefixed";
@@ -31,7 +33,7 @@ import { DefaultPubSubTopic } from "../constants.js";
 import {
   FilterPushRpc,
   FilterSubscribeResponse,
-  FilterSubscribeRpc,
+  FilterSubscribeRpc
 } from "./filter_rpc.js";
 
 const log = debug("waku:filter:v2");
@@ -41,14 +43,14 @@ type SubscriptionCallback<T extends IDecodedMessage> = {
   callback: Callback<T>;
 };
 
-const FilterCodecs = {
+export const FilterCodecs = {
   SUBSCRIBE: "/vac/waku/filter-subscribe/2.0.0-beta1",
-  PUSH: "/vac/waku/filter-push/2.0.0-beta1",
+  PUSH: "/vac/waku/filter-push/2.0.0-beta1"
 };
 
 class Subscription {
   private readonly peer: Peer;
-  private readonly pubSubTopic: PubSubTopic;
+  private readonly pubsubTopic: PubSubTopic;
   private newStream: (peer: Peer) => Promise<Stream>;
 
   private subscriptionCallbacks: Map<
@@ -57,12 +59,12 @@ class Subscription {
   >;
 
   constructor(
-    pubSubTopic: PubSubTopic,
+    pubsubTopic: PubSubTopic,
     remotePeer: Peer,
     newStream: (peer: Peer) => Promise<Stream>
   ) {
     this.peer = remotePeer;
-    this.pubSubTopic = pubSubTopic;
+    this.pubsubTopic = pubsubTopic;
     this.newStream = newStream;
     this.subscriptionCallbacks = new Map();
   }
@@ -78,7 +80,7 @@ class Subscription {
     const stream = await this.newStream(this.peer);
 
     const request = FilterSubscribeRpc.createSubscribeRequest(
-      this.pubSubTopic,
+      this.pubsubTopic,
       contentTopics
     );
 
@@ -91,6 +93,12 @@ class Subscription {
         async (source) => await all(source)
       );
 
+      if (!res || !res.length) {
+        throw Error(
+          `No response received for request ${request.requestId}: ${res}`
+        );
+      }
+
       const { statusCode, requestId, statusDesc } =
         FilterSubscribeResponse.decode(res[0].slice());
 
@@ -125,7 +133,7 @@ class Subscription {
       // Decoder that decode to different implementations of `IDecodedMessage`
       const subscriptionCallback = {
         decoders,
-        callback,
+        callback
       } as unknown as SubscriptionCallback<IDecodedMessage>;
 
       // The callback and decoder may override previous values, this is on
@@ -137,7 +145,7 @@ class Subscription {
   async unsubscribe(contentTopics: ContentTopic[]): Promise<void> {
     const stream = await this.newStream(this.peer);
     const unsubscribeRequest = FilterSubscribeRpc.createUnsubscribeRequest(
-      this.pubSubTopic,
+      this.pubsubTopic,
       contentTopics
     );
 
@@ -186,7 +194,7 @@ class Subscription {
     const stream = await this.newStream(this.peer);
 
     const request = FilterSubscribeRpc.createUnsubscribeAllRequest(
-      this.pubSubTopic
+      this.pubsubTopic
     );
 
     try {
@@ -221,67 +229,72 @@ class Subscription {
       log("No subscription callback available for ", contentTopic);
       return;
     }
-    await pushMessage(subscriptionCallback, this.pubSubTopic, message);
+    await pushMessage(subscriptionCallback, this.pubsubTopic, message);
   }
 }
 
 class Filter extends BaseProtocol implements IReceiver {
-  private readonly options: ProtocolCreateOptions;
+  private readonly pubsubTopics: PubSubTopic[] = [];
   private activeSubscriptions = new Map<string, Subscription>();
+  private readonly NUM_PEERS_PROTOCOL = 1;
 
   private getActiveSubscription(
-    pubSubTopic: PubSubTopic,
+    pubsubTopic: PubSubTopic,
     peerIdStr: PeerIdStr
   ): Subscription | undefined {
-    return this.activeSubscriptions.get(`${pubSubTopic}_${peerIdStr}`);
+    return this.activeSubscriptions.get(`${pubsubTopic}_${peerIdStr}`);
   }
 
   private setActiveSubscription(
-    pubSubTopic: PubSubTopic,
+    pubsubTopic: PubSubTopic,
     peerIdStr: PeerIdStr,
     subscription: Subscription
   ): Subscription {
-    this.activeSubscriptions.set(`${pubSubTopic}_${peerIdStr}`, subscription);
+    this.activeSubscriptions.set(`${pubsubTopic}_${peerIdStr}`, subscription);
     return subscription;
   }
 
   constructor(libp2p: Libp2p, options?: ProtocolCreateOptions) {
     super(FilterCodecs.SUBSCRIBE, libp2p.components);
 
+    this.pubsubTopics = options?.pubsubTopics || [DefaultPubSubTopic];
+
     libp2p.handle(FilterCodecs.PUSH, this.onRequest.bind(this)).catch((e) => {
       log("Failed to register ", FilterCodecs.PUSH, e);
     });
 
     this.activeSubscriptions = new Map();
-
-    this.options = options ?? {};
   }
 
   async createSubscription(
-    pubSubTopic?: string,
-    peerId?: PeerId
+    pubsubTopic: string = DefaultPubSubTopic
   ): Promise<Subscription> {
-    const _pubSubTopic =
-      pubSubTopic ?? this.options.pubSubTopic ?? DefaultPubSubTopic;
+    ensurePubsubTopicIsConfigured(pubsubTopic, this.pubsubTopics);
 
-    const peer = await this.getPeer(peerId);
+    //TODO: get a relevant peer for the topic/shard
+    // https://github.com/waku-org/js-waku/pull/1586#discussion_r1336428230
+    const peer = (
+      await this.getPeers({
+        maxBootstrapPeers: 1,
+        numPeers: this.NUM_PEERS_PROTOCOL
+      })
+    )[0];
 
     const subscription =
-      this.getActiveSubscription(_pubSubTopic, peer.id.toString()) ??
+      this.getActiveSubscription(pubsubTopic, peer.id.toString()) ??
       this.setActiveSubscription(
-        _pubSubTopic,
+        pubsubTopic,
         peer.id.toString(),
-        new Subscription(_pubSubTopic, peer, this.newStream.bind(this, peer))
+        new Subscription(pubsubTopic, peer, this.getStream.bind(this, peer))
       );
 
     return subscription;
   }
 
   public toSubscriptionIterator<T extends IDecodedMessage>(
-    decoders: IDecoder<T> | IDecoder<T>[],
-    opts?: ProtocolOptions | undefined
+    decoders: IDecoder<T> | IDecoder<T>[]
   ): Promise<IAsyncIterator<T>> {
-    return toAsyncIterator(this, decoders, opts);
+    return toAsyncIterator(this, decoders);
   }
 
   /**
@@ -301,10 +314,9 @@ class Filter extends BaseProtocol implements IReceiver {
    */
   async subscribe<T extends IDecodedMessage>(
     decoders: IDecoder<T> | IDecoder<T>[],
-    callback: Callback<T>,
-    opts?: ProtocolOptions
+    callback: Callback<T>
   ): Promise<Unsubscribe> {
-    const subscription = await this.createSubscription(undefined, opts?.peerId);
+    const subscription = await this.createSubscription();
 
     await subscription.subscribe(decoders, callback);
 
@@ -373,7 +385,7 @@ export function wakuFilter(
 
 async function pushMessage<T extends IDecodedMessage>(
   subscriptionCallback: SubscriptionCallback<T>,
-  pubSubTopic: PubSubTopic,
+  pubsubTopic: PubSubTopic,
   message: WakuMessage
 ): Promise<void> {
   const { decoders, callback } = subscriptionCallback;
@@ -387,7 +399,7 @@ async function pushMessage<T extends IDecodedMessage>(
   try {
     const decodePromises = decoders.map((dec) =>
       dec
-        .fromProtoObj(pubSubTopic, message as IProtoMessage)
+        .fromProtoObj(pubsubTopic, message as IProtoMessage)
         .then((decoded) => decoded || Promise.reject("Decoding failed"))
     );