@waku/core 0.0.23 → 0.0.24

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@waku/core",
-  "version": "0.0.23",
+  "version": "0.0.24",
   "description": "TypeScript implementation of the Waku v2 protocol",
   "types": "./dist/index.d.ts",
   "module": "./dist/index.js",
@@ -69,18 +69,19 @@
     "reset-hard": "git clean -dfx -e .idea && git reset --hard && npm i && npm run build"
   },
   "engines": {
-    "node": ">=16"
+    "node": ">=18"
   },
   "dependencies": {
     "@noble/hashes": "^1.3.2",
-    "@waku/interfaces": "0.0.18",
+    "@waku/enr": "^0.0.18",
+    "@waku/interfaces": "0.0.19",
     "@waku/proto": "0.0.5",
-    "@waku/utils": "0.0.11",
+    "@waku/utils": "0.0.12",
     "debug": "^4.3.4",
     "it-all": "^3.0.3",
     "it-length-prefixed": "^9.0.1",
     "it-pipe": "^3.0.1",
-    "p-event": "^5.0.1",
+    "p-event": "^6.0.0",
     "uint8arraylist": "^2.4.3",
     "uuid": "^9.0.0"
   },
@@ -96,21 +97,13 @@
     "@waku/build-utils": "*",
     "chai": "^4.3.7",
     "cspell": "^7.3.2",
-    "fast-check": "^3.12.0",
+    "fast-check": "^3.13.1",
     "ignore-loader": "^0.1.2",
     "isomorphic-fetch": "^3.0.0",
-    "karma": "^6.4.1",
-    "karma-chrome-launcher": "^3.2.0",
-    "karma-mocha": "^2.0.1",
-    "karma-webpack": "^5.0.0",
     "mocha": "^10.2.0",
     "npm-run-all": "^4.1.5",
     "process": "^0.11.10",
-    "puppeteer": "^21.1.1",
-    "rollup": "^3.29.0",
-    "ts-loader": "^9.4.2",
-    "ts-node": "^10.9.1",
-    "typescript": "^5.0.4"
+    "rollup": "^3.29.2"
   },
   "peerDependencies": {
     "@multiformats/multiaddr": "^12.0.0",
@@ -121,9 +114,6 @@
       "optional": true
     }
   },
-  "typedoc": {
-    "entryPoint": "./src/index.ts"
-  },
   "files": [
     "dist",
     "bundle",

package/src/lib/connection_manager.ts

@@ -1,7 +1,9 @@
 import type { PeerId } from "@libp2p/interface/peer-id";
 import type { PeerInfo } from "@libp2p/interface/peer-info";
 import type { Peer } from "@libp2p/interface/peer-store";
+import type { PeerStore } from "@libp2p/interface/peer-store";
 import { CustomEvent, EventEmitter } from "@libp2p/interfaces/events";
+import { decodeRelayShard } from "@waku/enr";
 import {
   ConnectionManagerOptions,
   EPeersByDiscoveryEvents,
@@ -9,9 +11,12 @@ import {
   IPeersByDiscoveryEvents,
   IRelay,
   KeepAliveOptions,
-  PeersByDiscoveryResult
+  PeersByDiscoveryResult,
+  PubSubTopic,
+  ShardInfo
 } from "@waku/interfaces";
 import { Libp2p, Tags } from "@waku/interfaces";
+import { shardInfoToPubSubTopics } from "@waku/utils";
 import debug from "debug";
 
 import { KeepAliveManager } from "./keep_alive_manager.js";
@@ -40,6 +45,7 @@ export class ConnectionManager
     peerId: string,
     libp2p: Libp2p,
     keepAliveOptions: KeepAliveOptions,
+    pubsubTopics: PubSubTopic[],
     relay?: IRelay,
     options?: ConnectionManagerOptions
   ): ConnectionManager {
@@ -48,6 +54,7 @@ export class ConnectionManager
       instance = new ConnectionManager(
         libp2p,
         keepAliveOptions,
+        pubsubTopics,
         relay,
         options
       );
@@ -104,11 +111,13 @@ export class ConnectionManager
   private constructor(
     libp2p: Libp2p,
     keepAliveOptions: KeepAliveOptions,
+    private configuredPubSubTopics: PubSubTopic[],
     relay?: IRelay,
     options?: Partial<ConnectionManagerOptions>
   ) {
     super();
     this.libp2p = libp2p;
+    this.configuredPubSubTopics = configuredPubSubTopics;
     this.options = {
       maxDialAttemptsForPeer: DEFAULT_MAX_DIAL_ATTEMPTS_FOR_PEER,
       maxBootstrapPeersAllowed: DEFAULT_MAX_BOOTSTRAP_PEERS_ALLOWED,
@@ -217,16 +226,24 @@ export class ConnectionManager
       try {
         const error = this.dialErrorsForPeer.get(peerId.toString());
 
-        let errorMessage;
-        if (error instanceof AggregateError) {
-          errorMessage = JSON.stringify(error.errors[0]);
-        } else {
-          errorMessage = error.message;
-        }
+        if (error) {
+          let errorMessage;
+          if (error instanceof AggregateError) {
+            if (!error.errors) {
+              log(`No errors array found for AggregateError`);
+            } else if (error.errors.length === 0) {
+              log(`Errors array is empty for AggregateError`);
+            } else {
+              errorMessage = JSON.stringify(error.errors[0]);
+            }
+          } else {
+            errorMessage = error.message;
+          }
 
-        log(
-          `Deleting undialable peer ${peerId.toString()} from peer store. Error: ${errorMessage}`
-        );
+          log(
+            `Deleting undialable peer ${peerId.toString()} from peer store. Error: ${errorMessage}`
+          );
+        }
 
         this.dialErrorsForPeer.delete(peerId.toString());
         await this.libp2p.peerStore.delete(peerId);
@@ -297,15 +314,15 @@ export class ConnectionManager
   }
 
   private async attemptDial(peerId: PeerId): Promise<void> {
+    if (!(await this.shouldDialPeer(peerId))) return;
+
     if (this.currentActiveDialCount >= this.options.maxParallelDials) {
       this.pendingPeerDialQueue.push(peerId);
       return;
     }
 
-    if (!(await this.shouldDialPeer(peerId))) return;
-
     this.dialPeer(peerId).catch((err) => {
-      throw `Error dialing peer ${peerId.toString()} : ${err}`;
+      log(`Error dialing peer ${peerId.toString()} : ${err}`);
     });
   }
 
@@ -314,20 +331,7 @@ export class ConnectionManager
       void (async () => {
         const { id: peerId } = evt.detail;
 
-        const isBootstrap = (await this.getTagNamesForPeer(peerId)).includes(
-          Tags.BOOTSTRAP
-        );
-
-        this.dispatchEvent(
-          new CustomEvent<PeerId>(
-            isBootstrap
-              ? EPeersByDiscoveryEvents.PEER_DISCOVERY_BOOTSTRAP
-              : EPeersByDiscoveryEvents.PEER_DISCOVERY_PEER_EXCHANGE,
-            {
-              detail: peerId
-            }
-          )
-        );
+        await this.dispatchDiscoveryEvent(peerId);
 
         try {
           await this.attemptDial(peerId);
@@ -390,15 +394,54 @@ export class ConnectionManager
   };
 
   /**
-   * Checks if the peer is dialable based on the following conditions:
-   * 1. If the peer is a bootstrap peer, it is only dialable if the number of current bootstrap connections is less than the max allowed.
-   * 2. If the peer is not a bootstrap peer
+   * Checks if the peer should be dialed based on the following conditions:
+   * 1. If the peer is already connected, don't dial
+   * 2. If the peer is not part of any of the configured pubsub topics, don't dial
+   * 3. If the peer is not dialable based on bootstrap status, don't dial
+   * @returns true if the peer should be dialed, false otherwise
    */
   private async shouldDialPeer(peerId: PeerId): Promise<boolean> {
+    // if we're already connected to the peer, don't dial
     const isConnected = this.libp2p.getConnections(peerId).length > 0;
+    if (isConnected) {
+      log(`Already connected to peer ${peerId.toString()}. Not dialing.`);
+      return false;
+    }
 
-    if (isConnected) return false;
+    // if the peer is not part of any of the configured pubsub topics, don't dial
+    if (!(await this.isPeerTopicConfigured(peerId))) {
+      const shardInfo = await this.getPeerShardInfo(
+        peerId,
+        this.libp2p.peerStore
+      );
+      log(
+        `Discovered peer ${peerId.toString()} with ShardInfo ${shardInfo} is not part of any of the configured pubsub topics (${
+          this.configuredPubSubTopics
+        }). 
+            Not dialing.`
+      );
+      return false;
+    }
 
+    // if the peer is not dialable based on bootstrap status, don't dial
+    if (!(await this.isPeerDialableBasedOnBootstrapStatus(peerId))) {
+      log(
+        `Peer ${peerId.toString()} is not dialable based on bootstrap status. Not dialing.`
+      );
+      return false;
+    }
+
+    return true;
+  }
+
+  /**
+   * Checks if the peer is dialable based on the following conditions:
+   * 1. If the peer is a bootstrap peer, it is only dialable if the number of current bootstrap connections is less than the max allowed.
+   * 2. If the peer is not a bootstrap peer
+   */
+  private async isPeerDialableBasedOnBootstrapStatus(
+    peerId: PeerId
+  ): Promise<boolean> {
     const tagNames = await this.getTagNamesForPeer(peerId);
 
     const isBootstrap = tagNames.some((tagName) => tagName === Tags.BOOTSTRAP);
@@ -418,6 +461,23 @@ export class ConnectionManager
     return false;
   }
 
+  private async dispatchDiscoveryEvent(peerId: PeerId): Promise<void> {
+    const isBootstrap = (await this.getTagNamesForPeer(peerId)).includes(
+      Tags.BOOTSTRAP
+    );
+
+    this.dispatchEvent(
+      new CustomEvent<PeerId>(
+        isBootstrap
+          ? EPeersByDiscoveryEvents.PEER_DISCOVERY_BOOTSTRAP
+          : EPeersByDiscoveryEvents.PEER_DISCOVERY_PEER_EXCHANGE,
+        {
+          detail: peerId
+        }
+      )
+    );
+  }
+
   /**
    * Fetches the tag names for a given peer
    */
@@ -430,4 +490,31 @@ export class ConnectionManager
       return [];
     }
   }
+
+  private async isPeerTopicConfigured(peerId: PeerId): Promise<boolean> {
+    const shardInfo = await this.getPeerShardInfo(
+      peerId,
+      this.libp2p.peerStore
+    );
+
+    // If there's no shard information, simply return true
+    if (!shardInfo) return true;
+
+    const pubsubTopics = shardInfoToPubSubTopics(shardInfo);
+
+    const isTopicConfigured = pubsubTopics.some((topic) =>
+      this.configuredPubSubTopics.includes(topic)
+    );
+    return isTopicConfigured;
+  }
+
+  private async getPeerShardInfo(
+    peerId: PeerId,
+    peerStore: PeerStore
+  ): Promise<ShardInfo | undefined> {
+    const peer = await peerStore.get(peerId);
+    const shardInfoBytes = peer.metadata.get("shardInfo");
+    if (!shardInfoBytes) return undefined;
+    return decodeRelayShard(shardInfoBytes);
+  }
 }

package/src/lib/filter/index.ts

@@ -17,7 +17,11 @@ import type {
   Unsubscribe
 } from "@waku/interfaces";
 import { WakuMessage } from "@waku/proto";
-import { groupByContentTopic, toAsyncIterator } from "@waku/utils";
+import {
+  ensurePubsubTopicIsConfigured,
+  groupByContentTopic,
+  toAsyncIterator
+} from "@waku/utils";
 import debug from "debug";
 import all from "it-all";
 import * as lp from "it-length-prefixed";
@@ -46,7 +50,7 @@ export const FilterCodecs = {
 
 class Subscription {
   private readonly peer: Peer;
-  private readonly pubSubTopic: PubSubTopic;
+  private readonly pubsubTopic: PubSubTopic;
   private newStream: (peer: Peer) => Promise<Stream>;
 
   private subscriptionCallbacks: Map<
@@ -55,12 +59,12 @@ class Subscription {
   >;
 
   constructor(
-    pubSubTopic: PubSubTopic,
+    pubsubTopic: PubSubTopic,
     remotePeer: Peer,
     newStream: (peer: Peer) => Promise<Stream>
   ) {
     this.peer = remotePeer;
-    this.pubSubTopic = pubSubTopic;
+    this.pubsubTopic = pubsubTopic;
     this.newStream = newStream;
     this.subscriptionCallbacks = new Map();
   }
@@ -76,7 +80,7 @@ class Subscription {
     const stream = await this.newStream(this.peer);
 
     const request = FilterSubscribeRpc.createSubscribeRequest(
-      this.pubSubTopic,
+      this.pubsubTopic,
       contentTopics
     );
 
@@ -89,6 +93,12 @@ class Subscription {
         async (source) => await all(source)
       );
 
+      if (!res || !res.length) {
+        throw Error(
+          `No response received for request ${request.requestId}: ${res}`
+        );
+      }
+
       const { statusCode, requestId, statusDesc } =
         FilterSubscribeResponse.decode(res[0].slice());
 
@@ -135,7 +145,7 @@ class Subscription {
   async unsubscribe(contentTopics: ContentTopic[]): Promise<void> {
     const stream = await this.newStream(this.peer);
     const unsubscribeRequest = FilterSubscribeRpc.createUnsubscribeRequest(
-      this.pubSubTopic,
+      this.pubsubTopic,
       contentTopics
     );
 
@@ -184,7 +194,7 @@ class Subscription {
     const stream = await this.newStream(this.peer);
 
     const request = FilterSubscribeRpc.createUnsubscribeAllRequest(
-      this.pubSubTopic
+      this.pubsubTopic
     );
 
     try {
@@ -219,47 +229,50 @@ class Subscription {
       log("No subscription callback available for ", contentTopic);
       return;
     }
-    await pushMessage(subscriptionCallback, this.pubSubTopic, message);
+    await pushMessage(subscriptionCallback, this.pubsubTopic, message);
   }
 }
 
 class Filter extends BaseProtocol implements IReceiver {
-  private readonly options: ProtocolCreateOptions;
+  private readonly pubsubTopics: PubSubTopic[] = [];
   private activeSubscriptions = new Map<string, Subscription>();
   private readonly NUM_PEERS_PROTOCOL = 1;
 
   private getActiveSubscription(
-    pubSubTopic: PubSubTopic,
+    pubsubTopic: PubSubTopic,
     peerIdStr: PeerIdStr
   ): Subscription | undefined {
-    return this.activeSubscriptions.get(`${pubSubTopic}_${peerIdStr}`);
+    return this.activeSubscriptions.get(`${pubsubTopic}_${peerIdStr}`);
   }
 
   private setActiveSubscription(
-    pubSubTopic: PubSubTopic,
+    pubsubTopic: PubSubTopic,
     peerIdStr: PeerIdStr,
     subscription: Subscription
   ): Subscription {
-    this.activeSubscriptions.set(`${pubSubTopic}_${peerIdStr}`, subscription);
+    this.activeSubscriptions.set(`${pubsubTopic}_${peerIdStr}`, subscription);
     return subscription;
   }
 
   constructor(libp2p: Libp2p, options?: ProtocolCreateOptions) {
     super(FilterCodecs.SUBSCRIBE, libp2p.components);
 
+    this.pubsubTopics = options?.pubsubTopics || [DefaultPubSubTopic];
+
     libp2p.handle(FilterCodecs.PUSH, this.onRequest.bind(this)).catch((e) => {
       log("Failed to register ", FilterCodecs.PUSH, e);
     });
 
     this.activeSubscriptions = new Map();
-
-    this.options = options ?? {};
   }
 
-  async createSubscription(pubSubTopic?: string): Promise<Subscription> {
-    const _pubSubTopic =
-      pubSubTopic ?? this.options.pubSubTopic ?? DefaultPubSubTopic;
+  async createSubscription(
+    pubsubTopic: string = DefaultPubSubTopic
+  ): Promise<Subscription> {
+    ensurePubsubTopicIsConfigured(pubsubTopic, this.pubsubTopics);
 
+    //TODO: get a relevant peer for the topic/shard
+    // https://github.com/waku-org/js-waku/pull/1586#discussion_r1336428230
     const peer = (
       await this.getPeers({
         maxBootstrapPeers: 1,
@@ -268,11 +281,11 @@ class Filter extends BaseProtocol implements IReceiver {
     )[0];
 
     const subscription =
-      this.getActiveSubscription(_pubSubTopic, peer.id.toString()) ??
+      this.getActiveSubscription(pubsubTopic, peer.id.toString()) ??
       this.setActiveSubscription(
-        _pubSubTopic,
+        pubsubTopic,
         peer.id.toString(),
-        new Subscription(_pubSubTopic, peer, this.getStream.bind(this, peer))
+        new Subscription(pubsubTopic, peer, this.getStream.bind(this, peer))
       );
 
     return subscription;
@@ -372,7 +385,7 @@ export function wakuFilter(
 
 async function pushMessage<T extends IDecodedMessage>(
   subscriptionCallback: SubscriptionCallback<T>,
-  pubSubTopic: PubSubTopic,
+  pubsubTopic: PubSubTopic,
   message: WakuMessage
 ): Promise<void> {
   const { decoders, callback } = subscriptionCallback;
@@ -386,7 +399,7 @@ async function pushMessage<T extends IDecodedMessage>(
   try {
     const decodePromises = decoders.map((dec) =>
       dec
-        .fromProtoObj(pubSubTopic, message as IProtoMessage)
+        .fromProtoObj(pubsubTopic, message as IProtoMessage)
         .then((decoded) => decoded || Promise.reject("Decoding failed"))
     );
 

package/src/lib/keep_alive_manager.ts

@@ -1,6 +1,6 @@
 import type { PeerId } from "@libp2p/interface/peer-id";
 import type { PeerStore } from "@libp2p/interface/peer-store";
-import type { IRelay } from "@waku/interfaces";
+import type { IRelay, PeerIdStr } from "@waku/interfaces";
 import type { KeepAliveOptions } from "@waku/interfaces";
 import { utf8ToBytes } from "@waku/utils/bytes";
 import debug from "debug";
@@ -13,7 +13,7 @@ const log = debug("waku:keep-alive");
 
 export class KeepAliveManager {
   private pingKeepAliveTimers: Map<string, ReturnType<typeof setInterval>>;
-  private relayKeepAliveTimers: Map<PeerId, ReturnType<typeof setInterval>>;
+  private relayKeepAliveTimers: Map<PeerId, ReturnType<typeof setInterval>[]>;
   private options: KeepAliveOptions;
   private relay?: IRelay;
 
@@ -37,14 +37,24 @@ export class KeepAliveManager {
 
     const peerIdStr = peerId.toString();
 
+    // Ping the peer every pingPeriodSecs seconds
+    // if pingPeriodSecs is 0, don't ping the peer
     if (pingPeriodSecs !== 0) {
       const interval = setInterval(() => {
         void (async () => {
+          let ping: number;
           try {
             // ping the peer for keep alive
             // also update the peer store with the latency
-            const ping = await libp2pPing.ping(peerId);
-            log(`Ping succeeded (${peerIdStr})`, ping);
+            try {
+              ping = await libp2pPing.ping(peerId);
+              log(`Ping succeeded (${peerIdStr})`, ping);
+            } catch (error) {
+              log(`Ping failed for peer (${peerIdStr}).
+                Next ping will be attempted in ${pingPeriodSecs} seconds.
+              `);
+              return;
+            }
 
             try {
               await peerStore.patch(peerId, {
@@ -66,17 +76,12 @@ export class KeepAliveManager {
 
     const relay = this.relay;
     if (relay && relayPeriodSecs !== 0) {
-      const encoder = createEncoder({
-        contentTopic: RelayPingContentTopic,
-        ephemeral: true
-      });
-      const interval = setInterval(() => {
-        log("Sending Waku Relay ping message");
-        relay
-          .send(encoder, { payload: new Uint8Array([1]) })
-          .catch((e) => log("Failed to send relay ping", e));
-      }, relayPeriodSecs * 1000);
-      this.relayKeepAliveTimers.set(peerId, interval);
+      const intervals = this.scheduleRelayPings(
+        relay,
+        relayPeriodSecs,
+        peerId.toString()
+      );
+      this.relayKeepAliveTimers.set(peerId, intervals);
     }
   }
 
@@ -89,7 +94,7 @@ export class KeepAliveManager {
     }
 
     if (this.relayKeepAliveTimers.has(peerId)) {
-      clearInterval(this.relayKeepAliveTimers.get(peerId));
+      this.relayKeepAliveTimers.get(peerId)?.map(clearInterval);
       this.relayKeepAliveTimers.delete(peerId);
     }
   }
@@ -105,4 +110,32 @@ export class KeepAliveManager {
     this.pingKeepAliveTimers.clear();
     this.relayKeepAliveTimers.clear();
   }
+
+  private scheduleRelayPings(
+    relay: IRelay,
+    relayPeriodSecs: number,
+    peerIdStr: PeerIdStr
+  ): NodeJS.Timeout[] {
+    // send a ping message to each PubSubTopic the peer is part of
+    const intervals: NodeJS.Timeout[] = [];
+    for (const topic of relay.pubsubTopics) {
+      const meshPeers = relay.getMeshPeers(topic);
+      if (!meshPeers.includes(peerIdStr)) continue;
+
+      const encoder = createEncoder({
+        pubsubTopic: topic,
+        contentTopic: RelayPingContentTopic,
+        ephemeral: true
+      });
+      const interval = setInterval(() => {
+        log("Sending Waku Relay ping message");
+        relay
+          .send(encoder, { payload: new Uint8Array([1]) })
+          .catch((e) => log("Failed to send relay ping", e));
+      }, relayPeriodSecs * 1000);
+      intervals.push(interval);
+    }
+
+    return intervals;
+  }
 }