@waku/core 0.0.26-7eb3375.0 → 0.0.27

package/src/lib/filter/index.ts

@@ -1,6 +1,6 @@
-import { Stream } from "@libp2p/interface/connection";
-import type { Peer } from "@libp2p/interface/peer-store";
-import type { IncomingStreamData } from "@libp2p/interface-internal/registrar";
+import { Stream } from "@libp2p/interface";
+import type { Peer } from "@libp2p/interface";
+import type { IncomingStreamData } from "@libp2p/interface-internal";
 import type {
   Callback,
   ContentTopic,
@@ -11,15 +11,18 @@ import type {
   IProtoMessage,
   IReceiver,
   Libp2p,
-  PeerIdStr,
   ProtocolCreateOptions,
   PubsubTopic,
+  SingleShardInfo,
   Unsubscribe
 } from "@waku/interfaces";
+import { DefaultPubsubTopic } from "@waku/interfaces";
+import { messageHashStr } from "@waku/message-hash";
 import { WakuMessage } from "@waku/proto";
 import {
   ensurePubsubTopicIsConfigured,
   groupByContentTopic,
+  singleShardInfoToPubsubTopic,
   toAsyncIterator
 } from "@waku/utils";
 import { Logger } from "@waku/utils";
@@ -28,7 +31,6 @@ import * as lp from "it-length-prefixed";
 import { pipe } from "it-pipe";
 
 import { BaseProtocol } from "../base_protocol.js";
-import { DefaultPubsubTopic } from "../constants.js";
 
 import {
   FilterPushRpc,
@@ -48,10 +50,14 @@ export const FilterCodecs = {
   PUSH: "/vac/waku/filter-push/2.0.0-beta1"
 };
 
+/**
+ * A subscription object refers to a subscription to a given pubsub topic.
+ */
 class Subscription {
-  private readonly peer: Peer;
+  readonly peers: Peer[];
   private readonly pubsubTopic: PubsubTopic;
   private newStream: (peer: Peer) => Promise<Stream>;
+  readonly receivedMessagesHashStr: string[] = [];
 
   private subscriptionCallbacks: Map<
     ContentTopic,
@@ -60,10 +66,10 @@ class Subscription {
 
   constructor(
     pubsubTopic: PubsubTopic,
-    remotePeer: Peer,
+    remotePeers: Peer[],
     newStream: (peer: Peer) => Promise<Stream>
   ) {
-    this.peer = remotePeer;
+    this.peers = remotePeers;
     this.pubsubTopic = pubsubTopic;
     this.newStream = newStream;
     this.subscriptionCallbacks = new Map();
@@ -87,53 +93,59 @@ class Subscription {
     const decodersGroupedByCT = groupByContentTopic(decodersArray);
     const contentTopics = Array.from(decodersGroupedByCT.keys());
 
-    const stream = await this.newStream(this.peer);
+    const promises = this.peers.map(async (peer) => {
+      const stream = await this.newStream(peer);
 
-    const request = FilterSubscribeRpc.createSubscribeRequest(
-      this.pubsubTopic,
-      contentTopics
-    );
-
-    try {
-      const res = await pipe(
-        [request.encode()],
-        lp.encode,
-        stream,
-        lp.decode,
-        async (source) => await all(source)
+      const request = FilterSubscribeRpc.createSubscribeRequest(
+        this.pubsubTopic,
+        contentTopics
       );
 
-      if (!res || !res.length) {
-        throw Error(
-          `No response received for request ${request.requestId}: ${res}`
+      try {
+        const res = await pipe(
+          [request.encode()],
+          lp.encode,
+          stream,
+          lp.decode,
+          async (source) => await all(source)
         );
-      }
 
-      const { statusCode, requestId, statusDesc } =
-        FilterSubscribeResponse.decode(res[0].slice());
+        if (!res || !res.length) {
+          throw Error(
+            `No response received for request ${request.requestId}: ${res}`
+          );
+        }
+
+        const { statusCode, requestId, statusDesc } =
+          FilterSubscribeResponse.decode(res[0].slice());
+
+        if (statusCode < 200 || statusCode >= 300) {
+          throw new Error(
+            `Filter subscribe request ${requestId} failed with status code ${statusCode}: ${statusDesc}`
+          );
+        }
 
-      if (statusCode < 200 || statusCode >= 300) {
+        log.info(
+          "Subscribed to peer ",
+          peer.id.toString(),
+          "for content topics",
+          contentTopics
+        );
+      } catch (e) {
         throw new Error(
-          `Filter subscribe request ${requestId} failed with status code ${statusCode}: ${statusDesc}`
+          "Error subscribing to peer: " +
+            peer.id.toString() +
+            " for content topics: " +
+            contentTopics +
+            ": " +
+            e
         );
       }
+    });
 
-      log.info(
-        "Subscribed to peer ",
-        this.peer.id.toString(),
-        "for content topics",
-        contentTopics
-      );
-    } catch (e) {
-      throw new Error(
-        "Error subscribing to peer: " +
-          this.peer.id.toString() +
-          " for content topics: " +
-          contentTopics +
-          ": " +
-          e
-      );
-    }
+    const results = await Promise.allSettled(promises);
+
+    this.handleErrors(results, "subscribe");
 
     // Save the callback functions by content topics so they
     // can easily be removed (reciprocally replaced) if `unsubscribe` (reciprocally `subscribe`)
@@ -153,133 +165,206 @@ class Subscription {
   }
 
   async unsubscribe(contentTopics: ContentTopic[]): Promise<void> {
-    const stream = await this.newStream(this.peer);
-    const unsubscribeRequest = FilterSubscribeRpc.createUnsubscribeRequest(
-      this.pubsubTopic,
-      contentTopics
-    );
+    const promises = this.peers.map(async (peer) => {
+      const stream = await this.newStream(peer);
+      const unsubscribeRequest = FilterSubscribeRpc.createUnsubscribeRequest(
+        this.pubsubTopic,
+        contentTopics
+      );
 
-    try {
-      await pipe([unsubscribeRequest.encode()], lp.encode, stream.sink);
-    } catch (error) {
-      throw new Error("Error subscribing: " + error);
-    }
+      try {
+        await pipe([unsubscribeRequest.encode()], lp.encode, stream.sink);
+      } catch (error) {
+        throw new Error("Error unsubscribing: " + error);
+      }
 
-    contentTopics.forEach((contentTopic: string) => {
-      this.subscriptionCallbacks.delete(contentTopic);
+      contentTopics.forEach((contentTopic: string) => {
+        this.subscriptionCallbacks.delete(contentTopic);
+      });
     });
+
+    const results = await Promise.allSettled(promises);
+
+    this.handleErrors(results, "unsubscribe");
   }
 
   async ping(): Promise<void> {
-    const stream = await this.newStream(this.peer);
+    const promises = this.peers.map(async (peer) => {
+      const stream = await this.newStream(peer);
+
+      const request = FilterSubscribeRpc.createSubscriberPingRequest();
+
+      try {
+        const res = await pipe(
+          [request.encode()],
+          lp.encode,
+          stream,
+          lp.decode,
+          async (source) => await all(source)
+        );
 
-    const request = FilterSubscribeRpc.createSubscriberPingRequest();
+        if (!res || !res.length) {
+          throw Error(
+            `No response received for request ${request.requestId}: ${res}`
+          );
+        }
 
-    try {
-      const res = await pipe(
-        [request.encode()],
-        lp.encode,
-        stream,
-        lp.decode,
-        async (source) => await all(source)
-      );
+        const { statusCode, requestId, statusDesc } =
+          FilterSubscribeResponse.decode(res[0].slice());
 
-      if (!res || !res.length) {
-        throw Error(
-          `No response received for request ${request.requestId}: ${res}`
-        );
+        if (statusCode < 200 || statusCode >= 300) {
+          throw new Error(
+            `Filter ping request ${requestId} failed with status code ${statusCode}: ${statusDesc}`
+          );
+        }
+        log.info(`Ping successful for peer ${peer.id.toString()}`);
+      } catch (error) {
+        log.error("Error pinging: ", error);
+        throw error; // Rethrow the actual error instead of wrapping it
       }
+    });
 
-      const { statusCode, requestId, statusDesc } =
-        FilterSubscribeResponse.decode(res[0].slice());
-
-      if (statusCode < 200 || statusCode >= 300) {
-        throw new Error(
-          `Filter ping request ${requestId} failed with status code ${statusCode}: ${statusDesc}`
-        );
-      }
+    const results = await Promise.allSettled(promises);
 
-      log.info("Ping successful");
-    } catch (error) {
-      log.error("Error pinging: ", error);
-      throw new Error("Error pinging: " + error);
-    }
+    this.handleErrors(results, "ping");
   }
 
   async unsubscribeAll(): Promise<void> {
-    const stream = await this.newStream(this.peer);
-
-    const request = FilterSubscribeRpc.createUnsubscribeAllRequest(
-      this.pubsubTopic
-    );
+    const promises = this.peers.map(async (peer) => {
+      const stream = await this.newStream(peer);
 
-    try {
-      const res = await pipe(
-        [request.encode()],
-        lp.encode,
-        stream,
-        lp.decode,
-        async (source) => await all(source)
+      const request = FilterSubscribeRpc.createUnsubscribeAllRequest(
+        this.pubsubTopic
       );
 
-      if (!res || !res.length) {
-        throw Error(
-          `No response received for request ${request.requestId}: ${res}`
+      try {
+        const res = await pipe(
+          [request.encode()],
+          lp.encode,
+          stream,
+          lp.decode,
+          async (source) => await all(source)
         );
-      }
 
-      const { statusCode, requestId, statusDesc } =
-        FilterSubscribeResponse.decode(res[0].slice());
+        if (!res || !res.length) {
+          throw Error(
+            `No response received for request ${request.requestId}: ${res}`
+          );
+        }
+
+        const { statusCode, requestId, statusDesc } =
+          FilterSubscribeResponse.decode(res[0].slice());
 
-      if (statusCode < 200 || statusCode >= 300) {
+        if (statusCode < 200 || statusCode >= 300) {
+          throw new Error(
+            `Filter unsubscribe all request ${requestId} failed with status code ${statusCode}: ${statusDesc}`
+          );
+        }
+
+        this.subscriptionCallbacks.clear();
+        log.info(
+          `Unsubscribed from all content topics for pubsub topic ${this.pubsubTopic}`
+        );
+      } catch (error) {
         throw new Error(
-          `Filter unsubscribe all request ${requestId} failed with status code ${statusCode}: ${statusDesc}`
+          "Error unsubscribing from all content topics: " + error
         );
       }
+    });
 
-      this.subscriptionCallbacks.clear();
-      log.info("Unsubscribed from all content topics");
-    } catch (error) {
-      throw new Error("Error unsubscribing from all content topics: " + error);
-    }
+    const results = await Promise.allSettled(promises);
+
+    this.handleErrors(results, "unsubscribeAll");
   }
 
   async processMessage(message: WakuMessage): Promise<void> {
-    const contentTopic = message.contentTopic;
+    const hashedMessageStr = messageHashStr(
+      this.pubsubTopic,
+      message as IProtoMessage
+    );
+    if (this.receivedMessagesHashStr.includes(hashedMessageStr)) {
+      log.info("Message already received, skipping");
+      return;
+    }
+    this.receivedMessagesHashStr.push(hashedMessageStr);
+
+    const { contentTopic } = message;
     const subscriptionCallback = this.subscriptionCallbacks.get(contentTopic);
     if (!subscriptionCallback) {
       log.error("No subscription callback available for ", contentTopic);
       return;
     }
+    log.info(
+      "Processing message with content topic ",
+      contentTopic,
+      " on pubsub topic ",
+      this.pubsubTopic
+    );
     await pushMessage(subscriptionCallback, this.pubsubTopic, message);
   }
+
+  // Filter out only the rejected promises and extract & handle their reasons
+  private handleErrors(
+    results: PromiseSettledResult<void>[],
+    type: "ping" | "subscribe" | "unsubscribe" | "unsubscribeAll"
+  ): void {
+    const errors = results
+      .filter(
+        (result): result is PromiseRejectedResult =>
+          result.status === "rejected"
+      )
+      .map((rejectedResult) => rejectedResult.reason);
+
+    if (errors.length === this.peers.length) {
+      const errorCounts = new Map<string, number>();
+      // TODO: streamline error logging with https://github.com/orgs/waku-org/projects/2/views/1?pane=issue&itemId=42849952
+      errors.forEach((error) => {
+        const message = error instanceof Error ? error.message : String(error);
+        errorCounts.set(message, (errorCounts.get(message) || 0) + 1);
+      });
+
+      const uniqueErrorMessages = Array.from(
+        errorCounts,
+        ([message, count]) => `${message} (occurred ${count} times)`
+      ).join(", ");
+      throw new Error(`Error ${type} all peers: ${uniqueErrorMessages}`);
+    } else if (errors.length > 0) {
+      // TODO: handle renewing faulty peers with new peers (https://github.com/waku-org/js-waku/issues/1463)
+      log.warn(
+        `Some ${type} failed. These will be refreshed with new peers`,
+        errors
+      );
+    } else {
+      log.info(`${type} successful for all peers`);
+    }
+  }
 }
 
 class Filter extends BaseProtocol implements IReceiver {
-  private readonly pubsubTopics: PubsubTopic[] = [];
   private activeSubscriptions = new Map<string, Subscription>();
-  private readonly NUM_PEERS_PROTOCOL = 1;
 
   private getActiveSubscription(
-    pubsubTopic: PubsubTopic,
-    peerIdStr: PeerIdStr
+    pubsubTopic: PubsubTopic
   ): Subscription | undefined {
-    return this.activeSubscriptions.get(`${pubsubTopic}_${peerIdStr}`);
+    return this.activeSubscriptions.get(pubsubTopic);
   }
 
   private setActiveSubscription(
     pubsubTopic: PubsubTopic,
-    peerIdStr: PeerIdStr,
     subscription: Subscription
   ): Subscription {
-    this.activeSubscriptions.set(`${pubsubTopic}_${peerIdStr}`, subscription);
+    this.activeSubscriptions.set(pubsubTopic, subscription);
     return subscription;
   }
 
   constructor(libp2p: Libp2p, options?: ProtocolCreateOptions) {
-    super(FilterCodecs.SUBSCRIBE, libp2p.components);
-
-    this.pubsubTopics = options?.pubsubTopics || [DefaultPubsubTopic];
+    super(
+      FilterCodecs.SUBSCRIBE,
+      libp2p.components,
+      log,
+      options!.pubsubTopics!,
+      options
+    );
 
     libp2p.handle(FilterCodecs.PUSH, this.onRequest.bind(this)).catch((e) => {
       log.error("Failed to register ", FilterCodecs.PUSH, e);
@@ -288,26 +373,40 @@ class Filter extends BaseProtocol implements IReceiver {
     this.activeSubscriptions = new Map();
   }
 
+  /**
+   * Creates a new subscription to the given pubsub topic.
+   * The subscription is made to multiple peers for decentralization.
+   * @param pubsubTopicShardInfo The pubsub topic to subscribe to.
+   * @returns The subscription object.
+   */
   async createSubscription(
-    pubsubTopic: string = DefaultPubsubTopic
+    pubsubTopicShardInfo: SingleShardInfo | PubsubTopic = DefaultPubsubTopic
   ): Promise<Subscription> {
+    const pubsubTopic =
+      typeof pubsubTopicShardInfo == "string"
+        ? pubsubTopicShardInfo
+        : singleShardInfoToPubsubTopic(pubsubTopicShardInfo);
+
     ensurePubsubTopicIsConfigured(pubsubTopic, this.pubsubTopics);
 
-    //TODO: get a relevant peer for the topic/shard
-    // https://github.com/waku-org/js-waku/pull/1586#discussion_r1336428230
-    const peer = (
-      await this.getPeers({
-        maxBootstrapPeers: 1,
-        numPeers: this.NUM_PEERS_PROTOCOL
-      })
-    )[0];
+    const peers = await this.getPeers({
+      maxBootstrapPeers: 1,
+      numPeers: this.numPeersToUse
+    });
+    if (peers.length === 0) {
+      throw new Error("No peer found to initiate subscription.");
+    }
+
+    log.info(
+      `Creating filter subscription with ${peers.length} peers: `,
+      peers.map((peer) => peer.id.toString())
+    );
 
     const subscription =
-      this.getActiveSubscription(pubsubTopic, peer.id.toString()) ??
+      this.getActiveSubscription(pubsubTopic) ??
       this.setActiveSubscription(
         pubsubTopic,
-        peer.id.toString(),
-        new Subscription(pubsubTopic, peer, this.getStream.bind(this, peer))
+        new Subscription(pubsubTopic, peers, this.getStream.bind(this))
       );
 
     return subscription;
@@ -354,8 +453,11 @@ class Filter extends BaseProtocol implements IReceiver {
   }
 
   private onRequest(streamData: IncomingStreamData): void {
+    const { connection, stream } = streamData;
+    const { remotePeer } = connection;
+    log.info(`Received message from ${remotePeer.toString()}`);
     try {
-      pipe(streamData.stream, lp.decode, async (source) => {
+      pipe(stream, lp.decode, async (source) => {
         for await (const bytes of source) {
           const response = FilterPushRpc.decode(bytes.slice());
 
@@ -371,11 +473,7 @@ class Filter extends BaseProtocol implements IReceiver {
             return;
           }
 
-          const peerIdStr = streamData.connection.remotePeer.toString();
-          const subscription = this.getActiveSubscription(
-            pubsubTopic,
-            peerIdStr
-          );
+          const subscription = this.getActiveSubscription(pubsubTopic);
 
           if (!subscription) {
             log.error(
@@ -401,7 +499,7 @@ class Filter extends BaseProtocol implements IReceiver {
 }
 
 export function wakuFilter(
-  init: Partial<ProtocolCreateOptions> = {}
+  init: ProtocolCreateOptions = { pubsubTopics: [] }
 ): (libp2p: Libp2p) => IFilter {
   return (libp2p: Libp2p) => new Filter(libp2p, init);
 }

package/src/lib/filterPeers.ts

@@ -1,24 +1,32 @@
-import { Peer } from "@libp2p/interface/peer-store";
+import { Peer } from "@libp2p/interface";
 import { Tags } from "@waku/interfaces";
 
 /**
- * Retrieves a list of peers based on the specified criteria.
+ * Retrieves a list of peers based on the specified criteria:
+ * 1. If numPeers is 0, return all peers
+ * 2. Bootstrap peers are prioritized
+ * 3. Non-bootstrap peers are randomly selected to fill up to numPeers
  *
  * @param peers - The list of peers to filter from.
- * @param numPeers - The total number of peers to retrieve. If 0, all peers are returned.
+ * @param numPeers - The total number of peers to retrieve. If 0, all peers are returned, irrespective of `maxBootstrapPeers`.
  * @param maxBootstrapPeers - The maximum number of bootstrap peers to retrieve.
- * @returns A Promise that resolves to an array of peers based on the specified criteria.
+ * @returns An array of peers based on the specified criteria.
  */
-export async function filterPeers(
+export function filterPeersByDiscovery(
   peers: Peer[],
   numPeers: number,
   maxBootstrapPeers: number
-): Promise<Peer[]> {
+): Peer[] {
   // Collect the bootstrap peers up to the specified maximum
-  const bootstrapPeers = peers
+  let bootstrapPeers = peers
     .filter((peer) => peer.tags.has(Tags.BOOTSTRAP))
     .slice(0, maxBootstrapPeers);
 
+  // If numPeers is less than the number of bootstrap peers, adjust the bootstrapPeers array
+  if (numPeers > 0 && numPeers < bootstrapPeers.length) {
+    bootstrapPeers = bootstrapPeers.slice(0, numPeers);
+  }
+
   // Collect non-bootstrap peers
   const nonBootstrapPeers = peers.filter(
     (peer) => !peer.tags.has(Tags.BOOTSTRAP)

package/src/lib/keep_alive_manager.ts

@@ -1,10 +1,9 @@
-import type { PeerId } from "@libp2p/interface/peer-id";
-import type { PeerStore } from "@libp2p/interface/peer-store";
+import type { PeerId, PeerStore } from "@libp2p/interface";
+import type { PingService } from "@libp2p/ping";
 import type { IRelay, PeerIdStr } from "@waku/interfaces";
 import type { KeepAliveOptions } from "@waku/interfaces";
-import { Logger } from "@waku/utils";
+import { Logger, pubsubTopicToSingleShardInfo } from "@waku/utils";
 import { utf8ToBytes } from "@waku/utils/bytes";
-import type { PingService } from "libp2p/ping";
 
 import { createEncoder } from "./message/version_0.js";
 
@@ -129,7 +128,7 @@ export class KeepAliveManager {
       if (!meshPeers.includes(peerIdStr)) continue;
 
       const encoder = createEncoder({
-        pubsubTopic: topic,
+        pubsubTopicShardInfo: pubsubTopicToSingleShardInfo(topic),
         contentTopic: RelayPingContentTopic,
         ephemeral: true
       });

package/src/lib/light_push/index.ts

@@ -1,12 +1,10 @@
-import type { Stream } from "@libp2p/interface/connection";
-import type { PeerId } from "@libp2p/interface/peer-id";
+import type { PeerId, Stream } from "@libp2p/interface";
 import {
   IEncoder,
   ILightPush,
   IMessage,
   Libp2p,
   ProtocolCreateOptions,
-  PubsubTopic,
   SendError,
   SendResult
 } from "@waku/interfaces";
@@ -22,7 +20,6 @@ import { pipe } from "it-pipe";
 import { Uint8ArrayList } from "uint8arraylist";
 
 import { BaseProtocol } from "../base_protocol.js";
-import { DefaultPubsubTopic } from "../constants.js";
 
 import { PushRpc } from "./push_rpc.js";
 
@@ -45,12 +42,14 @@ type PreparePushMessageResult =
  * Implements the [Waku v2 Light Push protocol](https://rfc.vac.dev/spec/19/).
  */
 class LightPush extends BaseProtocol implements ILightPush {
-  private readonly pubsubTopics: PubsubTopic[];
-  private readonly NUM_PEERS_PROTOCOL = 1;
-
   constructor(libp2p: Libp2p, options?: ProtocolCreateOptions) {
-    super(LightPushCodec, libp2p.components);
-    this.pubsubTopics = options?.pubsubTopics ?? [DefaultPubsubTopic];
+    super(
+      LightPushCodec,
+      libp2p.components,
+      log,
+      options!.pubsubTopics!,
+      options
+    );
   }
 
   private async preparePushMessage(
@@ -109,10 +108,9 @@ class LightPush extends BaseProtocol implements ILightPush {
       };
     }
 
-    //TODO: get a relevant peer for the topic/shard
     const peers = await this.getPeers({
       maxBootstrapPeers: 1,
-      numPeers: this.NUM_PEERS_PROTOCOL
+      numPeers: this.numPeersToUse
     });
 
     if (!peers.length) {
@@ -178,6 +176,8 @@ class LightPush extends BaseProtocol implements ILightPush {
     });
 
     const results = await Promise.allSettled(promises);
+
+    // TODO: handle renewing faulty peers with new peers (https://github.com/waku-org/js-waku/issues/1463)
     const errors = results
       .filter(
         (