@waku/core 0.0.29-fd60cc2.0 → 0.0.30-682cc66.0

package/src/lib/filter/index.ts

@@ -1,30 +1,13 @@
-import { Stream } from "@libp2p/interface";
 import type { Peer } from "@libp2p/interface";
 import type { IncomingStreamData } from "@libp2p/interface-internal";
 import type {
-  Callback,
   ContentTopic,
-  IAsyncIterator,
-  IDecodedMessage,
-  IDecoder,
-  IFilter,
-  IProtoMessage,
-  IReceiver,
+  IBaseProtocolCore,
   Libp2p,
   ProtocolCreateOptions,
-  PubsubTopic,
-  SingleShardInfo,
-  Unsubscribe
+  PubsubTopic
 } from "@waku/interfaces";
-import { DefaultPubsubTopic } from "@waku/interfaces";
-import { messageHashStr } from "@waku/message-hash";
 import { WakuMessage } from "@waku/proto";
-import {
-  ensurePubsubTopicIsConfigured,
-  groupByContentTopic,
-  singleShardInfoToPubsubTopic,
-  toAsyncIterator
-} from "@waku/utils";
 import { Logger } from "@waku/utils";
 import all from "it-all";
 import * as lp from "it-length-prefixed";
@@ -40,329 +23,20 @@ import {
 
 const log = new Logger("filter:v2");
 
-type SubscriptionCallback<T extends IDecodedMessage> = {
-  decoders: IDecoder<T>[];
-  callback: Callback<T>;
-};
-
 export const FilterCodecs = {
   SUBSCRIBE: "/vac/waku/filter-subscribe/2.0.0-beta1",
   PUSH: "/vac/waku/filter-push/2.0.0-beta1"
 };
 
-/**
- * A subscription object refers to a subscription to a given pubsub topic.
- */
-class Subscription {
-  readonly peers: Peer[];
-  private readonly pubsubTopic: PubsubTopic;
-  private newStream: (peer: Peer) => Promise<Stream>;
-  readonly receivedMessagesHashStr: string[] = [];
-
-  private subscriptionCallbacks: Map<
-    ContentTopic,
-    SubscriptionCallback<IDecodedMessage>
-  >;
-
+export class FilterCore extends BaseProtocol implements IBaseProtocolCore {
   constructor(
-    pubsubTopic: PubsubTopic,
-    remotePeers: Peer[],
-    newStream: (peer: Peer) => Promise<Stream>
+    private handleIncomingMessage: (
+      pubsubTopic: PubsubTopic,
+      wakuMessage: WakuMessage
+    ) => Promise<void>,
+    libp2p: Libp2p,
+    options?: ProtocolCreateOptions
   ) {
-    this.peers = remotePeers;
-    this.pubsubTopic = pubsubTopic;
-    this.newStream = newStream;
-    this.subscriptionCallbacks = new Map();
-  }
-
-  async subscribe<T extends IDecodedMessage>(
-    decoders: IDecoder<T> | IDecoder<T>[],
-    callback: Callback<T>
-  ): Promise<void> {
-    const decodersArray = Array.isArray(decoders) ? decoders : [decoders];
-
-    // check that all decoders are configured for the same pubsub topic as this subscription
-    decodersArray.forEach((decoder) => {
-      if (decoder.pubsubTopic !== this.pubsubTopic) {
-        throw new Error(
-          `Pubsub topic not configured: decoder is configured for pubsub topic ${decoder.pubsubTopic} but this subscription is for pubsub topic ${this.pubsubTopic}. Please create a new Subscription for the different pubsub topic.`
-        );
-      }
-    });
-
-    const decodersGroupedByCT = groupByContentTopic(decodersArray);
-    const contentTopics = Array.from(decodersGroupedByCT.keys());
-
-    const promises = this.peers.map(async (peer) => {
-      const stream = await this.newStream(peer);
-
-      const request = FilterSubscribeRpc.createSubscribeRequest(
-        this.pubsubTopic,
-        contentTopics
-      );
-
-      try {
-        const res = await pipe(
-          [request.encode()],
-          lp.encode,
-          stream,
-          lp.decode,
-          async (source) => await all(source)
-        );
-
-        if (!res || !res.length) {
-          throw Error(
-            `No response received for request ${request.requestId}: ${res}`
-          );
-        }
-
-        const { statusCode, requestId, statusDesc } =
-          FilterSubscribeResponse.decode(res[0].slice());
-
-        if (statusCode < 200 || statusCode >= 300) {
-          throw new Error(
-            `Filter subscribe request ${requestId} failed with status code ${statusCode}: ${statusDesc}`
-          );
-        }
-
-        log.info(
-          "Subscribed to peer ",
-          peer.id.toString(),
-          "for content topics",
-          contentTopics
-        );
-      } catch (e) {
-        throw new Error(
-          "Error subscribing to peer: " +
-            peer.id.toString() +
-            " for content topics: " +
-            contentTopics +
-            ": " +
-            e
-        );
-      }
-    });
-
-    const results = await Promise.allSettled(promises);
-
-    this.handleErrors(results, "subscribe");
-
-    // Save the callback functions by content topics so they
-    // can easily be removed (reciprocally replaced) if `unsubscribe` (reciprocally `subscribe`)
-    // is called for those content topics
-    decodersGroupedByCT.forEach((decoders, contentTopic) => {
-      // Cast the type because a given `subscriptionCallbacks` map may hold
-      // Decoder that decode to different implementations of `IDecodedMessage`
-      const subscriptionCallback = {
-        decoders,
-        callback
-      } as unknown as SubscriptionCallback<IDecodedMessage>;
-
-      // The callback and decoder may override previous values, this is on
-      // purpose as the user may call `subscribe` to refresh the subscription
-      this.subscriptionCallbacks.set(contentTopic, subscriptionCallback);
-    });
-  }
-
-  async unsubscribe(contentTopics: ContentTopic[]): Promise<void> {
-    const promises = this.peers.map(async (peer) => {
-      const stream = await this.newStream(peer);
-      const unsubscribeRequest = FilterSubscribeRpc.createUnsubscribeRequest(
-        this.pubsubTopic,
-        contentTopics
-      );
-
-      try {
-        await pipe([unsubscribeRequest.encode()], lp.encode, stream.sink);
-      } catch (error) {
-        throw new Error("Error unsubscribing: " + error);
-      }
-
-      contentTopics.forEach((contentTopic: string) => {
-        this.subscriptionCallbacks.delete(contentTopic);
-      });
-    });
-
-    const results = await Promise.allSettled(promises);
-
-    this.handleErrors(results, "unsubscribe");
-  }
-
-  async ping(): Promise<void> {
-    const promises = this.peers.map(async (peer) => {
-      const stream = await this.newStream(peer);
-
-      const request = FilterSubscribeRpc.createSubscriberPingRequest();
-
-      try {
-        const res = await pipe(
-          [request.encode()],
-          lp.encode,
-          stream,
-          lp.decode,
-          async (source) => await all(source)
-        );
-
-        if (!res || !res.length) {
-          throw Error(
-            `No response received for request ${request.requestId}: ${res}`
-          );
-        }
-
-        const { statusCode, requestId, statusDesc } =
-          FilterSubscribeResponse.decode(res[0].slice());
-
-        if (statusCode < 200 || statusCode >= 300) {
-          throw new Error(
-            `Filter ping request ${requestId} failed with status code ${statusCode}: ${statusDesc}`
-          );
-        }
-        log.info(`Ping successful for peer ${peer.id.toString()}`);
-      } catch (error) {
-        log.error("Error pinging: ", error);
-        throw error; // Rethrow the actual error instead of wrapping it
-      }
-    });
-
-    const results = await Promise.allSettled(promises);
-
-    this.handleErrors(results, "ping");
-  }
-
-  async unsubscribeAll(): Promise<void> {
-    const promises = this.peers.map(async (peer) => {
-      const stream = await this.newStream(peer);
-
-      const request = FilterSubscribeRpc.createUnsubscribeAllRequest(
-        this.pubsubTopic
-      );
-
-      try {
-        const res = await pipe(
-          [request.encode()],
-          lp.encode,
-          stream,
-          lp.decode,
-          async (source) => await all(source)
-        );
-
-        if (!res || !res.length) {
-          throw Error(
-            `No response received for request ${request.requestId}: ${res}`
-          );
-        }
-
-        const { statusCode, requestId, statusDesc } =
-          FilterSubscribeResponse.decode(res[0].slice());
-
-        if (statusCode < 200 || statusCode >= 300) {
-          throw new Error(
-            `Filter unsubscribe all request ${requestId} failed with status code ${statusCode}: ${statusDesc}`
-          );
-        }
-
-        this.subscriptionCallbacks.clear();
-        log.info(
-          `Unsubscribed from all content topics for pubsub topic ${this.pubsubTopic}`
-        );
-      } catch (error) {
-        throw new Error(
-          "Error unsubscribing from all content topics: " + error
-        );
-      }
-    });
-
-    const results = await Promise.allSettled(promises);
-
-    this.handleErrors(results, "unsubscribeAll");
-  }
-
-  async processMessage(message: WakuMessage): Promise<void> {
-    const hashedMessageStr = messageHashStr(
-      this.pubsubTopic,
-      message as IProtoMessage
-    );
-    if (this.receivedMessagesHashStr.includes(hashedMessageStr)) {
-      log.info("Message already received, skipping");
-      return;
-    }
-    this.receivedMessagesHashStr.push(hashedMessageStr);
-
-    const { contentTopic } = message;
-    const subscriptionCallback = this.subscriptionCallbacks.get(contentTopic);
-    if (!subscriptionCallback) {
-      log.error("No subscription callback available for ", contentTopic);
-      return;
-    }
-    log.info(
-      "Processing message with content topic ",
-      contentTopic,
-      " on pubsub topic ",
-      this.pubsubTopic
-    );
-    await pushMessage(subscriptionCallback, this.pubsubTopic, message);
-  }
-
-  // Filter out only the rejected promises and extract & handle their reasons
-  private handleErrors(
-    results: PromiseSettledResult<void>[],
-    type: "ping" | "subscribe" | "unsubscribe" | "unsubscribeAll"
-  ): void {
-    const errors = results
-      .filter(
-        (result): result is PromiseRejectedResult =>
-          result.status === "rejected"
-      )
-      .map((rejectedResult) => rejectedResult.reason);
-
-    if (errors.length === this.peers.length) {
-      const errorCounts = new Map<string, number>();
-      // TODO: streamline error logging with https://github.com/orgs/waku-org/projects/2/views/1?pane=issue&itemId=42849952
-      errors.forEach((error) => {
-        const message = error instanceof Error ? error.message : String(error);
-        errorCounts.set(message, (errorCounts.get(message) || 0) + 1);
-      });
-
-      const uniqueErrorMessages = Array.from(
-        errorCounts,
-        ([message, count]) => `${message} (occurred ${count} times)`
-      ).join(", ");
-      throw new Error(`Error ${type} all peers: ${uniqueErrorMessages}`);
-    } else if (errors.length > 0) {
-      // TODO: handle renewing faulty peers with new peers (https://github.com/waku-org/js-waku/issues/1463)
-      log.warn(
-        `Some ${type} failed. These will be refreshed with new peers`,
-        errors
-      );
-    } else {
-      log.info(`${type} successful for all peers`);
-    }
-  }
-}
-
-const DEFAULT_NUM_PEERS = 3;
-
-class Filter extends BaseProtocol implements IReceiver {
-  private activeSubscriptions = new Map<string, Subscription>();
-
-  private getActiveSubscription(
-    pubsubTopic: PubsubTopic
-  ): Subscription | undefined {
-    return this.activeSubscriptions.get(pubsubTopic);
-  }
-
-  private setActiveSubscription(
-    pubsubTopic: PubsubTopic,
-    subscription: Subscription
-  ): Subscription {
-    this.activeSubscriptions.set(pubsubTopic, subscription);
-    return subscription;
-  }
-
-  //TODO: Remove when FilterCore and FilterSDK are introduced
-  private readonly numPeersToUse: number;
-
-  constructor(libp2p: Libp2p, options?: ProtocolCreateOptions) {
     super(
       FilterCodecs.SUBSCRIBE,
       libp2p.components,
@@ -371,106 +45,9 @@ class Filter extends BaseProtocol implements IReceiver {
       options
     );
 
-    this.numPeersToUse = options?.numPeersToUse ?? DEFAULT_NUM_PEERS;
-
     libp2p.handle(FilterCodecs.PUSH, this.onRequest.bind(this)).catch((e) => {
       log.error("Failed to register ", FilterCodecs.PUSH, e);
     });
-
-    this.activeSubscriptions = new Map();
-  }
-
-  /**
-   * Creates a new subscription to the given pubsub topic.
-   * The subscription is made to multiple peers for decentralization.
-   * @param pubsubTopicShardInfo The pubsub topic to subscribe to.
-   * @returns The subscription object.
-   */
-  async createSubscription(
-    pubsubTopicShardInfo: SingleShardInfo | PubsubTopic = DefaultPubsubTopic
-  ): Promise<Subscription> {
-    const pubsubTopic =
-      typeof pubsubTopicShardInfo == "string"
-        ? pubsubTopicShardInfo
-        : singleShardInfoToPubsubTopic(pubsubTopicShardInfo);
-
-    ensurePubsubTopicIsConfigured(pubsubTopic, this.pubsubTopics);
-
-    const peers = await this.getPeers({
-      maxBootstrapPeers: 1,
-      numPeers: this.numPeersToUse
-    });
-    if (peers.length === 0) {
-      throw new Error("No peer found to initiate subscription.");
-    }
-
-    log.info(
-      `Creating filter subscription with ${peers.length} peers: `,
-      peers.map((peer) => peer.id.toString())
-    );
-
-    const subscription =
-      this.getActiveSubscription(pubsubTopic) ??
-      this.setActiveSubscription(
-        pubsubTopic,
-        new Subscription(pubsubTopic, peers, this.getStream.bind(this))
-      );
-
-    return subscription;
-  }
-
-  public toSubscriptionIterator<T extends IDecodedMessage>(
-    decoders: IDecoder<T> | IDecoder<T>[]
-  ): Promise<IAsyncIterator<T>> {
-    return toAsyncIterator(this, decoders);
-  }
-
-  /**
-   * This method is used to satisfy the `IReceiver` interface.
-   *
-   * @hidden
-   *
-   * @param decoders The decoders to use for the subscription.
-   * @param callback The callback function to use for the subscription.
-   * @param opts Optional protocol options for the subscription.
-   *
-   * @returns A Promise that resolves to a function that unsubscribes from the subscription.
-   *
-   * @remarks
-   * This method should not be used directly.
-   * Instead, use `createSubscription` to create a new subscription.
-   */
-  async subscribe<T extends IDecodedMessage>(
-    decoders: IDecoder<T> | IDecoder<T>[],
-    callback: Callback<T>
-  ): Promise<Unsubscribe> {
-    const pubsubTopics = this.getPubsubTopics(decoders);
-
-    if (pubsubTopics.length === 0) {
-      throw Error(
-        "Failed to subscribe: no pubsubTopic found on decoders provided."
-      );
-    }
-
-    if (pubsubTopics.length > 1) {
-      throw Error(
-        "Failed to subscribe: all decoders should have the same pubsub topic. Use createSubscription to be more agile."
-      );
-    }
-
-    const subscription = await this.createSubscription(pubsubTopics[0]);
-
-    await subscription.subscribe(decoders, callback);
-
-    const contentTopics = Array.from(
-      groupByContentTopic(
-        Array.isArray(decoders) ? decoders : [decoders]
-      ).keys()
-    );
-
-    return async () => {
-      await subscription.unsubscribe(contentTopics);
-    };
   }
 
   private onRequest(streamData: IncomingStreamData): void {
@@ -494,16 +71,7 @@ class Filter extends BaseProtocol implements IReceiver {
             return;
           }
 
-          const subscription = this.getActiveSubscription(pubsubTopic);
-
-          if (!subscription) {
-            log.error(
-              `No subscription locally registered for topic ${pubsubTopic}`
-            );
-            return;
-          }
-
-          await subscription.processMessage(wakuMessage);
+          await this.handleIncomingMessage(pubsubTopic, wakuMessage);
         }
       }).then(
         () => {
@@ -518,51 +86,117 @@ class Filter extends BaseProtocol implements IReceiver {
     }
   }
 
-  private getPubsubTopics(decoders: IDecoder<any> | IDecoder<any>[]): string[] {
-    if (!Array.isArray(decoders)) {
-      return [decoders.pubsubTopic];
+  async subscribe(
+    pubsubTopic: PubsubTopic,
+    peer: Peer,
+    contentTopics: ContentTopic[]
+  ): Promise<void> {
+    const stream = await this.getStream(peer);
+
+    const request = FilterSubscribeRpc.createSubscribeRequest(
+      pubsubTopic,
+      contentTopics
+    );
+
+    const res = await pipe(
+      [request.encode()],
+      lp.encode,
+      stream,
+      lp.decode,
+      async (source) => await all(source)
+    );
+
+    if (!res || !res.length) {
+      throw Error(
+        `No response received for request ${request.requestId}: ${res}`
+      );
     }
 
-    if (decoders.length === 0) {
-      return [];
+    const { statusCode, requestId, statusDesc } =
+      FilterSubscribeResponse.decode(res[0].slice());
+
+    if (statusCode < 200 || statusCode >= 300) {
+      throw new Error(
+        `Filter subscribe request ${requestId} failed with status code ${statusCode}: ${statusDesc}`
+      );
     }
+  }
 
-    const pubsubTopics = new Set(decoders.map((d) => d.pubsubTopic));
+  async unsubscribe(
+    pubsubTopic: PubsubTopic,
+    peer: Peer,
+    contentTopics: ContentTopic[]
+  ): Promise<void> {
+    const stream = await this.getStream(peer);
+    const unsubscribeRequest = FilterSubscribeRpc.createUnsubscribeRequest(
+      pubsubTopic,
+      contentTopics
+    );
 
-    return [...pubsubTopics];
+    await pipe([unsubscribeRequest.encode()], lp.encode, stream.sink);
   }
-}
 
-export function wakuFilter(
-  init: ProtocolCreateOptions = { pubsubTopics: [] }
-): (libp2p: Libp2p) => IFilter {
-  return (libp2p: Libp2p) => new Filter(libp2p, init);
-}
+  async unsubscribeAll(pubsubTopic: PubsubTopic, peer: Peer): Promise<void> {
+    const stream = await this.getStream(peer);
 
-async function pushMessage<T extends IDecodedMessage>(
-  subscriptionCallback: SubscriptionCallback<T>,
-  pubsubTopic: PubsubTopic,
-  message: WakuMessage
-): Promise<void> {
-  const { decoders, callback } = subscriptionCallback;
+    const request = FilterSubscribeRpc.createUnsubscribeAllRequest(pubsubTopic);
 
-  const { contentTopic } = message;
-  if (!contentTopic) {
-    log.warn("Message has no content topic, skipping");
-    return;
+    const res = await pipe(
+      [request.encode()],
+      lp.encode,
+      stream,
+      lp.decode,
+      async (source) => await all(source)
+    );
+
+    if (!res || !res.length) {
+      throw Error(
+        `No response received for request ${request.requestId}: ${res}`
+      );
+    }
+
+    const { statusCode, requestId, statusDesc } =
+      FilterSubscribeResponse.decode(res[0].slice());
+
+    if (statusCode < 200 || statusCode >= 300) {
+      throw new Error(
+        `Filter unsubscribe all request ${requestId} failed with status code ${statusCode}: ${statusDesc}`
+      );
+    }
   }
 
-  try {
-    const decodePromises = decoders.map((dec) =>
-      dec
-        .fromProtoObj(pubsubTopic, message as IProtoMessage)
-        .then((decoded) => decoded || Promise.reject("Decoding failed"))
-    );
+  async ping(peer: Peer): Promise<void> {
+    const stream = await this.getStream(peer);
 
-    const decodedMessage = await Promise.any(decodePromises);
+    const request = FilterSubscribeRpc.createSubscriberPingRequest();
 
-    await callback(decodedMessage);
-  } catch (e) {
-    log.error("Error decoding message", e);
+    try {
+      const res = await pipe(
+        [request.encode()],
+        lp.encode,
+        stream,
+        lp.decode,
+        async (source) => await all(source)
+      );
+
+      if (!res || !res.length) {
+        throw Error(
+          `No response received for request ${request.requestId}: ${res}`
+        );
+      }
+
+      const { statusCode, requestId, statusDesc } =
+        FilterSubscribeResponse.decode(res[0].slice());
+
+      if (statusCode < 200 || statusCode >= 300) {
+        throw new Error(
+          `Filter ping request ${requestId} failed with status code ${statusCode}: ${statusDesc}`
+        );
+      }
+      log.info(`Ping successful for peer ${peer.id.toString()}`);
+    } catch (error) {
+      log.error("Error pinging: ", error);
+      throw error; // Rethrow the actual error instead of wrapping it
+    }
   }
 }

package/src/lib/keep_alive_manager.ts

@@ -56,7 +56,7 @@ export class KeepAliveManager {
             }
 
             try {
-              await peerStore.patch(peerId, {
+              await peerStore.merge(peerId, {
                 metadata: {
                   ping: utf8ToBytes(ping.toString())
                 }

package/src/lib/wait_for_remote_peer.ts

@@ -70,7 +70,7 @@ export async function waitForRemotePeer(
     if (!waku.filter)
       throw new Error("Cannot wait for Filter peer: protocol not mounted");
     promises.push(
-      waitForConnectedPeer(waku.filter, waku.libp2p.services.metadata)
+      waitForConnectedPeer(waku.filter.protocol, waku.libp2p.services.metadata)
     );
   }