@waku/core 0.0.22 → 0.0.24

package/src/lib/store/index.ts

@@ -1,5 +1,4 @@
-import type { Stream } from "@libp2p/interface-connection";
-import type { PeerId } from "@libp2p/interface-peer-id";
+import type { Stream } from "@libp2p/interface/connection";
 import { sha256 } from "@noble/hashes/sha256";
 import {
   Cursor,
@@ -8,9 +7,10 @@ import {
   IStore,
   Libp2p,
   ProtocolCreateOptions,
+  PubSubTopic
 } from "@waku/interfaces";
 import { proto_store as proto } from "@waku/proto";
-import { isDefined } from "@waku/utils";
+import { ensurePubsubTopicIsConfigured, isDefined } from "@waku/utils";
 import { concat, utf8ToBytes } from "@waku/utils/bytes";
 import debug from "debug";
 import all from "it-all";
@@ -40,10 +40,6 @@ export interface TimeFilter {
 }
 
 export interface QueryOptions {
-  /**
-   * The peer to query. If undefined, a pseudo-random peer is selected from the connected Waku Store peers.
-   */
-  peerId?: PeerId;
   /**
    * The direction in which pages are retrieved:
    * - { @link PageDirection.BACKWARD }: Most recent page first.
@@ -79,13 +75,62 @@ export interface QueryOptions {
  * The Waku Store protocol can be used to retrieved historical messages.
  */
 class Store extends BaseProtocol implements IStore {
-  options: ProtocolCreateOptions;
+  private readonly pubsubTopics: PubSubTopic[];
+  private readonly NUM_PEERS_PROTOCOL = 1;
 
   constructor(libp2p: Libp2p, options?: ProtocolCreateOptions) {
     super(StoreCodec, libp2p.components);
-    this.options = options ?? {};
+    this.pubsubTopics = options?.pubsubTopics ?? [DefaultPubSubTopic];
+  }
+
+  /**
+   * Processes messages based on the provided callback and options.
+   * @private
+   */
+  private async processMessages<T extends IDecodedMessage>(
+    messages: Promise<T | undefined>[],
+    callback: (message: T) => Promise<void | boolean> | boolean | void,
+    options?: QueryOptions
+  ): Promise<boolean> {
+    let abort = false;
+    const messagesOrUndef: Array<T | undefined> = await Promise.all(messages);
+    let processedMessages: Array<T> = messagesOrUndef.filter(isDefined);
+
+    if (this.shouldReverseOrder(options)) {
+      processedMessages = processedMessages.reverse();
+    }
+
+    await Promise.all(
+      processedMessages.map(async (msg) => {
+        if (msg && !abort) {
+          abort = Boolean(await callback(msg));
+        }
+      })
+    );
+
+    return abort;
+  }
+
+  /**
+   * Determines whether to reverse the order of messages based on the provided options.
+   *
+   * Messages in pages are ordered from oldest (first) to most recent (last).
+   * https://github.com/vacp2p/rfc/issues/533
+   *
+   * @private
+   */
+  private shouldReverseOrder(options?: QueryOptions): boolean {
+    return (
+      typeof options?.pageDirection === "undefined" ||
+      options?.pageDirection === PageDirection.BACKWARD
+    );
   }
 
+  /**
+   * @deprecated Use `queryWithOrderedCallback` instead
+   **/
+  queryOrderedCallback = this.queryWithOrderedCallback;
+
   /**
    * Do a query to a Waku Store to retrieve historical/missed messages.
    *
@@ -103,42 +148,20 @@ class Store extends BaseProtocol implements IStore {
    * or if an error is encountered when processing the reply,
    * or if two decoders with the same content topic are passed.
    */
-  async queryOrderedCallback<T extends IDecodedMessage>(
+  async queryWithOrderedCallback<T extends IDecodedMessage>(
     decoders: IDecoder<T>[],
     callback: (message: T) => Promise<void | boolean> | boolean | void,
     options?: QueryOptions
   ): Promise<void> {
-    let abort = false;
     for await (const promises of this.queryGenerator(decoders, options)) {
-      if (abort) break;
-      const messagesOrUndef: Array<T | undefined> = await Promise.all(promises);
-
-      let messages: Array<T> = messagesOrUndef.filter(isDefined);
-
-      // Messages in pages are ordered from oldest (first) to most recent (last).
-      // https://github.com/vacp2p/rfc/issues/533
-      if (
-        typeof options?.pageDirection === "undefined" ||
-        options?.pageDirection === PageDirection.BACKWARD
-      ) {
-        messages = messages.reverse();
-      }
-
-      await Promise.all(
-        messages.map(async (msg) => {
-          if (msg && !abort) {
-            abort = Boolean(await callback(msg));
-          }
-        })
-      );
+      if (await this.processMessages(promises, callback, options)) break;
     }
   }
 
   /**
    * Do a query to a Waku Store to retrieve historical/missed messages.
-   *
    * The callback function takes a `Promise<WakuMessage>` in input,
-   * useful if messages needs to be decrypted and performance matters.
+   * useful if messages need to be decrypted and performance matters.
    *
    * The order of the messages passed to the callback is as follows:
    * - within a page, messages are expected to be ordered from oldest to most recent
@@ -152,7 +175,7 @@ class Store extends BaseProtocol implements IStore {
    * or if an error is encountered when processing the reply,
    * or if two decoders with the same content topic are passed.
    */
-  async queryCallbackOnPromise<T extends IDecodedMessage>(
+  async queryWithPromiseCallback<T extends IDecodedMessage>(
     decoders: IDecoder<T>[],
     callback: (
       message: Promise<T | undefined>
@@ -160,17 +183,15 @@ class Store extends BaseProtocol implements IStore {
     options?: QueryOptions
   ): Promise<void> {
     let abort = false;
-    let promises: Promise<void>[] = [];
     for await (const page of this.queryGenerator(decoders, options)) {
-      const _promises = page.map(async (msg) => {
-        if (!abort) {
-          abort = Boolean(await callback(msg));
-        }
+      const _promises = page.map(async (msgPromise) => {
+        if (abort) return;
+        abort = Boolean(await callback(msgPromise));
       });
 
-      promises = promises.concat(_promises);
+      await Promise.all(_promises);
+      if (abort) break;
     }
-    await Promise.all(promises);
   }
 
   /**
@@ -183,18 +204,23 @@ class Store extends BaseProtocol implements IStore {
    * as follows:
    * - within a page, messages SHOULD be ordered from oldest to most recent
    * - pages direction depends on { @link QueryOptions.pageDirection }
-   *
-   * However, there is no way to guarantee the behavior of the remote node.
-   *
    * @throws If not able to reach a Waku Store peer to query,
    * or if an error is encountered when processing the reply,
    * or if two decoders with the same content topic are passed.
+   *
+   * This API only supports querying a single pubsub topic at a time.
+   * If multiple decoders are provided, they must all have the same pubsub topic.
+   * @throws If multiple decoders with different pubsub topics are provided.
+   * @throws If no decoders are provided.
+   * @throws If no decoders are found for the provided pubsub topic.
    */
   async *queryGenerator<T extends IDecodedMessage>(
     decoders: IDecoder<T>[],
     options?: QueryOptions
   ): AsyncGenerator<Promise<T | undefined>[]> {
-    const { pubSubTopic = DefaultPubSubTopic } = this.options;
+    if (decoders.length === 0) {
+      throw new Error("No decoders provided");
+    }
 
     let startTime, endTime;
 
@@ -203,6 +229,33 @@ class Store extends BaseProtocol implements IStore {
       endTime = options.timeFilter.endTime;
     }
 
+    // convert array to set to remove duplicates
+    const uniquePubSubTopicsInQuery = Array.from(
+      new Set(decoders.map((decoder) => decoder.pubsubTopic))
+    );
+
+    // If multiple pubsub topics are provided, throw an error
+    if (uniquePubSubTopicsInQuery.length > 1) {
+      throw new Error(
+        "API does not support querying multiple pubsub topics at once"
+      );
+    }
+
+    // we can be certain that there is only one pubsub topic in the query
+    const pubSubTopicForQuery = uniquePubSubTopicsInQuery[0];
+
+    ensurePubsubTopicIsConfigured(pubSubTopicForQuery, this.pubsubTopics);
+
+    // check that the pubsubTopic from the Cursor and Decoder match
+    if (
+      options?.cursor?.pubsubTopic &&
+      options.cursor.pubsubTopic !== pubSubTopicForQuery
+    ) {
+      throw new Error(
+        `Cursor pubsub topic (${options?.cursor?.pubsubTopic}) does not match decoder pubsub topic (${pubSubTopicForQuery})`
+      );
+    }
+
     const decodersAsMap = new Map();
     decoders.forEach((dec) => {
       if (decodersAsMap.has(dec.contentTopic)) {
@@ -213,27 +266,35 @@ class Store extends BaseProtocol implements IStore {
       decodersAsMap.set(dec.contentTopic, dec);
     });
 
-    const contentTopics = decoders.map((dec) => dec.contentTopic);
+    const contentTopics = decoders
+      .filter((decoder) => decoder.pubsubTopic === pubSubTopicForQuery)
+      .map((dec) => dec.contentTopic);
+
+    if (contentTopics.length === 0) {
+      throw new Error("No decoders found for topic " + pubSubTopicForQuery);
+    }
 
     const queryOpts = Object.assign(
       {
-        pubSubTopic: pubSubTopic,
+        pubsubTopic: pubSubTopicForQuery,
         pageDirection: PageDirection.BACKWARD,
-        pageSize: DefaultPageSize,
+        pageSize: DefaultPageSize
       },
       options,
       { contentTopics, startTime, endTime }
     );
 
-    log("Querying history with the following options", {
-      ...options,
-      peerId: options?.peerId?.toString(),
-    });
+    log("Querying history with the following options", options);
 
-    const peer = await this.getPeer(options?.peerId);
+    const peer = (
+      await this.getPeers({
+        numPeers: this.NUM_PEERS_PROTOCOL,
+        maxBootstrapPeers: 1
+      })
+    )[0];
 
     for await (const messages of paginate<T>(
-      this.newStream.bind(this, peer),
+      this.getStream.bind(this, peer),
       queryOpts,
       decodersAsMap,
       options?.cursor
@@ -266,7 +327,7 @@ async function* paginate<T extends IDecodedMessage>(
 
     log(
       "Querying store peer",
-      `for (${queryOpts.pubSubTopic})`,
+      `for (${queryOpts.pubsubTopic})`,
       queryOpts.contentTopics
     );
 
@@ -313,7 +374,7 @@ async function* paginate<T extends IDecodedMessage>(
         const decoder = decoders.get(contentTopic);
         if (decoder) {
           return decoder.fromProtoObj(
-            queryOpts.pubSubTopic,
+            queryOpts.pubsubTopic,
             toProtoMessage(protoMsg)
           );
         }
@@ -346,10 +407,7 @@ async function* paginate<T extends IDecodedMessage>(
   }
 }
 
-export async function createCursor(
-  message: IDecodedMessage,
-  pubsubTopic: string = DefaultPubSubTopic
-): Promise<Cursor> {
+export async function createCursor(message: IDecodedMessage): Promise<Cursor> {
   if (
     !message ||
     !message.timestamp ||
@@ -367,9 +425,9 @@ export async function createCursor(
 
   return {
     digest,
-    pubsubTopic,
+    pubsubTopic: message.pubsubTopic,
     senderTime: messageTime,
-    receiverTime: messageTime,
+    receiverTime: messageTime
   };
 }
 

package/src/lib/stream_manager.ts

@@ -0,0 +1,72 @@
+import type { PeerUpdate } from "@libp2p/interface";
+import type { Stream } from "@libp2p/interface/connection";
+import { Peer } from "@libp2p/interface/peer-store";
+import { Libp2p } from "@waku/interfaces";
+import { selectConnection } from "@waku/utils/libp2p";
+import debug from "debug";
+
+export class StreamManager {
+  private streamPool: Map<string, Promise<Stream | void>>;
+  private readonly log: debug.Debugger;
+
+  constructor(
+    public multicodec: string,
+    public getConnections: Libp2p["getConnections"],
+    public addEventListener: Libp2p["addEventListener"]
+  ) {
+    this.log = debug(`waku:stream-manager:${multicodec}`);
+    this.addEventListener(
+      "peer:update",
+      this.handlePeerUpdateStreamPool.bind(this)
+    );
+    this.getStream = this.getStream.bind(this);
+    this.streamPool = new Map();
+  }
+
+  public async getStream(peer: Peer): Promise<Stream> {
+    const peerIdStr = peer.id.toString();
+    const streamPromise = this.streamPool.get(peerIdStr);
+
+    if (!streamPromise) {
+      return this.newStream(peer); // fallback by creating a new stream on the spot
+    }
+
+    // We have the stream, let's remove it from the map
+    this.streamPool.delete(peerIdStr);
+
+    this.prepareNewStream(peer);
+
+    const stream = await streamPromise;
+
+    if (!stream || stream.status === "closed") {
+      return this.newStream(peer); // fallback by creating a new stream on the spot
+    }
+
+    return stream;
+  }
+
+  private async newStream(peer: Peer): Promise<Stream> {
+    const connections = this.getConnections(peer.id);
+    const connection = selectConnection(connections);
+    if (!connection) {
+      throw new Error("Failed to get a connection to the peer");
+    }
+    return connection.newStream(this.multicodec);
+  }
+
+  private prepareNewStream(peer: Peer): void {
+    const streamPromise = this.newStream(peer).catch(() => {
+      // No error thrown as this call is not triggered by the user
+      this.log(`Failed to prepare a new stream for ${peer.id.toString()}`);
+    });
+    this.streamPool.set(peer.id.toString(), streamPromise);
+  }
+
+  private handlePeerUpdateStreamPool = (evt: CustomEvent<PeerUpdate>): void => {
+    const peer = evt.detail.peer;
+    if (peer.protocols.includes(this.multicodec)) {
+      this.log(`Preemptively opening a stream to ${peer.id.toString()}`);
+      this.prepareNewStream(peer);
+    }
+  };
+}

package/src/lib/to_proto_message.ts

@@ -8,7 +8,7 @@ const EmptyMessage: IProtoMessage = {
   timestamp: undefined,
   meta: undefined,
   rateLimitProof: undefined,
-  ephemeral: undefined,
+  ephemeral: undefined
 };
 
 export function toProtoMessage(wire: WakuMessageProto): IProtoMessage {

package/src/lib/wait_for_remote_peer.ts

@@ -1,4 +1,4 @@
-import type { IdentifyResult } from "@libp2p/interface-libp2p";
+import type { IdentifyResult } from "@libp2p/interface";
 import type { IBaseProtocol, IRelay, Waku } from "@waku/interfaces";
 import { Protocols } from "@waku/interfaces";
 import debug from "debug";
@@ -9,8 +9,8 @@ const log = debug("waku:wait-for-remote-peer");
 /**
  * Wait for a remote peer to be ready given the passed protocols.
  * Must be used after attempting to connect to nodes, using
- * {@link @waku/core.WakuNode.dial} or a bootstrap method with
- * {@link @waku/sdk.createLightNode}.
+ * {@link @waku/core!WakuNode.dial} or a bootstrap method with
+ * {@link @waku/sdk!createLightNode}.
  *
  * If the passed protocols is a GossipSub protocol, then it resolves only once
  * a peer is in a mesh, to help ensure that other peers will send and receive
@@ -96,15 +96,18 @@ async function waitForConnectedPeer(protocol: IBaseProtocol): Promise<void> {
 }
 
 /**
- * Wait for a peer with the given protocol to be connected and in the gossipsub
- * mesh.
+ * Wait for at least one peer with the given protocol to be connected and in the gossipsub
+ * mesh for all pubsubTopics.
  */
 async function waitForGossipSubPeerInMesh(waku: IRelay): Promise<void> {
   let peers = waku.getMeshPeers();
+  const pubsubTopics = waku.pubsubTopics;
 
-  while (peers.length == 0) {
-    await pEvent(waku.gossipSub, "gossipsub:heartbeat");
-    peers = waku.getMeshPeers();
+  for (const topic of pubsubTopics) {
+    while (peers.length == 0) {
+      await pEvent(waku.gossipSub, "gossipsub:heartbeat");
+      peers = waku.getMeshPeers(topic);
+    }
   }
 }
 

package/src/lib/waku.ts

@@ -1,5 +1,5 @@
-import type { Stream } from "@libp2p/interface-connection";
-import { isPeerId, PeerId } from "@libp2p/interface-peer-id";
+import type { Stream } from "@libp2p/interface/connection";
+import { isPeerId, PeerId } from "@libp2p/interface/peer-id";
 import { multiaddr, Multiaddr, MultiaddrInput } from "@multiformats/multiaddr";
 import type {
   IFilter,
@@ -7,14 +7,15 @@ import type {
   IRelay,
   IStore,
   Libp2p,
-  Waku,
+  PubSubTopic,
+  Waku
 } from "@waku/interfaces";
 import { Protocols } from "@waku/interfaces";
 import debug from "debug";
 
 import { ConnectionManager } from "./connection_manager.js";
 
-export const DefaultPingKeepAliveValueSecs = 0;
+export const DefaultPingKeepAliveValueSecs = 5 * 60;
 export const DefaultRelayKeepAliveValueSecs = 5 * 60;
 export const DefaultUserAgent = "js-waku";
 
@@ -52,6 +53,7 @@ export class WakuNode implements Waku {
 
   constructor(
     options: WakuOptions,
+    public readonly pubsubTopics: PubSubTopic[],
     libp2p: Libp2p,
     store?: (libp2p: Libp2p) => IStore,
     lightPush?: (libp2p: Libp2p) => ILightPush,
@@ -86,6 +88,7 @@ export class WakuNode implements Waku {
       peerId,
       libp2p,
       { pingKeepAlive, relayKeepAlive },
+      pubsubTopics,
       this.relay
     );
 

package/dist/lib/push_or_init_map.d.ts

@@ -1 +0,0 @@
-export declare function pushOrInitMapSet<K, V>(map: Map<K, Set<V>>, key: K, newValue: V): void;

package/dist/lib/push_or_init_map.js

@@ -1,9 +0,0 @@
-export function pushOrInitMapSet(map, key, newValue) {
-    let arr = map.get(key);
-    if (typeof arr === "undefined") {
-        map.set(key, new Set());
-        arr = map.get(key);
-    }
-    arr.add(newValue);
-}
-//# sourceMappingURL=push_or_init_map.js.map

package/dist/lib/push_or_init_map.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"push_or_init_map.js","sourceRoot":"","sources":["../../src/lib/push_or_init_map.ts"],"names":[],"mappings":"AAAA,MAAM,UAAU,gBAAgB,CAC9B,GAAmB,EACnB,GAAM,EACN,QAAW;IAEX,IAAI,GAAG,GAAG,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACvB,IAAI,OAAO,GAAG,KAAK,WAAW,EAAE;QAC9B,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,GAAG,EAAE,CAAC,CAAC;QACxB,GAAG,GAAG,GAAG,CAAC,GAAG,CAAC,GAAG,CAAW,CAAC;KAC9B;IAED,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;AACpB,CAAC"}

package/src/lib/push_or_init_map.ts

@@ -1,13 +0,0 @@
-export function pushOrInitMapSet<K, V>(
-  map: Map<K, Set<V>>,
-  key: K,
-  newValue: V
-): void {
-  let arr = map.get(key);
-  if (typeof arr === "undefined") {
-    map.set(key, new Set());
-    arr = map.get(key) as Set<V>;
-  }
-
-  arr.add(newValue);
-}