@waku/core 0.0.22 → 0.0.23

package/src/lib/keep_alive_manager.ts

@@ -1,10 +1,12 @@
-import type { PeerId } from "@libp2p/interface-peer-id";
+import type { PeerId } from "@libp2p/interface/peer-id";
+import type { PeerStore } from "@libp2p/interface/peer-store";
 import type { IRelay } from "@waku/interfaces";
 import type { KeepAliveOptions } from "@waku/interfaces";
+import { utf8ToBytes } from "@waku/utils/bytes";
 import debug from "debug";
 import type { PingService } from "libp2p/ping";
 
-import { createEncoder } from "../index.js";
+import { createEncoder } from "./message/version_0.js";
 
 export const RelayPingContentTopic = "/relay-ping/1/ping/null";
 const log = debug("waku:keep-alive");
@@ -22,8 +24,12 @@ export class KeepAliveManager {
     this.relay = relay;
   }
 
-  public start(peerId: PeerId, libp2pPing: PingService): void {
-    // Just in case a timer already exist for this peer
+  public start(
+    peerId: PeerId,
+    libp2pPing: PingService,
+    peerStore: PeerStore
+  ): void {
+    // Just in case a timer already exists for this peer
     this.stop(peerId);
 
     const { pingKeepAlive: pingPeriodSecs, relayKeepAlive: relayPeriodSecs } =
@@ -33,10 +39,28 @@ export class KeepAliveManager {
 
     if (pingPeriodSecs !== 0) {
       const interval = setInterval(() => {
-        libp2pPing.ping(peerId).catch((e) => {
-          log(`Ping failed (${peerIdStr})`, e);
-        });
+        void (async () => {
+          try {
+            // ping the peer for keep alive
+            // also update the peer store with the latency
+            const ping = await libp2pPing.ping(peerId);
+            log(`Ping succeeded (${peerIdStr})`, ping);
+
+            try {
+              await peerStore.patch(peerId, {
+                metadata: {
+                  ping: utf8ToBytes(ping.toString())
+                }
+              });
+            } catch (e) {
+              log("Failed to update ping", e);
+            }
+          } catch (e) {
+            log(`Ping failed (${peerIdStr})`, e);
+          }
+        })();
       }, pingPeriodSecs * 1000);
+
       this.pingKeepAliveTimers.set(peerIdStr, interval);
     }
 
@@ -44,7 +68,7 @@ export class KeepAliveManager {
     if (relay && relayPeriodSecs !== 0) {
       const encoder = createEncoder({
         contentTopic: RelayPingContentTopic,
-        ephemeral: true,
+        ephemeral: true
       });
       const interval = setInterval(() => {
         log("Sending Waku Relay ping message");
@@ -73,7 +97,7 @@ export class KeepAliveManager {
   public stopAll(): void {
     for (const timer of [
       ...Object.values(this.pingKeepAliveTimers),
-      ...Object.values(this.relayKeepAliveTimers),
+      ...Object.values(this.relayKeepAliveTimers)
     ]) {
       clearInterval(timer);
     }

package/src/lib/light_push/index.ts

@@ -1,13 +1,12 @@
-import type { PeerId } from "@libp2p/interface-peer-id";
+import type { PeerId } from "@libp2p/interface/peer-id";
 import {
   IEncoder,
   ILightPush,
   IMessage,
   Libp2p,
   ProtocolCreateOptions,
-  ProtocolOptions,
   SendError,
-  SendResult,
+  SendResult
 } from "@waku/interfaces";
 import { PushResponse } from "@waku/proto";
 import { isSizeValid } from "@waku/utils";
@@ -27,80 +26,137 @@ const log = debug("waku:light-push");
 export const LightPushCodec = "/vac/waku/lightpush/2.0.0-beta1";
 export { PushResponse };
 
+type PreparePushMessageResult =
+  | {
+      query: PushRpc;
+      error: null;
+    }
+  | {
+      query: null;
+      error: SendError;
+    };
+
 /**
  * Implements the [Waku v2 Light Push protocol](https://rfc.vac.dev/spec/19/).
  */
 class LightPush extends BaseProtocol implements ILightPush {
   options: ProtocolCreateOptions;
+  private readonly NUM_PEERS_PROTOCOL = 1;
 
   constructor(libp2p: Libp2p, options?: ProtocolCreateOptions) {
     super(LightPushCodec, libp2p.components);
     this.options = options || {};
   }
 
-  async send(
+  private async preparePushMessage(
     encoder: IEncoder,
     message: IMessage,
-    opts?: ProtocolOptions
-  ): Promise<SendResult> {
-    const { pubSubTopic = DefaultPubSubTopic } = this.options;
-
-    const peer = await this.getPeer(opts?.peerId);
-    const stream = await this.newStream(peer);
-
-    const recipients: PeerId[] = [];
-    let error: undefined | SendError = undefined;
-
+    pubSubTopic: string
+  ): Promise<PreparePushMessageResult> {
     try {
       if (!isSizeValid(message.payload)) {
-        log("Failed to send waku light push: message is bigger that 1MB");
-        return {
-          recipients,
-          error: SendError.SIZE_TOO_BIG,
-        };
+        log("Failed to send waku light push: message is bigger than 1MB");
+        return { query: null, error: SendError.SIZE_TOO_BIG };
       }
 
       const protoMessage = await encoder.toProtoObj(message);
       if (!protoMessage) {
         log("Failed to encode to protoMessage, aborting push");
         return {
-          recipients,
-          error: SendError.ENCODE_FAILED,
+          query: null,
+          error: SendError.ENCODE_FAILED
         };
       }
+
       const query = PushRpc.createRequest(protoMessage, pubSubTopic);
-      const res = await pipe(
-        [query.encode()],
-        lp.encode,
-        stream,
-        lp.decode,
-        async (source) => await all(source)
-      );
+      return { query, error: null };
+    } catch (error) {
+      log("Failed to prepare push message", error);
+
+      return {
+        query: null,
+        error: SendError.GENERIC_FAIL
+      };
+    }
+  }
+
+  async send(encoder: IEncoder, message: IMessage): Promise<SendResult> {
+    const { pubSubTopic = DefaultPubSubTopic } = this.options;
+    const recipients: PeerId[] = [];
+
+    const { query, error: preparationError } = await this.preparePushMessage(
+      encoder,
+      message,
+      pubSubTopic
+    );
+
+    if (preparationError || !query) {
+      return {
+        recipients,
+        errors: [preparationError]
+      };
+    }
+
+    const peers = await this.getPeers({
+      maxBootstrapPeers: 1,
+      numPeers: this.NUM_PEERS_PROTOCOL
+    });
+
+    const promises = peers.map(async (peer) => {
+      let error: SendError | undefined;
+      const stream = await this.getStream(peer);
+
       try {
-        const bytes = new Uint8ArrayList();
-        res.forEach((chunk) => {
-          bytes.append(chunk);
-        });
-
-        const response = PushRpc.decode(bytes).response;
-
-        if (response?.isSuccess) {
-          recipients.push(peer.id);
-        } else {
-          log("No response in PushRPC");
-          error = SendError.NO_RPC_RESPONSE;
+        const res = await pipe(
+          [query.encode()],
+          lp.encode,
+          stream,
+          lp.decode,
+          async (source) => await all(source)
+        );
+        try {
+          const bytes = new Uint8ArrayList();
+          res.forEach((chunk) => {
+            bytes.append(chunk);
+          });
+
+          const response = PushRpc.decode(bytes).response;
+
+          if (response?.isSuccess) {
+            recipients.some((recipient) => recipient.equals(peer.id)) ||
+              recipients.push(peer.id);
+          } else {
+            log("No response in PushRPC");
+            error = SendError.NO_RPC_RESPONSE;
+          }
+        } catch (err) {
+          log("Failed to decode push reply", err);
+          error = SendError.DECODE_FAILED;
         }
       } catch (err) {
-        log("Failed to decode push reply", err);
-        error = SendError.DECODE_FAILED;
+        log("Failed to send waku light push request", err);
+        error = SendError.GENERIC_FAIL;
       }
-    } catch (err) {
-      log("Failed to send waku light push request", err);
-      error = SendError.GENERIC_FAIL;
-    }
+
+      return { recipients, error };
+    });
+
+    const results = await Promise.allSettled(promises);
+    const errors = results
+      .filter(
+        (
+          result
+        ): result is PromiseFulfilledResult<{
+          recipients: PeerId[];
+          error: SendError | undefined;
+        }> => result.status === "fulfilled"
+      )
+      .map((result) => result.value.error)
+      .filter((error) => error !== undefined) as SendError[];
+
     return {
-      error,
       recipients,
+      errors
     };
   }
 }

package/src/lib/light_push/push_rpc.ts

@@ -13,9 +13,9 @@ export class PushRpc {
       requestId: uuid(),
       request: {
         message: message,
-        pubsubTopic: pubSubTopic,
+        pubsubTopic: pubSubTopic
       },
-      response: undefined,
+      response: undefined
     });
   }
 

package/src/lib/message/version_0.ts

@@ -6,7 +6,7 @@ import type {
   IMessage,
   IMetaSetter,
   IProtoMessage,
-  IRateLimitProof,
+  IRateLimitProof
 } from "@waku/interfaces";
 import { proto_message as proto } from "@waku/proto";
 import debug from "debug";
@@ -18,7 +18,10 @@ export const Version = 0;
 export { proto };
 
 export class DecodedMessage implements IDecodedMessage {
-  constructor(public pubSubTopic: string, protected proto: proto.WakuMessage) {}
+  constructor(
+    public pubSubTopic: string,
+    protected proto: proto.WakuMessage
+  ) {}
 
   get ephemeral(): boolean {
     return Boolean(this.proto.ephemeral);
@@ -91,7 +94,7 @@ export class Encoder implements IEncoder {
       timestamp: BigInt(timestamp.valueOf()) * OneMillion,
       meta: undefined,
       rateLimitProof: message.rateLimitProof,
-      ephemeral: this.ephemeral,
+      ephemeral: this.ephemeral
     };
 
     if (this.metaSetter) {
@@ -115,7 +118,7 @@ export class Encoder implements IEncoder {
 export function createEncoder({
   contentTopic,
   ephemeral,
-  metaSetter,
+  metaSetter
 }: EncoderOptions): Encoder {
   return new Encoder(contentTopic, ephemeral, metaSetter);
 }
@@ -137,7 +140,7 @@ export class Decoder implements IDecoder<DecodedMessage> {
       timestamp: protoMessage.timestamp ?? undefined,
       meta: protoMessage.meta ?? undefined,
       rateLimitProof: protoMessage.rateLimitProof ?? undefined,
-      ephemeral: protoMessage.ephemeral ?? false,
+      ephemeral: protoMessage.ephemeral ?? false
     });
   }
 

package/src/lib/predefined_bootstrap_nodes.ts

@@ -4,7 +4,7 @@ export const DefaultWantedNumber = 1;
 
 export enum Fleet {
   Prod = "prod",
-  Test = "test",
+  Test = "test"
 }
 
 /**
@@ -51,8 +51,8 @@ export const fleets = {
         "node-01.do-ams3.wakuv2.prod":
           "/dns4/node-01.do-ams3.wakuv2.prod.statusim.net/tcp/8000/wss/p2p/16Uiu2HAmL5okWopX7NqZWBUKVqW8iUxCEmd5GMHLVPwCgzYzQv3e",
         "node-01.gc-us-central1-a.wakuv2.prod":
-          "/dns4/node-01.gc-us-central1-a.wakuv2.prod.statusim.net/tcp/8000/wss/p2p/16Uiu2HAmVkKntsECaYfefR1V2yCR79CegLATuTPE6B9TxgxBiiiA",
-      },
+          "/dns4/node-01.gc-us-central1-a.wakuv2.prod.statusim.net/tcp/8000/wss/p2p/16Uiu2HAmVkKntsECaYfefR1V2yCR79CegLATuTPE6B9TxgxBiiiA"
+      }
     },
     "wakuv2.test": {
       "waku-websocket": {
@@ -61,8 +61,8 @@ export const fleets = {
         "node-01.do-ams3.wakuv2.test":
           "/dns4/node-01.do-ams3.wakuv2.test.statusim.net/tcp/8000/wss/p2p/16Uiu2HAmPLe7Mzm8TsYUubgCAW1aJoeFScxrLj8ppHFivPo97bUZ",
         "node-01.gc-us-central1-a.wakuv2.test":
-          "/dns4/node-01.gc-us-central1-a.wakuv2.test.statusim.net/tcp/8000/wss/p2p/16Uiu2HAmJb2e28qLXxT5kZxVUUoJt72EMzNGXB47Rxx5hw3q4YjS",
-      },
-    },
-  },
+          "/dns4/node-01.gc-us-central1-a.wakuv2.test.statusim.net/tcp/8000/wss/p2p/16Uiu2HAmJb2e28qLXxT5kZxVUUoJt72EMzNGXB47Rxx5hw3q4YjS"
+      }
+    }
+  }
 };

package/src/lib/store/history_rpc.ts

@@ -6,7 +6,7 @@ const OneMillion = BigInt(1_000_000);
 
 export enum PageDirection {
   BACKWARD = "backward",
-  FORWARD = "forward",
+  FORWARD = "forward"
 }
 
 export interface Params {
@@ -43,7 +43,7 @@ export class HistoryRpc {
     const pagingInfo = {
       pageSize: BigInt(params.pageSize),
       cursor: params.cursor,
-      direction,
+      direction
     } as proto.PagingInfo;
 
     let startTime, endTime;
@@ -63,9 +63,9 @@ export class HistoryRpc {
         contentFilters,
         pagingInfo,
         startTime,
-        endTime,
+        endTime
       },
-      response: undefined,
+      response: undefined
     });
   }
 

package/src/lib/store/index.ts

@@ -1,5 +1,4 @@
-import type { Stream } from "@libp2p/interface-connection";
-import type { PeerId } from "@libp2p/interface-peer-id";
+import type { Stream } from "@libp2p/interface/connection";
 import { sha256 } from "@noble/hashes/sha256";
 import {
   Cursor,
@@ -7,7 +6,7 @@ import {
   IDecoder,
   IStore,
   Libp2p,
-  ProtocolCreateOptions,
+  ProtocolCreateOptions
 } from "@waku/interfaces";
 import { proto_store as proto } from "@waku/proto";
 import { isDefined } from "@waku/utils";
@@ -40,10 +39,6 @@ export interface TimeFilter {
 }
 
 export interface QueryOptions {
-  /**
-   * The peer to query. If undefined, a pseudo-random peer is selected from the connected Waku Store peers.
-   */
-  peerId?: PeerId;
   /**
    * The direction in which pages are retrieved:
    * - { @link PageDirection.BACKWARD }: Most recent page first.
@@ -80,12 +75,61 @@ export interface QueryOptions {
  */
 class Store extends BaseProtocol implements IStore {
   options: ProtocolCreateOptions;
+  private readonly NUM_PEERS_PROTOCOL = 1;
 
   constructor(libp2p: Libp2p, options?: ProtocolCreateOptions) {
     super(StoreCodec, libp2p.components);
     this.options = options ?? {};
   }
 
+  /**
+   * Processes messages based on the provided callback and options.
+   * @private
+   */
+  private async processMessages<T extends IDecodedMessage>(
+    messages: Promise<T | undefined>[],
+    callback: (message: T) => Promise<void | boolean> | boolean | void,
+    options?: QueryOptions
+  ): Promise<boolean> {
+    let abort = false;
+    const messagesOrUndef: Array<T | undefined> = await Promise.all(messages);
+    let processedMessages: Array<T> = messagesOrUndef.filter(isDefined);
+
+    if (this.shouldReverseOrder(options)) {
+      processedMessages = processedMessages.reverse();
+    }
+
+    await Promise.all(
+      processedMessages.map(async (msg) => {
+        if (msg && !abort) {
+          abort = Boolean(await callback(msg));
+        }
+      })
+    );
+
+    return abort;
+  }
+
+  /**
+   * Determines whether to reverse the order of messages based on the provided options.
+   *
+   * Messages in pages are ordered from oldest (first) to most recent (last).
+   * https://github.com/vacp2p/rfc/issues/533
+   *
+   * @private
+   */
+  private shouldReverseOrder(options?: QueryOptions): boolean {
+    return (
+      typeof options?.pageDirection === "undefined" ||
+      options?.pageDirection === PageDirection.BACKWARD
+    );
+  }
+
+  /**
+   * @deprecated Use `queryWithOrderedCallback` instead
+   **/
+  queryOrderedCallback = this.queryWithOrderedCallback;
+
   /**
    * Do a query to a Waku Store to retrieve historical/missed messages.
    *
@@ -103,42 +147,20 @@ class Store extends BaseProtocol implements IStore {
    * or if an error is encountered when processing the reply,
    * or if two decoders with the same content topic are passed.
    */
-  async queryOrderedCallback<T extends IDecodedMessage>(
+  async queryWithOrderedCallback<T extends IDecodedMessage>(
     decoders: IDecoder<T>[],
     callback: (message: T) => Promise<void | boolean> | boolean | void,
     options?: QueryOptions
   ): Promise<void> {
-    let abort = false;
     for await (const promises of this.queryGenerator(decoders, options)) {
-      if (abort) break;
-      const messagesOrUndef: Array<T | undefined> = await Promise.all(promises);
-
-      let messages: Array<T> = messagesOrUndef.filter(isDefined);
-
-      // Messages in pages are ordered from oldest (first) to most recent (last).
-      // https://github.com/vacp2p/rfc/issues/533
-      if (
-        typeof options?.pageDirection === "undefined" ||
-        options?.pageDirection === PageDirection.BACKWARD
-      ) {
-        messages = messages.reverse();
-      }
-
-      await Promise.all(
-        messages.map(async (msg) => {
-          if (msg && !abort) {
-            abort = Boolean(await callback(msg));
-          }
-        })
-      );
+      if (await this.processMessages(promises, callback, options)) break;
     }
   }
 
   /**
    * Do a query to a Waku Store to retrieve historical/missed messages.
-   *
    * The callback function takes a `Promise<WakuMessage>` in input,
-   * useful if messages needs to be decrypted and performance matters.
+   * useful if messages need to be decrypted and performance matters.
    *
    * The order of the messages passed to the callback is as follows:
    * - within a page, messages are expected to be ordered from oldest to most recent
@@ -152,7 +174,7 @@ class Store extends BaseProtocol implements IStore {
    * or if an error is encountered when processing the reply,
    * or if two decoders with the same content topic are passed.
    */
-  async queryCallbackOnPromise<T extends IDecodedMessage>(
+  async queryWithPromiseCallback<T extends IDecodedMessage>(
     decoders: IDecoder<T>[],
     callback: (
       message: Promise<T | undefined>
@@ -160,17 +182,15 @@ class Store extends BaseProtocol implements IStore {
     options?: QueryOptions
   ): Promise<void> {
     let abort = false;
-    let promises: Promise<void>[] = [];
     for await (const page of this.queryGenerator(decoders, options)) {
-      const _promises = page.map(async (msg) => {
-        if (!abort) {
-          abort = Boolean(await callback(msg));
-        }
+      const _promises = page.map(async (msgPromise) => {
+        if (abort) return;
+        abort = Boolean(await callback(msgPromise));
       });
 
-      promises = promises.concat(_promises);
+      await Promise.all(_promises);
+      if (abort) break;
     }
-    await Promise.all(promises);
   }
 
   /**
@@ -183,9 +203,6 @@ class Store extends BaseProtocol implements IStore {
    * as follows:
    * - within a page, messages SHOULD be ordered from oldest to most recent
    * - pages direction depends on { @link QueryOptions.pageDirection }
-   *
-   * However, there is no way to guarantee the behavior of the remote node.
-   *
    * @throws If not able to reach a Waku Store peer to query,
    * or if an error is encountered when processing the reply,
    * or if two decoders with the same content topic are passed.
@@ -219,21 +236,23 @@ class Store extends BaseProtocol implements IStore {
       {
         pubSubTopic: pubSubTopic,
         pageDirection: PageDirection.BACKWARD,
-        pageSize: DefaultPageSize,
+        pageSize: DefaultPageSize
       },
       options,
       { contentTopics, startTime, endTime }
     );
 
-    log("Querying history with the following options", {
-      ...options,
-      peerId: options?.peerId?.toString(),
-    });
+    log("Querying history with the following options", options);
 
-    const peer = await this.getPeer(options?.peerId);
+    const peer = (
+      await this.getPeers({
+        numPeers: this.NUM_PEERS_PROTOCOL,
+        maxBootstrapPeers: 1
+      })
+    )[0];
 
     for await (const messages of paginate<T>(
-      this.newStream.bind(this, peer),
+      this.getStream.bind(this, peer),
       queryOpts,
       decodersAsMap,
       options?.cursor
@@ -369,7 +388,7 @@ export async function createCursor(
     digest,
     pubsubTopic,
     senderTime: messageTime,
-    receiverTime: messageTime,
+    receiverTime: messageTime
   };
 }
 

package/src/lib/stream_manager.ts

@@ -0,0 +1,69 @@
+import type { PeerUpdate } from "@libp2p/interface";
+import type { Stream } from "@libp2p/interface/connection";
+import { Peer } from "@libp2p/interface/peer-store";
+import { Libp2p } from "@waku/interfaces";
+import { selectConnection } from "@waku/utils/libp2p";
+import debug from "debug";
+
+export class StreamManager {
+  private streamPool: Map<string, Promise<Stream>>;
+  private log: debug.Debugger;
+
+  constructor(
+    public multicodec: string,
+    public getConnections: Libp2p["getConnections"],
+    public addEventListener: Libp2p["addEventListener"]
+  ) {
+    this.log = debug(`waku:stream-manager:${multicodec}`);
+    this.addEventListener(
+      "peer:update",
+      this.handlePeerUpdateStreamPool.bind(this)
+    );
+    this.getStream = this.getStream.bind(this);
+    this.streamPool = new Map();
+  }
+
+  public async getStream(peer: Peer): Promise<Stream> {
+    const peerIdStr = peer.id.toString();
+    const streamPromise = this.streamPool.get(peerIdStr);
+
+    if (!streamPromise) {
+      return this.newStream(peer); // fallback by creating a new stream on the spot
+    }
+
+    // We have the stream, let's remove it from the map
+    this.streamPool.delete(peerIdStr);
+
+    this.prepareNewStream(peer);
+
+    const stream = await streamPromise;
+
+    if (stream.status === "closed") {
+      return this.newStream(peer); // fallback by creating a new stream on the spot
+    }
+
+    return stream;
+  }
+
+  private async newStream(peer: Peer): Promise<Stream> {
+    const connections = this.getConnections(peer.id);
+    const connection = selectConnection(connections);
+    if (!connection) {
+      throw new Error("Failed to get a connection to the peer");
+    }
+    return connection.newStream(this.multicodec);
+  }
+
+  private prepareNewStream(peer: Peer): void {
+    const streamPromise = this.newStream(peer);
+    this.streamPool.set(peer.id.toString(), streamPromise);
+  }
+
+  private handlePeerUpdateStreamPool = (evt: CustomEvent<PeerUpdate>): void => {
+    const peer = evt.detail.peer;
+    if (peer.protocols.includes(this.multicodec)) {
+      this.log(`Preemptively opening a stream to ${peer.id.toString()}`);
+      this.prepareNewStream(peer);
+    }
+  };
+}

package/src/lib/to_proto_message.ts

@@ -8,7 +8,7 @@ const EmptyMessage: IProtoMessage = {
   timestamp: undefined,
   meta: undefined,
   rateLimitProof: undefined,
-  ephemeral: undefined,
+  ephemeral: undefined
 };
 
 export function toProtoMessage(wire: WakuMessageProto): IProtoMessage {