@waku/core 0.0.23 → 0.0.25-a42b7be.0

package/src/lib/keep_alive_manager.ts

@@ -1,19 +1,19 @@
 import type { PeerId } from "@libp2p/interface/peer-id";
 import type { PeerStore } from "@libp2p/interface/peer-store";
-import type { IRelay } from "@waku/interfaces";
+import type { IRelay, PeerIdStr } from "@waku/interfaces";
 import type { KeepAliveOptions } from "@waku/interfaces";
+import { Logger } from "@waku/utils";
 import { utf8ToBytes } from "@waku/utils/bytes";
-import debug from "debug";
 import type { PingService } from "libp2p/ping";
 
 import { createEncoder } from "./message/version_0.js";
 
 export const RelayPingContentTopic = "/relay-ping/1/ping/null";
-const log = debug("waku:keep-alive");
+const log = new Logger("keep-alive");
 
 export class KeepAliveManager {
   private pingKeepAliveTimers: Map<string, ReturnType<typeof setInterval>>;
-  private relayKeepAliveTimers: Map<PeerId, ReturnType<typeof setInterval>>;
+  private relayKeepAliveTimers: Map<PeerId, ReturnType<typeof setInterval>[]>;
   private options: KeepAliveOptions;
   private relay?: IRelay;
 
@@ -37,14 +37,24 @@ export class KeepAliveManager {
 
     const peerIdStr = peerId.toString();
 
+    // Ping the peer every pingPeriodSecs seconds
+    // if pingPeriodSecs is 0, don't ping the peer
     if (pingPeriodSecs !== 0) {
       const interval = setInterval(() => {
         void (async () => {
+          let ping: number;
           try {
             // ping the peer for keep alive
             // also update the peer store with the latency
-            const ping = await libp2pPing.ping(peerId);
-            log(`Ping succeeded (${peerIdStr})`, ping);
+            try {
+              ping = await libp2pPing.ping(peerId);
+              log.info(`Ping succeeded (${peerIdStr})`, ping);
+            } catch (error) {
+              log.error(`Ping failed for peer (${peerIdStr}).
+                Next ping will be attempted in ${pingPeriodSecs} seconds.
+              `);
+              return;
+            }
 
             try {
               await peerStore.patch(peerId, {
@@ -53,10 +63,10 @@ export class KeepAliveManager {
                 }
               });
             } catch (e) {
-              log("Failed to update ping", e);
+              log.error("Failed to update ping", e);
             }
           } catch (e) {
-            log(`Ping failed (${peerIdStr})`, e);
+            log.error(`Ping failed (${peerIdStr})`, e);
           }
         })();
       }, pingPeriodSecs * 1000);
@@ -66,17 +76,12 @@ export class KeepAliveManager {
 
     const relay = this.relay;
     if (relay && relayPeriodSecs !== 0) {
-      const encoder = createEncoder({
-        contentTopic: RelayPingContentTopic,
-        ephemeral: true
-      });
-      const interval = setInterval(() => {
-        log("Sending Waku Relay ping message");
-        relay
-          .send(encoder, { payload: new Uint8Array([1]) })
-          .catch((e) => log("Failed to send relay ping", e));
-      }, relayPeriodSecs * 1000);
-      this.relayKeepAliveTimers.set(peerId, interval);
+      const intervals = this.scheduleRelayPings(
+        relay,
+        relayPeriodSecs,
+        peerId.toString()
+      );
+      this.relayKeepAliveTimers.set(peerId, intervals);
     }
   }
 
@@ -89,7 +94,7 @@ export class KeepAliveManager {
     }
 
     if (this.relayKeepAliveTimers.has(peerId)) {
-      clearInterval(this.relayKeepAliveTimers.get(peerId));
+      this.relayKeepAliveTimers.get(peerId)?.map(clearInterval);
       this.relayKeepAliveTimers.delete(peerId);
     }
   }
@@ -105,4 +110,32 @@ export class KeepAliveManager {
     this.pingKeepAliveTimers.clear();
     this.relayKeepAliveTimers.clear();
   }
+
+  private scheduleRelayPings(
+    relay: IRelay,
+    relayPeriodSecs: number,
+    peerIdStr: PeerIdStr
+  ): NodeJS.Timeout[] {
+    // send a ping message to each PubSubTopic the peer is part of
+    const intervals: NodeJS.Timeout[] = [];
+    for (const topic of relay.pubsubTopics) {
+      const meshPeers = relay.getMeshPeers(topic);
+      if (!meshPeers.includes(peerIdStr)) continue;
+
+      const encoder = createEncoder({
+        pubsubTopic: topic,
+        contentTopic: RelayPingContentTopic,
+        ephemeral: true
+      });
+      const interval = setInterval(() => {
+        log.info("Sending Waku Relay ping message");
+        relay
+          .send(encoder, { payload: new Uint8Array([1]) })
+          .catch((e) => log.error("Failed to send relay ping", e));
+      }, relayPeriodSecs * 1000);
+      intervals.push(interval);
+    }
+
+    return intervals;
+  }
 }

package/src/lib/light_push/index.ts

@@ -1,3 +1,4 @@
+import type { Stream } from "@libp2p/interface/connection";
 import type { PeerId } from "@libp2p/interface/peer-id";
 import {
   IEncoder,
@@ -5,12 +6,16 @@ import {
   IMessage,
   Libp2p,
   ProtocolCreateOptions,
+  PubSubTopic,
   SendError,
   SendResult
 } from "@waku/interfaces";
 import { PushResponse } from "@waku/proto";
-import { isSizeValid } from "@waku/utils";
-import debug from "debug";
+import {
+  ensurePubsubTopicIsConfigured,
+  isMessageSizeUnderCap
+} from "@waku/utils";
+import { Logger } from "@waku/utils";
 import all from "it-all";
 import * as lp from "it-length-prefixed";
 import { pipe } from "it-pipe";
@@ -21,7 +26,7 @@ import { DefaultPubSubTopic } from "../constants.js";
 
 import { PushRpc } from "./push_rpc.js";
 
-const log = debug("waku:light-push");
+const log = new Logger("light-push");
 
 export const LightPushCodec = "/vac/waku/lightpush/2.0.0-beta1";
 export { PushResponse };
@@ -40,38 +45,43 @@ type PreparePushMessageResult =
  * Implements the [Waku v2 Light Push protocol](https://rfc.vac.dev/spec/19/).
  */
 class LightPush extends BaseProtocol implements ILightPush {
-  options: ProtocolCreateOptions;
+  private readonly pubsubTopics: PubSubTopic[];
   private readonly NUM_PEERS_PROTOCOL = 1;
 
   constructor(libp2p: Libp2p, options?: ProtocolCreateOptions) {
     super(LightPushCodec, libp2p.components);
-    this.options = options || {};
+    this.pubsubTopics = options?.pubsubTopics ?? [DefaultPubSubTopic];
   }
 
   private async preparePushMessage(
     encoder: IEncoder,
     message: IMessage,
-    pubSubTopic: string
+    pubsubTopic: string
   ): Promise<PreparePushMessageResult> {
     try {
-      if (!isSizeValid(message.payload)) {
-        log("Failed to send waku light push: message is bigger than 1MB");
+      if (!message.payload || message.payload.length === 0) {
+        log.error("Failed to send waku light push: payload is empty");
+        return { query: null, error: SendError.EMPTY_PAYLOAD };
+      }
+
+      if (!(await isMessageSizeUnderCap(encoder, message))) {
+        log.error("Failed to send waku light push: message is bigger than 1MB");
         return { query: null, error: SendError.SIZE_TOO_BIG };
       }
 
       const protoMessage = await encoder.toProtoObj(message);
       if (!protoMessage) {
-        log("Failed to encode to protoMessage, aborting push");
+        log.error("Failed to encode to protoMessage, aborting push");
         return {
           query: null,
           error: SendError.ENCODE_FAILED
         };
       }
 
-      const query = PushRpc.createRequest(protoMessage, pubSubTopic);
+      const query = PushRpc.createRequest(protoMessage, pubsubTopic);
       return { query, error: null };
     } catch (error) {
-      log("Failed to prepare push message", error);
+      log.error("Failed to prepare push message", error);
 
       return {
         query: null,
@@ -81,13 +91,15 @@ class LightPush extends BaseProtocol implements ILightPush {
   }
 
   async send(encoder: IEncoder, message: IMessage): Promise<SendResult> {
-    const { pubSubTopic = DefaultPubSubTopic } = this.options;
+    const { pubsubTopic } = encoder;
+    ensurePubsubTopicIsConfigured(pubsubTopic, this.pubsubTopics);
+
     const recipients: PeerId[] = [];
 
     const { query, error: preparationError } = await this.preparePushMessage(
       encoder,
       message,
-      pubSubTopic
+      pubsubTopic
     );
 
     if (preparationError || !query) {
@@ -97,48 +109,72 @@ class LightPush extends BaseProtocol implements ILightPush {
       };
     }
 
+    //TODO: get a relevant peer for the topic/shard
     const peers = await this.getPeers({
       maxBootstrapPeers: 1,
       numPeers: this.NUM_PEERS_PROTOCOL
     });
 
+    if (!peers.length) {
+      return {
+        recipients,
+        errors: [SendError.NO_PEER_AVAILABLE]
+      };
+    }
+
     const promises = peers.map(async (peer) => {
-      let error: SendError | undefined;
-      const stream = await this.getStream(peer);
+      let stream: Stream | undefined;
+      try {
+        stream = await this.getStream(peer);
+      } catch (err) {
+        log.error(
+          `Failed to get a stream for remote peer${peer.id.toString()}`,
+          err
+        );
+        return { recipients, error: SendError.REMOTE_PEER_FAULT };
+      }
 
+      let res: Uint8ArrayList[] | undefined;
       try {
-        const res = await pipe(
+        res = await pipe(
           [query.encode()],
           lp.encode,
           stream,
           lp.decode,
           async (source) => await all(source)
         );
-        try {
-          const bytes = new Uint8ArrayList();
-          res.forEach((chunk) => {
-            bytes.append(chunk);
-          });
-
-          const response = PushRpc.decode(bytes).response;
-
-          if (response?.isSuccess) {
-            recipients.some((recipient) => recipient.equals(peer.id)) ||
-              recipients.push(peer.id);
-          } else {
-            log("No response in PushRPC");
-            error = SendError.NO_RPC_RESPONSE;
-          }
-        } catch (err) {
-          log("Failed to decode push reply", err);
-          error = SendError.DECODE_FAILED;
-        }
       } catch (err) {
-        log("Failed to send waku light push request", err);
-        error = SendError.GENERIC_FAIL;
+        log.error("Failed to send waku light push request", err);
+        return { recipients, error: SendError.GENERIC_FAIL };
+      }
+
+      const bytes = new Uint8ArrayList();
+      res.forEach((chunk) => {
+        bytes.append(chunk);
+      });
+
+      let response: PushResponse | undefined;
+      try {
+        response = PushRpc.decode(bytes).response;
+      } catch (err) {
+        log.error("Failed to decode push reply", err);
+        return { recipients, error: SendError.DECODE_FAILED };
       }
 
-      return { recipients, error };
+      if (!response) {
+        log.error("Remote peer fault: No response in PushRPC");
+        return { recipients, error: SendError.REMOTE_PEER_FAULT };
+      }
+
+      if (!response.isSuccess) {
+        log.error("Remote peer rejected the message: ", response.info);
+        return { recipients, error: SendError.REMOTE_PEER_REJECTED };
+      }
+
+      recipients.some((recipient) => recipient.equals(peer.id)) ||
+        recipients.push(peer.id);
+
+      return { recipients };
     });
 
     const results = await Promise.allSettled(promises);

package/src/lib/light_push/push_rpc.ts

@@ -7,13 +7,13 @@ export class PushRpc {
 
   static createRequest(
     message: proto.WakuMessage,
-    pubSubTopic: string
+    pubsubTopic: string
   ): PushRpc {
     return new PushRpc({
       requestId: uuid(),
       request: {
         message: message,
-        pubsubTopic: pubSubTopic
+        pubsubTopic: pubsubTopic
       },
       response: undefined
     });

package/src/lib/message/version_0.ts

@@ -6,12 +6,15 @@ import type {
   IMessage,
   IMetaSetter,
   IProtoMessage,
-  IRateLimitProof
+  IRateLimitProof,
+  PubSubTopic
 } from "@waku/interfaces";
 import { proto_message as proto } from "@waku/proto";
-import debug from "debug";
+import { Logger } from "@waku/utils";
 
-const log = debug("waku:message:version-0");
+import { DefaultPubSubTopic } from "../constants.js";
+
+const log = new Logger("message:version-0");
 const OneMillion = BigInt(1_000_000);
 
 export const Version = 0;
@@ -19,7 +22,7 @@ export { proto };
 
 export class DecodedMessage implements IDecodedMessage {
   constructor(
-    public pubSubTopic: string,
+    public pubsubTopic: string,
     protected proto: proto.WakuMessage
   ) {}
 
@@ -73,6 +76,7 @@ export class Encoder implements IEncoder {
   constructor(
     public contentTopic: string,
     public ephemeral: boolean = false,
+    public pubsubTopic: PubSubTopic,
     public metaSetter?: IMetaSetter
   ) {
     if (!contentTopic || contentTopic === "") {
@@ -109,22 +113,25 @@ export class Encoder implements IEncoder {
 /**
  * Creates an encoder that encode messages without Waku level encryption or signature.
  *
- * An encoder is used to encode messages in the [`14/WAKU2-MESSAGE](https://rfc.vac.dev/spec/14/)
+ * An encoder is used to encode messages in the [14/WAKU2-MESSAGE](https://rfc.vac.dev/spec/14/)
  * format to be sent over the Waku network. The resulting encoder can then be
- * pass to { @link @waku/interfaces.LightPush.push } or
- * { @link @waku/interfaces.Relay.send } to automatically encode outgoing
+ * pass to { @link @waku/interfaces!ISender.send } to automatically encode outgoing
  * messages.
  */
 export function createEncoder({
+  pubsubTopic = DefaultPubSubTopic,
   contentTopic,
   ephemeral,
   metaSetter
 }: EncoderOptions): Encoder {
-  return new Encoder(contentTopic, ephemeral, metaSetter);
+  return new Encoder(contentTopic, ephemeral, pubsubTopic, metaSetter);
 }
 
 export class Decoder implements IDecoder<DecodedMessage> {
-  constructor(public contentTopic: string) {
+  constructor(
+    public pubsubTopic: PubSubTopic,
+    public contentTopic: string
+  ) {
     if (!contentTopic || contentTopic === "") {
       throw new Error("Content topic must be specified");
     }
@@ -132,7 +139,6 @@ export class Decoder implements IDecoder<DecodedMessage> {
 
   fromWireToProtoObj(bytes: Uint8Array): Promise<IProtoMessage | undefined> {
     const protoMessage = proto.WakuMessage.decode(bytes);
-    log("Message decoded", protoMessage);
     return Promise.resolve({
       payload: protoMessage.payload,
       contentTopic: protoMessage.contentTopic,
@@ -145,13 +151,13 @@ export class Decoder implements IDecoder<DecodedMessage> {
   }
 
   async fromProtoObj(
-    pubSubTopic: string,
+    pubsubTopic: string,
     proto: IProtoMessage
   ): Promise<DecodedMessage | undefined> {
     // https://rfc.vac.dev/spec/14/
     // > If omitted, the value SHOULD be interpreted as version 0.
     if (proto.version ?? 0 !== Version) {
-      log(
+      log.error(
         "Failed to decode due to incorrect version, expected:",
         Version,
         ", actual:",
@@ -160,7 +166,7 @@ export class Decoder implements IDecoder<DecodedMessage> {
       return Promise.resolve(undefined);
     }
 
-    return new DecodedMessage(pubSubTopic, proto);
+    return new DecodedMessage(pubsubTopic, proto);
   }
 }
 
@@ -169,12 +175,14 @@ export class Decoder implements IDecoder<DecodedMessage> {
  *
  * A decoder is used to decode messages from the [14/WAKU2-MESSAGE](https://rfc.vac.dev/spec/14/)
  * format when received from the Waku network. The resulting decoder can then be
- * pass to { @link @waku/interfaces.Filter.subscribe } or
- * { @link @waku/interfaces.Relay.subscribe } to automatically decode incoming
+ * pass to { @link @waku/interfaces!IReceiver.subscribe } to automatically decode incoming
  * messages.
  *
  * @param contentTopic The resulting decoder will only decode messages with this content topic.
  */
-export function createDecoder(contentTopic: string): Decoder {
-  return new Decoder(contentTopic);
+export function createDecoder(
+  contentTopic: string,
+  pubsubTopic: PubSubTopic = DefaultPubSubTopic
+): Decoder {
+  return new Decoder(pubsubTopic, contentTopic);
 }

package/src/lib/store/history_rpc.ts

@@ -11,7 +11,7 @@ export enum PageDirection {
 
 export interface Params {
   contentTopics: string[];
-  pubSubTopic: string;
+  pubsubTopic: string;
   pageDirection: PageDirection;
   pageSize: number;
   startTime?: Date;
@@ -59,7 +59,7 @@ export class HistoryRpc {
     return new HistoryRpc({
       requestId: uuid(),
       query: {
-        pubsubTopic: params.pubSubTopic,
+        pubsubTopic: params.pubsubTopic,
         contentFilters,
         pagingInfo,
         startTime,

package/src/lib/store/index.ts

@@ -6,12 +6,13 @@ import {
   IDecoder,
   IStore,
   Libp2p,
-  ProtocolCreateOptions
+  ProtocolCreateOptions,
+  PubSubTopic
 } from "@waku/interfaces";
 import { proto_store as proto } from "@waku/proto";
-import { isDefined } from "@waku/utils";
+import { ensurePubsubTopicIsConfigured, isDefined } from "@waku/utils";
+import { Logger } from "@waku/utils";
 import { concat, utf8ToBytes } from "@waku/utils/bytes";
-import debug from "debug";
 import all from "it-all";
 import * as lp from "it-length-prefixed";
 import { pipe } from "it-pipe";
@@ -25,7 +26,7 @@ import { HistoryRpc, PageDirection, Params } from "./history_rpc.js";
 
 import HistoryError = proto.HistoryResponse.HistoryError;
 
-const log = debug("waku:store");
+const log = new Logger("store");
 
 export const StoreCodec = "/vac/waku/store/2.0.0-beta4";
 
@@ -74,12 +75,12 @@ export interface QueryOptions {
  * The Waku Store protocol can be used to retrieved historical messages.
  */
 class Store extends BaseProtocol implements IStore {
-  options: ProtocolCreateOptions;
+  private readonly pubsubTopics: PubSubTopic[];
   private readonly NUM_PEERS_PROTOCOL = 1;
 
   constructor(libp2p: Libp2p, options?: ProtocolCreateOptions) {
     super(StoreCodec, libp2p.components);
-    this.options = options ?? {};
+    this.pubsubTopics = options?.pubsubTopics ?? [DefaultPubSubTopic];
   }
 
   /**
@@ -206,12 +207,20 @@ class Store extends BaseProtocol implements IStore {
    * @throws If not able to reach a Waku Store peer to query,
    * or if an error is encountered when processing the reply,
    * or if two decoders with the same content topic are passed.
+   *
+   * This API only supports querying a single pubsub topic at a time.
+   * If multiple decoders are provided, they must all have the same pubsub topic.
+   * @throws If multiple decoders with different pubsub topics are provided.
+   * @throws If no decoders are provided.
+   * @throws If no decoders are found for the provided pubsub topic.
    */
   async *queryGenerator<T extends IDecodedMessage>(
     decoders: IDecoder<T>[],
     options?: QueryOptions
   ): AsyncGenerator<Promise<T | undefined>[]> {
-    const { pubSubTopic = DefaultPubSubTopic } = this.options;
+    if (decoders.length === 0) {
+      throw new Error("No decoders provided");
+    }
 
     let startTime, endTime;
 
@@ -220,6 +229,33 @@ class Store extends BaseProtocol implements IStore {
       endTime = options.timeFilter.endTime;
     }
 
+    // convert array to set to remove duplicates
+    const uniquePubSubTopicsInQuery = Array.from(
+      new Set(decoders.map((decoder) => decoder.pubsubTopic))
+    );
+
+    // If multiple pubsub topics are provided, throw an error
+    if (uniquePubSubTopicsInQuery.length > 1) {
+      throw new Error(
+        "API does not support querying multiple pubsub topics at once"
+      );
+    }
+
+    // we can be certain that there is only one pubsub topic in the query
+    const pubSubTopicForQuery = uniquePubSubTopicsInQuery[0];
+
+    ensurePubsubTopicIsConfigured(pubSubTopicForQuery, this.pubsubTopics);
+
+    // check that the pubsubTopic from the Cursor and Decoder match
+    if (
+      options?.cursor?.pubsubTopic &&
+      options.cursor.pubsubTopic !== pubSubTopicForQuery
+    ) {
+      throw new Error(
+        `Cursor pubsub topic (${options?.cursor?.pubsubTopic}) does not match decoder pubsub topic (${pubSubTopicForQuery})`
+      );
+    }
+
     const decodersAsMap = new Map();
     decoders.forEach((dec) => {
       if (decodersAsMap.has(dec.contentTopic)) {
@@ -230,11 +266,17 @@ class Store extends BaseProtocol implements IStore {
       decodersAsMap.set(dec.contentTopic, dec);
     });
 
-    const contentTopics = decoders.map((dec) => dec.contentTopic);
+    const contentTopics = decoders
+      .filter((decoder) => decoder.pubsubTopic === pubSubTopicForQuery)
+      .map((dec) => dec.contentTopic);
+
+    if (contentTopics.length === 0) {
+      throw new Error("No decoders found for topic " + pubSubTopicForQuery);
+    }
 
     const queryOpts = Object.assign(
       {
-        pubSubTopic: pubSubTopic,
+        pubsubTopic: pubSubTopicForQuery,
         pageDirection: PageDirection.BACKWARD,
         pageSize: DefaultPageSize
       },
@@ -242,8 +284,6 @@ class Store extends BaseProtocol implements IStore {
       { contentTopics, startTime, endTime }
     );
 
-    log("Querying history with the following options", options);
-
     const peer = (
       await this.getPeers({
         numPeers: this.NUM_PEERS_PROTOCOL,
@@ -283,9 +323,9 @@ async function* paginate<T extends IDecodedMessage>(
 
     const historyRpcQuery = HistoryRpc.createQuery(queryOpts);
 
-    log(
+    log.info(
       "Querying store peer",
-      `for (${queryOpts.pubSubTopic})`,
+      `for (${queryOpts.pubsubTopic})`,
       queryOpts.contentTopics
     );
 
@@ -307,7 +347,7 @@ async function* paginate<T extends IDecodedMessage>(
     const reply = historyRpcQuery.decode(bytes);
 
     if (!reply.response) {
-      log("Stopping pagination due to store `response` field missing");
+      log.warn("Stopping pagination due to store `response` field missing");
       break;
     }
 
@@ -318,13 +358,13 @@ async function* paginate<T extends IDecodedMessage>(
     }
 
     if (!response.messages || !response.messages.length) {
-      log(
+      log.warn(
         "Stopping pagination due to store `response.messages` field missing or empty"
       );
       break;
     }
 
-    log(`${response.messages.length} messages retrieved from store`);
+    log.error(`${response.messages.length} messages retrieved from store`);
 
     yield response.messages.map((protoMsg) => {
       const contentTopic = protoMsg.contentTopic;
@@ -332,7 +372,7 @@ async function* paginate<T extends IDecodedMessage>(
         const decoder = decoders.get(contentTopic);
         if (decoder) {
           return decoder.fromProtoObj(
-            queryOpts.pubSubTopic,
+            queryOpts.pubsubTopic,
             toProtoMessage(protoMsg)
           );
         }
@@ -344,7 +384,7 @@ async function* paginate<T extends IDecodedMessage>(
     if (typeof nextCursor === "undefined") {
       // If the server does not return cursor then there is an issue,
       // Need to abort, or we end up in an infinite loop
-      log(
+      log.warn(
         "Stopping pagination due to `response.pagingInfo.cursor` missing from store response"
       );
       break;
@@ -365,10 +405,7 @@ async function* paginate<T extends IDecodedMessage>(
   }
 }
 
-export async function createCursor(
-  message: IDecodedMessage,
-  pubsubTopic: string = DefaultPubSubTopic
-): Promise<Cursor> {
+export async function createCursor(message: IDecodedMessage): Promise<Cursor> {
   if (
     !message ||
     !message.timestamp ||
@@ -386,7 +423,7 @@ export async function createCursor(
 
   return {
     digest,
-    pubsubTopic,
+    pubsubTopic: message.pubsubTopic,
     senderTime: messageTime,
     receiverTime: messageTime
   };