@waku/core 0.0.34-9f1d8ca.0 → 0.0.34-c41b319.0

package/src/lib/metadata/metadata.ts

@@ -0,0 +1,182 @@
+import type { PeerId } from "@libp2p/interface";
+import { IncomingStreamData } from "@libp2p/interface";
+import {
+  type IMetadata,
+  type Libp2pComponents,
+  type MetadataQueryResult,
+  type PeerIdStr,
+  ProtocolError,
+  PubsubTopic,
+  type ShardInfo
+} from "@waku/interfaces";
+import { proto_metadata } from "@waku/proto";
+import { encodeRelayShard, Logger, pubsubTopicsToShardInfo } from "@waku/utils";
+import all from "it-all";
+import * as lp from "it-length-prefixed";
+import { pipe } from "it-pipe";
+import { Uint8ArrayList } from "uint8arraylist";
+
+import { BaseProtocol } from "../base_protocol.js";
+
+const log = new Logger("metadata");
+
+export const MetadataCodec = "/vac/waku/metadata/1.0.0";
+
+class Metadata extends BaseProtocol implements IMetadata {
+  private libp2pComponents: Libp2pComponents;
+  protected handshakesConfirmed: Map<PeerIdStr, ShardInfo> = new Map();
+
+  public constructor(
+    public pubsubTopics: PubsubTopic[],
+    libp2p: Libp2pComponents
+  ) {
+    super(MetadataCodec, libp2p.components, pubsubTopics);
+    this.libp2pComponents = libp2p;
+    void libp2p.registrar.handle(MetadataCodec, (streamData) => {
+      void this.onRequest(streamData);
+    });
+  }
+
+  /**
+   * Make a metadata query to a peer
+   */
+  public async query(peerId: PeerId): Promise<MetadataQueryResult> {
+    const request = proto_metadata.WakuMetadataRequest.encode(
+      pubsubTopicsToShardInfo(this.pubsubTopics)
+    );
+
+    const peer = await this.libp2pComponents.peerStore.get(peerId);
+    if (!peer) {
+      return {
+        shardInfo: null,
+        error: ProtocolError.NO_PEER_AVAILABLE
+      };
+    }
+
+    let stream;
+    try {
+      stream = await this.getStream(peerId);
+    } catch (error) {
+      log.error("Failed to get stream", error);
+      return {
+        shardInfo: null,
+        error: ProtocolError.NO_STREAM_AVAILABLE
+      };
+    }
+
+    const encodedResponse = await pipe(
+      [request],
+      lp.encode,
+      stream,
+      lp.decode,
+      async (source) => await all(source)
+    );
+
+    const { error, shardInfo } = this.decodeMetadataResponse(encodedResponse);
+
+    if (error) {
+      return {
+        shardInfo: null,
+        error
+      };
+    }
+
+    await this.savePeerShardInfo(peerId, shardInfo);
+
+    return {
+      shardInfo,
+      error: null
+    };
+  }
+
+  public async confirmOrAttemptHandshake(
+    peerId: PeerId
+  ): Promise<MetadataQueryResult> {
+    const shardInfo = this.handshakesConfirmed.get(peerId.toString());
+    if (shardInfo) {
+      return {
+        shardInfo,
+        error: null
+      };
+    }
+
+    return await this.query(peerId);
+  }
+
+  /**
+   * Handle an incoming metadata request
+   */
+  private async onRequest(streamData: IncomingStreamData): Promise<void> {
+    try {
+      const { stream, connection } = streamData;
+      const encodedShardInfo = proto_metadata.WakuMetadataResponse.encode(
+        pubsubTopicsToShardInfo(this.pubsubTopics)
+      );
+
+      const encodedResponse = await pipe(
+        [encodedShardInfo],
+        lp.encode,
+        stream,
+        lp.decode,
+        async (source) => await all(source)
+      );
+
+      const { error, shardInfo } = this.decodeMetadataResponse(encodedResponse);
+
+      if (error) {
+        return;
+      }
+
+      await this.savePeerShardInfo(connection.remotePeer, shardInfo);
+    } catch (error) {
+      log.error("Error handling metadata request", error);
+    }
+  }
+
+  private decodeMetadataResponse(
+    encodedResponse: Uint8ArrayList[]
+  ): MetadataQueryResult {
+    const bytes = new Uint8ArrayList();
+
+    encodedResponse.forEach((chunk) => {
+      bytes.append(chunk);
+    });
+    const response = proto_metadata.WakuMetadataResponse.decode(
+      bytes
+    ) as ShardInfo;
+
+    if (!response) {
+      log.error("Error decoding metadata response");
+      return {
+        shardInfo: null,
+        error: ProtocolError.DECODE_FAILED
+      };
+    }
+
+    return {
+      shardInfo: response,
+      error: null
+    };
+  }
+
+  private async savePeerShardInfo(
+    peerId: PeerId,
+    shardInfo: ShardInfo
+  ): Promise<void> {
+    // add or update the shardInfo to peer store
+    await this.libp2pComponents.peerStore.merge(peerId, {
+      metadata: {
+        shardInfo: encodeRelayShard(shardInfo)
+      }
+    });
+
+    this.handshakesConfirmed.set(peerId.toString(), shardInfo);
+  }
+}
+
+export function wakuMetadata(
+  pubsubTopics: PubsubTopic[]
+): (components: Libp2pComponents) => IMetadata {
+  return (components: Libp2pComponents) =>
+    new Metadata(pubsubTopics, components);
+}

package/src/lib/store/index.ts

@@ -1,136 +1 @@
-import type { Peer } from "@libp2p/interface";
-import {
-  IDecodedMessage,
-  IDecoder,
-  IStoreCore,
-  Libp2p,
-  PubsubTopic,
-  QueryRequestParams
-} from "@waku/interfaces";
-import { Logger } from "@waku/utils";
-import all from "it-all";
-import * as lp from "it-length-prefixed";
-import { pipe } from "it-pipe";
-import { Uint8ArrayList } from "uint8arraylist";
-
-import { BaseProtocol } from "../base_protocol.js";
-import { toProtoMessage } from "../to_proto_message.js";
-
-import {
-  DEFAULT_PAGE_SIZE,
-  MAX_PAGE_SIZE,
-  StoreQueryRequest,
-  StoreQueryResponse
-} from "./rpc.js";
-
-const log = new Logger("store");
-
-export const StoreCodec = "/vac/waku/store-query/3.0.0";
-
-export class StoreCore extends BaseProtocol implements IStoreCore {
-  public constructor(
-    public readonly pubsubTopics: PubsubTopic[],
-    libp2p: Libp2p
-  ) {
-    super(StoreCodec, libp2p.components, log, pubsubTopics);
-  }
-
-  public async *queryPerPage<T extends IDecodedMessage>(
-    queryOpts: QueryRequestParams,
-    decoders: Map<string, IDecoder<T>>,
-    peer: Peer
-  ): AsyncGenerator<Promise<T | undefined>[]> {
-    if (
-      queryOpts.contentTopics.toString() !==
-      Array.from(decoders.keys()).toString()
-    ) {
-      throw new Error(
-        "Internal error, the decoders should match the query's content topics"
-      );
-    }
-
-    let currentCursor = queryOpts.paginationCursor;
-    while (true) {
-      const storeQueryRequest = StoreQueryRequest.create({
-        ...queryOpts,
-        paginationCursor: currentCursor
-      });
-
-      let stream;
-      try {
-        stream = await this.getStream(peer);
-      } catch (e) {
-        log.error("Failed to get stream", e);
-        break;
-      }
-
-      const res = await pipe(
-        [storeQueryRequest.encode()],
-        lp.encode,
-        stream,
-        lp.decode,
-        async (source) => await all(source)
-      );
-
-      const bytes = new Uint8ArrayList();
-      res.forEach((chunk) => {
-        bytes.append(chunk);
-      });
-
-      const storeQueryResponse = StoreQueryResponse.decode(bytes);
-
-      if (
-        !storeQueryResponse.statusCode ||
-        storeQueryResponse.statusCode >= 300
-      ) {
-        const errorMessage = `Store query failed with status code: ${storeQueryResponse.statusCode}, description: ${storeQueryResponse.statusDesc}`;
-        log.error(errorMessage);
-        throw new Error(errorMessage);
-      }
-
-      if (!storeQueryResponse.messages || !storeQueryResponse.messages.length) {
-        log.warn("Stopping pagination due to empty messages in response");
-        break;
-      }
-
-      log.info(
-        `${storeQueryResponse.messages.length} messages retrieved from store`
-      );
-
-      const decodedMessages = storeQueryResponse.messages.map((protoMsg) => {
-        if (!protoMsg.message) {
-          return Promise.resolve(undefined);
-        }
-        const contentTopic = protoMsg.message.contentTopic;
-        if (contentTopic) {
-          const decoder = decoders.get(contentTopic);
-          if (decoder) {
-            return decoder.fromProtoObj(
-              protoMsg.pubsubTopic || "",
-              toProtoMessage(protoMsg.message)
-            );
-          }
-        }
-        return Promise.resolve(undefined);
-      });
-
-      yield decodedMessages;
-
-      if (queryOpts.paginationForward) {
-        currentCursor =
-          storeQueryResponse.messages[storeQueryResponse.messages.length - 1]
-            .messageHash;
-      } else {
-        currentCursor = storeQueryResponse.messages[0].messageHash;
-      }
-
-      if (
-        storeQueryResponse.messages.length > MAX_PAGE_SIZE &&
-        storeQueryResponse.messages.length <
-          (queryOpts.paginationLimit || DEFAULT_PAGE_SIZE)
-      ) {
-        break;
-      }
-    }
-  }
-}
+export { StoreCore, StoreCodec } from "./store.js";

package/src/lib/store/store.ts

@@ -0,0 +1,136 @@
+import type { PeerId } from "@libp2p/interface";
+import {
+  IDecodedMessage,
+  IDecoder,
+  IStoreCore,
+  Libp2p,
+  PubsubTopic,
+  QueryRequestParams
+} from "@waku/interfaces";
+import { Logger } from "@waku/utils";
+import all from "it-all";
+import * as lp from "it-length-prefixed";
+import { pipe } from "it-pipe";
+import { Uint8ArrayList } from "uint8arraylist";
+
+import { BaseProtocol } from "../base_protocol.js";
+import { toProtoMessage } from "../to_proto_message.js";
+
+import {
+  DEFAULT_PAGE_SIZE,
+  MAX_PAGE_SIZE,
+  StoreQueryRequest,
+  StoreQueryResponse
+} from "./rpc.js";
+
+const log = new Logger("store");
+
+export const StoreCodec = "/vac/waku/store-query/3.0.0";
+
+export class StoreCore extends BaseProtocol implements IStoreCore {
+  public constructor(
+    public readonly pubsubTopics: PubsubTopic[],
+    libp2p: Libp2p
+  ) {
+    super(StoreCodec, libp2p.components, pubsubTopics);
+  }
+
+  public async *queryPerPage<T extends IDecodedMessage>(
+    queryOpts: QueryRequestParams,
+    decoders: Map<string, IDecoder<T>>,
+    peerId: PeerId
+  ): AsyncGenerator<Promise<T | undefined>[]> {
+    if (
+      queryOpts.contentTopics.toString() !==
+      Array.from(decoders.keys()).toString()
+    ) {
+      throw new Error(
+        "Internal error, the decoders should match the query's content topics"
+      );
+    }
+
+    let currentCursor = queryOpts.paginationCursor;
+    while (true) {
+      const storeQueryRequest = StoreQueryRequest.create({
+        ...queryOpts,
+        paginationCursor: currentCursor
+      });
+
+      let stream;
+      try {
+        stream = await this.getStream(peerId);
+      } catch (e) {
+        log.error("Failed to get stream", e);
+        break;
+      }
+
+      const res = await pipe(
+        [storeQueryRequest.encode()],
+        lp.encode,
+        stream,
+        lp.decode,
+        async (source) => await all(source)
+      );
+
+      const bytes = new Uint8ArrayList();
+      res.forEach((chunk) => {
+        bytes.append(chunk);
+      });
+
+      const storeQueryResponse = StoreQueryResponse.decode(bytes);
+
+      if (
+        !storeQueryResponse.statusCode ||
+        storeQueryResponse.statusCode >= 300
+      ) {
+        const errorMessage = `Store query failed with status code: ${storeQueryResponse.statusCode}, description: ${storeQueryResponse.statusDesc}`;
+        log.error(errorMessage);
+        throw new Error(errorMessage);
+      }
+
+      if (!storeQueryResponse.messages || !storeQueryResponse.messages.length) {
+        log.warn("Stopping pagination due to empty messages in response");
+        break;
+      }
+
+      log.info(
+        `${storeQueryResponse.messages.length} messages retrieved from store`
+      );
+
+      const decodedMessages = storeQueryResponse.messages.map((protoMsg) => {
+        if (!protoMsg.message) {
+          return Promise.resolve(undefined);
+        }
+        const contentTopic = protoMsg.message.contentTopic;
+        if (contentTopic) {
+          const decoder = decoders.get(contentTopic);
+          if (decoder) {
+            return decoder.fromProtoObj(
+              protoMsg.pubsubTopic || "",
+              toProtoMessage(protoMsg.message)
+            );
+          }
+        }
+        return Promise.resolve(undefined);
+      });
+
+      yield decodedMessages;
+
+      if (queryOpts.paginationForward) {
+        currentCursor =
+          storeQueryResponse.messages[storeQueryResponse.messages.length - 1]
+            .messageHash;
+      } else {
+        currentCursor = storeQueryResponse.messages[0].messageHash;
+      }
+
+      if (
+        storeQueryResponse.messages.length > MAX_PAGE_SIZE &&
+        storeQueryResponse.messages.length <
+          (queryOpts.paginationLimit || DEFAULT_PAGE_SIZE)
+      ) {
+        break;
+      }
+    }
+  }
+}

package/src/lib/stream_manager/stream_manager.ts

@@ -21,39 +21,38 @@ export class StreamManager {
     this.addEventListener("peer:update", this.handlePeerUpdateStreamPool);
   }
 
-  public async getStream(peer: Peer): Promise<Stream> {
-    const peerId = peer.id.toString();
-
-    const scheduledStream = this.streamPool.get(peerId);
+  public async getStream(peerId: PeerId): Promise<Stream> {
+    const peerIdStr = peerId.toString();
+    const scheduledStream = this.streamPool.get(peerIdStr);
 
     if (scheduledStream) {
-      this.streamPool.delete(peerId);
+      this.streamPool.delete(peerIdStr);
       await scheduledStream;
     }
 
-    let stream = this.getOpenStreamForCodec(peer.id);
+    let stream = this.getOpenStreamForCodec(peerId);
 
     if (stream) {
       this.log.info(
-        `Found existing stream peerId=${peer.id.toString()} multicodec=${this.multicodec}`
+        `Found existing stream peerId=${peerIdStr} multicodec=${this.multicodec}`
       );
-      this.lockStream(peer.id.toString(), stream);
+      this.lockStream(peerIdStr, stream);
       return stream;
     }
 
-    stream = await this.createStream(peer);
-    this.lockStream(peer.id.toString(), stream);
+    stream = await this.createStream(peerId);
+    this.lockStream(peerIdStr, stream);
 
     return stream;
   }
 
-  private async createStream(peer: Peer, retries = 0): Promise<Stream> {
-    const connections = this.getConnections(peer.id);
+  private async createStream(peerId: PeerId, retries = 0): Promise<Stream> {
+    const connections = this.getConnections(peerId);
     const connection = selectOpenConnection(connections);
 
     if (!connection) {
       throw new Error(
-        `Failed to get a connection to the peer peerId=${peer.id.toString()} multicodec=${this.multicodec}`
+        `Failed to get a connection to the peer peerId=${peerId.toString()} multicodec=${this.multicodec}`
       );
     }
 
@@ -63,11 +62,11 @@ export class StreamManager {
     for (let i = 0; i < retries + 1; i++) {
       try {
         this.log.info(
-          `Attempting to create a stream for peerId=${peer.id.toString()} multicodec=${this.multicodec}`
+          `Attempting to create a stream for peerId=${peerId.toString()} multicodec=${this.multicodec}`
         );
         stream = await connection.newStream(this.multicodec);
         this.log.info(
-          `Created stream for peerId=${peer.id.toString()} multicodec=${this.multicodec}`
+          `Created stream for peerId=${peerId.toString()} multicodec=${this.multicodec}`
         );
         break;
       } catch (error) {
@@ -77,8 +76,7 @@ export class StreamManager {
 
     if (!stream) {
       throw new Error(
-        `Failed to create a new stream for ${peer.id.toString()} -- ` +
-          lastError
+        `Failed to create a new stream for ${peerId.toString()} -- ` + lastError
       );
     }
 
@@ -97,7 +95,7 @@ export class StreamManager {
 
     try {
       this.ongoingCreation.add(peerId);
-      await this.createStream(peer);
+      await this.createStream(peer.id);
     } catch (error) {
       this.log.error(`Failed to createStreamWithLock:`, error);
     } finally {