@waku/core 0.0.34-c41b319.0 → 0.0.34-c43cec2.0

package/src/lib/light_push/index.ts

@@ -1 +1,189 @@
-export { LightPushCore, LightPushCodec, PushResponse } from "./light_push.js";
+import type { Peer, Stream } from "@libp2p/interface";
+import {
+  type CoreProtocolResult,
+  type IBaseProtocolCore,
+  type IEncoder,
+  type IMessage,
+  type Libp2p,
+  ProtocolError,
+  PubsubTopic,
+  type ThisOrThat
+} from "@waku/interfaces";
+import { PushResponse } from "@waku/proto";
+import { isMessageSizeUnderCap } from "@waku/utils";
+import { Logger } from "@waku/utils";
+import all from "it-all";
+import * as lp from "it-length-prefixed";
+import { pipe } from "it-pipe";
+import { Uint8ArrayList } from "uint8arraylist";
+
+import { BaseProtocol } from "../base_protocol.js";
+
+import { PushRpc } from "./push_rpc.js";
+import { isRLNResponseError, matchRLNErrorMessage } from "./utils.js";
+
+const log = new Logger("light-push");
+
+export const LightPushCodec = "/vac/waku/lightpush/2.0.0-beta1";
+export { PushResponse };
+
+type PreparePushMessageResult = ThisOrThat<"query", PushRpc>;
+
+/**
+ * Implements the [Waku v2 Light Push protocol](https://rfc.vac.dev/spec/19/).
+ */
+export class LightPushCore extends BaseProtocol implements IBaseProtocolCore {
+  public constructor(
+    public readonly pubsubTopics: PubsubTopic[],
+    libp2p: Libp2p
+  ) {
+    super(LightPushCodec, libp2p.components, pubsubTopics);
+  }
+
+  private async preparePushMessage(
+    encoder: IEncoder,
+    message: IMessage
+  ): Promise<PreparePushMessageResult> {
+    try {
+      if (!message.payload || message.payload.length === 0) {
+        log.error("Failed to send waku light push: payload is empty");
+        return { query: null, error: ProtocolError.EMPTY_PAYLOAD };
+      }
+
+      if (!(await isMessageSizeUnderCap(encoder, message))) {
+        log.error("Failed to send waku light push: message is bigger than 1MB");
+        return { query: null, error: ProtocolError.SIZE_TOO_BIG };
+      }
+
+      const protoMessage = await encoder.toProtoObj(message);
+      if (!protoMessage) {
+        log.error("Failed to encode to protoMessage, aborting push");
+        return {
+          query: null,
+          error: ProtocolError.ENCODE_FAILED
+        };
+      }
+
+      const query = PushRpc.createRequest(protoMessage, encoder.pubsubTopic);
+      return { query, error: null };
+    } catch (error) {
+      log.error("Failed to prepare push message", error);
+
+      return {
+        query: null,
+        error: ProtocolError.GENERIC_FAIL
+      };
+    }
+  }
+
+  // TODO(weboko): use peer.id as parameter instead
+  public async send(
+    encoder: IEncoder,
+    message: IMessage,
+    peer: Peer
+  ): Promise<CoreProtocolResult> {
+    const { query, error: preparationError } = await this.preparePushMessage(
+      encoder,
+      message
+    );
+
+    if (preparationError || !query) {
+      return {
+        success: null,
+        failure: {
+          error: preparationError,
+          peerId: peer.id
+        }
+      };
+    }
+
+    let stream: Stream;
+    try {
+      stream = await this.getStream(peer);
+    } catch (error) {
+      log.error("Failed to get stream", error);
+      return {
+        success: null,
+        failure: {
+          error: ProtocolError.NO_STREAM_AVAILABLE,
+          peerId: peer.id
+        }
+      };
+    }
+
+    let res: Uint8ArrayList[] | undefined;
+    try {
+      res = await pipe(
+        [query.encode()],
+        lp.encode,
+        stream,
+        lp.decode,
+        async (source) => await all(source)
+      );
+    } catch (err) {
+      log.error("Failed to send waku light push request", err);
+      return {
+        success: null,
+        failure: {
+          error: ProtocolError.GENERIC_FAIL,
+          peerId: peer.id
+        }
+      };
+    }
+
+    const bytes = new Uint8ArrayList();
+    res.forEach((chunk) => {
+      bytes.append(chunk);
+    });
+
+    let response: PushResponse | undefined;
+    try {
+      response = PushRpc.decode(bytes).response;
+    } catch (err) {
+      log.error("Failed to decode push reply", err);
+      return {
+        success: null,
+        failure: {
+          error: ProtocolError.DECODE_FAILED,
+          peerId: peer.id
+        }
+      };
+    }
+
+    if (!response) {
+      log.error("Remote peer fault: No response in PushRPC");
+      return {
+        success: null,
+        failure: {
+          error: ProtocolError.NO_RESPONSE,
+          peerId: peer.id
+        }
+      };
+    }
+
+    if (isRLNResponseError(response.info)) {
+      const rlnErrorCase = matchRLNErrorMessage(response.info!);
+      log.error("Remote peer rejected the message: ", rlnErrorCase);
+      return {
+        success: null,
+        failure: {
+          error: rlnErrorCase,
+          peerId: peer.id
+        }
+      };
+    }
+
+    if (!response.isSuccess) {
+      log.error("Remote peer rejected the message: ", response.info);
+      return {
+        success: null,
+        failure: {
+          error: ProtocolError.REMOTE_PEER_REJECTED,
+          peerId: peer.id
+        }
+      };
+    }
+
+    return { success: peer.id, failure: null };
+  }
+}

package/src/lib/metadata/index.ts

@@ -1 +1,182 @@
-export { wakuMetadata, MetadataCodec } from "./metadata.js";
+import type { PeerId } from "@libp2p/interface";
+import { IncomingStreamData } from "@libp2p/interface";
+import {
+  type IMetadata,
+  type Libp2pComponents,
+  type MetadataQueryResult,
+  type PeerIdStr,
+  ProtocolError,
+  PubsubTopic,
+  type ShardInfo
+} from "@waku/interfaces";
+import { proto_metadata } from "@waku/proto";
+import { encodeRelayShard, Logger, pubsubTopicsToShardInfo } from "@waku/utils";
+import all from "it-all";
+import * as lp from "it-length-prefixed";
+import { pipe } from "it-pipe";
+import { Uint8ArrayList } from "uint8arraylist";
+
+import { BaseProtocol } from "../base_protocol.js";
+
+const log = new Logger("metadata");
+
+export const MetadataCodec = "/vac/waku/metadata/1.0.0";
+
+class Metadata extends BaseProtocol implements IMetadata {
+  private libp2pComponents: Libp2pComponents;
+  protected handshakesConfirmed: Map<PeerIdStr, ShardInfo> = new Map();
+
+  public constructor(
+    public pubsubTopics: PubsubTopic[],
+    libp2p: Libp2pComponents
+  ) {
+    super(MetadataCodec, libp2p.components, pubsubTopics);
+    this.libp2pComponents = libp2p;
+    void libp2p.registrar.handle(MetadataCodec, (streamData) => {
+      void this.onRequest(streamData);
+    });
+  }
+
+  /**
+   * Make a metadata query to a peer
+   */
+  public async query(peerId: PeerId): Promise<MetadataQueryResult> {
+    const request = proto_metadata.WakuMetadataRequest.encode(
+      pubsubTopicsToShardInfo(this.pubsubTopics)
+    );
+
+    const peer = await this.libp2pComponents.peerStore.get(peerId);
+    if (!peer) {
+      return {
+        shardInfo: null,
+        error: ProtocolError.NO_PEER_AVAILABLE
+      };
+    }
+
+    let stream;
+    try {
+      stream = await this.getStream(peer);
+    } catch (error) {
+      log.error("Failed to get stream", error);
+      return {
+        shardInfo: null,
+        error: ProtocolError.NO_STREAM_AVAILABLE
+      };
+    }
+
+    const encodedResponse = await pipe(
+      [request],
+      lp.encode,
+      stream,
+      lp.decode,
+      async (source) => await all(source)
+    );
+
+    const { error, shardInfo } = this.decodeMetadataResponse(encodedResponse);
+
+    if (error) {
+      return {
+        shardInfo: null,
+        error
+      };
+    }
+
+    await this.savePeerShardInfo(peerId, shardInfo);
+
+    return {
+      shardInfo,
+      error: null
+    };
+  }
+
+  public async confirmOrAttemptHandshake(
+    peerId: PeerId
+  ): Promise<MetadataQueryResult> {
+    const shardInfo = this.handshakesConfirmed.get(peerId.toString());
+    if (shardInfo) {
+      return {
+        shardInfo,
+        error: null
+      };
+    }
+
+    return await this.query(peerId);
+  }
+
+  /**
+   * Handle an incoming metadata request
+   */
+  private async onRequest(streamData: IncomingStreamData): Promise<void> {
+    try {
+      const { stream, connection } = streamData;
+      const encodedShardInfo = proto_metadata.WakuMetadataResponse.encode(
+        pubsubTopicsToShardInfo(this.pubsubTopics)
+      );
+
+      const encodedResponse = await pipe(
+        [encodedShardInfo],
+        lp.encode,
+        stream,
+        lp.decode,
+        async (source) => await all(source)
+      );
+
+      const { error, shardInfo } = this.decodeMetadataResponse(encodedResponse);
+
+      if (error) {
+        return;
+      }
+
+      await this.savePeerShardInfo(connection.remotePeer, shardInfo);
+    } catch (error) {
+      log.error("Error handling metadata request", error);
+    }
+  }
+
+  private decodeMetadataResponse(
+    encodedResponse: Uint8ArrayList[]
+  ): MetadataQueryResult {
+    const bytes = new Uint8ArrayList();
+
+    encodedResponse.forEach((chunk) => {
+      bytes.append(chunk);
+    });
+    const response = proto_metadata.WakuMetadataResponse.decode(
+      bytes
+    ) as ShardInfo;
+
+    if (!response) {
+      log.error("Error decoding metadata response");
+      return {
+        shardInfo: null,
+        error: ProtocolError.DECODE_FAILED
+      };
+    }
+
+    return {
+      shardInfo: response,
+      error: null
+    };
+  }
+
+  private async savePeerShardInfo(
+    peerId: PeerId,
+    shardInfo: ShardInfo
+  ): Promise<void> {
+    // add or update the shardInfo to peer store
+    await this.libp2pComponents.peerStore.merge(peerId, {
+      metadata: {
+        shardInfo: encodeRelayShard(shardInfo)
+      }
+    });
+
+    this.handshakesConfirmed.set(peerId.toString(), shardInfo);
+  }
+}
+
+export function wakuMetadata(
+  pubsubTopics: PubsubTopic[]
+): (components: Libp2pComponents) => IMetadata {
+  return (components: Libp2pComponents) =>
+    new Metadata(pubsubTopics, components);
+}

package/src/lib/store/index.ts

@@ -1 +1,136 @@
-export { StoreCore, StoreCodec } from "./store.js";
+import type { Peer } from "@libp2p/interface";
+import {
+  IDecodedMessage,
+  IDecoder,
+  IStoreCore,
+  Libp2p,
+  PubsubTopic,
+  QueryRequestParams
+} from "@waku/interfaces";
+import { Logger } from "@waku/utils";
+import all from "it-all";
+import * as lp from "it-length-prefixed";
+import { pipe } from "it-pipe";
+import { Uint8ArrayList } from "uint8arraylist";
+
+import { BaseProtocol } from "../base_protocol.js";
+import { toProtoMessage } from "../to_proto_message.js";
+
+import {
+  DEFAULT_PAGE_SIZE,
+  MAX_PAGE_SIZE,
+  StoreQueryRequest,
+  StoreQueryResponse
+} from "./rpc.js";
+
+const log = new Logger("store");
+
+export const StoreCodec = "/vac/waku/store-query/3.0.0";
+
+export class StoreCore extends BaseProtocol implements IStoreCore {
+  public constructor(
+    public readonly pubsubTopics: PubsubTopic[],
+    libp2p: Libp2p
+  ) {
+    super(StoreCodec, libp2p.components, pubsubTopics);
+  }
+
+  public async *queryPerPage<T extends IDecodedMessage>(
+    queryOpts: QueryRequestParams,
+    decoders: Map<string, IDecoder<T>>,
+    peer: Peer
+  ): AsyncGenerator<Promise<T | undefined>[]> {
+    if (
+      queryOpts.contentTopics.toString() !==
+      Array.from(decoders.keys()).toString()
+    ) {
+      throw new Error(
+        "Internal error, the decoders should match the query's content topics"
+      );
+    }
+
+    let currentCursor = queryOpts.paginationCursor;
+    while (true) {
+      const storeQueryRequest = StoreQueryRequest.create({
+        ...queryOpts,
+        paginationCursor: currentCursor
+      });
+
+      let stream;
+      try {
+        stream = await this.getStream(peer);
+      } catch (e) {
+        log.error("Failed to get stream", e);
+        break;
+      }
+
+      const res = await pipe(
+        [storeQueryRequest.encode()],
+        lp.encode,
+        stream,
+        lp.decode,
+        async (source) => await all(source)
+      );
+
+      const bytes = new Uint8ArrayList();
+      res.forEach((chunk) => {
+        bytes.append(chunk);
+      });
+
+      const storeQueryResponse = StoreQueryResponse.decode(bytes);
+
+      if (
+        !storeQueryResponse.statusCode ||
+        storeQueryResponse.statusCode >= 300
+      ) {
+        const errorMessage = `Store query failed with status code: ${storeQueryResponse.statusCode}, description: ${storeQueryResponse.statusDesc}`;
+        log.error(errorMessage);
+        throw new Error(errorMessage);
+      }
+
+      if (!storeQueryResponse.messages || !storeQueryResponse.messages.length) {
+        log.warn("Stopping pagination due to empty messages in response");
+        break;
+      }
+
+      log.info(
+        `${storeQueryResponse.messages.length} messages retrieved from store`
+      );
+
+      const decodedMessages = storeQueryResponse.messages.map((protoMsg) => {
+        if (!protoMsg.message) {
+          return Promise.resolve(undefined);
+        }
+        const contentTopic = protoMsg.message.contentTopic;
+        if (contentTopic) {
+          const decoder = decoders.get(contentTopic);
+          if (decoder) {
+            return decoder.fromProtoObj(
+              protoMsg.pubsubTopic || "",
+              toProtoMessage(protoMsg.message)
+            );
+          }
+        }
+        return Promise.resolve(undefined);
+      });
+
+      yield decodedMessages;
+
+      if (queryOpts.paginationForward) {
+        currentCursor =
+          storeQueryResponse.messages[storeQueryResponse.messages.length - 1]
+            .messageHash;
+      } else {
+        currentCursor = storeQueryResponse.messages[0].messageHash;
+      }
+
+      if (
+        storeQueryResponse.messages.length > MAX_PAGE_SIZE &&
+        storeQueryResponse.messages.length <
+          (queryOpts.paginationLimit || DEFAULT_PAGE_SIZE)
+      ) {
+        break;
+      }
+    }
+  }
+}

package/src/lib/stream_manager/stream_manager.ts

@@ -21,38 +21,39 @@ export class StreamManager {
     this.addEventListener("peer:update", this.handlePeerUpdateStreamPool);
   }
 
-  public async getStream(peerId: PeerId): Promise<Stream> {
-    const peerIdStr = peerId.toString();
-    const scheduledStream = this.streamPool.get(peerIdStr);
+  public async getStream(peer: Peer): Promise<Stream> {
+    const peerId = peer.id.toString();
+
+    const scheduledStream = this.streamPool.get(peerId);
 
     if (scheduledStream) {
-      this.streamPool.delete(peerIdStr);
+      this.streamPool.delete(peerId);
       await scheduledStream;
     }
 
-    let stream = this.getOpenStreamForCodec(peerId);
+    let stream = this.getOpenStreamForCodec(peer.id);
 
     if (stream) {
       this.log.info(
-        `Found existing stream peerId=${peerIdStr} multicodec=${this.multicodec}`
+        `Found existing stream peerId=${peer.id.toString()} multicodec=${this.multicodec}`
       );
-      this.lockStream(peerIdStr, stream);
+      this.lockStream(peer.id.toString(), stream);
       return stream;
     }
 
-    stream = await this.createStream(peerId);
-    this.lockStream(peerIdStr, stream);
+    stream = await this.createStream(peer);
+    this.lockStream(peer.id.toString(), stream);
 
     return stream;
   }
 
-  private async createStream(peerId: PeerId, retries = 0): Promise<Stream> {
-    const connections = this.getConnections(peerId);
+  private async createStream(peer: Peer, retries = 0): Promise<Stream> {
+    const connections = this.getConnections(peer.id);
     const connection = selectOpenConnection(connections);
 
     if (!connection) {
       throw new Error(
-        `Failed to get a connection to the peer peerId=${peerId.toString()} multicodec=${this.multicodec}`
+        `Failed to get a connection to the peer peerId=${peer.id.toString()} multicodec=${this.multicodec}`
       );
     }
 
@@ -62,11 +63,11 @@ export class StreamManager {
     for (let i = 0; i < retries + 1; i++) {
       try {
         this.log.info(
-          `Attempting to create a stream for peerId=${peerId.toString()} multicodec=${this.multicodec}`
+          `Attempting to create a stream for peerId=${peer.id.toString()} multicodec=${this.multicodec}`
         );
         stream = await connection.newStream(this.multicodec);
         this.log.info(
-          `Created stream for peerId=${peerId.toString()} multicodec=${this.multicodec}`
+          `Created stream for peerId=${peer.id.toString()} multicodec=${this.multicodec}`
         );
         break;
       } catch (error) {
@@ -76,7 +77,8 @@ export class StreamManager {
 
     if (!stream) {
       throw new Error(
-        `Failed to create a new stream for ${peerId.toString()} -- ` + lastError
+        `Failed to create a new stream for ${peer.id.toString()} -- ` +
+          lastError
       );
     }
 
@@ -95,7 +97,7 @@ export class StreamManager {
 
     try {
       this.ongoingCreation.add(peerId);
-      await this.createStream(peer.id);
+      await this.createStream(peer);
     } catch (error) {
       this.log.error(`Failed to createStreamWithLock:`, error);
     } finally {

package/dist/lib/filter/filter.d.ts

@@ -1,18 +0,0 @@
-import type { PeerId } from "@libp2p/interface";
-import { type ContentTopic, type CoreProtocolResult, type IBaseProtocolCore, type Libp2p, type PubsubTopic } from "@waku/interfaces";
-import { WakuMessage } from "@waku/proto";
-import { BaseProtocol } from "../base_protocol.js";
-export declare const FilterCodecs: {
-    SUBSCRIBE: string;
-    PUSH: string;
-};
-export declare class FilterCore extends BaseProtocol implements IBaseProtocolCore {
-    private handleIncomingMessage;
-    readonly pubsubTopics: PubsubTopic[];
-    constructor(handleIncomingMessage: (pubsubTopic: PubsubTopic, wakuMessage: WakuMessage, peerIdStr: string) => Promise<void>, pubsubTopics: PubsubTopic[], libp2p: Libp2p);
-    subscribe(pubsubTopic: PubsubTopic, peerId: PeerId, contentTopics: ContentTopic[]): Promise<CoreProtocolResult>;
-    unsubscribe(pubsubTopic: PubsubTopic, peerId: PeerId, contentTopics: ContentTopic[]): Promise<CoreProtocolResult>;
-    unsubscribeAll(pubsubTopic: PubsubTopic, peerId: PeerId): Promise<CoreProtocolResult>;
-    ping(peerId: PeerId): Promise<CoreProtocolResult>;
-    private onRequest;
-}