@waku/core 0.0.33-d3301ff.0 → 0.0.33-de10ff4.0

package/src/lib/stream_manager/stream_manager.ts

@@ -1,115 +1,173 @@
-import type { PeerUpdate, Stream } from "@libp2p/interface";
-import type { Peer, PeerId } from "@libp2p/interface";
-import { Libp2p } from "@waku/interfaces";
+import type { Peer, PeerId, PeerUpdate, Stream } from "@libp2p/interface";
+import type { Libp2p } from "@waku/interfaces";
 import { Logger } from "@waku/utils";
 
-import { selectConnection } from "./utils.js";
+import { selectOpenConnection } from "./utils.js";
 
-const CONNECTION_TIMEOUT = 5_000;
-const RETRY_BACKOFF_BASE = 1_000;
-const MAX_RETRIES = 3;
+const STREAM_LOCK_KEY = "consumed";
 
 export class StreamManager {
-  private readonly streamPool: Map<string, Promise<Stream | void>>;
   private readonly log: Logger;
 
+  private ongoingCreation: Set<string> = new Set();
+  private streamPool: Map<string, Promise<void>> = new Map();
+
   public constructor(
-    public multicodec: string,
-    public getConnections: Libp2p["getConnections"],
-    public addEventListener: Libp2p["addEventListener"]
+    private multicodec: string,
+    private getConnections: Libp2p["getConnections"],
+    private addEventListener: Libp2p["addEventListener"]
   ) {
     this.log = new Logger(`stream-manager:${multicodec}`);
-    this.streamPool = new Map();
     this.addEventListener("peer:update", this.handlePeerUpdateStreamPool);
   }
 
   public async getStream(peer: Peer): Promise<Stream> {
-    const peerIdStr = peer.id.toString();
-    const streamPromise = this.streamPool.get(peerIdStr);
+    const peerId = peer.id.toString();
+
+    const scheduledStream = this.streamPool.get(peerId);
 
-    if (!streamPromise) {
-      return this.createStream(peer);
+    if (scheduledStream) {
+      this.streamPool.delete(peerId);
+      await scheduledStream;
     }
 
-    this.streamPool.delete(peerIdStr);
-    this.prepareStream(peer);
+    let stream = this.getOpenStreamForCodec(peer.id);
 
-    try {
-      const stream = await streamPromise;
-      if (stream && stream.status !== "closed") {
-        return stream;
-      }
-    } catch (error) {
-      this.log.warn(`Failed to get stream for ${peerIdStr} -- `, error);
-      this.log.warn("Attempting to create a new stream for the peer");
+    if (stream) {
+      this.log.info(
+        `Found existing stream peerId=${peer.id.toString()} multicodec=${this.multicodec}`
+      );
+      this.lockStream(peer.id.toString(), stream);
+      return stream;
     }
 
-    return this.createStream(peer);
+    stream = await this.createStream(peer);
+    this.lockStream(peer.id.toString(), stream);
+
+    return stream;
   }
 
   private async createStream(peer: Peer, retries = 0): Promise<Stream> {
     const connections = this.getConnections(peer.id);
-    const connection = selectConnection(connections);
+    const connection = selectOpenConnection(connections);
 
     if (!connection) {
-      throw new Error("Failed to get a connection to the peer");
+      throw new Error(
+        `Failed to get a connection to the peer peerId=${peer.id.toString()} multicodec=${this.multicodec}`
+      );
     }
 
-    try {
-      return await connection.newStream(this.multicodec);
-    } catch (error) {
-      if (retries < MAX_RETRIES) {
-        const backoff = RETRY_BACKOFF_BASE * Math.pow(2, retries);
-        await new Promise((resolve) => setTimeout(resolve, backoff));
-        return this.createStream(peer, retries + 1);
+    let lastError: unknown;
+    let stream: Stream | undefined;
+
+    for (let i = 0; i < retries + 1; i++) {
+      try {
+        this.log.info(
+          `Attempting to create a stream for peerId=${peer.id.toString()} multicodec=${this.multicodec}`
+        );
+        stream = await connection.newStream(this.multicodec);
+        this.log.info(
+          `Created stream for peerId=${peer.id.toString()} multicodec=${this.multicodec}`
+        );
+        break;
+      } catch (error) {
+        lastError = error;
       }
+    }
+
+    if (!stream) {
       throw new Error(
-        `Failed to create a new stream for ${peer.id.toString()} -- ` + error
+        `Failed to create a new stream for ${peer.id.toString()} -- ` +
+          lastError
       );
     }
+
+    return stream;
   }
 
-  private prepareStream(peer: Peer): void {
-    const timeoutPromise = new Promise<void>((resolve) =>
-      setTimeout(resolve, CONNECTION_TIMEOUT)
-    );
+  private async createStreamWithLock(peer: Peer): Promise<void> {
+    const peerId = peer.id.toString();
 
-    const streamPromise = Promise.race([
-      this.createStream(peer),
-      timeoutPromise.then(() => {
-        throw new Error("Connection timeout");
-      })
-    ]).catch((error) => {
-      this.log.error(
-        `Failed to prepare a new stream for ${peer.id.toString()} -- `,
-        error
+    if (this.ongoingCreation.has(peerId)) {
+      this.log.info(
+        `Skipping creation of a stream due to lock for peerId=${peerId} multicodec=${this.multicodec}`
       );
-    });
+      return;
+    }
+
+    try {
+      this.ongoingCreation.add(peerId);
+      await this.createStream(peer);
+    } catch (error) {
+      this.log.error(`Failed to createStreamWithLock:`, error);
+    } finally {
+      this.ongoingCreation.delete(peerId);
+    }
 
-    this.streamPool.set(peer.id.toString(), streamPromise);
+    return;
   }
 
   private handlePeerUpdateStreamPool = (evt: CustomEvent<PeerUpdate>): void => {
     const { peer } = evt.detail;
 
-    if (peer.protocols.includes(this.multicodec)) {
-      const isConnected = this.isConnectedTo(peer.id);
+    if (!peer.protocols.includes(this.multicodec)) {
+      return;
+    }
+
+    const stream = this.getOpenStreamForCodec(peer.id);
 
-      if (isConnected) {
-        this.log.info(`Preemptively opening a stream to ${peer.id.toString()}`);
-        this.prepareStream(peer);
-      } else {
-        const peerIdStr = peer.id.toString();
-        this.streamPool.delete(peerIdStr);
-        this.log.info(
-          `Removed pending stream for disconnected peer ${peerIdStr}`
-        );
-      }
+    if (stream) {
+      return;
     }
+
+    this.scheduleNewStream(peer);
   };
 
-  private isConnectedTo(peerId: PeerId): boolean {
+  private scheduleNewStream(peer: Peer): void {
+    this.log.info(
+      `Scheduling creation of a stream for peerId=${peer.id.toString()} multicodec=${this.multicodec}`
+    );
+
+    // abandon previous attempt
+    if (this.streamPool.has(peer.id.toString())) {
+      this.streamPool.delete(peer.id.toString());
+    }
+
+    this.streamPool.set(peer.id.toString(), this.createStreamWithLock(peer));
+  }
+
+  private getOpenStreamForCodec(peerId: PeerId): Stream | undefined {
     const connections = this.getConnections(peerId);
-    return connections.some((connection) => connection.status === "open");
+    const connection = selectOpenConnection(connections);
+
+    if (!connection) {
+      return;
+    }
+
+    const stream = connection.streams.find(
+      (s) => s.protocol === this.multicodec
+    );
+
+    if (!stream) {
+      return;
+    }
+
+    const isStreamUnusable = ["done", "closed", "closing"].includes(
+      stream.writeStatus || ""
+    );
+    if (isStreamUnusable || this.isStreamLocked(stream)) {
+      return;
+    }
+
+    return stream;
+  }
+
+  private lockStream(peerId: string, stream: Stream): void {
+    this.log.info(`Locking stream for peerId:${peerId}\tstreamId:${stream.id}`);
+    stream.metadata[STREAM_LOCK_KEY] = true;
+  }
+
+  private isStreamLocked(stream: Stream): boolean {
+    return !!stream.metadata[STREAM_LOCK_KEY];
   }
 }

package/src/lib/stream_manager/utils.ts

@@ -1,22 +1,10 @@
 import type { Connection } from "@libp2p/interface";
 
-export function selectConnection(
+export function selectOpenConnection(
   connections: Connection[]
 ): Connection | undefined {
-  if (!connections.length) return;
-  if (connections.length === 1) return connections[0];
-
-  let latestConnection: Connection | undefined;
-
-  connections.forEach((connection) => {
-    if (connection.status === "open") {
-      if (!latestConnection) {
-        latestConnection = connection;
-      } else if (connection.timeline.open > latestConnection.timeline.open) {
-        latestConnection = connection;
-      }
-    }
-  });
-
-  return latestConnection;
+  return connections
+    .filter((c) => c.status === "open")
+    .sort((left, right) => right.timeline.open - left.timeline.open)
+    .at(0);
 }

package/bundle/base_protocol-C47QkJ2o.js

@@ -1,335 +0,0 @@
-import { h as bytesToUtf8, T as Tags, L as Logger, e as pubsubTopicsToShardInfo } from './index-tdQNdKHx.js';
-
-const decodeRelayShard = (bytes) => {
-    // explicitly converting to Uint8Array to avoid Buffer
-    // https://github.com/libp2p/js-libp2p/issues/2146
-    bytes = new Uint8Array(bytes);
-    if (bytes.length < 3)
-        throw new Error("Insufficient data");
-    const view = new DataView(bytes.buffer);
-    const clusterId = view.getUint16(0);
-    const shards = [];
-    if (bytes.length === 130) {
-        // rsv format (Bit Vector)
-        for (let i = 0; i < 1024; i++) {
-            const byteIndex = Math.floor(i / 8) + 2; // Adjusted for the 2-byte cluster field
-            const bitIndex = 7 - (i % 8);
-            if (view.getUint8(byteIndex) & (1 << bitIndex)) {
-                shards.push(i);
-            }
-        }
-    }
-    else {
-        // rs format (Index List)
-        const numIndices = view.getUint8(2);
-        for (let i = 0, offset = 3; i < numIndices; i++, offset += 2) {
-            if (offset + 1 >= bytes.length)
-                throw new Error("Unexpected end of data");
-            shards.push(view.getUint16(offset));
-        }
-    }
-    return { clusterId, shards };
-};
-const encodeRelayShard = (shardInfo) => {
-    const { clusterId, shards } = shardInfo;
-    const totalLength = shards.length >= 64 ? 130 : 3 + 2 * shards.length;
-    const buffer = new ArrayBuffer(totalLength);
-    const view = new DataView(buffer);
-    view.setUint16(0, clusterId);
-    if (shards.length >= 64) {
-        // rsv format (Bit Vector)
-        for (const index of shards) {
-            const byteIndex = Math.floor(index / 8) + 2; // Adjusted for the 2-byte cluster field
-            const bitIndex = 7 - (index % 8);
-            view.setUint8(byteIndex, view.getUint8(byteIndex) | (1 << bitIndex));
-        }
-    }
-    else {
-        // rs format (Index List)
-        view.setUint8(2, shards.length);
-        for (let i = 0, offset = 3; i < shards.length; i++, offset += 2) {
-            view.setUint16(offset, shards[i]);
-        }
-    }
-    return new Uint8Array(buffer);
-};
-
-/**
- * Function to sort peers by latency from lowest to highest
- * @param peerStore - The Libp2p PeerStore
- * @param peers - The list of peers to choose from
- * @returns Sorted array of peers by latency
- */
-async function sortPeersByLatency(peerStore, peers) {
-    if (peers.length === 0)
-        return [];
-    const results = await Promise.all(peers.map(async (peer) => {
-        try {
-            const pingBytes = (await peerStore.get(peer.id)).metadata.get("ping");
-            if (!pingBytes)
-                return { peer, ping: Infinity };
-            const ping = Number(bytesToUtf8(pingBytes));
-            return { peer, ping };
-        }
-        catch (error) {
-            return { peer, ping: Infinity };
-        }
-    }));
-    // filter out null values
-    const validResults = results.filter((result) => result !== null);
-    return validResults
-        .sort((a, b) => a.ping - b.ping)
-        .map((result) => result.peer);
-}
-/**
- * Returns the list of peers that supports the given protocol.
- */
-async function getPeersForProtocol(peerStore, protocols) {
-    const peers = [];
-    await peerStore.forEach((peer) => {
-        for (let i = 0; i < protocols.length; i++) {
-            if (peer.protocols.includes(protocols[i])) {
-                peers.push(peer);
-                break;
-            }
-        }
-    });
-    return peers;
-}
-async function getConnectedPeersForProtocolAndShard(connections, peerStore, protocols, shardInfo) {
-    const openConnections = connections.filter((connection) => connection.status === "open");
-    const peerPromises = openConnections.map(async (connection) => {
-        const peer = await peerStore.get(connection.remotePeer);
-        const supportsProtocol = protocols.some((protocol) => peer.protocols.includes(protocol));
-        if (supportsProtocol) {
-            if (shardInfo) {
-                const encodedPeerShardInfo = peer.metadata.get("shardInfo");
-                const peerShardInfo = encodedPeerShardInfo && decodeRelayShard(encodedPeerShardInfo);
-                if (peerShardInfo && shardInfo.clusterId === peerShardInfo.clusterId) {
-                    return peer;
-                }
-            }
-            else {
-                return peer;
-            }
-        }
-        return null;
-    });
-    const peersWithNulls = await Promise.all(peerPromises);
-    return peersWithNulls.filter((peer) => peer !== null);
-}
-
-/**
- * Retrieves a list of peers based on the specified criteria:
- * 1. If numPeers is 0, return all peers
- * 2. Bootstrap peers are prioritized
- * 3. Non-bootstrap peers are randomly selected to fill up to numPeers
- *
- * @param peers - The list of peers to filter from.
- * @param numPeers - The total number of peers to retrieve. If 0, all peers are returned, irrespective of `maxBootstrapPeers`.
- * @param maxBootstrapPeers - The maximum number of bootstrap peers to retrieve.
- * @returns An array of peers based on the specified criteria.
- */
-function filterPeersByDiscovery(peers, numPeers, maxBootstrapPeers) {
-    // Collect the bootstrap peers up to the specified maximum
-    let bootstrapPeers = peers
-        .filter((peer) => peer.tags.has(Tags.BOOTSTRAP))
-        .slice(0, maxBootstrapPeers);
-    // If numPeers is less than the number of bootstrap peers, adjust the bootstrapPeers array
-    if (numPeers > 0 && numPeers < bootstrapPeers.length) {
-        bootstrapPeers = bootstrapPeers.slice(0, numPeers);
-    }
-    // Collect non-bootstrap peers
-    const nonBootstrapPeers = peers.filter((peer) => !peer.tags.has(Tags.BOOTSTRAP));
-    // If numPeers is 0, return all peers
-    if (numPeers === 0) {
-        return [...bootstrapPeers, ...nonBootstrapPeers];
-    }
-    // Initialize the list of selected peers with the bootstrap peers
-    const selectedPeers = [...bootstrapPeers];
-    // Fill up to numPeers with remaining random peers if needed
-    while (selectedPeers.length < numPeers && nonBootstrapPeers.length > 0) {
-        const randomIndex = Math.floor(Math.random() * nonBootstrapPeers.length);
-        const randomPeer = nonBootstrapPeers.splice(randomIndex, 1)[0];
-        selectedPeers.push(randomPeer);
-    }
-    return selectedPeers;
-}
-
-function selectConnection(connections) {
-    if (!connections.length)
-        return;
-    if (connections.length === 1)
-        return connections[0];
-    let latestConnection;
-    connections.forEach((connection) => {
-        if (connection.status === "open") {
-            if (!latestConnection) {
-                latestConnection = connection;
-            }
-            else if (connection.timeline.open > latestConnection.timeline.open) {
-                latestConnection = connection;
-            }
-        }
-    });
-    return latestConnection;
-}
-
-const CONNECTION_TIMEOUT = 5_000;
-const RETRY_BACKOFF_BASE = 1_000;
-const MAX_RETRIES = 3;
-class StreamManager {
-    multicodec;
-    getConnections;
-    addEventListener;
-    streamPool;
-    log;
-    constructor(multicodec, getConnections, addEventListener) {
-        this.multicodec = multicodec;
-        this.getConnections = getConnections;
-        this.addEventListener = addEventListener;
-        this.log = new Logger(`stream-manager:${multicodec}`);
-        this.streamPool = new Map();
-        this.addEventListener("peer:update", this.handlePeerUpdateStreamPool);
-    }
-    async getStream(peer) {
-        const peerIdStr = peer.id.toString();
-        const streamPromise = this.streamPool.get(peerIdStr);
-        if (!streamPromise) {
-            return this.createStream(peer);
-        }
-        this.streamPool.delete(peerIdStr);
-        this.prepareStream(peer);
-        try {
-            const stream = await streamPromise;
-            if (stream && stream.status !== "closed") {
-                return stream;
-            }
-        }
-        catch (error) {
-            this.log.warn(`Failed to get stream for ${peerIdStr} -- `, error);
-            this.log.warn("Attempting to create a new stream for the peer");
-        }
-        return this.createStream(peer);
-    }
-    async createStream(peer, retries = 0) {
-        const connections = this.getConnections(peer.id);
-        const connection = selectConnection(connections);
-        if (!connection) {
-            throw new Error("Failed to get a connection to the peer");
-        }
-        try {
-            return await connection.newStream(this.multicodec);
-        }
-        catch (error) {
-            if (retries < MAX_RETRIES) {
-                const backoff = RETRY_BACKOFF_BASE * Math.pow(2, retries);
-                await new Promise((resolve) => setTimeout(resolve, backoff));
-                return this.createStream(peer, retries + 1);
-            }
-            throw new Error(`Failed to create a new stream for ${peer.id.toString()} -- ` + error);
-        }
-    }
-    prepareStream(peer) {
-        const timeoutPromise = new Promise((resolve) => setTimeout(resolve, CONNECTION_TIMEOUT));
-        const streamPromise = Promise.race([
-            this.createStream(peer),
-            timeoutPromise.then(() => {
-                throw new Error("Connection timeout");
-            })
-        ]).catch((error) => {
-            this.log.error(`Failed to prepare a new stream for ${peer.id.toString()} -- `, error);
-        });
-        this.streamPool.set(peer.id.toString(), streamPromise);
-    }
-    handlePeerUpdateStreamPool = (evt) => {
-        const { peer } = evt.detail;
-        if (peer.protocols.includes(this.multicodec)) {
-            const isConnected = this.isConnectedTo(peer.id);
-            if (isConnected) {
-                this.log.info(`Preemptively opening a stream to ${peer.id.toString()}`);
-                this.prepareStream(peer);
-            }
-            else {
-                const peerIdStr = peer.id.toString();
-                this.streamPool.delete(peerIdStr);
-                this.log.info(`Removed pending stream for disconnected peer ${peerIdStr}`);
-            }
-        }
-    };
-    isConnectedTo(peerId) {
-        const connections = this.getConnections(peerId);
-        return connections.some((connection) => connection.status === "open");
-    }
-}
-
-/**
- * A class with predefined helpers, to be used as a base to implement Waku
- * Protocols.
- */
-class BaseProtocol {
-    multicodec;
-    components;
-    log;
-    pubsubTopics;
-    addLibp2pEventListener;
-    removeLibp2pEventListener;
-    streamManager;
-    constructor(multicodec, components, log, pubsubTopics) {
-        this.multicodec = multicodec;
-        this.components = components;
-        this.log = log;
-        this.pubsubTopics = pubsubTopics;
-        this.addLibp2pEventListener = components.events.addEventListener.bind(components.events);
-        this.removeLibp2pEventListener = components.events.removeEventListener.bind(components.events);
-        this.streamManager = new StreamManager(multicodec, components.connectionManager.getConnections.bind(components.connectionManager), this.addLibp2pEventListener);
-    }
-    async getStream(peer) {
-        return this.streamManager.getStream(peer);
-    }
-    get peerStore() {
-        return this.components.peerStore;
-    }
-    /**
-     * Returns known peers from the address book (`libp2p.peerStore`) that support
-     * the class protocol. Waku may or may not be currently connected to these
-     * peers.
-     */
-    async allPeers() {
-        return getPeersForProtocol(this.peerStore, [this.multicodec]);
-    }
-    async connectedPeers() {
-        const peers = await this.allPeers();
-        return peers.filter((peer) => {
-            return (this.components.connectionManager.getConnections(peer.id).length > 0);
-        });
-    }
-    /**
-     * Retrieves a list of connected peers that support the protocol. The list is sorted by latency.
-     *
-     * @param numPeers - The total number of peers to retrieve. If 0, all peers are returned.
-     * @param maxBootstrapPeers - The maximum number of bootstrap peers to retrieve.
-  
-    * @returns A list of peers that support the protocol sorted by latency.
-    */
-    async getPeers({ numPeers, maxBootstrapPeers } = {
-        maxBootstrapPeers: 1,
-        numPeers: 0
-    }) {
-        // Retrieve all connected peers that support the protocol & shard (if configured)
-        const connectedPeersForProtocolAndShard = await getConnectedPeersForProtocolAndShard(this.components.connectionManager.getConnections(), this.peerStore, [this.multicodec], pubsubTopicsToShardInfo(this.pubsubTopics));
-        // Filter the peers based on discovery & number of peers requested
-        const filteredPeers = filterPeersByDiscovery(connectedPeersForProtocolAndShard, numPeers, maxBootstrapPeers);
-        // Sort the peers by latency
-        const sortedFilteredPeers = await sortPeersByLatency(this.peerStore, filteredPeers);
-        if (sortedFilteredPeers.length === 0) {
-            this.log.warn("No peers found. Ensure you have a connection to the network.");
-        }
-        if (sortedFilteredPeers.length < numPeers) {
-            this.log.warn(`Only ${sortedFilteredPeers.length} peers found. Requested ${numPeers}.`);
-        }
-        return sortedFilteredPeers;
-    }
-}
-
-export { BaseProtocol as B, StreamManager as S, decodeRelayShard as d, encodeRelayShard as e };

package/dist/lib/wait_for_remote_peer.d.ts

@@ -1,22 +0,0 @@
-import type { Waku } from "@waku/interfaces";
-import { Protocols } from "@waku/interfaces";
-/**
- * Wait for a remote peer to be ready given the passed protocols.
- * Must be used after attempting to connect to nodes, using
- * {@link @waku/sdk!WakuNode.dial} or a bootstrap method with
- * {@link @waku/sdk!createLightNode}.
- *
- * If the passed protocols is a GossipSub protocol, then it resolves only once
- * a peer is in a mesh, to help ensure that other peers will send and receive
- * message to us.
- *
- * @param waku The Waku Node
- * @param protocols The protocols that need to be enabled by remote peers.
- * @param timeoutMs A timeout value in milliseconds..
- *
- * @returns A promise that **resolves** if all desired protocols are fulfilled by
- * remote nodes, **rejects** if the timeoutMs is reached.
- * @throws If passing a protocol that is not mounted
- * @default Wait for remote peers with protocols enabled locally and no time out is applied.
- */
-export declare function waitForRemotePeer(waku: Waku, protocols?: Protocols[], timeoutMs?: number): Promise<void>;