@waku/core 0.0.26-7eb3375.0 → 0.0.27

package/CHANGELOG.md

@@ -5,6 +5,62 @@ All notable changes to this project will be documented in this file.
 The file is maintained by [Release Please](https://github.com/googleapis/release-please) based on [Conventional Commits](https://www.conventionalcommits.org) specification,
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.0.27](https://github.com/waku-org/js-waku/compare/core-v0.0.26...core-v0.0.27) (2024-03-04)
+
+
+### ⚠ BREAKING CHANGES
+
+* protocols filter peers as per configured shard ([#1756](https://github.com/waku-org/js-waku/issues/1756))
+
+### Features
+
+* Create node and subscription by content topic ([ee2d417](https://github.com/waku-org/js-waku/commit/ee2d4176f8cca45a51b7dac0009f0eb01952f540))
+* Decouple sharding params out of core ([e138b4f](https://github.com/waku-org/js-waku/commit/e138b4f5c49a35a37830e31e8be87d824f53249f))
+* Lightpush & filter send requests to multiple peers ([#1779](https://github.com/waku-org/js-waku/issues/1779)) ([7affbe2](https://github.com/waku-org/js-waku/commit/7affbe222dd30ccb6619839f4bc5eb86433a80f7))
+* Local discovery ([#1811](https://github.com/waku-org/js-waku/issues/1811)) ([199f6ab](https://github.com/waku-org/js-waku/commit/199f6ab2ff83694b93e94e935e2925537e01e281))
+* Make ShardingParams optional in sdk, required internally ([68d3229](https://github.com/waku-org/js-waku/commit/68d3229644f395bd84b2e2a7067c7b51e9da3dd0))
+* Protocols filter peers as per configured shard ([#1756](https://github.com/waku-org/js-waku/issues/1756)) ([477c2a5](https://github.com/waku-org/js-waku/commit/477c2a5918f2f75cd2c14bc6ed75e1687c5a09b4))
+
+
+### Dependencies
+
+* The following workspace dependencies were updated
+  * dependencies
+    * @waku/enr bumped from ^0.0.20 to ^0.0.21
+    * @waku/interfaces bumped from 0.0.21 to 0.0.22
+    * @waku/message-hash bumped from ^0.1.10 to ^0.1.11
+    * @waku/utils bumped from 0.0.14 to 0.0.15
+
+## [0.0.26](https://github.com/waku-org/js-waku/compare/core-v0.0.25...core-v0.0.26) (2024-01-10)
+
+
+### ⚠ BREAKING CHANGES
+
+* add support for sharded pubsub topics & remove support for named pubsub topics ([#1697](https://github.com/waku-org/js-waku/issues/1697))
+* change all instances of `PubSubTopic` to `PubsubTopic` ([#1703](https://github.com/waku-org/js-waku/issues/1703))
+
+### Features
+
+* Add support for autosharded pubsub topics ([2bc3735](https://github.com/waku-org/js-waku/commit/2bc3735e4dcf85f06b3dee542024d7f20a40fac2))
+* Add support for sharded pubsub topics & remove support for named pubsub topics ([#1697](https://github.com/waku-org/js-waku/issues/1697)) ([4cf2ffe](https://github.com/waku-org/js-waku/commit/4cf2ffefa75e0571805036b71644d2cdd4fe3192))
+* Metadata protocol ([#1732](https://github.com/waku-org/js-waku/issues/1732)) ([9ac2a3f](https://github.com/waku-org/js-waku/commit/9ac2a3f36352523b79fcd8f8a94bd6e0e109fc30))
+* Track node connection state ([#1719](https://github.com/waku-org/js-waku/issues/1719)) ([1d0e2ac](https://github.com/waku-org/js-waku/commit/1d0e2ace7fa5b44ab192505c7ebce01a7ce343e0))
+
+
+### Miscellaneous Chores
+
+* Change all instances of `PubSubTopic` to `PubsubTopic` ([#1703](https://github.com/waku-org/js-waku/issues/1703)) ([3166a51](https://github.com/waku-org/js-waku/commit/3166a5135e77583da4fa722ee2aa47c785854a38))
+
+
+### Dependencies
+
+* The following workspace dependencies were updated
+  * dependencies
+    * @waku/enr bumped from ^0.0.19 to ^0.0.20
+    * @waku/interfaces bumped from 0.0.20 to 0.0.21
+    * @waku/proto bumped from 0.0.5 to 0.0.6
+    * @waku/utils bumped from 0.0.13 to 0.0.14
+
 ## [0.0.25](https://github.com/waku-org/js-waku/compare/core-v0.0.24...core-v0.0.25) (2023-11-01)
 
 

package/bundle/base_protocol-LhsIWF3-.js

@@ -0,0 +1,308 @@
+import { b as bytesToUtf8, L as Logger, k as ensureShardingConfigured } from './index-8YyfzF9R.js';
+import { T as Tags } from './browser-BQyFvtq6.js';
+
+const decodeRelayShard = (bytes) => {
+    // explicitly converting to Uint8Array to avoid Buffer
+    // https://github.com/libp2p/js-libp2p/issues/2146
+    bytes = new Uint8Array(bytes);
+    if (bytes.length < 3)
+        throw new Error("Insufficient data");
+    const view = new DataView(bytes.buffer);
+    const clusterId = view.getUint16(0);
+    const shards = [];
+    if (bytes.length === 130) {
+        // rsv format (Bit Vector)
+        for (let i = 0; i < 1024; i++) {
+            const byteIndex = Math.floor(i / 8) + 2; // Adjusted for the 2-byte cluster field
+            const bitIndex = 7 - (i % 8);
+            if (view.getUint8(byteIndex) & (1 << bitIndex)) {
+                shards.push(i);
+            }
+        }
+    }
+    else {
+        // rs format (Index List)
+        const numIndices = view.getUint8(2);
+        for (let i = 0, offset = 3; i < numIndices; i++, offset += 2) {
+            if (offset + 1 >= bytes.length)
+                throw new Error("Unexpected end of data");
+            shards.push(view.getUint16(offset));
+        }
+    }
+    return { clusterId, shards };
+};
+const encodeRelayShard = (shardInfo) => {
+    const { clusterId, shards } = shardInfo;
+    const totalLength = shards.length >= 64 ? 130 : 3 + 2 * shards.length;
+    const buffer = new ArrayBuffer(totalLength);
+    const view = new DataView(buffer);
+    view.setUint16(0, clusterId);
+    if (shards.length >= 64) {
+        // rsv format (Bit Vector)
+        for (const index of shards) {
+            const byteIndex = Math.floor(index / 8) + 2; // Adjusted for the 2-byte cluster field
+            const bitIndex = 7 - (index % 8);
+            view.setUint8(byteIndex, view.getUint8(byteIndex) | (1 << bitIndex));
+        }
+    }
+    else {
+        // rs format (Index List)
+        view.setUint8(2, shards.length);
+        for (let i = 0, offset = 3; i < shards.length; i++, offset += 2) {
+            view.setUint16(offset, shards[i]);
+        }
+    }
+    return new Uint8Array(buffer);
+};
+
+/**
+ * Function to sort peers by latency from lowest to highest
+ * @param peerStore - The Libp2p PeerStore
+ * @param peers - The list of peers to choose from
+ * @returns Sorted array of peers by latency
+ */
+async function sortPeersByLatency(peerStore, peers) {
+    if (peers.length === 0)
+        return [];
+    const results = await Promise.all(peers.map(async (peer) => {
+        try {
+            const pingBytes = (await peerStore.get(peer.id)).metadata.get("ping");
+            if (!pingBytes)
+                return { peer, ping: Infinity };
+            const ping = Number(bytesToUtf8(pingBytes));
+            return { peer, ping };
+        }
+        catch (error) {
+            return { peer, ping: Infinity };
+        }
+    }));
+    // filter out null values
+    const validResults = results.filter((result) => result !== null);
+    return validResults
+        .sort((a, b) => a.ping - b.ping)
+        .map((result) => result.peer);
+}
+/**
+ * Returns the list of peers that supports the given protocol.
+ */
+async function getPeersForProtocol(peerStore, protocols) {
+    const peers = [];
+    await peerStore.forEach((peer) => {
+        for (let i = 0; i < protocols.length; i++) {
+            if (peer.protocols.includes(protocols[i])) {
+                peers.push(peer);
+                break;
+            }
+        }
+    });
+    return peers;
+}
+async function getConnectedPeersForProtocolAndShard(connections, peerStore, protocols, shardInfo) {
+    const openConnections = connections.filter((connection) => connection.status === "open");
+    const peerPromises = openConnections.map(async (connection) => {
+        const peer = await peerStore.get(connection.remotePeer);
+        const supportsProtocol = protocols.some((protocol) => peer.protocols.includes(protocol));
+        if (supportsProtocol) {
+            if (shardInfo) {
+                const encodedPeerShardInfo = peer.metadata.get("shardInfo");
+                const peerShardInfo = encodedPeerShardInfo && decodeRelayShard(encodedPeerShardInfo);
+                if (peerShardInfo && shardInfo.clusterId === peerShardInfo.clusterId) {
+                    return peer;
+                }
+            }
+            else {
+                return peer;
+            }
+        }
+        return null;
+    });
+    const peersWithNulls = await Promise.all(peerPromises);
+    return peersWithNulls.filter((peer) => peer !== null);
+}
+function selectConnection(connections) {
+    if (!connections.length)
+        return;
+    if (connections.length === 1)
+        return connections[0];
+    let latestConnection;
+    connections.forEach((connection) => {
+        if (connection.status === "open") {
+            if (!latestConnection) {
+                latestConnection = connection;
+            }
+            else if (connection.timeline.open > latestConnection.timeline.open) {
+                latestConnection = connection;
+            }
+        }
+    });
+    return latestConnection;
+}
+
+/**
+ * Retrieves a list of peers based on the specified criteria:
+ * 1. If numPeers is 0, return all peers
+ * 2. Bootstrap peers are prioritized
+ * 3. Non-bootstrap peers are randomly selected to fill up to numPeers
+ *
+ * @param peers - The list of peers to filter from.
+ * @param numPeers - The total number of peers to retrieve. If 0, all peers are returned, irrespective of `maxBootstrapPeers`.
+ * @param maxBootstrapPeers - The maximum number of bootstrap peers to retrieve.
+ * @returns An array of peers based on the specified criteria.
+ */
+function filterPeersByDiscovery(peers, numPeers, maxBootstrapPeers) {
+    // Collect the bootstrap peers up to the specified maximum
+    let bootstrapPeers = peers
+        .filter((peer) => peer.tags.has(Tags.BOOTSTRAP))
+        .slice(0, maxBootstrapPeers);
+    // If numPeers is less than the number of bootstrap peers, adjust the bootstrapPeers array
+    if (numPeers > 0 && numPeers < bootstrapPeers.length) {
+        bootstrapPeers = bootstrapPeers.slice(0, numPeers);
+    }
+    // Collect non-bootstrap peers
+    const nonBootstrapPeers = peers.filter((peer) => !peer.tags.has(Tags.BOOTSTRAP));
+    // If numPeers is 0, return all peers
+    if (numPeers === 0) {
+        return [...bootstrapPeers, ...nonBootstrapPeers];
+    }
+    // Initialize the list of selected peers with the bootstrap peers
+    const selectedPeers = [...bootstrapPeers];
+    // Fill up to numPeers with remaining random peers if needed
+    while (selectedPeers.length < numPeers && nonBootstrapPeers.length > 0) {
+        const randomIndex = Math.floor(Math.random() * nonBootstrapPeers.length);
+        const randomPeer = nonBootstrapPeers.splice(randomIndex, 1)[0];
+        selectedPeers.push(randomPeer);
+    }
+    return selectedPeers;
+}
+
+class StreamManager {
+    multicodec;
+    getConnections;
+    addEventListener;
+    streamPool;
+    log;
+    constructor(multicodec, getConnections, addEventListener) {
+        this.multicodec = multicodec;
+        this.getConnections = getConnections;
+        this.addEventListener = addEventListener;
+        this.log = new Logger(`stream-manager:${multicodec}`);
+        this.addEventListener("peer:update", this.handlePeerUpdateStreamPool.bind(this));
+        this.getStream = this.getStream.bind(this);
+        this.streamPool = new Map();
+    }
+    async getStream(peer) {
+        const peerIdStr = peer.id.toString();
+        const streamPromise = this.streamPool.get(peerIdStr);
+        if (!streamPromise) {
+            return this.newStream(peer); // fallback by creating a new stream on the spot
+        }
+        // We have the stream, let's remove it from the map
+        this.streamPool.delete(peerIdStr);
+        this.prepareNewStream(peer);
+        const stream = await streamPromise;
+        if (!stream || stream.status === "closed") {
+            return this.newStream(peer); // fallback by creating a new stream on the spot
+        }
+        return stream;
+    }
+    async newStream(peer) {
+        const connections = this.getConnections(peer.id);
+        const connection = selectConnection(connections);
+        if (!connection) {
+            throw new Error("Failed to get a connection to the peer");
+        }
+        return connection.newStream(this.multicodec);
+    }
+    prepareNewStream(peer) {
+        const streamPromise = this.newStream(peer).catch(() => {
+            // No error thrown as this call is not triggered by the user
+            this.log.error(`Failed to prepare a new stream for ${peer.id.toString()}`);
+        });
+        this.streamPool.set(peer.id.toString(), streamPromise);
+    }
+    handlePeerUpdateStreamPool = (evt) => {
+        const peer = evt.detail.peer;
+        if (peer.protocols.includes(this.multicodec)) {
+            this.log.info(`Preemptively opening a stream to ${peer.id.toString()}`);
+            this.prepareNewStream(peer);
+        }
+    };
+}
+
+const DEFAULT_NUM_PEERS_TO_USE = 3;
+/**
+ * A class with predefined helpers, to be used as a base to implement Waku
+ * Protocols.
+ */
+class BaseProtocol {
+    multicodec;
+    components;
+    log;
+    pubsubTopics;
+    options;
+    addLibp2pEventListener;
+    removeLibp2pEventListener;
+    numPeersToUse;
+    streamManager;
+    constructor(multicodec, components, log, pubsubTopics, options) {
+        this.multicodec = multicodec;
+        this.components = components;
+        this.log = log;
+        this.pubsubTopics = pubsubTopics;
+        this.options = options;
+        this.numPeersToUse = options?.numPeersToUse ?? DEFAULT_NUM_PEERS_TO_USE;
+        this.addLibp2pEventListener = components.events.addEventListener.bind(components.events);
+        this.removeLibp2pEventListener = components.events.removeEventListener.bind(components.events);
+        this.streamManager = new StreamManager(multicodec, components.connectionManager.getConnections.bind(components.connectionManager), this.addLibp2pEventListener);
+    }
+    async getStream(peer) {
+        return this.streamManager.getStream(peer);
+    }
+    get peerStore() {
+        return this.components.peerStore;
+    }
+    /**
+     * Returns known peers from the address book (`libp2p.peerStore`) that support
+     * the class protocol. Waku may or may not be currently connected to these
+     * peers.
+     */
+    async allPeers() {
+        return getPeersForProtocol(this.peerStore, [this.multicodec]);
+    }
+    async connectedPeers() {
+        const peers = await this.allPeers();
+        return peers.filter((peer) => {
+            return (this.components.connectionManager.getConnections(peer.id).length > 0);
+        });
+    }
+    /**
+     * Retrieves a list of connected peers that support the protocol. The list is sorted by latency.
+     *
+     * @param numPeers - The total number of peers to retrieve. If 0, all peers are returned.
+     * @param maxBootstrapPeers - The maximum number of bootstrap peers to retrieve.
+  
+    * @returns A list of peers that support the protocol sorted by latency.
+    */
+    async getPeers({ numPeers, maxBootstrapPeers } = {
+        maxBootstrapPeers: 1,
+        numPeers: 0
+    }) {
+        // Retrieve all connected peers that support the protocol & shard (if configured)
+        const connectedPeersForProtocolAndShard = await getConnectedPeersForProtocolAndShard(this.components.connectionManager.getConnections(), this.peerStore, [this.multicodec], this.options?.shardInfo
+            ? ensureShardingConfigured(this.options.shardInfo).shardInfo
+            : undefined);
+        // Filter the peers based on discovery & number of peers requested
+        const filteredPeers = filterPeersByDiscovery(connectedPeersForProtocolAndShard, numPeers, maxBootstrapPeers);
+        // Sort the peers by latency
+        const sortedFilteredPeers = await sortPeersByLatency(this.peerStore, filteredPeers);
+        if (sortedFilteredPeers.length === 0) {
+            this.log.warn("No peers found. Ensure you have a connection to the network.");
+        }
+        if (sortedFilteredPeers.length < numPeers) {
+            this.log.warn(`Only ${sortedFilteredPeers.length} peers found. Requested ${numPeers}.`);
+        }
+        return sortedFilteredPeers;
+    }
+}
+
+export { BaseProtocol as B, StreamManager as S, decodeRelayShard as d, encodeRelayShard as e };