@dignetwork/dig-sdk 0.0.1-alpha.150 → 0.0.1-alpha.151

package/dist/DigNetwork/DigNetwork.d.ts

@@ -7,17 +7,17 @@ export declare class DigNetwork {
     private static networkSyncMap;
     constructor(storeId: string);
     static subscribeToStore(storeId: string): Promise<void>;
+    static getAllNetworkDataStoreIds(): string[];
     static getUdiContent(udi: string): Promise<void>;
     /**
-     * Find a peer that has the store key and root hash.
+     * Find a peer that has the store key and root hash, using ranked peers first and searching in groups of 5.
      *
      * @param {string} storeId - The ID of the store.
      * @param {string} rootHash - The root hash of the store.
      * @param {string} [key] - Optional key to check for in the store.
-     * @param {string[]} [initialBlackList] - Initial list of blacklisted peer IPs.
      * @returns {Promise<DigPeer | null>} - A valid peer or null if none found.
      */
-    static findPeerWithStoreKey(storeId: string, rootHash: string, key?: string, initialBlackList?: string[]): Promise<DigPeer | null>;
+    static findPeerWithStoreKey(storeId: string, rootHash: string, key?: string): Promise<DigPeer | null>;
     static unsubscribeFromStore(storeId: string): void;
     static pingNetworkOfUpdate(storeId: string, rootHash: string): Promise<void>;
     syncStoreFromPeers(prioritizedPeer?: DigPeer, maxRootsToProcess?: number): Promise<void>;

package/dist/DigNetwork/DigNetwork.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"DigNetwork.d.ts","sourceRoot":"","sources":["../../src/DigNetwork/DigNetwork.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAKpC,qBAAa,UAAU;IACrB,OAAO,CAAC,SAAS,CAAY;IAC7B,OAAO,CAAC,UAAU,CAAa;IAC/B,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,aAAa,CAA2B;IAChD,OAAO,CAAC,MAAM,CAAC,cAAc,CAAmC;gBAEpD,OAAO,EAAE,MAAM;WAOP,gBAAgB,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;WAQhD,aAAa,CAAC,GAAG,EAAE,MAAM;IAI7C;;;;;;;;OAQG;WACiB,oBAAoB,CACtC,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,MAAM,EAChB,GAAG,CAAC,EAAE,MAAM,EACZ,gBAAgB,GAAE,MAAM,EAAO,GAC9B,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC;WA6EZ,oBAAoB,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI;WAOrC,mBAAmB,CACrC,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,IAAI,CAAC;IAcH,kBAAkB,CAC7B,eAAe,CAAC,EAAE,OAAO,EACzB,iBAAiB,CAAC,EAAE,MAAM,GACzB,OAAO,CAAC,IAAI,CAAC;IAgHH,mBAAmB,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;CAUvD"}
+{"version":3,"file":"DigNetwork.d.ts","sourceRoot":"","sources":["../../src/DigNetwork/DigNetwork.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAKpC,qBAAa,UAAU;IACrB,OAAO,CAAC,SAAS,CAAY;IAC7B,OAAO,CAAC,UAAU,CAAa;IAC/B,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,aAAa,CAA2B;IAChD,OAAO,CAAC,MAAM,CAAC,cAAc,CAAmC;gBAEpD,OAAO,EAAE,MAAM;WAOP,gBAAgB,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;WAQtD,yBAAyB,IAAI,MAAM,EAAE;WAI/B,aAAa,CAAC,GAAG,EAAE,MAAM;IAI7C;;;;;;;OAOG;WACiB,oBAAoB,CACtC,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,MAAM,EAChB,GAAG,CAAC,EAAE,MAAM,GACX,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC;WAsFZ,oBAAoB,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI;WAOrC,mBAAmB,CACrC,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,IAAI,CAAC;IAcH,kBAAkB,CAC7B,eAAe,CAAC,EAAE,OAAO,EACzB,iBAAiB,CAAC,EAAE,MAAM,GACzB,OAAO,CAAC,IAAI,CAAC;IAqFH,mBAAmB,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;CAUvD"}

package/dist/DigNetwork/DigNetwork.js

@@ -44,74 +44,75 @@ class DigNetwork {
         const digNetwork = new DigNetwork(storeId);
         await digNetwork.syncStoreFromPeers();
     }
+    static getAllNetworkDataStoreIds() {
+        throw new Error("Method not implemented.");
+    }
     static async getUdiContent(udi) {
-        // TODO: Implement this method
+        throw new Error("Method not implemented.");
     }
     /**
-     * Find a peer that has the store key and root hash.
+     * Find a peer that has the store key and root hash, using ranked peers first and searching in groups of 5.
      *
      * @param {string} storeId - The ID of the store.
      * @param {string} rootHash - The root hash of the store.
      * @param {string} [key] - Optional key to check for in the store.
-     * @param {string[]} [initialBlackList] - Initial list of blacklisted peer IPs.
      * @returns {Promise<DigPeer | null>} - A valid peer or null if none found.
      */
-    static async findPeerWithStoreKey(storeId, rootHash, key, initialBlackList = []) {
-        const peerBlackList = new Set(initialBlackList);
+    static async findPeerWithStoreKey(storeId, rootHash, key) {
         const serverCoin = new blockchain_1.ServerCoin(storeId);
-        while (true) {
-            try {
-                // Sample 10 peers from the current epoch excluding blacklisted peers
-                const digPeers = await serverCoin.sampleCurrentEpoch(10, Array.from(peerBlackList));
-                // If no peers are returned, break out of the loop
-                if (digPeers.length === 0) {
-                    console.log("No more peers found.");
-                    break;
-                }
-                // Create a race of promises for all peers
-                const peerPromises = digPeers.map((peerIp) => {
-                    return new Promise(async (resolve) => {
-                        try {
-                            const digPeer = new DigPeer_1.DigPeer(peerIp, storeId);
-                            const { storeExists, rootHashExists } = await digPeer.propagationServer.checkStoreExists(rootHash);
-                            // Check if the store and root hash exist on the peer
-                            if (storeExists && rootHashExists) {
-                                console.log(`Found Peer at ${peerIp} for storeId: ${storeId}, root hash ${rootHash}`);
-                                // If no key is provided, resolve the peer
-                                if (!key) {
-                                    return resolve(digPeer);
-                                }
-                                // If key is provided, check if the peer has it
-                                const keyResponse = await digPeer.contentServer.headKey(key, rootHash);
-                                if (keyResponse.headers?.["x-key-exists"] === "true") {
-                                    return resolve(digPeer);
-                                }
-                            }
+        try {
+            // Fetch all active peers for the current epoch
+            const digPeers = await serverCoin.getActiveEpochPeers();
+            // If no peers are returned, exit early
+            if (digPeers.length === 0) {
+                console.log("No peers found.");
+                return null;
+            }
+            // Initialize PeerRanker with the list of digPeers (IP addresses)
+            const peerRanker = new utils_1.PeerRanker(digPeers);
+            // Rank the peers based on latency and bandwidth
+            const rankedPeers = await peerRanker.rankPeers();
+            // If no peers are returned after ranking, exit early
+            if (rankedPeers.length === 0) {
+                console.log("No valid peers found after ranking.");
+                return null;
+            }
+            // Define the iterator function to process each peer
+            const iteratorFn = async (peerMetrics) => {
+                const peerIp = peerMetrics.ip;
+                try {
+                    const digPeer = new DigPeer_1.DigPeer(peerIp, storeId);
+                    // Wrap the store check with a 10-second timeout
+                    const { storeExists, rootHashExists } = await (0, utils_1.withTimeout)(digPeer.propagationServer.checkStoreExists(rootHash), 10000, `Timeout while checking store on peer ${peerIp}`);
+                    // Check if the store and root hash exist on the peer
+                    if (storeExists && rootHashExists) {
+                        console.log(`Found Peer at ${peerIp} for storeId: ${storeId}, root hash ${rootHash}`);
+                        // If no key is provided, return the peer
+                        if (!key) {
+                            return digPeer;
                         }
-                        catch (error) {
-                            console.error(`Error connecting to DIG Peer ${peerIp}.`);
+                        // If key is provided, wrap key check with a 10-second timeout
+                        const keyResponse = await (0, utils_1.withTimeout)(digPeer.contentServer.headKey(key, rootHash), 10000, `Timeout while checking key on peer ${peerIp}`);
+                        if (keyResponse.headers?.["x-key-exists"] === "true") {
+                            return digPeer;
                         }
-                        // If the peer does not meet the criteria, resolve with null
-                        resolve(null);
-                    });
-                });
-                // Wait for the first valid peer that resolves
-                const firstValidPeer = await Promise.race(peerPromises);
-                // If a valid peer is found, return it
-                if (firstValidPeer) {
-                    return firstValidPeer;
+                    }
                 }
-                // If none of the peers were valid, add them to the blacklist
-                digPeers.forEach((peerIp) => peerBlackList.add(peerIp));
-                // Retry with the next set of peers
-                console.log("No valid peers found, retrying with new peers...");
-            }
-            catch (error) {
-                console.error("Error sampling peers. Resampling...");
-            }
+                catch (error) {
+                    console.error(`Error connecting to DIG Peer ${peerIp}:`, error.message);
+                }
+                // If the peer does not meet the criteria, return null
+                return null;
+            };
+            // Use Promise.race to return the first valid peer found
+            const validPeer = await Promise.race(rankedPeers.map((peer) => iteratorFn(peer)));
+            // Return the first valid peer or null if none is found
+            return validPeer || null;
+        }
+        catch (error) {
+            console.error("Error sampling peers:", error);
+            return null;
         }
-        // Return null if no valid peer was found after all attempts
-        return null;
     }
     static unsubscribeFromStore(storeId) {
         fs.rmdirSync(path.join(config_1.DIG_FOLDER_PATH, "stores", storeId), {
@@ -135,7 +136,6 @@ class DigNetwork {
         }
         console.log("Starting network sync for store:", this.dataStore.StoreId);
         DigNetwork.networkSyncMap.set(this.dataStore.StoreId, true);
-        let peerBlackList = [];
         try {
             const rootHistory = await this.dataStore.getRootHistory();
             if (!rootHistory.length) {
@@ -156,41 +156,24 @@ class DigNetwork {
                 : rootHistoryFiltered;
             // Process the root hashes sequentially
             for (const rootInfo of rootsToProcess) {
-                let selectedPeer = null;
-                while (true) {
-                    try {
-                        // Find a peer with the store and root hash
-                        if (prioritizedPeer) {
-                            selectedPeer = prioritizedPeer;
-                        }
-                        else {
-                            selectedPeer = await DigNetwork.findPeerWithStoreKey(this.dataStore.StoreId, rootInfo.root_hash, undefined, peerBlackList);
-                        }
-                        if (!selectedPeer) {
-                            console.error(`No peer found with root hash ${rootInfo.root_hash}. Moving to next root.`);
-                            break; // Exit the while loop to proceed to the next rootInfo
-                        }
-                        // Check if the selected peer has the store and root hash
-                        const { storeExists, rootHashExists } = await selectedPeer.propagationServer.checkStoreExists(rootInfo.root_hash);
-                        if (!storeExists || !rootHashExists) {
-                            console.warn(`Peer ${selectedPeer.IpAddress} does not have the required store or root hash. Trying another peer...`);
-                            peerBlackList.push(selectedPeer.IpAddress); // Blacklist and retry
-                            continue;
-                        }
-                        // Download the store root and associated data
-                        await selectedPeer.downloadStoreRoot(rootInfo.root_hash);
-                        // Clear the blacklist upon successful download
-                        peerBlackList = [];
-                        // Break after successful download to proceed to next root hash
-                        break;
+                try {
+                    let selectedPeer = prioritizedPeer || null;
+                    if (!selectedPeer) {
+                        // Use the `findPeerWithStoreKey` method to find a peer with the store and root hash
+                        selectedPeer = await DigNetwork.findPeerWithStoreKey(this.dataStore.StoreId, rootInfo.root_hash);
                     }
-                    catch (error) {
-                        if (error.message)
-                            console.error(`Error downloading from peer ${selectedPeer?.IpAddress}. Retrying with another peer.`, error);
-                        if (selectedPeer) {
-                            peerBlackList.push(selectedPeer.IpAddress); // Blacklist and retry
-                        }
+                    if (!selectedPeer) {
+                        console.error(`No peer found with root hash ${rootInfo.root_hash}. Moving to next root.`);
+                        continue; // Move to the next rootInfo
                     }
+                    // Download the store root and associated data
+                    await selectedPeer.downloadStoreRoot(rootInfo.root_hash);
+                    // Break after successful download to proceed to next root hash
+                }
+                catch (error) {
+                    if (error.message)
+                        console.error(`Error downloading from peer ${prioritizedPeer?.IpAddress}. Retrying with another peer.`, error);
+                    // Continue to next rootInfo in case of error
                 }
             }
             console.log("Syncing store complete.");

package/dist/blockchain/StoreMonitorRegistry.js

@@ -102,7 +102,7 @@ class StoreMonitorRegistry {
      */
     async startMonitor(storeId, callback) {
         let retryCount = 0;
-        const maxRetryDelay = 60000; // 60 seconds
+        const maxRetryDelay = 1000; // 60 seconds
         while (this.activeMonitors.has(storeId)) {
             try {
                 await this.monitorStore(storeId, callback);

package/dist/utils/PeerRanker.d.ts

@@ -22,12 +22,14 @@ export declare class PeerRanker {
     /**
      * Measures the latency of a given IP address using an HTTPS request.
      * Tries HEAD first, then falls back to GET if HEAD is not supported.
+     * If the latency is cached, returns the cached value.
      * @param ip - The IP address of the peer.
      * @returns Promise resolving to the latency in milliseconds or rejecting if the peer fails.
      */
     private measureLatency;
     /**
      * Measures the upload bandwidth of a given IP address by sending random data.
+     * If the bandwidth is cached, returns the cached value.
      * @param ip - The IP address of the peer.
      * @returns Promise resolving to the upload bandwidth in bytes per second or rejecting if the peer fails.
      */

package/dist/utils/PeerRanker.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"PeerRanker.d.ts","sourceRoot":"","sources":["../../src/utils/PeerRanker.ts"],"names":[],"mappings":"AAMA,MAAM,WAAW,WAAW;IAC1B,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,qBAAa,UAAU;IACrB,OAAO,CAAC,WAAW,CAAW;IAC9B,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAS;IAChC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAS;IAC/B,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,cAAc,CAAS;IAG/B,OAAO,CAAC,WAAW,CAAqB;IACxC,OAAO,CAAC,YAAY,CAAa;IAEjC;;;OAGG;gBACS,WAAW,EAAE,MAAM,EAAE,EAAE,OAAO,GAAE,MAAa,EAAE,cAAc,GAAE,MAAoB;IAW/F;;;;;OAKG;YACW,cAAc;IA2C5B;;;;OAIG;YACW,gBAAgB;IAmC9B;;;;;OAKG;IACU,SAAS,CAAC,QAAQ,GAAE,MAAY,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IAmCtE;;;;OAIG;IACI,eAAe,IAAI,WAAW,GAAG,IAAI;IAW5C;;OAEG;IACI,aAAa,IAAI,IAAI;CAG7B"}
+{"version":3,"file":"PeerRanker.d.ts","sourceRoot":"","sources":["../../src/utils/PeerRanker.ts"],"names":[],"mappings":"AAUA,MAAM,WAAW,WAAW;IAC1B,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,qBAAa,UAAU;IACrB,OAAO,CAAC,WAAW,CAAW;IAC9B,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAS;IAChC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAS;IAC/B,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,cAAc,CAAS;IAG/B,OAAO,CAAC,WAAW,CAAqB;IACxC,OAAO,CAAC,YAAY,CAAa;IAEjC;;;OAGG;gBACS,WAAW,EAAE,MAAM,EAAE,EAAE,OAAO,GAAE,MAAa,EAAE,cAAc,GAAE,MAAoB;IAW/F;;;;;;OAMG;YACW,cAAc;IAoD5B;;;;;OAKG;YACW,gBAAgB;IA6C9B;;;;;OAKG;IACU,SAAS,CAAC,QAAQ,GAAE,MAAY,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IAmCtE;;;;OAIG;IACI,eAAe,IAAI,WAAW,GAAG,IAAI;IAW5C;;OAEG;IACI,aAAa,IAAI,IAAI;CAG7B"}

package/dist/utils/PeerRanker.js

@@ -7,8 +7,11 @@ exports.PeerRanker = void 0;
 const axios_1 = __importDefault(require("axios"));
 const fs_1 = __importDefault(require("fs"));
 const https_1 = __importDefault(require("https"));
+const node_cache_1 = __importDefault(require("node-cache")); // Import node-cache
 const ssl_1 = require("./ssl");
 const promiseUtils_1 = require("./promiseUtils");
+// Cache with TTL of 1 day (86400 seconds)
+const peerCache = new node_cache_1.default({ stdTTL: 86400 });
 /**
  * Utility class to rank peers based on latency and upload bandwidth using HTTPS with mTLS.
  */
@@ -32,10 +35,16 @@ class PeerRanker {
     /**
      * Measures the latency of a given IP address using an HTTPS request.
      * Tries HEAD first, then falls back to GET if HEAD is not supported.
+     * If the latency is cached, returns the cached value.
      * @param ip - The IP address of the peer.
      * @returns Promise resolving to the latency in milliseconds or rejecting if the peer fails.
      */
     async measureLatency(ip) {
+        const cachedMetrics = peerCache.get(ip);
+        if (cachedMetrics && cachedMetrics.latency) {
+            console.log(`Latency for IP ${ip} retrieved from cache.`);
+            return cachedMetrics.latency;
+        }
         const url = `https://${ip}:4159/diagnostics/ping`;
         const configHead = {
             url: url,
@@ -69,6 +78,8 @@ class PeerRanker {
                 await (0, axios_1.default)(configGet);
             }
             const latency = Date.now() - startTime;
+            // Cache the latency for the IP
+            peerCache.set(ip, { ...cachedMetrics, latency });
             return latency;
         }
         catch (error) {
@@ -78,10 +89,16 @@ class PeerRanker {
     }
     /**
      * Measures the upload bandwidth of a given IP address by sending random data.
+     * If the bandwidth is cached, returns the cached value.
      * @param ip - The IP address of the peer.
      * @returns Promise resolving to the upload bandwidth in bytes per second or rejecting if the peer fails.
      */
     async measureBandwidth(ip) {
+        const cachedMetrics = peerCache.get(ip);
+        if (cachedMetrics && cachedMetrics.bandwidth) {
+            console.log(`Bandwidth for IP ${ip} retrieved from cache.`);
+            return cachedMetrics.bandwidth;
+        }
         const url = `https://${ip}:4159/diagnostics/bandwidth`;
         const randomData = Buffer.alloc(this.uploadTestSize, 'a');
         const config = {
@@ -106,6 +123,8 @@ class PeerRanker {
             await (0, axios_1.default)(config);
             const timeElapsed = (Date.now() - startTime) / 1000;
             const bandwidth = this.uploadTestSize / timeElapsed;
+            // Cache the bandwidth for the IP
+            peerCache.set(ip, { ...cachedMetrics, bandwidth });
             return bandwidth;
         }
         catch (error) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dignetwork/dig-sdk",
-  "version": "0.0.1-alpha.150",
+  "version": "0.0.1-alpha.151",
   "description": "",
   "type": "commonjs",
   "main": "./dist/index.js",