@dignetwork/dig-sdk 0.0.1-alpha.71 → 0.0.1-alpha.72

package/dist/DigNetwork/DigPeer.js

@@ -112,7 +112,7 @@ class DigPeer {
         await PropagationServer_1.PropagationServer.downloadStore(this.storeId, rootHash, this.IpAddress);
     }
     async downloadData(dataPath) {
-        await this.propagationServer.downloadFile(dataPath);
+        await this.propagationServer.downloadFile(dataPath, dataPath);
     }
 }
 exports.DigPeer = DigPeer;

package/dist/DigNetwork/PropagationServer.d.ts

@@ -10,7 +10,6 @@ export declare class PropagationServer {
     username: string | undefined;
     password: string | undefined;
     private static readonly port;
-    private static multiBar;
     constructor(ipAddress: string, storeId: string);
     /**
      * Initialize the Wallet instance.
@@ -22,9 +21,6 @@ export declare class PropagationServer {
     createHttpsAgent(): https.Agent;
     /**
      * Check if the store and optional root hash exist by making a HEAD request.
-     *
-     * @param {string} [rootHash] - Optional root hash to check for existence.
-     * @returns {Promise<{ storeExists: boolean, rootHashExists: boolean }>} - An object indicating if the store and root hash exist.
      */
     checkStoreExists(rootHash?: string): Promise<{
         storeExists: boolean;
@@ -32,14 +28,10 @@ export declare class PropagationServer {
     }>;
     /**
      * Start an upload session by sending a POST request with the rootHash.dat file.
-     *
-     * @param {string} rootHash - The root hash used to name the .dat file.
-     * @param {string} datFilePath - The full path to the rootHash.dat file.
      */
     startUploadSession(rootHash: string): Promise<void>;
     /**
      * Request a nonce for a file by sending a HEAD request to the server.
-     * Checks if the file already exists based on the 'x-file-exists' header.
      */
     getFileNonce(filename: string): Promise<{
         nonce: string;
@@ -47,36 +39,24 @@ export declare class PropagationServer {
     }>;
     /**
      * Upload a file to the server by sending a PUT request.
-     * Logs progress using cli-progress for each file.
+     * Logs progress using a local cli-progress bar.
      */
-    uploadFile(label: string, dataPath: string): Promise<any>;
+    uploadFile(label: string, dataPath: string): Promise<void>;
     /**
      * Commit the upload session by sending a POST request to the server.
-     * This finalizes the upload and moves files from the temporary session directory to the permanent location.
      */
     commitUploadSession(): Promise<any>;
     /**
      * Static function to handle the entire upload process for multiple files based on rootHash.
-     * @param {string} storeId - The ID of the DataStore.
-     * @param {string} rootHash - The root hash used to derive the file set.
-     * @param {string} publicKey - The public key of the user.
-     * @param {string} ipAddress - The IP address of the server.
      */
     static uploadStore(storeId: string, rootHash: string, ipAddress: string): Promise<void>;
     /**
      * Fetch a file from the server by sending a GET request and return its content in memory.
-     * Logs progress using cli-progress.
-     * @param {string} dataPath - The data path of the file to download.
-     * @returns {Promise<Buffer>} - The file content in memory as a Buffer.
+     * Logs progress using a local cli-progress bar.
      */
     fetchFile(dataPath: string): Promise<Buffer>;
     /**
      * Get details of a file, including whether it exists and its size.
-     * Makes a HEAD request to the server and checks the response headers.
-     *
-     * @param {string} dataPath - The path of the file within the DataStore.
-     * @param {string} rootHash - The root hash associated with the DataStore.
-     * @returns {Promise<{ exists: boolean; size: number }>} - An object containing file existence and size information.
      */
     getFileDetails(dataPath: string, rootHash: string): Promise<{
         exists: boolean;
@@ -84,15 +64,11 @@ export declare class PropagationServer {
     }>;
     /**
      * Download a file from the server by sending a GET request.
-     * Logs progress using cli-progress.
-     * @param {string} dataPath - The data path of the file to download.
+     * Logs progress using a local cli-progress bar.
      */
-    downloadFile(dataPath: string): Promise<void>;
+    downloadFile(label: string, dataPath: string): Promise<void>;
     /**
      * Static function to handle downloading multiple files from a DataStore based on file paths.
-     * @param {string} storeId - The ID of the DataStore.
-     * @param {string[]} dataPaths - The list of data paths to download.
-     * @param {string} ipAddress - The IP address of the server.
      */
     static downloadStore(storeId: string, rootHash: string, ipAddress: string): Promise<void>;
 }

package/dist/DigNetwork/PropagationServer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"PropagationServer.d.ts","sourceRoot":"","sources":["../../src/DigNetwork/PropagationServer.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,MAAM,OAAO,CAAC;AAiB1B,qBAAa,iBAAiB;IAC5B,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,GAAG,CAAC;IACZ,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;IAE7B,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAQ;IAIpC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAqCrB;gBAEU,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM;IAY9C;;OAEG;IACG,gBAAgB;IAKtB;;OAEG;IACH,gBAAgB;IAQhB;;;;;OAKG;IACG,gBAAgB,CACpB,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO,CAAC;QAAE,WAAW,EAAE,OAAO,CAAC;QAAC,cAAc,EAAE,OAAO,CAAA;KAAE,CAAC;IAwC7D;;;;;OAKG;IACG,kBAAkB,CAAC,QAAQ,EAAE,MAAM;IAqDzC;;;OAGG;IACG,YAAY,CAChB,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,OAAO,CAAA;KAAE,CAAC;IAyBlD;;;OAGG;IACG,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM;IA+DhD;;;OAGG;IACG,mBAAmB;IAgCzB;;;;;;OAMG;WACU,WAAW,CACtB,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,MAAM,EAChB,SAAS,EAAE,MAAM;IAiDnB;;;;;OAKG;IACG,SAAS,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAuClD;;;;;;;OAOG;IACG,cAAc,CAClB,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC;QAAE,MAAM,EAAE,OAAO,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC;IA2B7C;;;;OAIG;IACG,YAAY,CAAC,QAAQ,EAAE,MAAM;IAgEnC;;;;;OAKG;WACU,aAAa,CACxB,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,MAAM,EAChB,SAAS,EAAE,MAAM;CAgCpB"}
+{"version":3,"file":"PropagationServer.d.ts","sourceRoot":"","sources":["../../src/DigNetwork/PropagationServer.ts"],"names":[],"mappings":"AAQA,OAAO,KAAK,MAAM,OAAO,CAAC;AAoB1B,qBAAa,iBAAiB;IAC5B,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,GAAG,CAAC;IACZ,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;IAE7B,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAQ;gBAExB,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM;IAY9C;;OAEG;IACG,gBAAgB;IAOtB;;OAEG;IACH,gBAAgB;IAQhB;;OAEG;IACG,gBAAgB,CACpB,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO,CAAC;QAAE,WAAW,EAAE,OAAO,CAAC;QAAC,cAAc,EAAE,OAAO,CAAA;KAAE,CAAC;IAwC7D;;OAEG;IACG,kBAAkB,CAAC,QAAQ,EAAE,MAAM;IAiDzC;;OAEG;IACG,YAAY,CAChB,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,OAAO,CAAA;KAAE,CAAC;IAyBlD;;;OAGG;IACG,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM;IAyFhD;;OAEG;IACG,mBAAmB;IAgCzB;;OAEG;WACU,WAAW,CACtB,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,MAAM,EAChB,SAAS,EAAE,MAAM;IA4DnB;;;OAGG;IACG,SAAS,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAwElD;;OAEG;IACG,cAAc,CAClB,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC;QAAE,MAAM,EAAE,OAAO,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC;IA0B7C;;;OAGG;IACG,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM;IA6ElD;;OAEG;WACU,aAAa,CACxB,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,MAAM,EAChB,SAAS,EAAE,MAAM;CAuDpB"}

package/dist/DigNetwork/PropagationServer.js

@@ -1,9 +1,33 @@
 "use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
 var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.PropagationServer = void 0;
+// Import necessary modules and dependencies
 const axios_1 = __importDefault(require("axios"));
 const fs_1 = __importDefault(require("fs"));
 const path_1 = __importDefault(require("path"));
@@ -13,24 +37,26 @@ const ssl_1 = require("../utils/ssl");
 const credentialsUtils_1 = require("../utils/credentialsUtils");
 const https_1 = __importDefault(require("https"));
 const cli_progress_1 = __importDefault(require("cli-progress"));
-const colorette_1 = require("colorette"); // For colored output
+const colorette_1 = require("colorette");
 const config_1 = require("../utils/config");
 const hashUtils_1 = require("../utils/hashUtils");
 const nanospinner_1 = require("nanospinner");
+const progress_stream_1 = __importDefault(require("progress-stream"));
 // Helper function to trim long filenames with ellipsis and ensure consistent padding
 function formatFilename(filename, maxLength = 30) {
+    if (!filename) {
+        return "Unknown File".padEnd(maxLength, " ");
+    }
     if (filename.length > maxLength) {
-        // Trim the filename and add ellipsis
         return `...${filename.slice(-(maxLength - 3))}`.padEnd(maxLength, " ");
     }
-    // For shorter filenames, just pad to the maxLength
     return filename.padEnd(maxLength, " ");
 }
 class PropagationServer {
     constructor(ipAddress, storeId) {
         this.storeId = storeId;
-        this.sessionId = ""; // Session ID will be set after starting the upload session
-        this.publicKey = ""; // Public key will be set after initializing the wallet
+        this.sessionId = "";
+        this.publicKey = "";
         this.ipAddress = ipAddress;
         // Get or create SSL certificates
         const { certPath, keyPath } = (0, ssl_1.getOrCreateSSLCerts)();
@@ -42,7 +68,7 @@ class PropagationServer {
      */
     async initializeWallet() {
         this.wallet = await blockchain_1.Wallet.load("default");
-        this.publicKey = await this.wallet.getPublicSyntheticKey().toString("hex");
+        this.publicKey = (await this.wallet.getPublicSyntheticKey()).toString("hex");
     }
     /**
      * Create an Axios HTTPS Agent with self-signed certificate allowance.
@@ -51,14 +77,11 @@ class PropagationServer {
         return new https_1.default.Agent({
             cert: fs_1.default.readFileSync(this.certPath),
             key: fs_1.default.readFileSync(this.keyPath),
-            rejectUnauthorized: false, // Allow self-signed certificates
+            rejectUnauthorized: false,
         });
     }
     /**
      * Check if the store and optional root hash exist by making a HEAD request.
-     *
-     * @param {string} [rootHash] - Optional root hash to check for existence.
-     * @returns {Promise<{ storeExists: boolean, rootHashExists: boolean }>} - An object indicating if the store and root hash exist.
      */
     async checkStoreExists(rootHash) {
         const spinner = (0, nanospinner_1.createSpinner)(`Checking if store ${this.storeId} exists...`).start();
@@ -94,9 +117,6 @@ class PropagationServer {
     }
     /**
      * Start an upload session by sending a POST request with the rootHash.dat file.
-     *
-     * @param {string} rootHash - The root hash used to name the .dat file.
-     * @param {string} datFilePath - The full path to the rootHash.dat file.
      */
     async startUploadSession(rootHash) {
         const spinner = (0, nanospinner_1.createSpinner)(`Starting upload session for store ${this.storeId}...`).start();
@@ -138,7 +158,6 @@ class PropagationServer {
     }
     /**
      * Request a nonce for a file by sending a HEAD request to the server.
-     * Checks if the file already exists based on the 'x-file-exists' header.
      */
     async getFileNonce(filename) {
         try {
@@ -160,7 +179,7 @@ class PropagationServer {
     }
     /**
      * Upload a file to the server by sending a PUT request.
-     * Logs progress using cli-progress for each file.
+     * Logs progress using a local cli-progress bar.
      */
     async uploadFile(label, dataPath) {
         const filePath = path_1.default.join(config_1.STORE_PATH, this.storeId, dataPath);
@@ -172,48 +191,70 @@ class PropagationServer {
         const wallet = await blockchain_1.Wallet.load("default");
         const keyOwnershipSig = await wallet.createKeyOwnershipSignature(nonce);
         const publicKey = await wallet.getPublicSyntheticKey();
-        const formData = new form_data_1.default();
-        formData.append("file", fs_1.default.createReadStream(filePath));
+        // Get the file size
         const fileSize = fs_1.default.statSync(filePath).size;
-        // Create a new progress bar for each file
-        const progressBar = PropagationServer.multiBar.create(fileSize, 0, {
-            filename: (0, colorette_1.yellow)(path_1.default.basename(label)),
-            percentage: 0,
-        });
-        let uploadedBytes = 0;
-        const config = {
-            headers: {
-                "Content-Type": "multipart/form-data",
+        let progressBar;
+        try {
+            // Create a new progress bar for the file
+            progressBar = new cli_progress_1.default.SingleBar({
+                format: `${(0, colorette_1.blue)('[{bar}]')} | ${(0, colorette_1.yellow)('{filename}')} | {percentage}% | {value}/{total} bytes`,
+                hideCursor: true,
+                barsize: 30,
+                align: "left",
+                autopadding: true,
+                noTTYOutput: false,
+                stopOnComplete: true,
+                clearOnComplete: false,
+            }, cli_progress_1.default.Presets.legacy);
+            progressBar.start(fileSize, 0, {
+                filename: formatFilename(path_1.default.basename(label)),
+            });
+            // Create a progress stream
+            const progressStream = (0, progress_stream_1.default)({
+                length: fileSize,
+                time: 500, // Adjust as needed
+            });
+            progressStream.on("progress", (progress) => {
+                progressBar.update(progress.transferred);
+            });
+            // Create a read stream and pipe it through the progress stream
+            const fileReadStream = fs_1.default
+                .createReadStream(filePath)
+                .pipe(progressStream);
+            // Use form-data to construct the request body
+            const formData = new form_data_1.default();
+            formData.append("file", fileReadStream);
+            const headers = {
+                ...formData.getHeaders(),
                 "x-nonce": nonce,
                 "x-public-key": publicKey.toString("hex"),
                 "x-key-ownership-sig": keyOwnershipSig,
-                ...formData.getHeaders(),
-            },
-            httpsAgent: this.createHttpsAgent(),
-            // Tracking upload progress and updating the progress bar
-            onUploadProgress: (progressEvent) => {
-                const uploadedBytes = progressEvent.loaded;
-                const percentage = Math.round((100 * progressEvent.loaded) / progressEvent.total);
-                progressBar.update(uploadedBytes, { percentage });
-            },
-        };
-        try {
+            };
+            const config = {
+                headers: headers,
+                httpsAgent: this.createHttpsAgent(),
+                maxContentLength: Infinity,
+                maxBodyLength: Infinity,
+            };
             const url = `https://${this.ipAddress}:${PropagationServer.port}/upload/${this.storeId}/${this.sessionId}/${dataPath}`;
             const response = await axios_1.default.put(url, formData, config);
-            // Complete the progress bar
-            progressBar.update(fileSize, { percentage: 100 });
-            progressBar.stop();
-            return response.data;
+            // Wait for the progress stream to finish
+            await new Promise((resolve, reject) => {
+                progressStream.on("end", resolve);
+                progressStream.on("error", reject);
+            });
         }
         catch (error) {
-            console.error((0, colorette_1.red)(`✖ Error uploading file ${label}:`), error.message);
-            progressBar.stop(); // Stop the progress bar in case of error
             throw error;
         }
+        finally {
+            if (progressBar) {
+                progressBar.stop();
+            }
+        }
     }
     /**
      * Commit the upload session by sending a POST request to the server.
-     * This finalizes the upload and moves files from the temporary session directory to the permanent location.
      */
     async commitUploadSession() {
         const spinner = (0, nanospinner_1.createSpinner)(`Committing upload session for store ${this.storeId}...`).start();
@@ -242,10 +283,6 @@ class PropagationServer {
     }
     /**
      * Static function to handle the entire upload process for multiple files based on rootHash.
-     * @param {string} storeId - The ID of the DataStore.
-     * @param {string} rootHash - The root hash used to derive the file set.
-     * @param {string} publicKey - The public key of the user.
-     * @param {string} ipAddress - The IP address of the server.
      */
     static async uploadStore(storeId, rootHash, ipAddress) {
         const propagationServer = new PropagationServer(ipAddress, storeId);
@@ -268,67 +305,92 @@ class PropagationServer {
         await propagationServer.startUploadSession(rootHash);
         const dataStore = blockchain_1.DataStore.from(storeId);
         const files = await dataStore.getFileSetForRootHash(rootHash);
-        // Upload each file
-        for (const file of files) {
-            await propagationServer.uploadFile(file.name, file.path);
-        }
+        // Prepare upload tasks
+        const uploadTasks = files.map((file) => ({
+            label: file.name,
+            dataPath: file.path,
+        }));
+        // Limit the number of concurrent uploads
+        const concurrencyLimit = 3; // Adjust this number as needed
+        // Import asyncPool from your utilities
+        const { asyncPool } = await Promise.resolve().then(() => __importStar(require("../utils/asyncPool")));
+        await asyncPool(concurrencyLimit, uploadTasks, async (task) => {
+            await propagationServer.uploadFile(task.label, task.dataPath);
+        });
         // Commit the session after all files have been uploaded
         await propagationServer.commitUploadSession();
         console.log((0, colorette_1.green)(`✔ All files have been uploaded to DataStore ${storeId}.`));
     }
     /**
      * Fetch a file from the server by sending a GET request and return its content in memory.
-     * Logs progress using cli-progress.
-     * @param {string} dataPath - The data path of the file to download.
-     * @returns {Promise<Buffer>} - The file content in memory as a Buffer.
+     * Logs progress using a local cli-progress bar.
      */
     async fetchFile(dataPath) {
+        const url = `https://${this.ipAddress}:${PropagationServer.port}/fetch/${this.storeId}/${dataPath}`;
         const config = {
-            responseType: "arraybuffer", // To store the file content in memory
+            responseType: "stream",
             httpsAgent: this.createHttpsAgent(),
-            onDownloadProgress: (progressEvent) => {
-                const totalLength = progressEvent.total || 0;
-                const downloadedBytes = progressEvent.loaded;
-                // Update progress bar
-                const progressBar = PropagationServer.multiBar.create(totalLength, 0, {
-                    dataPath: (0, colorette_1.yellow)(dataPath),
-                    percentage: 0,
-                });
-                progressBar.update(downloadedBytes, {
-                    percentage: Math.round((downloadedBytes / totalLength) * 100),
-                });
-                if (downloadedBytes === totalLength) {
-                    progressBar.update(totalLength, { percentage: 100 });
-                    progressBar.stop();
-                    console.log((0, colorette_1.green)(`✔ File ${dataPath} fetched successfully.`));
-                }
-            },
         };
-        const url = `https://${this.ipAddress}:${PropagationServer.port}/fetch/${this.storeId}/${dataPath}`;
+        let progressBar;
         try {
             const response = await axios_1.default.get(url, config);
-            // Return the file contents as a Buffer
-            return Buffer.from(response.data);
+            const totalLengthHeader = response.headers["content-length"];
+            const totalLength = totalLengthHeader
+                ? parseInt(totalLengthHeader, 10)
+                : null;
+            if (!totalLength) {
+                throw new Error("Content-Length header is missing");
+            }
+            // Create a new progress bar for the file
+            progressBar = new cli_progress_1.default.SingleBar({
+                format: `${(0, colorette_1.blue)('[{bar}]')} | ${(0, colorette_1.yellow)('{filename}')} | {percentage}% | {value}/{total} bytes`,
+                hideCursor: true,
+                barsize: 30,
+                align: "left",
+                autopadding: true,
+                noTTYOutput: false,
+                stopOnComplete: true,
+                clearOnComplete: false,
+            }, cli_progress_1.default.Presets.legacy);
+            progressBar.start(totalLength, 0, {
+                filename: formatFilename(dataPath),
+            });
+            let dataBuffers = [];
+            const progressStream = (0, progress_stream_1.default)({
+                length: totalLength,
+                time: 500, // Adjust as needed
+            });
+            progressStream.on("progress", (progress) => {
+                progressBar.update(progress.transferred);
+            });
+            response.data.pipe(progressStream);
+            progressStream.on("data", (chunk) => {
+                dataBuffers.push(chunk);
+            });
+            // Wait for the progress stream to finish
+            await new Promise((resolve, reject) => {
+                progressStream.on("end", resolve);
+                progressStream.on("error", reject);
+            });
+            return Buffer.concat(dataBuffers);
         }
         catch (error) {
-            console.error((0, colorette_1.red)(`✖ Error fetching file ${dataPath}:`), error);
             throw error;
         }
+        finally {
+            if (progressBar) {
+                progressBar.stop();
+            }
+        }
     }
     /**
      * Get details of a file, including whether it exists and its size.
-     * Makes a HEAD request to the server and checks the response headers.
-     *
-     * @param {string} dataPath - The path of the file within the DataStore.
-     * @param {string} rootHash - The root hash associated with the DataStore.
-     * @returns {Promise<{ exists: boolean; size: number }>} - An object containing file existence and size information.
      */
     async getFileDetails(dataPath, rootHash) {
         try {
             const config = {
                 httpsAgent: this.createHttpsAgent(),
             };
-            // Construct the URL for the HEAD request to check file details
             const url = `https://${this.ipAddress}:${PropagationServer.port}/store/${this.storeId}/${rootHash}/${dataPath}`;
             const response = await axios_1.default.head(url, config);
             // Check the headers for file existence and size
@@ -336,7 +398,7 @@ class PropagationServer {
             const fileSize = parseInt(response.headers["x-file-size"], 10);
             return {
                 exists: fileExists,
-                size: fileExists ? fileSize : 0, // Return 0 size if file doesn't exist
+                size: fileExists ? fileSize : 0,
             };
         }
         catch (error) {
@@ -346,63 +408,73 @@ class PropagationServer {
     }
     /**
      * Download a file from the server by sending a GET request.
-     * Logs progress using cli-progress.
-     * @param {string} dataPath - The data path of the file to download.
+     * Logs progress using a local cli-progress bar.
      */
-    async downloadFile(dataPath) {
-        const config = {
-            responseType: "stream", // Make sure the response is streamed
-            httpsAgent: this.createHttpsAgent(),
-        };
+    async downloadFile(label, dataPath) {
         const url = `https://${this.ipAddress}:${PropagationServer.port}/fetch/${this.storeId}/${dataPath}`;
-        const downloadPath = path_1.default.join(config_1.STORE_PATH, this.storeId, dataPath); // Save to the correct store directory
+        const downloadPath = path_1.default.join(config_1.STORE_PATH, this.storeId, dataPath);
         // Ensure that the directory for the file exists
         fs_1.default.mkdirSync(path_1.default.dirname(downloadPath), { recursive: true });
-        const fileWriteStream = fs_1.default.createWriteStream(downloadPath);
+        const config = {
+            responseType: "stream",
+            httpsAgent: this.createHttpsAgent(),
+        };
+        let progressBar;
         try {
             const response = await axios_1.default.get(url, config);
-            const totalLength = parseInt(response.headers["content-length"], 10);
-            console.log((0, colorette_1.cyan)(`Starting download for ${dataPath}...`));
-            // Create a progress bar for the download
-            const progressBar = PropagationServer.multiBar.create(totalLength, 0, {
-                dataPath: (0, colorette_1.yellow)(dataPath),
-                percentage: 0,
+            const totalLengthHeader = response.headers["content-length"];
+            const totalLength = totalLengthHeader
+                ? parseInt(totalLengthHeader, 10)
+                : null;
+            if (!totalLength) {
+                throw new Error("Content-Length header is missing");
+            }
+            // Create a new progress bar for the file
+            progressBar = new cli_progress_1.default.SingleBar({
+                format: `${(0, colorette_1.blue)('[{bar}]')} | ${(0, colorette_1.yellow)('{filename}')} | {percentage}% | {value}/{total} bytes`,
+                hideCursor: true,
+                barsize: 30,
+                align: "left",
+                autopadding: true,
+                noTTYOutput: false,
+                stopOnComplete: true,
+                clearOnComplete: false,
+            }, cli_progress_1.default.Presets.legacy);
+            progressBar.start(totalLength, 0, {
+                filename: formatFilename(label),
             });
-            let downloadedBytes = 0;
-            // Pipe the response data to the file stream and track progress
-            response.data.on("data", (chunk) => {
-                downloadedBytes += chunk.length;
-                progressBar.update(downloadedBytes, {
-                    percentage: Math.round((downloadedBytes / totalLength) * 100),
-                });
+            const fileWriteStream = fs_1.default.createWriteStream(downloadPath);
+            const progressStream = (0, progress_stream_1.default)({
+                length: totalLength,
+                time: 500, // Adjust as needed
             });
-            // Pipe the data into the file and finalize
-            response.data.pipe(fileWriteStream);
-            return new Promise((resolve, reject) => {
-                fileWriteStream.on("finish", () => {
-                    progressBar.update(totalLength, { percentage: 100 });
-                    progressBar.stop();
-                    console.log((0, colorette_1.green)(`
-              File ${dataPath} downloaded successfully to ${downloadPath}.`));
-                    resolve();
-                });
-                fileWriteStream.on("error", (error) => {
-                    progressBar.stop();
-                    console.error((0, colorette_1.red)(`✖ Error downloading file ${dataPath}:`), error);
-                    reject(error);
-                });
+            progressStream.on("progress", (progress) => {
+                progressBar.update(progress.transferred);
             });
+            response.data.pipe(progressStream).pipe(fileWriteStream);
+            // Wait for both the file write stream and the progress stream to finish
+            await Promise.all([
+                new Promise((resolve, reject) => {
+                    fileWriteStream.on("finish", resolve);
+                    fileWriteStream.on("error", reject);
+                }),
+                new Promise((resolve, reject) => {
+                    progressStream.on("end", resolve);
+                    progressStream.on("error", reject);
+                }),
+            ]);
         }
         catch (error) {
-            console.error((0, colorette_1.red)(`✖ Error downloading file ${dataPath}:`), error);
             throw error;
         }
+        finally {
+            if (progressBar) {
+                progressBar.stop();
+            }
+        }
     }
     /**
      * Static function to handle downloading multiple files from a DataStore based on file paths.
-     * @param {string} storeId - The ID of the DataStore.
-     * @param {string[]} dataPaths - The list of data paths to download.
-     * @param {string} ipAddress - The IP address of the server.
      */
     static async downloadStore(storeId, rootHash, ipAddress) {
         const propagationServer = new PropagationServer(ipAddress, storeId);
@@ -413,13 +485,25 @@ class PropagationServer {
         if (!storeExists || !rootHashExists) {
             throw new Error(`Store ${storeId} does not exist.`);
         }
-        await propagationServer.downloadFile("height.json");
+        // Fetch the rootHash.dat file
         const datFileContent = await propagationServer.fetchFile(`${rootHash}.dat`);
         const root = JSON.parse(datFileContent.toString());
+        // Prepare download tasks
+        const downloadTasks = [];
         for (const [fileKey, fileData] of Object.entries(root.files)) {
-            const dataPath = (0, hashUtils_1.getFilePathFromSha256)(root.files[fileKey].sha256, "data");
-            await propagationServer.downloadFile(dataPath);
+            const dataPath = (0, hashUtils_1.getFilePathFromSha256)(fileData.sha256, "data");
+            const label = Buffer.from(fileKey, "hex").toString("utf-8");
+            downloadTasks.push({ label, dataPath });
         }
+        // Limit the number of concurrent downloads
+        const concurrencyLimit = 5; // Adjust this number as needed
+        // Import asyncPool from your utilities
+        const { asyncPool } = await Promise.resolve().then(() => __importStar(require("../utils/asyncPool")));
+        await asyncPool(concurrencyLimit, downloadTasks, async (task) => {
+            await propagationServer.downloadFile(task.label, task.dataPath);
+        });
+        // Save the rootHash.dat file
+        fs_1.default.writeFileSync(path_1.default.join(config_1.STORE_PATH, storeId, `${rootHash}.dat`), datFileContent);
         const dataStore = blockchain_1.DataStore.from(storeId);
         await dataStore.generateManifestFile();
         console.log((0, colorette_1.green)(`✔ All files have been downloaded to ${storeId}.`));
@@ -427,29 +511,3 @@ class PropagationServer {
 }
 exports.PropagationServer = PropagationServer;
 PropagationServer.port = 4159; // Static port used for all requests
-// Adjust cliProgress settings to align output properly and handle potential undefined values
-// MultiBar for handling multiple progress bars with green progress and margin between bars
-PropagationServer.multiBar = new cli_progress_1.default.MultiBar({
-    clearOnComplete: false,
-    hideCursor: true,
-    format: (options, params, payload) => {
-        // Bar length and padding settings
-        const barCompleteChar = (0, colorette_1.green)(options.barCompleteChar || "="); // Green bar for completed portion
-        const barIncompleteChar = options.barIncompleteChar || "-"; // Default incomplete character
-        const barSize = options.barsize || 40; // Default size of the bar
-        // Calculate the bar progress
-        const progressBar = `${barCompleteChar.repeat(Math.round(params.progress * barSize))}${barIncompleteChar.repeat(barSize - Math.round(params.progress * barSize))}`;
-        // Calculate the percentage manually
-        const percentage = Math.round((params.value / params.total) * 100);
-        // Format the filename to a fixed length
-        const formattedFilename = formatFilename(payload.filename, 30); // Trim to max 30 chars
-        // Padding the filename, percentage, and size
-        const percentageStr = `${percentage}%`.padEnd(4); // Ensure percentage is always 7 characters wide
-        const size = `${params.value}/${params.total} bytes`.padEnd(20); // Ensure size is always 20 characters wide
-        // Return the complete formatted progress bar with padding
-        return `${progressBar} | ${formattedFilename.padEnd(35)} | ${percentageStr} | ${size}`;
-    },
-    stopOnComplete: true,
-    barsize: 40,
-    align: "left",
-}, cli_progress_1.default.Presets.shades_classic);

package/dist/blockchain/DataStore.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"DataStore.d.ts","sourceRoot":"","sources":["../../src/blockchain/DataStore.ts"],"names":[],"mappings":"AAEA,OAAO,EAOL,SAAS,IAAI,eAAe,EAE5B,iBAAiB,EAIlB,MAAM,8BAA8B,CAAC;AAWtC,OAAO,EAAE,eAAe,EAAW,MAAM,UAAU,CAAC;AAGpD,OAAO,EACL,iBAAiB,EACjB,wBAAwB,EACzB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,qBAAqB,EAAE,MAAM,UAAU,CAAC;AAYjD,qBAAa,SAAS;IACpB,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,IAAI,CAAoB;gBAEpB,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,wBAAwB;IAiB/D,IAAW,OAAO,IAAI,MAAM,CAE3B;IAED,IAAW,IAAI,IAAI,iBAAiB,CAEnC;IAEM,QAAQ,IAAI,MAAM;IAIlB,QAAQ,IAAI,MAAM;IAIlB,SAAS,IAAI,MAAM;WAMN,cAAc,IAAI,OAAO,CAAC,SAAS,GAAG,SAAS,CAAC;WAOtD,WAAW,CAAC,UAAU,EAAE,MAAM,GAAG,SAAS;WAK1C,IAAI,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS;WAiBtD,MAAM,CACxB,MAAM,GAAE,qBAA0B,GACjC,OAAO,CAAC,SAAS,CAAC;mBAyCA,IAAI;IA4FzB;;;OAGG;IACU,YAAY,IAAI,OAAO,CAAC,MAAM,CAAC;IAK5C;;;OAGG;WACiB,iBAAiB,IAAI,OAAO,CAAC,MAAM,CAAC;IAIxD;;;;OAIG;mBACkB,mBAAmB;WAqB1B,YAAY,IAAI,SAAS,EAAE;IAW5B,aAAa,IAAI,OAAO,CAAC;QACpC,WAAW,EAAE,eAAe,CAAC;QAC7B,YAAY,EAAE,MAAM,CAAC;QACrB,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC;IAyGW,iBAAiB,IAAI,OAAO,CAAC;QACxC,eAAe,EAAE,MAAM,CAAC;QACxB,aAAa,EAAE,MAAM,CAAC;KACvB,CAAC;YA0BY,iBAAiB;IAYlB,cAAc,IAAI,OAAO,CAAC,eAAe,EAAE,CAAC;IAwC5C,oBAAoB,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAexD,WAAW,IAAI,OAAO,CAAC,iBAAiB,CAAC;IAKzC,QAAQ,IAAI,OAAO,CAAC,OAAO,CAAC;IAM5B,uBAAuB,CAClC,kBAAkB,CAAC,EAAE,MAAM,GAC1B,OAAO,CAAC,OAAO,CAAC;IAkBN,cAAc,CACzB,QAAQ,EAAE,iBAAiB,GAC1B,OAAO,CAAC,eAAe,CAAC;IA+C3B;;;;;OAKG;IACU,qBAAqB,CAChC,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;CAsD7C"}
+{"version":3,"file":"DataStore.d.ts","sourceRoot":"","sources":["../../src/blockchain/DataStore.ts"],"names":[],"mappings":"AAEA,OAAO,EAOL,SAAS,IAAI,eAAe,EAE5B,iBAAiB,EAIlB,MAAM,8BAA8B,CAAC;AAWtC,OAAO,EAAE,eAAe,EAAW,MAAM,UAAU,CAAC;AAGpD,OAAO,EACL,iBAAiB,EACjB,wBAAwB,EACzB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,qBAAqB,EAAE,MAAM,UAAU,CAAC;AAYjD,qBAAa,SAAS;IACpB,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,IAAI,CAAoB;gBAEpB,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,wBAAwB;IAiB/D,IAAW,OAAO,IAAI,MAAM,CAE3B;IAED,IAAW,IAAI,IAAI,iBAAiB,CAEnC;IAEM,QAAQ,IAAI,MAAM;IAIlB,QAAQ,IAAI,MAAM;IAIlB,SAAS,IAAI,MAAM;WAMN,cAAc,IAAI,OAAO,CAAC,SAAS,GAAG,SAAS,CAAC;WAOtD,WAAW,CAAC,UAAU,EAAE,MAAM,GAAG,SAAS;WAK1C,IAAI,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS;WAiBtD,MAAM,CACxB,MAAM,GAAE,qBAA0B,GACjC,OAAO,CAAC,SAAS,CAAC;mBAyCA,IAAI;IA4FzB;;;OAGG;IACU,YAAY,IAAI,OAAO,CAAC,MAAM,CAAC;IAK5C;;;OAGG;WACiB,iBAAiB,IAAI,OAAO,CAAC,MAAM,CAAC;IAIxD;;;;OAIG;mBACkB,mBAAmB;WAqB1B,YAAY,IAAI,SAAS,EAAE;IAW5B,aAAa,IAAI,OAAO,CAAC;QACpC,WAAW,EAAE,eAAe,CAAC;QAC7B,YAAY,EAAE,MAAM,CAAC;QACrB,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC;IAyGW,iBAAiB,IAAI,OAAO,CAAC;QACxC,eAAe,EAAE,MAAM,CAAC;QACxB,aAAa,EAAE,MAAM,CAAC;KACvB,CAAC;YA0BY,iBAAiB;IAYlB,cAAc,IAAI,OAAO,CAAC,eAAe,EAAE,CAAC;IAwC5C,oBAAoB,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAexD,WAAW,IAAI,OAAO,CAAC,iBAAiB,CAAC;IAKzC,QAAQ,IAAI,OAAO,CAAC,OAAO,CAAC;IAM5B,uBAAuB,CAClC,kBAAkB,CAAC,EAAE,MAAM,GAC1B,OAAO,CAAC,OAAO,CAAC;IAkBN,cAAc,CACzB,QAAQ,EAAE,iBAAiB,GAC1B,OAAO,CAAC,eAAe,CAAC;IA+C3B;;;;;OAKG;IACU,qBAAqB,CAChC,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;CA2C7C"}

package/dist/blockchain/DataStore.js

@@ -374,13 +374,8 @@ class DataStore {
     async getFileSetForRootHash(rootHash) {
         const datFilePath = path_1.default.join(config_1.STORE_PATH, this.storeId, `${rootHash}.dat`);
         const datFileContent = JSON.parse(fs_1.default.readFileSync(datFilePath, "utf-8"));
-        const heightDatFilePath = path_1.default.join(config_1.STORE_PATH, this.storeId, "height.json");
         // Use a Set to ensure uniqueness
         const filesInvolved = new Set();
-        filesInvolved.add({
-            name: "height.json",
-            path: "height.json",
-        });
         // Iterate over each file and perform an integrity check
         for (const [fileKey, fileData] of Object.entries(datFileContent.files)) {
             const filePath = (0, hashUtils_1.getFilePathFromSha256)(datFileContent.files[fileKey].sha256, "data");

package/dist/utils/asyncPool.d.ts

@@ -0,0 +1,9 @@
+/**
+ * Processes items in sequential batches with a concurrency limit.
+ * @param {number} limit - The maximum number of concurrent executions per batch.
+ * @param {Array<T>} items - The array of items to process.
+ * @param {(item: T) => Promise<R>} iteratorFn - The async function to apply to each item.
+ * @returns {Promise<Array<R>>} - A promise that resolves when all items have been processed.
+ */
+export declare function asyncPool<T, R>(limit: number, items: T[], iteratorFn: (item: T) => Promise<R>): Promise<R[]>;
+//# sourceMappingURL=asyncPool.d.ts.map

package/dist/utils/asyncPool.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"asyncPool.d.ts","sourceRoot":"","sources":["../../src/utils/asyncPool.ts"],"names":[],"mappings":"AACA;;;;;;GAMG;AACH,wBAAsB,SAAS,CAAC,CAAC,EAAE,CAAC,EAChC,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,CAAC,EAAE,EACV,UAAU,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,OAAO,CAAC,CAAC,CAAC,GAClC,OAAO,CAAC,CAAC,EAAE,CAAC,CAgBd"}

package/dist/utils/asyncPool.js

@@ -0,0 +1,23 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.asyncPool = asyncPool;
+/**
+ * Processes items in sequential batches with a concurrency limit.
+ * @param {number} limit - The maximum number of concurrent executions per batch.
+ * @param {Array<T>} items - The array of items to process.
+ * @param {(item: T) => Promise<R>} iteratorFn - The async function to apply to each item.
+ * @returns {Promise<Array<R>>} - A promise that resolves when all items have been processed.
+ */
+async function asyncPool(limit, items, iteratorFn) {
+    const ret = [];
+    for (let i = 0; i < items.length; i += limit) {
+        const batchItems = items.slice(i, i + limit);
+        const batchPromises = batchItems.map((item) => iteratorFn(item));
+        // Wait for the current batch to complete before starting the next one
+        const batchResults = await Promise.all(batchPromises);
+        ret.push(...batchResults);
+        // Optional: add a cooldown between batches
+        // await new Promise((resolve) => setTimeout(resolve, 500));
+    }
+    return ret;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dignetwork/dig-sdk",
-  "version": "0.0.1-alpha.71",
+  "version": "0.0.1-alpha.72",
   "description": "",
   "type": "commonjs",
   "main": "./dist/index.js",
@@ -46,7 +46,7 @@
     "nanospinner": "^1.1.0",
     "nconf": "^0.12.1",
     "node-cache": "^5.1.2",
-    "p-limit": "^6.1.0",
+    "progress-stream": "^2.0.0",
     "superagent": "^10.0.0",
     "unzipper": "^0.12.3"
   },
@@ -61,6 +61,7 @@
     "@types/mocha": "^10.0.7",
     "@types/nconf": "^0.10.7",
     "@types/node": "^22.1.0",
+    "@types/progress-stream": "^2.0.5",
     "@types/superagent": "^8.1.8",
     "@types/unzipper": "^0.10.10",
     "chai": "^5.1.1",