@dignetwork/dig-sdk 0.0.1-alpha.9 → 0.0.1-alpha.90

package/dist/DigNetwork/PropagationServer.js

@@ -4,412 +4,533 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.PropagationServer = void 0;
+// Import necessary modules and dependencies
+const axios_1 = __importDefault(require("axios"));
 const fs_1 = __importDefault(require("fs"));
-const https_1 = __importDefault(require("https"));
-const url_1 = require("url");
+const path_1 = __importDefault(require("path"));
+const form_data_1 = __importDefault(require("form-data"));
+const blockchain_1 = require("../blockchain");
 const ssl_1 = require("../utils/ssl");
 const credentialsUtils_1 = require("../utils/credentialsUtils");
-const spinnerUtils_1 = require("../utils/spinnerUtils");
-const blockchain_1 = require("../blockchain");
+const https_1 = __importDefault(require("https"));
+const cli_progress_1 = __importDefault(require("cli-progress"));
+const colorette_1 = require("colorette");
+const config_1 = require("../utils/config");
+const hashUtils_1 = require("../utils/hashUtils");
+const nanospinner_1 = require("nanospinner");
+const progress_stream_1 = __importDefault(require("progress-stream"));
+const stream_1 = require("stream");
+const asyncPool_1 = require("../utils/asyncPool");
+const fs_extra_1 = __importDefault(require("fs-extra"));
+const os_1 = __importDefault(require("os"));
+const merkle_1 = require("../utils/merkle");
+// Helper function to trim long filenames with ellipsis and ensure consistent padding
+function formatFilename(filename, maxLength = 30) {
+    if (!filename) {
+        return "Unknown File".padEnd(maxLength, " ");
+    }
+    if (filename.length > maxLength) {
+        return `...${filename.slice(-(maxLength - 3))}`.padEnd(maxLength, " ");
+    }
+    return filename.padEnd(maxLength, " ");
+}
 class PropagationServer {
     constructor(ipAddress, storeId) {
-        this.ipAddress = ipAddress;
         this.storeId = storeId;
-        if (!PropagationServer.certPath || !PropagationServer.keyPath) {
-            const { certPath, keyPath } = (0, ssl_1.getOrCreateSSLCerts)();
-            PropagationServer.certPath = certPath;
-            PropagationServer.keyPath = keyPath;
-        }
-    }
-    // Method to upload a file to the propagation server
-    async pushFile(filePath, relativePath) {
-        const { nonce, username, password } = await this.getUploadDetails();
-        const wallet = await blockchain_1.Wallet.load("default");
-        const keyOwnershipSig = await wallet.createKeyOwnershipSignature(nonce);
-        const publicKey = await wallet.getPublicSyntheticKey();
-        const uploadUrl = `https://${this.ipAddress}:${PropagationServer.port}/${this.storeId}/${relativePath}`;
-        await this.retryOperation(() => this.uploadFileDirect(filePath, uploadUrl, username, password, keyOwnershipSig, publicKey.toString("hex"), nonce), `Upload failed for ${relativePath}`);
-    }
-    // Method to subscribe to a store on the propagation server
-    async subscribeToStore() {
-        const url = `https://${this.ipAddress}:${PropagationServer.port}/subscribe`;
-        const data = JSON.stringify({ storeId: this.storeId });
-        await this.postRequest(url, data);
-    }
-    // Method to unsubscribe from a store on the propagation server
-    async unsubscribeFromStore() {
-        const url = `https://${this.ipAddress}:${PropagationServer.port}/unsubscribe`;
-        const data = JSON.stringify({ storeId: this.storeId });
-        await this.postRequest(url, data);
-    }
-    // Method to check the status of a store on the propagation server
-    async getStoreStatus() {
-        const url = `https://${this.ipAddress}:${PropagationServer.port}/status/${this.storeId}`;
-        return this.fetchJson(url);
-    }
-    async streamStoreData(dataPath) {
-        const url = `https://${this.ipAddress}:${PropagationServer.port}/${this.storeId}/${dataPath}`;
-        return this.createReadStreamWithRetries(url);
-    }
-    // Method to retrieve a store's key
-    async getStoreData(dataPath) {
-        const url = `https://${this.ipAddress}:${PropagationServer.port}/${this.storeId}/${dataPath}`;
-        return this.fetch(url);
-    }
-    async getStatus() {
-        const url = `https://${this.ipAddress}:${PropagationServer.port}/status/${this.storeId}`;
-        const statusJson = await this.fetch(url);
-        return JSON.parse(statusJson);
+        this.sessionId = "";
+        this.publicKey = "";
+        this.ipAddress = ipAddress;
+        // Get or create SSL certificates
+        const { certPath, keyPath } = (0, ssl_1.getOrCreateSSLCerts)();
+        this.certPath = certPath;
+        this.keyPath = keyPath;
     }
-    // Method to check if a specific store exists (HEAD request)
-    async headStore() {
-        const url = `https://${this.ipAddress}:${PropagationServer.port}/${this.storeId}`;
-        return this.head(url);
+    /**
+     * Initialize the Wallet instance.
+     */
+    async initializeWallet() {
+        this.wallet = await blockchain_1.Wallet.load("default");
+        this.publicKey = (await this.wallet.getPublicSyntheticKey()).toString("hex");
     }
-    // In PropagationServer.ts:
-    async isStoreSynced() {
-        const status = await this.getStatus();
-        return status.synced === true;
+    /**
+     * Create an Axios HTTPS Agent with self-signed certificate allowance.
+     */
+    createHttpsAgent() {
+        return new https_1.default.Agent({
+            cert: fs_1.default.readFileSync(this.certPath),
+            key: fs_1.default.readFileSync(this.keyPath),
+            rejectUnauthorized: false,
+        });
     }
-    // Method to handle upload details including nonce
-    async getUploadDetails() {
-        let username;
-        let password;
-        if (!username || !password) {
-            const credentials = await (0, credentialsUtils_1.promptCredentials)(this.ipAddress);
-            username = credentials.username;
-            password = credentials.password;
+    /**
+     * Check if the store and optional root hash exist by making a HEAD request.
+     */
+    async checkStoreExists(rootHash) {
+        const spinner = (0, nanospinner_1.createSpinner)(`Checking if store ${this.storeId} exists...`).start();
+        try {
+            const config = {
+                httpsAgent: this.createHttpsAgent(),
+            };
+            let url = `https://${this.ipAddress}:${PropagationServer.port}/${this.storeId}`;
+            if (rootHash) {
+                url += `?hasRootHash=${rootHash}`;
+            }
+            const response = await axios_1.default.head(url, config);
+            // Extract store existence and root hash existence from headers
+            const storeExists = response.headers["x-store-exists"] === "true";
+            const rootHashExists = response.headers["x-has-root-hash"] === "true";
+            if (storeExists) {
+                spinner.success({
+                    text: (0, colorette_1.green)(`Store ${this.storeId} exists on peer: ${this.ipAddress}`),
+                });
+            }
+            else {
+                spinner.error({
+                    text: (0, colorette_1.yellow)(`Store ${this.storeId} does not exist on ${this.ipAddress}. Credentials will be required to push.`),
+                });
+            }
+            return { storeExists, rootHashExists };
         }
-        const uploadDetails = await this.fetchUploadDetails(username, password);
-        if (!uploadDetails) {
-            throw new Error("Failed to retrieve upload details.");
+        catch (error) {
+            spinner.error({ text: (0, colorette_1.red)("Error checking if store exists:") });
+            console.error((0, colorette_1.red)(error.message));
+            throw error;
         }
-        return { ...uploadDetails, username, password };
     }
-    // Method to send a POST request
-    async postRequest(url, data) {
-        return new Promise((resolve, reject) => {
-            const urlObj = new url_1.URL(url);
-            const options = {
-                hostname: urlObj.hostname,
-                port: urlObj.port || PropagationServer.port,
-                path: urlObj.pathname,
-                method: "POST",
+    /**
+     * Start an upload session by sending a POST request with the rootHash.dat file.
+     */
+    async startUploadSession(rootHash) {
+        const spinner = (0, nanospinner_1.createSpinner)(`Starting upload session for store ${this.storeId}...`).start();
+        try {
+            const formData = new form_data_1.default();
+            const datFilePath = path_1.default.join(config_1.STORE_PATH, this.storeId, `${rootHash}.dat`);
+            // Ensure the rootHash.dat file exists
+            if (!fs_1.default.existsSync(datFilePath)) {
+                throw new Error(`File not found: ${datFilePath}`);
+            }
+            formData.append("file", fs_1.default.createReadStream(datFilePath), {
+                filename: `${rootHash}.dat`,
+            });
+            const config = {
+                httpsAgent: this.createHttpsAgent(),
                 headers: {
-                    "Content-Type": "application/json",
-                    "Content-Length": Buffer.byteLength(data),
+                    ...formData.getHeaders(),
                 },
-                key: fs_1.default.readFileSync(PropagationServer.keyPath),
-                cert: fs_1.default.readFileSync(PropagationServer.certPath),
-                rejectUnauthorized: false,
             };
-            const req = https_1.default.request(options, (res) => {
-                if (res.statusCode === 200) {
-                    resolve();
-                }
-                else {
-                    reject(new Error(`POST request failed with status ${res.statusCode}: ${res.statusMessage}`));
-                }
-            });
-            req.on("error", (err) => {
-                reject(err);
-            });
-            req.write(data);
-            req.end();
-        });
-    }
-    // Method to fetch upload details from the server
-    // Method to fetch upload details from the server
-    async fetchUploadDetails(username, password) {
-        const remote = `https://${this.ipAddress}:${PropagationServer.port}/${this.storeId}`;
-        return (0, spinnerUtils_1.waitForPromise)(() => {
-            return new Promise((resolve, reject) => {
-                const url = new url_1.URL(remote);
-                const options = {
-                    hostname: url.hostname,
-                    port: url.port,
-                    path: url.pathname,
-                    method: "HEAD",
-                    headers: {
-                        Authorization: `Basic ${Buffer.from(`${username}:${password}`).toString("base64")}`,
-                    },
-                    key: fs_1.default.readFileSync(PropagationServer.keyPath),
-                    cert: fs_1.default.readFileSync(PropagationServer.certPath),
-                    rejectUnauthorized: false, // Allow self-signed certificates
+            // Add Basic Auth if username and password are present
+            if (this.username && this.password) {
+                config.auth = {
+                    username: this.username,
+                    password: this.password,
                 };
-                const req = https_1.default.request(options, (res) => {
-                    if (res.statusCode === 200) {
-                        // Check if the headers are present and valid
-                        const nonce = res.headers["x-nonce"];
-                        const lastUploadedHash = res.headers?.["x-last-uploaded-hash"] ||
-                            "0000000000000000000000000000000000000000000000000000000000000000";
-                        const generationIndex = res.headers?.["x-generation-index"] || 0;
-                        console.log({ nonce, lastUploadedHash, generationIndex });
-                        if (nonce) {
-                            resolve({
-                                nonce: nonce,
-                                lastUploadedHash: lastUploadedHash,
-                                generationIndex: Number(generationIndex),
-                            });
-                        }
-                        else {
-                            reject(new Error("Missing required headers in the response."));
-                        }
-                    }
-                    else {
-                        reject(new Error(`Failed to perform preflight check: ${res.statusCode} ${res.statusMessage}`));
-                    }
-                });
-                req.on("error", (err) => {
-                    console.error(err.message);
-                    resolve(false); // Resolve to false if there is an error
-                });
-                req.end();
+            }
+            const url = `https://${this.ipAddress}:${PropagationServer.port}/upload/${this.storeId}?roothash=${rootHash}`;
+            const response = await axios_1.default.post(url, formData, config);
+            this.sessionId = response.data.sessionId;
+            spinner.success({
+                text: (0, colorette_1.green)(`Upload session started for Root Hash: ${(0, colorette_1.cyan)(rootHash)} with session ID ${this.sessionId}`),
             });
-        }, "Performing remote preflight", "Preflight succeeded.", "Error on preflight.");
+        }
+        catch (error) {
+            spinner.error({ text: (0, colorette_1.red)("Error starting upload session:") });
+            console.error((0, colorette_1.red)(error.message));
+            throw error;
+        }
     }
-    // Core method to upload a file directly to the server using a stream
-    async uploadFileDirect(filePath, uploadUrl, username, password, keyOwnershipSig, publicKey, nonce) {
-        const fileStream = fs_1.default.createReadStream(filePath);
-        const fileSize = fs_1.default.statSync(filePath).size;
-        return new Promise((resolve, reject) => {
-            const url = new url_1.URL(uploadUrl);
-            const options = {
-                hostname: url.hostname,
-                port: url.port,
-                path: url.pathname,
-                method: "PUT",
-                headers: {
-                    "Content-Type": "application/octet-stream",
-                    "Content-Length": fileSize,
-                    Authorization: `Basic ${Buffer.from(`${username}:${password}`).toString("base64")}`,
-                    "x-key-ownership-sig": keyOwnershipSig,
-                    "x-public-key": publicKey,
-                    "x-nonce": nonce,
-                },
-                key: fs_1.default.readFileSync(PropagationServer.keyPath),
-                cert: fs_1.default.readFileSync(PropagationServer.certPath),
-                rejectUnauthorized: false,
+    /**
+     * Request a nonce for a file by sending a HEAD request to the server.
+     */
+    async getFileNonce(filename) {
+        try {
+            const config = {
+                httpsAgent: this.createHttpsAgent(),
             };
-            const req = https_1.default.request(options, (res) => {
-                if (res.statusCode === 200) {
-                    resolve();
-                }
-                else {
-                    reject(new Error(`Upload failed with status ${res.statusCode}: ${res.statusMessage}`));
-                }
+            const url = `https://${this.ipAddress}:${PropagationServer.port}/upload/${this.storeId}/${this.sessionId}/${filename}`;
+            const response = await axios_1.default.head(url, config);
+            // Check for 'x-file-exists' header
+            const fileExists = response.headers["x-file-exists"] === "true";
+            // If file exists, no need to generate a nonce
+            const nonce = response.headers["x-nonce"];
+            return { nonce, fileExists };
+        }
+        catch (error) {
+            console.error((0, colorette_1.red)(`Error generating nonce for file ${filename}:`), error.message);
+            throw error;
+        }
+    }
+    /**
+     * Upload a file to the server by sending a PUT request.
+     * Logs progress using a local cli-progress bar.
+     */
+    async uploadFile(label, dataPath) {
+        const filePath = path_1.default.join(config_1.STORE_PATH, this.storeId, dataPath);
+        const { nonce, fileExists } = await this.getFileNonce(dataPath);
+        if (fileExists) {
+            console.log((0, colorette_1.blue)(`File ${label} already exists. Skipping upload.`));
+            return;
+        }
+        const wallet = await blockchain_1.Wallet.load("default");
+        const keyOwnershipSig = await wallet.createKeyOwnershipSignature(nonce);
+        const publicKey = await wallet.getPublicSyntheticKey();
+        // Get the file size
+        const fileSize = fs_1.default.statSync(filePath).size;
+        let progressBar;
+        try {
+            // Create a new progress bar for the file
+            progressBar = new cli_progress_1.default.SingleBar({
+                format: `${(0, colorette_1.blue)("[{bar}]")} | ${(0, colorette_1.yellow)("{filename}")} | {percentage}% | {value}/{total} bytes`,
+                hideCursor: true,
+                barsize: 30,
+                align: "left",
+                autopadding: true,
+                noTTYOutput: false,
+                stopOnComplete: true,
+                clearOnComplete: false,
+            }, cli_progress_1.default.Presets.legacy);
+            progressBar.start(fileSize, 0, {
+                filename: formatFilename(label),
             });
-            req.on("error", (err) => {
-                reject(err);
+            // Create the read stream
+            const fileReadStream = fs_1.default.createReadStream(filePath);
+            // Create a progress stream
+            const progressStream = (0, progress_stream_1.default)({
+                length: fileSize,
+                time: 500, // Adjust as needed
             });
-            fileStream.pipe(req);
-            fileStream.on("error", (err) => {
-                reject(err);
+            progressStream.on("progress", (progress) => {
+                progressBar.update(progress.transferred);
             });
-            req.on("end", () => {
-                resolve();
+            // Create a PassThrough stream
+            const passThroughStream = new stream_1.PassThrough();
+            // Pipe the read stream through the progress stream into the PassThrough stream
+            fileReadStream.pipe(progressStream).pipe(passThroughStream);
+            // Use form-data to construct the request body
+            const formData = new form_data_1.default();
+            formData.append("file", passThroughStream);
+            const headers = {
+                ...formData.getHeaders(),
+                "x-nonce": nonce,
+                "x-public-key": publicKey.toString("hex"),
+                "x-key-ownership-sig": keyOwnershipSig,
+            };
+            const config = {
+                headers: headers,
+                httpsAgent: this.createHttpsAgent(),
+                maxContentLength: Infinity,
+                maxBodyLength: Infinity,
+            };
+            const url = `https://${this.ipAddress}:${PropagationServer.port}/upload/${this.storeId}/${this.sessionId}/${dataPath}`;
+            // Create a promise that resolves when the progress stream ends
+            const progressPromise = new Promise((resolve, reject) => {
+                progressStream.on("end", resolve);
+                progressStream.on("error", reject);
             });
-        });
-    }
-    // Generic retry operation handler to reduce redundancy
-    async retryOperation(operation, errorMessage) {
-        let attempt = 0;
-        let delay = PropagationServer.initialDelay;
-        while (attempt < PropagationServer.maxRetries) {
-            try {
-                return await operation();
-            }
-            catch (error) {
-                if (attempt < PropagationServer.maxRetries - 1) {
-                    console.warn(`Attempt ${attempt + 1} failed: ${error.message}. Retrying in ${delay / 1000} seconds...`);
-                    await new Promise((resolve) => setTimeout(resolve, delay));
-                    delay = Math.min(PropagationServer.maxDelay, delay * PropagationServer.delayMultiplier);
-                }
-                else {
-                    console.error(`${errorMessage}. Aborting.`);
-                    throw error;
-                }
+            // Start the upload request
+            const uploadPromise = axios_1.default.put(url, formData, config);
+            // Wait for both the upload and the progress stream to finish
+            await Promise.all([uploadPromise, progressPromise]);
+        }
+        catch (error) {
+            throw error;
+        }
+        finally {
+            if (progressBar) {
+                progressBar.stop();
             }
-            attempt++;
         }
-        throw new Error(errorMessage);
     }
-    // Core method to fetch content from a URL
-    async fetch(url) {
-        return new Promise((resolve, reject) => {
-            const urlObj = new url_1.URL(url);
-            const options = {
-                hostname: urlObj.hostname,
-                port: urlObj.port,
-                path: urlObj.pathname + urlObj.search,
-                method: "GET",
-                key: fs_1.default.readFileSync(PropagationServer.keyPath),
-                cert: fs_1.default.readFileSync(PropagationServer.certPath),
-                rejectUnauthorized: false,
-            };
-            const request = https_1.default.request(options, (response) => {
-                let data = "";
-                if (response.statusCode === 200) {
-                    response.on("data", (chunk) => {
-                        data += chunk;
-                    });
-                    response.on("end", () => {
-                        resolve(data);
-                    });
-                }
-                else if (response.statusCode === 301 || response.statusCode === 302) {
-                    const redirectUrl = response.headers.location;
-                    if (redirectUrl) {
-                        this.fetch(redirectUrl).then(resolve).catch(reject);
-                    }
-                    else {
-                        reject(new Error("Redirected without a location header"));
+    /**
+     * Commit the upload session by sending a POST request to the server.
+     */
+    async commitUploadSession() {
+        const spinner = (0, nanospinner_1.createSpinner)(`Committing upload session for store ${this.storeId}...`).start();
+        try {
+            const config = {
+                httpsAgent: this.createHttpsAgent(),
+                auth: this.username && this.password
+                    ? {
+                        username: this.username,
+                        password: this.password,
                     }
-                }
-                else {
-                    reject(new Error(`Failed to retrieve data from ${url}. Status code: ${response.statusCode}`));
-                }
-            });
-            request.on("error", (error) => {
-                console.error(`Request error for ${url}:`, error);
-                reject(error);
+                    : undefined,
+            };
+            const url = `https://${this.ipAddress}:${PropagationServer.port}/commit/${this.storeId}/${this.sessionId}`;
+            const response = await axios_1.default.post(url, {}, config);
+            spinner.success({
+                text: (0, colorette_1.green)(`Upload session ${this.sessionId} successfully committed.`),
             });
-            request.end();
+            return response.data;
+        }
+        catch (error) {
+            spinner.error({ text: (0, colorette_1.red)("Error committing upload session:") });
+            console.error((0, colorette_1.red)(error.message));
+            throw error;
+        }
+    }
+    /**
+     * Static function to handle the entire upload process for multiple files based on rootHash.
+     */
+    static async uploadStore(storeId, rootHash, ipAddress) {
+        const propagationServer = new PropagationServer(ipAddress, storeId);
+        // Initialize wallet
+        await propagationServer.initializeWallet();
+        // Check if the store exists
+        const { storeExists, rootHashExists } = await propagationServer.checkStoreExists(rootHash);
+        // If the store does not exist, prompt for credentials
+        if (!storeExists) {
+            console.log((0, colorette_1.red)(`Store ${storeId} does not exist. Prompting for credentials...`));
+            const credentials = await (0, credentialsUtils_1.promptCredentials)(propagationServer.ipAddress);
+            propagationServer.username = credentials.username;
+            propagationServer.password = credentials.password;
+        }
+        if (rootHashExists) {
+            console.log((0, colorette_1.blue)(`Root hash ${rootHash} already exists in the store. Skipping upload.`));
+            return;
+        }
+        // Start the upload session
+        await propagationServer.startUploadSession(rootHash);
+        const dataStore = blockchain_1.DataStore.from(storeId);
+        const files = await dataStore.getFileSetForRootHash(rootHash);
+        // Prepare upload tasks
+        const uploadTasks = files.map((file) => ({
+            label: file.name,
+            dataPath: file.path,
+        }));
+        // Limit the number of concurrent uploads
+        const concurrencyLimit = 10; // Adjust this number as needed
+        await (0, asyncPool_1.asyncPool)(concurrencyLimit, uploadTasks, async (task) => {
+            await propagationServer.uploadFile(task.label, task.dataPath);
         });
+        // Commit the session after all files have been uploaded
+        await propagationServer.commitUploadSession();
+        console.log((0, colorette_1.green)(`✔ All files have been uploaded to for Root Hash: ${(0, colorette_1.cyan)(rootHash)}.`));
     }
-    // Helper method to perform HEAD requests
-    async head(url) {
-        return new Promise((resolve, reject) => {
-            const urlObj = new url_1.URL(url);
-            const options = {
-                hostname: urlObj.hostname,
-                port: urlObj.port || PropagationServer.port,
-                path: urlObj.pathname + urlObj.search,
-                method: "HEAD",
-                key: fs_1.default.readFileSync(PropagationServer.keyPath),
-                cert: fs_1.default.readFileSync(PropagationServer.certPath),
-                rejectUnauthorized: false,
-            };
-            const request = https_1.default.request(options, (response) => {
-                if (response.statusCode === 200) {
-                    resolve(true);
-                }
-                else {
-                    resolve(false);
-                }
+    /**
+     * Fetch a file from the server by sending a GET request and return its content in memory.
+     * Logs progress using a local cli-progress bar.
+     */
+    async fetchFile(dataPath) {
+        const url = `https://${this.ipAddress}:${PropagationServer.port}/fetch/${this.storeId}/${dataPath}`;
+        const config = {
+            responseType: "stream",
+            httpsAgent: this.createHttpsAgent(),
+        };
+        let progressBar;
+        try {
+            const response = await axios_1.default.get(url, config);
+            const totalLengthHeader = response.headers["content-length"];
+            const totalLength = totalLengthHeader
+                ? parseInt(totalLengthHeader, 10)
+                : null;
+            if (!totalLength) {
+                throw new Error("Content-Length header is missing");
+            }
+            // Create a new progress bar for the file
+            progressBar = new cli_progress_1.default.SingleBar({
+                format: `${(0, colorette_1.blue)("[{bar}]")} | ${(0, colorette_1.yellow)("{filename}")} | {percentage}% | {value}/{total} bytes`,
+                hideCursor: true,
+                barsize: 30,
+                align: "left",
+                autopadding: true,
+                noTTYOutput: false,
+                stopOnComplete: true,
+                clearOnComplete: false,
+            }, cli_progress_1.default.Presets.legacy);
+            progressBar.start(totalLength, 0, {
+                filename: formatFilename(dataPath),
             });
-            request.on("error", (error) => {
-                console.error(`Request error for ${url}:`, error);
-                reject(false);
+            let dataBuffers = [];
+            const progressStream = (0, progress_stream_1.default)({
+                length: totalLength,
+                time: 500, // Adjust as needed
             });
-            request.end();
-        });
-    }
-    // Helper method to fetch JSON data from a URL
-    async fetchJson(url) {
-        const response = await this.fetch(url);
-        return JSON.parse(response);
+            progressStream.on("progress", (progress) => {
+                progressBar.update(progress.transferred);
+            });
+            response.data.pipe(progressStream);
+            progressStream.on("data", (chunk) => {
+                dataBuffers.push(chunk);
+            });
+            // Wait for the progress stream to finish
+            await new Promise((resolve, reject) => {
+                progressStream.on("end", resolve);
+                progressStream.on("error", reject);
+            });
+            return Buffer.concat(dataBuffers);
+        }
+        catch (error) {
+            throw error;
+        }
+        finally {
+            if (progressBar) {
+                progressBar.stop();
+            }
+        }
     }
-    // Helper method to fetch content with retries and redirection handling
-    /*private async fetchWithRetries(url: string): Promise<string> {
-      let attempt = 0;
-      const maxRetries = 5;
-      const initialDelay = 2000; // 2 seconds
-      const maxDelay = 10000; // 10 seconds
-      const delayMultiplier = 1.5;
-      let delay = initialDelay;
-  
-      while (attempt < maxRetries) {
+    /**
+     * Get details of a file, including whether it exists and its size.
+     */
+    async getFileDetails(dataPath, rootHash) {
         try {
-          return await this.fetch(url);
-        } catch (error: any) {
-          if (attempt < maxRetries - 1) {
-            console.warn(
-              `Attempt ${attempt + 1} failed: ${error.message}. Retrying in ${
-                delay / 1000
-              } seconds...`
-            );
-            await new Promise((resolve) => setTimeout(resolve, delay));
-            delay = Math.min(maxDelay, delay * delayMultiplier);
-          } else {
-            console.error(`Failed to retrieve data from ${url}. Aborting.`);
-            throw new Error(`Failed to retrieve data: ${error.message}`);
-          }
+            const config = {
+                httpsAgent: this.createHttpsAgent(),
+            };
+            const url = `https://${this.ipAddress}:${PropagationServer.port}/store/${this.storeId}/${rootHash}/${dataPath}`;
+            const response = await axios_1.default.head(url, config);
+            // Check the headers for file existence and size
+            const fileExists = response.headers["x-file-exists"] === "true";
+            const fileSize = parseInt(response.headers["x-file-size"], 10);
+            return {
+                exists: fileExists,
+                size: fileExists ? fileSize : 0,
+            };
+        }
+        catch (error) {
+            console.error((0, colorette_1.red)(`✖ Error checking file details for ${dataPath}:`), error.message);
+            throw error;
         }
-        attempt++;
-      }
-      throw new Error(
-        `Failed to retrieve data from ${url} after ${maxRetries} attempts.`
-      );
-    }*/
-    // Helper method to create a stream with retry logic
-    async createReadStreamWithRetries(url) {
-        let attempt = 0;
-        const maxRetries = 1;
-        const initialDelay = PropagationServer.initialDelay;
-        const maxDelay = PropagationServer.maxDelay;
-        const delayMultiplier = PropagationServer.delayMultiplier;
-        let delay = initialDelay;
-        while (attempt < maxRetries) {
-            try {
-                return await this.createReadStream(url); // Stream data chunk by chunk
+    }
+    /**
+     * Download a file from the server by sending a GET request.
+     * Logs progress using a local cli-progress bar.
+     */
+    async downloadFile(label, dataPath, rootHash, baseDir) {
+        const url = `https://${this.ipAddress}:${PropagationServer.port}/fetch/${this.storeId}/${dataPath}`;
+        let downloadPath = path_1.default.join(baseDir, dataPath);
+        // Ensure that the directory for the file exists
+        fs_1.default.mkdirSync(path_1.default.dirname(downloadPath), { recursive: true });
+        const config = {
+            responseType: "stream",
+            httpsAgent: this.createHttpsAgent(),
+        };
+        let progressBar;
+        try {
+            const response = await axios_1.default.get(url, config);
+            const totalLengthHeader = response.headers["content-length"];
+            const totalLength = totalLengthHeader
+                ? parseInt(totalLengthHeader, 10)
+                : null;
+            if (!totalLength) {
+                throw new Error("Content-Length header is missing");
             }
-            catch (error) {
-                if (attempt < maxRetries - 1) {
-                    console.warn(`Attempt ${attempt + 1} failed: ${error.message}. Retrying in ${delay / 1000} seconds...`);
-                    await new Promise((resolve) => setTimeout(resolve, delay));
-                    delay = Math.min(maxDelay, delay * delayMultiplier);
-                }
-                else {
-                    console.error(`Failed to create stream from ${url}. Aborting.`);
-                    throw new Error(`Failed to create stream: ${error.message}`);
+            // Create a new progress bar for the file
+            progressBar = new cli_progress_1.default.SingleBar({
+                format: `${(0, colorette_1.blue)("[{bar}]")} | ${(0, colorette_1.yellow)("{filename}")} | {percentage}% | {value}/{total} bytes`,
+                hideCursor: true,
+                barsize: 30,
+                align: "left",
+                autopadding: true,
+                noTTYOutput: false,
+                stopOnComplete: true,
+                clearOnComplete: false,
+            }, cli_progress_1.default.Presets.legacy);
+            progressBar.start(totalLength, 0, {
+                filename: formatFilename(label),
+            });
+            const fileWriteStream = fs_1.default.createWriteStream(downloadPath);
+            const progressStream = (0, progress_stream_1.default)({
+                length: totalLength,
+                time: 500, // Adjust as needed
+            });
+            progressStream.on("progress", (progress) => {
+                progressBar.update(progress.transferred);
+            });
+            response.data.pipe(progressStream).pipe(fileWriteStream);
+            // Wait for both the file write stream and the progress stream to finish
+            await Promise.all([
+                new Promise((resolve, reject) => {
+                    fileWriteStream.on("finish", resolve);
+                    fileWriteStream.on("error", reject);
+                }),
+                new Promise((resolve, reject) => {
+                    progressStream.on("end", resolve);
+                    progressStream.on("error", reject);
+                }),
+            ]);
+            if (dataPath.includes("/data")) {
+                const integrity = await (0, merkle_1.merkleIntegrityCheck)(path_1.default.join(baseDir, `${rootHash}.dat`), baseDir, dataPath, rootHash);
+                if (!integrity) {
+                    throw new Error(`Integrity check failed for file: ${dataPath}`);
                 }
+                console.log("integrity check");
+            }
+        }
+        catch (error) {
+            throw error;
+        }
+        finally {
+            if (progressBar) {
+                progressBar.stop();
             }
-            attempt++;
         }
-        throw new Error(`Failed to create stream from ${url} after ${maxRetries} attempts.`);
     }
-    // Helper method to fetch a readable stream from a URL
-    async createReadStream(url) {
-        return new Promise((resolve, reject) => {
-            const urlObj = new url_1.URL(url);
-            const options = {
-                hostname: urlObj.hostname,
-                port: urlObj.port || PropagationServer.port,
-                path: urlObj.pathname + urlObj.search,
-                method: "GET",
-                key: fs_1.default.readFileSync(PropagationServer.keyPath),
-                cert: fs_1.default.readFileSync(PropagationServer.certPath),
-                rejectUnauthorized: false,
-            };
-            const request = https_1.default.request(options, (response) => {
-                if (response.statusCode === 200) {
-                    resolve(response); // Return the response stream chunk by chunk
-                }
-                else if (response.statusCode === 301 || response.statusCode === 302) {
-                    const redirectUrl = response.headers.location;
-                    if (redirectUrl) {
-                        this.createReadStream(redirectUrl).then(resolve).catch(reject);
-                    }
-                    else {
-                        reject(new Error("Redirected without a location header"));
+    /**
+     * Static function to handle downloading multiple files from a DataStore based on file paths.
+     */
+    static async downloadStore(storeId, rootHash, ipAddress) {
+        const propagationServer = new PropagationServer(ipAddress, storeId);
+        // Initialize wallet
+        await propagationServer.initializeWallet();
+        // Check if the store exists
+        const { storeExists, rootHashExists } = await propagationServer.checkStoreExists(rootHash);
+        if (!storeExists || !rootHashExists) {
+            throw new Error(`Store ${storeId} does not exist.`);
+        }
+        // Fetch the rootHash.dat file
+        const datFileContent = await propagationServer.fetchFile(`${rootHash}.dat`);
+        const root = JSON.parse(datFileContent.toString());
+        // Prepare download tasks
+        const downloadTasks = [];
+        for (const [fileKey, fileData] of Object.entries(root.files)) {
+            const dataPath = (0, hashUtils_1.getFilePathFromSha256)(root.files[fileKey].sha256, "data");
+            const label = Buffer.from(fileKey, "hex").toString("utf-8");
+            downloadTasks.push({ label, dataPath });
+        }
+        // Limit the number of concurrent downloads
+        const concurrencyLimit = 10; // Adjust this number as needed
+        // Create a temporary directory
+        const tempDir = fs_1.default.mkdtempSync(path_1.default.join(os_1.default.tmpdir(), "downloadStore-"));
+        try {
+            // Download files to the temporary directory
+            await (0, asyncPool_1.asyncPool)(concurrencyLimit, downloadTasks, async (task) => {
+                await propagationServer.downloadFile(task.label, task.dataPath, rootHash, tempDir);
+            });
+            // Save the rootHash.dat file to the temporary directory
+            if (!fs_1.default.existsSync(tempDir)) {
+                fs_1.default.mkdirSync(tempDir, { recursive: true });
+            }
+            fs_1.default.writeFileSync(path_1.default.join(tempDir, `${rootHash}.dat`), datFileContent);
+            // Integrity check for the downloaded files was done during the download
+            // Here we want to make sure we got all the files or we reject the download session
+            for (const [fileKey, fileData] of Object.entries(root.files)) {
+                const dataPath = (0, hashUtils_1.getFilePathFromSha256)(root.files[fileKey].sha256, "data");
+                if (!fs_1.default.existsSync(path_1.default.join(tempDir, dataPath))) {
+                    if (!fs_1.default.existsSync(path_1.default.join(config_1.STORE_PATH, storeId, dataPath))) {
+                        throw new Error(`Missing file: ${Buffer.from(fileKey, "hex")}, aborting session.`);
                     }
                 }
-                else {
-                    reject(new Error(`Failed to retrieve stream from ${url}. Status code: ${response.statusCode}`));
-                }
-            });
-            request.on("error", (error) => {
-                console.error(`Request error for ${url}:`, error);
-                reject(error);
+            }
+            // After all downloads are complete, copy from temp directory to the main directory
+            const destinationDir = path_1.default.join(config_1.STORE_PATH, storeId);
+            fs_extra_1.default.copySync(tempDir, destinationDir, {
+                overwrite: false, // Prevents overwriting existing files
+                errorOnExist: false, // No error if file already exists
             });
-            request.end();
-        });
+            // Generate the manifest file in the main directory
+            const dataStore = blockchain_1.DataStore.from(storeId);
+            await dataStore.cacheStoreCreationHeight();
+            await dataStore.generateManifestFile();
+            console.log((0, colorette_1.green)(`✔ All files have been downloaded to ${storeId}.`));
+        }
+        catch (error) {
+            console.log((0, colorette_1.red)("✖ Error downloading files:"), error);
+        }
+        finally {
+            // Clean up the temporary directory
+            fs_extra_1.default.removeSync(tempDir);
+        }
     }
 }
 exports.PropagationServer = PropagationServer;
-PropagationServer.port = 4159;
-PropagationServer.maxRetries = 5;
-PropagationServer.initialDelay = 2000; // 2 seconds
-PropagationServer.maxDelay = 10000; // 10 seconds
-PropagationServer.delayMultiplier = 1.5;
+PropagationServer.port = 4159; // Static port used for all requests