@trap_stevo/filetide 0.0.39 → 0.0.41

package/dist/cjs/HUDManagers/FileTransferManager.js

@@ -4,6 +4,7 @@ function _classPrivateMethodInitSpec(e, a) { _checkPrivateRedeclaration(e, a), a
 function _checkPrivateRedeclaration(e, t) { if (t.has(e)) throw new TypeError("Cannot initialize the same private elements twice on an object"); }
 function _assertClassBrand(e, t, n) { if ("function" == typeof e ? e === t : e.has(t)) return arguments.length < 3 ? t : n; throw new TypeError("Private element is not present on this object"); }
 const fs = require("fs");
+const FileTransferRecoveryManager = require("./FileTransferRecoveryManager.js");
 var _FileTransferManager_brand = /*#__PURE__*/new WeakSet();
 class FileTransferManager {
   constructor({
@@ -17,25 +18,135 @@ class FileTransferManager {
     this.maxRetries = maxRetries;
     this.chunkSize = chunkSize;
   }
-  sendFile(file, {
+  async sendFile(file, {
     fileDetails,
+    originDetails,
     onSendChunk,
     onComplete,
     onProgress,
     onStart
-  }) {
+  }, clientTide) {
     const transferId = _assertClassBrand(_FileTransferManager_brand, this, _generateTransferId).call(this, fileDetails);
-    return _assertClassBrand(_FileTransferManager_brand, this, _startTransfer).call(this, file, transferId, fileDetails, onSendChunk, onComplete, onProgress, onStart);
+    return await _assertClassBrand(_FileTransferManager_brand, this, _startTransfer).call(this, file, transferId, fileDetails, originDetails, onSendChunk, onComplete, onProgress, onStart, clientTide);
+  }
+  async sendLargeFile(fileHandle, {
+    fileDetails,
+    originDetails,
+    recipientID,
+    onSendChunk,
+    onComplete,
+    onProgress,
+    onStart,
+    onTiding
+  }, clientTide) {
+    const transferId = _assertClassBrand(_FileTransferManager_brand, this, _generateTransferId).call(this, fileDetails);
+    const recoveredTransferState = (await clientTide.emitEventWithResponse(originDetails.senderID, "get-client-transfer-state", "sent-current-transfer-state", {
+      recipientID,
+      senderID: originDetails.senderID,
+      filePath: fileDetails.destinationPath
+    }, 30000)) || {
+      transferredSize: 0,
+      lastChunkSent: 0
+    };
+    const totalChunks = Math.ceil(fileDetails.size / this.chunkSize);
+    this.activeTransfers.set(transferId, {
+      totalChunks,
+      transferredSize: recoveredTransferState.transferredSize,
+      completedChunks: recoveredTransferState.lastChunkSent,
+      activeChunks: 0,
+      failedChunks: new Map(),
+      currentChunk: recoveredTransferState.lastChunkSent,
+      size: fileDetails.size
+    });
+    const updateProgress = () => {
+      const transferState = this.activeTransfers.get(transferId);
+      if (onProgress) {
+        onProgress(transferState.completedChunks / transferState.totalChunks * 100, this.chunkSize);
+      }
+    };
+    const exponentialBackoff = retryCount => Math.min(1000 * 2 ** retryCount, 10000);
+    let nextExpectedChunk = recoveredTransferState.lastChunkSent;
+    const chunkBuffer = new Map();
+    const sendChunk = async chunkIndex => {
+      const start = chunkIndex * this.chunkSize;
+      const end = Math.min(start + this.chunkSize, fileDetails.size);
+      const {
+        buffer
+      } = await fileHandle.read(Buffer.alloc(end - start), 0, end - start, start);
+      const transferState = this.activeTransfers.get(transferId);
+      chunkBuffer.set(chunkIndex, buffer);
+      while (chunkBuffer.has(nextExpectedChunk)) {
+        const nextChunkBuffer = chunkBuffer.get(nextExpectedChunk);
+        await onSendChunk(transferId, nextExpectedChunk, this.chunkSize, nextChunkBuffer, totalChunks, this.activeTransfers.get(transferId).completedChunks + 1);
+        chunkBuffer.delete(nextExpectedChunk);
+        nextExpectedChunk++;
+        transferState.transferredSize += this.chunkSize;
+        transferState.completedChunks++;
+        updateProgress();
+      }
+      if (transferState.completedChunks === transferState.totalChunks) {
+        return onComplete(transferId).then(() => {
+          this.activeTransfers.delete(transferId);
+        });
+      }
+    };
+    const sendChunkBatch = async (startChunkIndex, batchSize = 100, miniBatchSize = 10) => {
+      for (let i = 0; i < batchSize && startChunkIndex + i < totalChunks; i += miniBatchSize) {
+        const miniBatchPromises = [];
+        for (let j = 0; j < miniBatchSize && startChunkIndex + i + j < totalChunks; j++) {
+          const chunkIndex = startChunkIndex + i + j;
+          miniBatchPromises.push(sendChunk(chunkIndex));
+        }
+        await Promise.all(miniBatchPromises);
+        await new Promise(resolve => setTimeout(resolve, 5));
+      }
+    };
+    const transferState = this.activeTransfers.get(transferId);
+    let currentBatchStart = transferState.currentChunk;
+    const keepAliveInterval = setInterval(() => {
+      if (onTiding) onTiding();
+    }, 5000);
+    try {
+      let adaptiveBatchSize = 100;
+      while (currentBatchStart < totalChunks) {
+        if (transferState.activeChunks < this.parallelChunks) {
+          adaptiveBatchSize = Math.min(adaptiveBatchSize + 10, 100);
+        } else {
+          adaptiveBatchSize = Math.max(adaptiveBatchSize - 10, 10);
+        }
+        await sendChunkBatch(currentBatchStart, adaptiveBatchSize);
+        currentBatchStart += adaptiveBatchSize;
+        await new Promise(resolve => setTimeout(resolve, 30));
+      }
+    } finally {
+      clearInterval(keepAliveInterval);
+    }
+    return new Promise((resolve, reject) => {
+      fileHandle.on("close", resolve);
+      fileHandle.on("error", error => {
+        this.activeTransfers.delete(transferId);
+        reject(error);
+      });
+    });
   }
 }
-function _startTransfer(file, transferId, fileDetails, onSendChunk, onComplete, onProgress, onStart) {
+async function _startTransfer(file, transferId, fileDetails, originDetails, onSendChunk, onComplete, onProgress, onStart, clientTide) {
+  const recoveredTransferState = (await clientTide.emitEventWithResponse(originDetails.senderID, "get-client-transfer-state", "sent-current-transfer-state", {
+    recipientID: originDetails.recipientID,
+    senderID: originDetails.senderID,
+    filePath: fileDetails.destinationPath
+  }, 30000)) || {
+    transferredSize: 0,
+    lastChunkSent: 0
+  };
   const totalChunks = Math.ceil(file.length / this.chunkSize);
   this.activeTransfers.set(transferId, {
     totalChunks,
-    completedChunks: 0,
+    transferredSize: recoveredTransferState.transferredSize,
+    completedChunks: recoveredTransferState.lastChunkSent,
     activeChunks: 0,
     failedChunks: new Map(),
-    currentChunk: 0,
+    currentChunk: recoveredTransferState.lastChunkSent,
     size: fileDetails.size
   });
   const updateProgress = () => {
@@ -44,10 +155,13 @@ function _startTransfer(file, transferId, fileDetails, onSendChunk, onComplete,
       onProgress(transferState.completedChunks / transferState.totalChunks * 100, this.chunkSize);
     }
   };
+  const exponentialBackoff = retryCount => Math.min(1000 * 2 ** retryCount, 10000);
   return new Promise((resolve, reject) => {
-    const sendChunk = chunkIndex => {
+    const sendChunk = async chunkIndex => {
       const transferState = this.activeTransfers.get(transferId);
-      if (chunkIndex >= transferState.totalChunks) return;
+      if (chunkIndex >= transferState.totalChunks) {
+        return;
+      }
       const start = chunkIndex * this.chunkSize;
       const end = Math.min(start + this.chunkSize, file.length);
       const chunk = file.slice(start, end);
@@ -60,7 +174,8 @@ function _startTransfer(file, transferId, fileDetails, onSendChunk, onComplete,
         });
       }
       transferState.activeChunks++;
-      onSendChunk(transferId, chunkIndex, this.chunkSize, chunk, transferState.totalChunks).then(() => {
+      try {
+        await onSendChunk(transferId, chunkIndex, this.chunkSize, chunk, transferState.totalChunks, transferState.completedChunks + 1);
         transferState.completedChunks++;
         transferState.activeChunks--;
         updateProgress();
@@ -68,23 +183,22 @@ function _startTransfer(file, transferId, fileDetails, onSendChunk, onComplete,
           sendChunk(transferState.currentChunk++);
         }
         if (transferState.completedChunks === transferState.totalChunks) {
-          onComplete(transferId).then(() => {
-            this.activeTransfers.delete(transferId);
-            resolve();
-          }).catch(reject);
+          await onComplete(transferId);
+          this.activeTransfers.delete(transferId);
+          resolve();
         }
-      }).catch(error => {
-        console.error(`Did not send chunk ${chunkIndex}: ${error.message}`);
+      } catch (error) {
+        console.error(`Did not send chunk ${chunkIndex} ~ ${error.message}`);
         transferState.activeChunks--;
-        if (transferState.failedChunks.get(chunkIndex) >= this.maxRetries) {
+        const retryCount = (transferState.failedChunks.get(chunkIndex) || 0) + 1;
+        if (retryCount > this.maxRetries) {
           this.activeTransfers.delete(transferId);
           reject(new Error(`Did not send chunk ${chunkIndex} after ${this.maxRetries} retries.`));
         } else {
-          const retryCount = (transferState.failedChunks.get(chunkIndex) || 0) + 1;
           transferState.failedChunks.set(chunkIndex, retryCount);
-          sendChunk(chunkIndex);
+          setTimeout(() => sendChunk(chunkIndex), exponentialBackoff(retryCount));
         }
-      });
+      }
     };
     const transferState = this.activeTransfers.get(transferId);
     for (let i = 0; i < Math.min(this.parallelChunks, transferState.totalChunks); i++) {

package/dist/cjs/HUDManagers/FileTransferReceiverManager.js

@@ -72,7 +72,7 @@ class FileTransferReceiverManager {
       }
     } catch (error) {
       if (onComplete) {
-        onComplete(data, this.activeTransfers, false);
+        onComplete(data, this.activeTransfers, false, error);
       }
     }
   }

package/dist/cjs/HUDManagers/FileTransferRecoveryManager.js

@@ -0,0 +1,94 @@
+"use strict";
+
+const path = require("path");
+const fs = require("fs");
+class FileTransferRecoveryManager {
+  constructor(storagePath = `${process.cwd()}/transfer_states`) {
+    if (FileTransferRecoveryManager.instance) {
+      return FileTransferRecoveryManager.instance;
+    }
+    this.storagePath = storagePath;
+    this.memoryStorage = new Map();
+    this.ensureStorageDirectory();
+    FileTransferRecoveryManager.instance = this;
+  }
+  ensureStorageDirectory() {
+    if (!fs.existsSync(this.storagePath)) {
+      fs.mkdirSync(this.storagePath, {
+        recursive: true
+      });
+    }
+  }
+  sanitizeFileName(fileName) {
+    return fileName.replace(/[^a-zA-Z0-9-_\.]/g, "-").replace(/-+/g, "-").trim();
+  }
+  generateKey(clientName, senderName, filePath) {
+    const normalizedPath = this.sanitizeFileName(filePath);
+    return `${clientName}_${senderName}_${normalizedPath}.json`;
+  }
+  getStorageFilePath(clientName, senderName, filePath) {
+    return path.join(this.storagePath, this.generateKey(clientName, senderName, filePath));
+  }
+  saveTransferState(clientName, senderName, filePath, chunkIndex, transferredSize) {
+    const key = this.generateKey(clientName, senderName, filePath);
+    const transferState = {
+      transferredSize,
+      senderName,
+      clientName,
+      filePath,
+      lastChunkSent: chunkIndex
+    };
+    this.memoryStorage.set(key, transferState);
+  }
+  saveToDisk(clientName, senderName, filePath) {
+    const key = this.generateKey(clientName, senderName, filePath);
+    const storageFile = this.getStorageFilePath(clientName, senderName, filePath);
+    const transferState = this.memoryStorage.get(key);
+    if (transferState) {
+      try {
+        fs.writeFileSync(storageFile, JSON.stringify(transferState, null, 2));
+        console.log(`Saved transfer state to disk: ${storageFile}`);
+      } catch (error) {
+        console.error(`Error saving transfer state for ${key}: ${error.message}`);
+      }
+    }
+  }
+  getTransferState(clientName, senderName, filePath) {
+    const key = this.generateKey(clientName, senderName, filePath);
+    if (this.memoryStorage.has(key)) {
+      return this.memoryStorage.get(key);
+    }
+    const storageFile = this.getStorageFilePath(clientName, senderName, filePath);
+    if (fs.existsSync(storageFile)) {
+      const transferState = JSON.parse(fs.readFileSync(storageFile, "utf-8"));
+      return transferState;
+    }
+    return null;
+  }
+  getLastChunk(clientName, senderName, filePath) {
+    const transferState = this.getTransferState(clientName, senderName, filePath);
+    return transferState ? transferState.lastChunkSent : null;
+  }
+  clearTransferState(clientName, senderName, filePath) {
+    const key = this.generateKey(clientName, senderName, filePath);
+    this.memoryStorage.delete(key);
+    const storageFile = this.getStorageFilePath(clientName, senderName, filePath);
+    if (fs.existsSync(storageFile)) {
+      fs.unlinkSync(storageFile);
+    }
+  }
+  flushToDisk() {
+    console.log(this.memoryStorage);
+    for (const [key, state] of this.memoryStorage.entries()) {
+      const {
+        clientName,
+        senderName,
+        filePath
+      } = state;
+      console.log("Saving current state ~ ", state);
+      this.saveToDisk(clientName, senderName, filePath);
+    }
+    ;
+  }
+}
+module.exports = new FileTransferRecoveryManager();

package/dist/cjs/HUDManagers/FileUtilityManager.js

@@ -67,15 +67,18 @@ class FileUtilityManager {
   /**
    * Function to list all paths across the system using wildcard '*'.
    * @param {number} depth - The maximum depth limit (from the root). Defaults to 1.
+   * @param {Function} onDirectory - Callback that outputs each directory's contents on the go.
    * @returns {Promise<Array>} - A promise that resolves to an array of file and directory metadata across the system.
    */
-  static async listAllPathsInOS(depth = 1) {
+  static async listAllPathsInOS(depth = 1, onDirectory, onFile) {
     const rootPaths = ["/", "C:/"];
     let allResults = [];
     const rootPromises = rootPaths.map(async rootPath => {
       if (await this.exists(rootPath)) {
-        const subResult = await this.listPaths(rootPath, depth);
-        allResults.push(...subResult.results);
+        const subResult = await this.listPaths(rootPath, depth, 0, onDirectory);
+        if (depth <= 3) {
+          allResults.push(...subResult.results);
+        }
       }
     });
     await Promise.all(rootPromises);
@@ -83,85 +86,114 @@ class FileUtilityManager {
   }
 
   /**
-    * Function to list all paths and their metadata recursively.
-    * @param {string} dir - The directory path to scan.
-    * @param {number} maxDepth - The maximum depth limit (from the root). Defaults to 1.
-    * @param {number} currentDepth - The current depth level. Starts at 0 (for the root).
-    * @returns {Promise<object>} - A promise that resolves to an object containing a list of files/directories with metadata.
-    */
-  static async listPaths(dir, maxDepth = 1, currentDepth = 0) {
+   * Function to list all paths and their metadata recursively.
+   * @param {string} dir - The directory path to scan.
+   * @param {number} maxDepth - The maximum depth limit (from the root). Defaults to 1.
+   * @param {number} currentDepth - The current depth level. Starts at 0 (for the root).
+   * @param {Function} onDirectory - Callback that outputs each directory's contents on the go.
+   * @returns {Promise<object>} - A promise that resolves to an object containing a list of files/directories with metadata.
+   */
+  static async listPaths(dir, maxDepth = 1, currentDepth = 0, onDirectory, onContent) {
     let results = [];
-    if (!maxDepth || currentDepth === undefined || currentDepth === null || currentDepth > maxDepth) {
+    if (currentDepth > maxDepth) {
       return {
         totalItems: 0,
         totalSize: 0,
         results
       };
     }
-    const list = await fsAsync.readdir(dir, {
-      withFileTypes: true
-    });
-    const stats = await fsAsync.stat(dir);
-    const dirMetadata = {
-      totalItems: 0,
-      totalSize: 0,
-      createdTime: stats.birthtime,
-      lastModified: stats.mtime,
-      type: "directory",
-      path: dir
-    };
-    results.push(dirMetadata);
-    const promises = list.map(async file => {
-      const filePath = this.getTidePath(path.join(dir, file.name));
-      try {
-        if (file.isDirectory()) {
-          const subDirResult = await this.listPaths(filePath, maxDepth, currentDepth + 1);
-          dirMetadata.totalItems += subDirResult.totalItems;
-          dirMetadata.totalSize += subDirResult.totalSize;
-          results.push(...subDirResult.results);
-        } else {
-          const fileStats = await fsAsync.stat(filePath);
-          const fileMetadata = {
-            createdTime: fileStats.birthtime,
-            lastModified: fileStats.mtime,
-            size: fileStats.size,
-            path: filePath,
-            type: "file"
-          };
-          dirMetadata.totalSize += fileStats.size;
-          dirMetadata.totalItems += 1;
-          results.push(fileMetadata);
+    try {
+      const list = await fsAsync.readdir(dir, {
+        withFileTypes: true
+      });
+      const dirStats = await fsAsync.stat(dir);
+      const dirMetadata = {
+        totalItems: 0,
+        totalSize: 0,
+        createdTime: dirStats.birthtime,
+        lastModified: dirStats.mtime,
+        type: "directory",
+        path: dir
+      };
+      results.push(dirMetadata);
+      const processEntries = list.map(async file => {
+        const filePath = path.join(dir, file.name);
+        try {
+          if (file.isDirectory()) {
+            console.log(`[FileTide] Depth >> ${currentDepth} | Including directory ~ ${filePath}`);
+            const subDirResult = await this.listPaths(filePath, maxDepth, currentDepth + 1, onDirectory);
+            dirMetadata.totalItems += subDirResult.totalItems;
+            dirMetadata.totalSize += subDirResult.totalSize;
+            results.push(...subDirResult.results);
+            if (onDirectory) {
+              onDirectory(subDirResult.totalItems, subDirResult.results);
+            }
+          } else {
+            console.log(`[FileTide] Depth >> ${currentDepth} | Including file ~ ${filePath}`);
+            const fileStats = await fsAsync.stat(filePath);
+            const fileMetadata = {
+              createdTime: fileStats.birthtime,
+              lastModified: fileStats.mtime,
+              size: fileStats.size,
+              path: filePath,
+              type: "file"
+            };
+            dirMetadata.totalItems += 1;
+            dirMetadata.totalSize += fileStats.size;
+            results.push(fileMetadata);
+          }
+        } catch (error) {
+          if (error.code === "ENOENT") {
+            console.log(`[FileTide] Warning ~ File or directory not found - ${filePath}`);
+          } else {
+            console.log(`[FileTide] Error processing path ~ ${filePath} ~ `, error);
+          }
         }
-      } catch (error) {
-        console.log(`[FileTide] Path not found: ${filePath}`);
+      });
+      const concurrencyLimit = 5;
+      for (let i = 0; i < processEntries.length; i += concurrencyLimit) {
+        await Promise.all(processEntries.slice(i, i + concurrencyLimit));
       }
-    });
-    await Promise.all(promises);
-    return {
-      totalItems: dirMetadata.totalItems,
-      totalSize: dirMetadata.totalSize,
-      results
-    };
+      return {
+        totalItems: dirMetadata.totalItems,
+        totalSize: dirMetadata.totalSize,
+        results
+      };
+    } catch (error) {
+      if (error.code === 'ENOENT') {
+        console.log(`[FileTide] Warning ~ Directory not found - ${dir}`);
+      } else {
+        console.log(`[FileTide] Error processing directory ~ ${dir} ~ `, error);
+      }
+      return {
+        totalItems: 0,
+        totalSize: 0,
+        results
+      };
+    }
   }
 
   /**
    * Allows for listing files in a specific directory or all files in the system '*'.
    * @param {Array} directories - Array of directories to scan or '*' for entire system.
    * @param {number} depth - The maximum depth limit (from the root). Defaults to 1.
+   * @param {Funtion} onDirectory - Callback that outputs each directory's contents on the go.
    * @returns {Promise<Array>} - A promise that resolves to an array of file and directory metadata from specified directories or the entire system.
    */
-  static async listFiles(directories, depth = 1) {
+  static async listFiles(directories, depth = 1, onDirectory) {
     let fileList = [];
     if (directories.includes("*")) {
-      fileList = await this.listAllPathsInOS(depth);
+      fileList = await this.listAllPathsInOS(depth, onDirectory);
     } else {
       const dirPromises = directories.map(async dir => {
         const currentDir = this.getTidePath(dir);
         if (await this.exists(currentDir)) {
-          const subResult = await this.listPaths(currentDir, depth);
-          fileList.push(...subResult.results);
+          const subResult = await this.listPaths(currentDir, depth, 0, onDirectory);
+          if (depth <= 3) {
+            fileList.push(...subResult.results);
+          }
         } else {
-          console.warn(`Directory does not exist: ${currentDir}`);
+          console.warn(`Directory does not exist ~ ${currentDir}`);
         }
       });
       await Promise.all(dirPromises);
@@ -286,15 +318,28 @@ class FileUtilityManager {
       console.log(`Directory at path ${dirPath} not found.`);
       return [];
     }
+    const largeFileThreshold = 2 * 1024 * 1024 * 1024;
     const filesData = [];
     async function readDirectory(currentPath) {
       const files = await FileUtilityManager.filterFilesAndDirectories(currentPath, filterContent);
       for (const file of files) {
         const stats = await fs.promises.stat(file.path);
         if (stats.isFile()) {
-          const fileData = await FileUtilityManager.getFileData(file.path);
-          if (fileData) {
-            filesData.push(fileData);
+          if (stats.size > largeFileThreshold) {
+            const currentFilePath = FileUtilityManager.getTidePath(file.path);
+            console.log(`Detected large file at ${currentFilePath}, size ~ ${stats.size} bytes.`);
+            filesData.push({
+              fileName: path.basename(currentFilePath),
+              filePath: currentFilePath,
+              fileSize: stats.size,
+              largeFile: true,
+              type: "file"
+            });
+          } else if (stats.size <= largeFileThreshold) {
+            const fileData = await FileUtilityManager.getFileData(file.path);
+            if (fileData) {
+              filesData.push(fileData);
+            }
           }
         } else if (stats.isDirectory()) {
           await readDirectory(file.path);
@@ -351,10 +396,22 @@ class FileUtilityManager {
       console.log(`Path at ${currentInputPath} not found.`);
       return null;
     }
+    const largeFileThreshold = 2 * 1024 * 1024 * 1024;
     const stats = fs.statSync(currentInputPath);
     if (stats.isFile()) {
+      if (stats.size > largeFileThreshold) {
+        console.log(`Detected large file at ${currentInputPath}, size ~ ${stats.size} bytes.`);
+        return {
+          fileName: path.basename(currentInputPath),
+          filePath: currentInputPath,
+          fileSize: stats.size,
+          largeFile: true,
+          type: "file"
+        };
+      }
       return {
         content: await this.getFileData(currentInputPath),
+        largeFile: false,
         type: "file"
       };
     } else if (stats.isDirectory()) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@trap_stevo/filetide",
-  "version": "0.0.39",
+  "version": "0.0.41",
   "description": "Revolutionizing real-time file transfer with seamless, instant communication across any device. Deliver files instantly, regardless of platform, and experience unparalleled speed and control in managing transfers. Elevate your file-sharing capabilities with a tool designed for precision, efficiency, and effortless connectivity.",
   "main": "dist/cjs/FileTide.js",
   "scripts": {
@@ -30,8 +30,8 @@
   "license": "See License in LICENSE.md",
   "dependencies": {
     "@trap_stevo/iotide": "^0.0.37",
-    "@trap_stevo/iotide-client": "^0.0.20",
-    "@trap_stevo/legendarybuilderpronodejs-utilities": "^1.0.40",
+    "@trap_stevo/iotide-client": "^0.0.23",
+    "@trap_stevo/legendarybuilderpronodejs-utilities": "^1.0.42",
     "chalk": "^4.1.2",
     "readline": "^1.3.0"
   },