@dignetwork/dig-sdk 0.0.1-alpha.10 → 0.0.1-alpha.11

package/dist/utils/directoryUtils.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"directoryUtils.d.ts","sourceRoot":"","sources":["../../src/utils/directoryUtils.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AAEzD;;;;;GAKG;AACH,eAAO,MAAM,YAAY,cACZ,iBAAiB,WACnB,MAAM,YACN,MAAM,KACd,OAAO,CAAC,IAAI,CAmCd,CAAC;AAEF;;;;GAIG;AACH,eAAO,MAAM,mBAAmB,eAAgB,MAAM,KAAG,MAiBxD,CAAC"}
+{"version":3,"file":"directoryUtils.d.ts","sourceRoot":"","sources":["../../src/utils/directoryUtils.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AAUzD;;;;;GAKG;AACH,eAAO,MAAM,YAAY,cACZ,iBAAiB,WACnB,MAAM,YACN,MAAM,KACd,OAAO,CAAC,IAAI,CA0Cd,CAAC;AAGF;;;;GAIG;AACH,eAAO,MAAM,mBAAmB,eAAgB,MAAM,KAAG,MAiBxD,CAAC"}

package/dist/utils/directoryUtils.js

@@ -29,7 +29,14 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.calculateFolderSize = exports.addDirectory = void 0;
 const fs = __importStar(require("fs"));
 const path = __importStar(require("path"));
+const util_1 = require("util");
 const ignore_1 = __importDefault(require("ignore"));
+const p_limit_1 = __importDefault(require("p-limit"));
+const limit = (0, p_limit_1.default)(10); // Limit the concurrency to 10 (adjust based on your system's file descriptor limit)
+// Promisify fs methods
+const readdir = (0, util_1.promisify)(fs.readdir);
+const stat = (0, util_1.promisify)(fs.stat);
+const readFile = (0, util_1.promisify)(fs.readFile);
 /**
  * Recursively add all files in a directory to the Merkle tree, skipping the .dig, .git folders, and files in .gitignore.
  * @param datalayer - The DataStoreManager instance.
@@ -41,31 +48,34 @@ const addDirectory = async (datalayer, dirPath, baseDir = dirPath) => {
     const gitignorePath = path.join(baseDir, ".gitignore");
     // Load .gitignore rules if the file exists
     if (fs.existsSync(gitignorePath)) {
-        const gitignoreContent = fs.readFileSync(gitignorePath, "utf-8");
+        const gitignoreContent = await readFile(gitignorePath, "utf-8");
         ig.add(gitignoreContent);
     }
-    const files = fs.readdirSync(dirPath);
-    for (const file of files) {
+    const files = await readdir(dirPath);
+    // Process each file or directory
+    await Promise.all(files.map(async (file) => {
         const filePath = path.join(dirPath, file);
         const relativePath = path.relative(baseDir, filePath).replace(/\\/g, "/");
         // Skip the .dig, .git folders and files or directories ignored by .gitignore
         if (file === ".dig" || file === ".git" || ig.ignores(relativePath)) {
-            continue;
+            return;
         }
-        const stat = fs.statSync(filePath);
-        if (stat.isDirectory()) {
-            await (0, exports.addDirectory)(datalayer, filePath, baseDir);
+        const fileStat = await stat(filePath);
+        if (fileStat.isDirectory()) {
+            // Recursively process the directory
+            return (0, exports.addDirectory)(datalayer, filePath, baseDir);
         }
         else {
-            await new Promise((resolve, reject) => {
+            // Process the file with limited concurrency
+            return limit(() => new Promise((resolve, reject) => {
                 const stream = fs.createReadStream(filePath);
                 datalayer
                     .upsertKey(stream, Buffer.from(relativePath).toString("hex"))
                     .then(resolve)
                     .catch(reject);
-            });
+            }));
         }
-    }
+    }));
 };
 exports.addDirectory = addDirectory;
 /**

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dignetwork/dig-sdk",
-  "version": "0.0.1-alpha.10",
+  "version": "0.0.1-alpha.11",
   "description": "",
   "type": "commonjs",
   "main": "./dist/index.js",
@@ -41,6 +41,7 @@
     "merkletreejs": "^0.4.0",
     "nanospinner": "^1.1.0",
     "nconf": "^0.12.1",
+    "p-limit": "^6.1.0",
     "superagent": "^10.0.0"
   },
   "devDependencies": {