@dignetwork/dig-sdk 0.0.1-alpha.15 → 0.0.1-alpha.17

package/dist/DataIntegrityTree/DataIntegrityTree.d.ts

@@ -106,7 +106,7 @@ declare class DataIntegrityTree {
      * @param rootHash - The root hash of the tree. Defaults to the latest root hash.
      * @returns The readable stream for the file.
      */
-    getValueStream(hexKey: string, rootHash?: string | null): Readable;
+    getValueStream(hexKey: string, rootHash?: string | null, byteOffset?: number | null, length?: number | null): Readable;
     /**
      * Delete all leaves from the Merkle tree.
      */

package/dist/DataIntegrityTree/DataIntegrityTree.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"DataIntegrityTree.d.ts","sourceRoot":"","sources":["../../src/DataIntegrityTree/DataIntegrityTree.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAC1C,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAuClC,MAAM,WAAW,wBAAwB;IACvC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAClC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAGlB,iBAAiB,CAAC,EAAE,OAAO,CAAC;CAC7B;AAED;;GAEG;AACH,cAAM,iBAAiB;IACrB,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,YAAY,CAAS;IAC7B,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,OAAO,CAAS;IACjB,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC5D,OAAO,CAAC,IAAI,CAAa;gBAEb,OAAO,EAAE,MAAM,EAAE,OAAO,GAAE,wBAA6B;WAiDrD,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,wBAAwB,GAAG,iBAAiB;IAIzF;;;OAGG;IACH,OAAO,CAAC,aAAa;IAQrB;;;OAGG;IACH,OAAO,CAAC,eAAe;IAUvB;;;;OAIG;IACH,OAAO,CAAC,kBAAkB;IAa1B;;;;OAIG;YACW,WAAW;IAazB;;;;;OAKG;IACG,SAAS,CAAC,UAAU,EAAE,QAAQ,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAmHjE;;;OAGG;IACH,SAAS,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI;IAW5B;;;;OAIG;IACH,QAAQ,CAAC,QAAQ,GAAE,MAAM,GAAG,IAAW,GAAG,MAAM,EAAE;IAalD;;;OAGG;IACH,OAAO,CAAC,YAAY;IAOpB;;;;OAIG;IACH,OAAO,IAAI,MAAM;IAIjB;;;;OAIG;IACH,SAAS,CAAC,QAAQ,GAAE,MAAM,GAAG,IAAW,GAAG,MAAM;IAqBjD;;;;OAIG;IACH,eAAe,CAAC,QAAQ,EAAE,MAAM,GAAG,UAAU;IAwB7C,OAAO,CAAC,wBAAwB;IAkBhC;;OAEG;IACH,MAAM,IAAI,MAAM,GAAG,SAAS;IAoC5B;;OAEG;IACH,gBAAgB,IAAI,IAAI;IAIxB;;;;;OAKG;IACH,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,GAAE,MAAM,GAAG,IAAW,GAAG,OAAO;IA+B/D;;;;;OAKG;IACH,cAAc,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,GAAE,MAAM,GAAG,IAAW,GAAG,QAAQ;IAuCxE;;OAEG;IACH,eAAe,IAAI,IAAI;IAKvB,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS;IAUhE;;;;;;OAMG;IACH,QAAQ,CACN,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,MAAM,EACd,QAAQ,GAAE,MAAM,GAAG,IAAW,GAC7B,MAAM;IAsCT;;;;;OAKG;IACH,WAAW,CAAC,cAAc,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO;IA4B5D;;;;;OAKG;IACH,WAAW,CACT,SAAS,EAAE,MAAM,EACjB,SAAS,EAAE,MAAM,GAChB;QAAE,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAAC,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;KAAE;IA+B/D;;;;;OAKG;IACG,kBAAkB,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IA0D5E;;;;;;;;;OASG;IACH,MAAM,CAAC,mCAAmC,CACxC,GAAG,EAAE,MAAM,EACX,MAAM,EAAE,MAAM,EACd,cAAc,EAAE,MAAM,EACtB,gBAAgB,EAAE,MAAM,GACvB,OAAO;CAgDX;AAED,OAAO,EAAE,iBAAiB,EAAE,CAAC"}
+{"version":3,"file":"DataIntegrityTree.d.ts","sourceRoot":"","sources":["../../src/DataIntegrityTree/DataIntegrityTree.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAC1C,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAuClC,MAAM,WAAW,wBAAwB;IACvC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAClC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAGlB,iBAAiB,CAAC,EAAE,OAAO,CAAC;CAC7B;AAED;;GAEG;AACH,cAAM,iBAAiB;IACrB,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,YAAY,CAAS;IAC7B,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,OAAO,CAAS;IACjB,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC5D,OAAO,CAAC,IAAI,CAAa;gBAEb,OAAO,EAAE,MAAM,EAAE,OAAO,GAAE,wBAA6B;WAiDrD,IAAI,CAChB,OAAO,EAAE,MAAM,EACf,OAAO,EAAE,wBAAwB,GAChC,iBAAiB;IAOpB;;;OAGG;IACH,OAAO,CAAC,aAAa;IAQrB;;;OAGG;IACH,OAAO,CAAC,eAAe;IAUvB;;;;OAIG;IACH,OAAO,CAAC,kBAAkB;IAa1B;;;;OAIG;YACW,WAAW;IAazB;;;;;OAKG;IACG,SAAS,CAAC,UAAU,EAAE,QAAQ,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAqFjE;;;OAGG;IACH,SAAS,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI;IAW5B;;;;OAIG;IACH,QAAQ,CAAC,QAAQ,GAAE,MAAM,GAAG,IAAW,GAAG,MAAM,EAAE;IAalD;;;OAGG;IACH,OAAO,CAAC,YAAY;IAOpB;;;;OAIG;IACH,OAAO,IAAI,MAAM;IAIjB;;;;OAIG;IACH,SAAS,CAAC,QAAQ,GAAE,MAAM,GAAG,IAAW,GAAG,MAAM;IAqBjD;;;;OAIG;IACH,eAAe,CAAC,QAAQ,EAAE,MAAM,GAAG,UAAU;IAwB7C,OAAO,CAAC,wBAAwB;IAuBhC;;OAEG;IACH,MAAM,IAAI,MAAM,GAAG,SAAS;IAoC5B;;OAEG;IACH,gBAAgB,IAAI,IAAI;IAIxB;;;;;OAKG;IACH,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,GAAE,MAAM,GAAG,IAAW,GAAG,OAAO;IA+B/D;;;;;OAKG;IACH,cAAc,CACZ,MAAM,EAAE,MAAM,EACd,QAAQ,GAAE,MAAM,GAAG,IAAW,EAC9B,UAAU,GAAE,MAAM,GAAG,IAAW,EAChC,MAAM,GAAE,MAAM,GAAG,IAAW,GAC3B,QAAQ;IA4DX;;OAEG;IACH,eAAe,IAAI,IAAI;IAKvB,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS;IAUhE;;;;;;OAMG;IACH,QAAQ,CACN,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,MAAM,EACd,QAAQ,GAAE,MAAM,GAAG,IAAW,GAC7B,MAAM;IAsCT;;;;;OAKG;IACH,WAAW,CAAC,cAAc,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO;IA4B5D;;;;;OAKG;IACH,WAAW,CACT,SAAS,EAAE,MAAM,EACjB,SAAS,EAAE,MAAM,GAChB;QAAE,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAAC,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;KAAE;IA+B/D;;;;;OAKG;IACG,kBAAkB,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IA0D5E;;;;;;;;;OASG;IACH,MAAM,CAAC,mCAAmC,CACxC,GAAG,EAAE,MAAM,EACX,MAAM,EAAE,MAAM,EACd,cAAc,EAAE,MAAM,EACtB,gBAAgB,EAAE,MAAM,GACvB,OAAO;CAgDX;AAED,OAAO,EAAE,iBAAiB,EAAE,CAAC"}

package/dist/DataIntegrityTree/DataIntegrityTree.js

@@ -111,7 +111,10 @@ class DataIntegrityTree {
         }
     }
     static from(storeId, options) {
-        return new DataIntegrityTree(storeId, { ...options, disableInitialize: true });
+        return new DataIntegrityTree(storeId, {
+            ...options,
+            disableInitialize: true,
+        });
     }
     /**
      * Load the manifest file.
@@ -188,81 +191,54 @@ class DataIntegrityTree {
         const tempFilePath = path.join(tempDir, `${crypto.randomUUID()}.gz`);
         return new Promise((resolve, reject) => {
             const tempWriteStream = fs.createWriteStream(tempFilePath);
-            // Update the hash with the original data
             readStream.on("data", (chunk) => {
                 uncompressedHash.update(chunk);
             });
-            // Pipe the read stream through gzip into the temporary write stream
             readStream.pipe(gzip).pipe(tempWriteStream);
             tempWriteStream.on("finish", async () => {
-                let finalWriteStream;
+                sha256 = uncompressedHash.digest("hex");
+                const finalWriteStream = this._createWriteStream(sha256);
+                const finalPath = finalWriteStream.path;
+                // Ensure the directory exists before copying the file
+                const finalDir = path.dirname(finalPath);
+                if (!fs.existsSync(finalDir)) {
+                    fs.mkdirSync(finalDir, { recursive: true });
+                }
                 try {
-                    sha256 = uncompressedHash.digest("hex");
-                    finalWriteStream = this._createWriteStream(sha256);
-                    const finalPath = finalWriteStream.path;
-                    // Ensure the directory exists
-                    const finalDir = path.dirname(finalPath);
-                    if (!fs.existsSync(finalDir)) {
-                        fs.mkdirSync(finalDir, { recursive: true });
-                    }
-                    // Copy the temporary gzipped file to the final destination
                     await this._streamFile(tempFilePath, finalPath);
-                    await unlink(tempFilePath); // Clean up the temporary file
+                    await unlink(tempFilePath);
                     const combinedHash = crypto
                         .createHash("sha256")
                         .update(`${key}/${sha256}`)
                         .digest("hex");
-                    // Check if the key already exists with the same hash
                     if (Array.from(this.files.values()).some((file) => file.hash === combinedHash)) {
                         console.log(`No changes detected for key: ${key}`);
                         return resolve();
                     }
-                    // Delete existing key if present
                     if (this.files.has(key)) {
                         this.deleteKey(key);
                     }
-                    // Insert the new key with the hash
                     console.log(`Inserted key: ${key}`);
                     this.files.set(key, {
                         hash: combinedHash,
                         sha256: sha256,
                     });
+                    await new Promise((resolve) => setTimeout(resolve, 100));
                     this._rebuildTree();
+                    await new Promise((resolve) => setTimeout(resolve, 100));
                     resolve();
                 }
                 catch (err) {
-                    // On error, cleanup the temporary file and reject
-                    await unlink(tempFilePath).catch(() => { });
                     reject(err);
                 }
-                finally {
-                    // Always close the final write stream if it exists
-                    if (finalWriteStream) {
-                        finalWriteStream.end();
-                    }
-                }
+                tempWriteStream.end();
+                finalWriteStream.end();
             });
-            tempWriteStream.on("error", async (err) => {
-                // Close streams and clean up in case of error
-                tempWriteStream.destroy();
-                gzip.destroy();
-                readStream.destroy();
-                await unlink(tempFilePath).catch(() => { }); // Clean up the temp file
+            tempWriteStream.on("error", (err) => {
+                tempWriteStream.end();
                 reject(err);
             });
-            readStream.on("error", async (err) => {
-                // Close streams and clean up in case of error
-                tempWriteStream.destroy();
-                gzip.destroy();
-                readStream.destroy();
-                await unlink(tempFilePath).catch(() => { }); // Clean up the temp file
-                reject(err);
-            });
-            gzip.on("error", (err) => {
-                // Handle errors in the gzip stream
-                tempWriteStream.destroy();
-                gzip.destroy();
-                readStream.destroy();
+            readStream.on("error", (err) => {
                 reject(err);
             });
         });
@@ -369,7 +345,9 @@ class DataIntegrityTree {
             ? fs.readFileSync(manifestPath, "utf-8").trim().split("\n")
             : [];
         // Check if the last entry is the same as the rootHash to avoid duplicates
-        const latestRootHash = manifestContent.length > 0 ? manifestContent[manifestContent.length - 1] : null;
+        const latestRootHash = manifestContent.length > 0
+            ? manifestContent[manifestContent.length - 1]
+            : null;
         if (latestRootHash !== rootHash) {
             // Append the new rootHash if it is not the same as the last one
             fs.appendFileSync(manifestPath, `${rootHash}\n`);
@@ -445,7 +423,7 @@ class DataIntegrityTree {
      * @param rootHash - The root hash of the tree. Defaults to the latest root hash.
      * @returns The readable stream for the file.
      */
-    getValueStream(hexKey, rootHash = null) {
+    getValueStream(hexKey, rootHash = null, byteOffset = null, length = null) {
         if (!isHexString(hexKey)) {
             throw new Error("key must be a valid hex string");
         }
@@ -468,8 +446,22 @@ class DataIntegrityTree {
         if (!fs.existsSync(filePath)) {
             throw new Error(`File at path ${filePath} does not exist`);
         }
-        // Create a read stream and pipe it through a decompression stream using the same algorithm (gzip)
-        const readStream = fs.createReadStream(filePath);
+        const fileSize = fs.statSync(filePath).size;
+        // Validate offset and length
+        if (byteOffset !== null && length !== null) {
+            if (byteOffset + length > fileSize) {
+                throw new Error(`Offset (${byteOffset}) and length (${length}) exceed the file size (${fileSize}).`);
+            }
+        }
+        // Create the read stream with optional byte range
+        const options = {};
+        if (byteOffset !== null) {
+            options.start = byteOffset;
+        }
+        if (length !== null && byteOffset !== null) {
+            options.end = byteOffset + length - 1; // `end` is inclusive, hence `byteOffset + length - 1`
+        }
+        const readStream = fs.createReadStream(filePath, options);
         const decompressStream = zlib.createGunzip();
         // Return the combined stream as a generic Readable stream
         return readStream.pipe(decompressStream);

package/dist/utils/directoryUtils.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"directoryUtils.d.ts","sourceRoot":"","sources":["../../src/utils/directoryUtils.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AA8BzD,eAAO,MAAM,YAAY,cACZ,iBAAiB,WACnB,MAAM,YACN,MAAM,KACd,OAAO,CAAC,IAAI,CAiDd,CAAC;AAEF;;;;GAIG;AACH,eAAO,MAAM,mBAAmB,eAAgB,MAAM,KAAG,MAiBxD,CAAC"}
+{"version":3,"file":"directoryUtils.d.ts","sourceRoot":"","sources":["../../src/utils/directoryUtils.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AA+BzD,eAAO,MAAM,YAAY,cACZ,iBAAiB,WACnB,MAAM,YACN,MAAM,KACd,OAAO,CAAC,IAAI,CA4Cd,CAAC;AAEF;;;;GAIG;AACH,eAAO,MAAM,mBAAmB,eAAgB,MAAM,KAAG,MAiBxD,CAAC"}

package/dist/utils/directoryUtils.js

@@ -32,22 +32,23 @@ const fs = __importStar(require("fs"));
 const ignore_1 = __importDefault(require("ignore"));
 // Custom concurrency handler
 const limitConcurrency = async (concurrencyLimit, tasks) => {
-    const results = [];
     const executing = [];
     for (const task of tasks) {
         const p = task();
-        results.push(p);
+        // Add the new task to the executing array
+        executing.push(p);
+        // When a task completes, remove it from the executing array
+        const cleanup = p.finally(() => {
+            executing.splice(executing.indexOf(cleanup), 1);
+        });
         // Once the limit is reached, wait for one to complete
         if (executing.length >= concurrencyLimit) {
+            await new Promise((resolve) => setTimeout(resolve, 100));
             await Promise.race(executing);
         }
-        // Add the new task to the executing array
-        executing.push(p);
-        // When a task completes, remove it from the executing array
-        p.finally(() => executing.splice(executing.indexOf(p), 1));
     }
     // Wait for all remaining tasks to complete
-    return Promise.all(results);
+    return Promise.all(executing);
 };
 const addDirectory = async (datalayer, dirPath, baseDir = dirPath) => {
     const ig = (0, ignore_1.default)();
@@ -77,14 +78,10 @@ const addDirectory = async (datalayer, dirPath, baseDir = dirPath) => {
                 const stream = fs.createReadStream(filePath);
                 datalayer
                     .upsertKey(stream, Buffer.from(relativePath).toString("hex"))
-                    .then(async () => {
-                    await new Promise((resolve) => setTimeout(resolve, 100));
-                    resolve();
-                })
+                    .then(() => resolve())
                     .catch(reject);
             }));
         }
-        await new Promise((resolve) => setTimeout(resolve, 100));
     }
     // Run tasks with limited concurrency (set the concurrency limit as needed)
     await limitConcurrency(10, tasks); // Adjust 10 based on your system limits

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dignetwork/dig-sdk",
-  "version": "0.0.1-alpha.15",
+  "version": "0.0.1-alpha.17",
   "description": "",
   "type": "commonjs",
   "main": "./dist/index.js",