@dignetwork/dig-sdk 0.0.1-alpha.11 → 0.0.1-alpha.111

package/dist/DataIntegrityTree/DataIntegrityTree.d.ts

@@ -78,6 +78,9 @@ declare class DataIntegrityTree {
      * @returns The serialized Merkle tree.
      */
     serialize(rootHash?: string | null): object;
+    static getRootOfForeignTree(treeData: {
+        leaves: string[];
+    }): string;
     /**
      * Deserialize a JSON object to a Merkle tree.
      * @param rootHash - The root hash of the tree.
@@ -106,7 +109,7 @@ declare class DataIntegrityTree {
      * @param rootHash - The root hash of the tree. Defaults to the latest root hash.
      * @returns The readable stream for the file.
      */
-    getValueStream(hexKey: string, rootHash?: string | null): Readable;
+    getValueStream(hexKey: string, rootHash?: string | null, byteOffset?: number | null, length?: number | null): Readable;
     /**
      * Delete all leaves from the Merkle tree.
      */
@@ -152,9 +155,14 @@ declare class DataIntegrityTree {
      * @param sha256 - The SHA-256 hash of the file.
      * @param serializedTree - The foreign serialized Merkle tree.
      * @param expectedRootHash - The expected root hash of the Merkle tree.
+     * @param dataDir - The directory where the data is stored.
+     * @param verifiedSha256 - A boolean indicating if the SHA-256 hash has already been verified.
      * @returns A boolean indicating if the SHA-256 is present in the foreign tree and the root hash matches.
      */
-    static validateKeyIntegrityWithForeignTree(key: string, sha256: string, serializedTree: object, expectedRootHash: string): boolean;
+    static validateKeyIntegrityWithForeignTree(hexkey: string, sha256: string, serializedTree: object, expectedRootHash: string, dataDir: string, verifiedSha256?: boolean): Promise<boolean>;
+    private static verifyFileHash;
+    private static deserializeAndVerifyTree;
+    private static checkHashInTree;
 }
 export { DataIntegrityTree };
 //# sourceMappingURL=DataIntegrityTree.d.ts.map

package/dist/DataIntegrityTree/DataIntegrityTree.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"DataIntegrityTree.d.ts","sourceRoot":"","sources":["../../src/DataIntegrityTree/DataIntegrityTree.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAC1C,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAuClC,MAAM,WAAW,wBAAwB;IACvC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAClC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAGlB,iBAAiB,CAAC,EAAE,OAAO,CAAC;CAC7B;AAED;;GAEG;AACH,cAAM,iBAAiB;IACrB,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,YAAY,CAAS;IAC7B,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,OAAO,CAAS;IACjB,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC5D,OAAO,CAAC,IAAI,CAAa;gBAEb,OAAO,EAAE,MAAM,EAAE,OAAO,GAAE,wBAA6B;WAiDrD,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,wBAAwB,GAAG,iBAAiB;IAIzF;;;OAGG;IACH,OAAO,CAAC,aAAa;IAQrB;;;OAGG;IACH,OAAO,CAAC,eAAe;IAUvB;;;;OAIG;IACH,OAAO,CAAC,kBAAkB;IAa1B;;;;OAIG;YACW,WAAW;IAazB;;;;;OAKG;IACG,SAAS,CAAC,UAAU,EAAE,QAAQ,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IA+EjE;;;OAGG;IACH,SAAS,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI;IAW5B;;;;OAIG;IACH,QAAQ,CAAC,QAAQ,GAAE,MAAM,GAAG,IAAW,GAAG,MAAM,EAAE;IAalD;;;OAGG;IACH,OAAO,CAAC,YAAY;IAOpB;;;;OAIG;IACH,OAAO,IAAI,MAAM;IAIjB;;;;OAIG;IACH,SAAS,CAAC,QAAQ,GAAE,MAAM,GAAG,IAAW,GAAG,MAAM;IAqBjD;;;;OAIG;IACH,eAAe,CAAC,QAAQ,EAAE,MAAM,GAAG,UAAU;IAwB7C,OAAO,CAAC,wBAAwB;IAkBhC;;OAEG;IACH,MAAM,IAAI,MAAM,GAAG,SAAS;IAoC5B;;OAEG;IACH,gBAAgB,IAAI,IAAI;IAIxB;;;;;OAKG;IACH,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,GAAE,MAAM,GAAG,IAAW,GAAG,OAAO;IA+B/D;;;;;OAKG;IACH,cAAc,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,GAAE,MAAM,GAAG,IAAW,GAAG,QAAQ;IAuCxE;;OAEG;IACH,eAAe,IAAI,IAAI;IAKvB,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS;IAUhE;;;;;;OAMG;IACH,QAAQ,CACN,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,MAAM,EACd,QAAQ,GAAE,MAAM,GAAG,IAAW,GAC7B,MAAM;IAsCT;;;;;OAKG;IACH,WAAW,CAAC,cAAc,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO;IA4B5D;;;;;OAKG;IACH,WAAW,CACT,SAAS,EAAE,MAAM,EACjB,SAAS,EAAE,MAAM,GAChB;QAAE,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAAC,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;KAAE;IA+B/D;;;;;OAKG;IACG,kBAAkB,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IA0D5E;;;;;;;;;OASG;IACH,MAAM,CAAC,mCAAmC,CACxC,GAAG,EAAE,MAAM,EACX,MAAM,EAAE,MAAM,EACd,cAAc,EAAE,MAAM,EACtB,gBAAgB,EAAE,MAAM,GACvB,OAAO;CAgDX;AAED,OAAO,EAAE,iBAAiB,EAAE,CAAC"}
+{"version":3,"file":"DataIntegrityTree.d.ts","sourceRoot":"","sources":["../../src/DataIntegrityTree/DataIntegrityTree.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAC1C,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAuClC,MAAM,WAAW,wBAAwB;IACvC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAClC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAGlB,iBAAiB,CAAC,EAAE,OAAO,CAAC;CAC7B;AAED;;GAEG;AACH,cAAM,iBAAiB;IACrB,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,YAAY,CAAS;IAC7B,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,OAAO,CAAS;IACjB,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC5D,OAAO,CAAC,IAAI,CAAa;gBAEb,OAAO,EAAE,MAAM,EAAE,OAAO,GAAE,wBAA6B;WAgDrD,IAAI,CAChB,OAAO,EAAE,MAAM,EACf,OAAO,EAAE,wBAAwB,GAChC,iBAAiB;IAOpB;;;OAGG;IACH,OAAO,CAAC,aAAa;IAQrB;;;OAGG;IACH,OAAO,CAAC,eAAe;IAevB;;;;OAIG;IACH,OAAO,CAAC,kBAAkB;IAa1B;;;;OAIG;YACW,WAAW;IAazB;;;;;OAKG;IACG,SAAS,CAAC,UAAU,EAAE,QAAQ,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAqFjE;;;OAGG;IACH,SAAS,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI;IAW5B;;;;OAIG;IACH,QAAQ,CAAC,QAAQ,GAAE,MAAM,GAAG,IAAW,GAAG,MAAM,EAAE;IAalD;;;OAGG;IACH,OAAO,CAAC,YAAY;IAOpB;;;;OAIG;IACH,OAAO,IAAI,MAAM;IAIjB;;;;OAIG;IACH,SAAS,CAAC,QAAQ,GAAE,MAAM,GAAG,IAAW,GAAG,MAAM;IAqBjD,MAAM,CAAC,oBAAoB,CAAC,QAAQ,EAAE;QAAE,MAAM,EAAE,MAAM,EAAE,CAAA;KAAE,GAAG,MAAM;IAQnE;;;;OAIG;IACH,eAAe,CAAC,QAAQ,EAAE,MAAM,GAAG,UAAU;IAwB7C,OAAO,CAAC,wBAAwB;IAuBhC;;OAEG;IACH,MAAM,IAAI,MAAM,GAAG,SAAS;IAoC5B;;OAEG;IACH,gBAAgB,IAAI,IAAI;IAIxB;;;;;OAKG;IACH,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,GAAE,MAAM,GAAG,IAAW,GAAG,OAAO;IA+B/D;;;;;OAKG;IACH,cAAc,CACZ,MAAM,EAAE,MAAM,EACd,QAAQ,GAAE,MAAM,GAAG,IAAW,EAC9B,UAAU,GAAE,MAAM,GAAG,IAAW,EAChC,MAAM,GAAE,MAAM,GAAG,IAAW,GAC3B,QAAQ;IA4DX;;OAEG;IACH,eAAe,IAAI,IAAI;IAKvB,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS;IAUhE;;;;;;OAMG;IACH,QAAQ,CACN,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,MAAM,EACd,QAAQ,GAAE,MAAM,GAAG,IAAW,GAC7B,MAAM;IAsCT;;;;;OAKG;IACH,WAAW,CAAC,cAAc,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO;IA4B5D;;;;;OAKG;IACH,WAAW,CACT,SAAS,EAAE,MAAM,EACjB,SAAS,EAAE,MAAM,GAChB;QAAE,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAAC,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;KAAE;IA+B/D;;;;;OAKG;IACG,kBAAkB,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAyD5E;;;;;;;;;;;OAWG;WACU,mCAAmC,CAC9C,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,MAAM,EACd,cAAc,EAAE,MAAM,EACtB,gBAAgB,EAAE,MAAM,EACxB,OAAO,EAAE,MAAM,EACf,cAAc,GAAE,OAAe,GAC9B,OAAO,CAAC,OAAO,CAAC;mBAgCE,cAAc;IA8CnC,OAAO,CAAC,MAAM,CAAC,wBAAwB;IAkCvC,OAAO,CAAC,MAAM,CAAC,eAAe;CAgB/B;AAED,OAAO,EAAE,iBAAiB,EAAE,CAAC"}

package/dist/DataIntegrityTree/DataIntegrityTree.js

@@ -94,8 +94,7 @@ class DataIntegrityTree {
         }
         this.files = new Map();
         if (options.rootHash) {
-            const manifest = this._loadManifest();
-            if (manifest.includes(options.rootHash)) {
+            if (fs.existsSync(path.join(this.storeDir, `${options.rootHash}.dat`))) {
                 this.tree = this.deserializeTree(options.rootHash);
             }
             else {
@@ -111,7 +110,10 @@ class DataIntegrityTree {
         }
     }
     static from(storeId, options) {
-        return new DataIntegrityTree(storeId, { ...options, disableInitialize: true });
+        return new DataIntegrityTree(storeId, {
+            ...options,
+            disableInitialize: true,
+        });
     }
     /**
      * Load the manifest file.
@@ -132,7 +134,10 @@ class DataIntegrityTree {
         const manifest = this._loadManifest();
         if (manifest.length > 0) {
             const latestRootHash = manifest[manifest.length - 1];
-            return this.deserializeTree(latestRootHash);
+            if (latestRootHash && isHexString(latestRootHash)) {
+                return this.deserializeTree(latestRootHash);
+            }
+            return new merkletreejs_1.MerkleTree([], crypto_js_1.SHA256, { sortPairs: true });
         }
         else {
             return new merkletreejs_1.MerkleTree([], crypto_js_1.SHA256, { sortPairs: true });
@@ -220,14 +225,19 @@ class DataIntegrityTree {
                         hash: combinedHash,
                         sha256: sha256,
                     });
+                    await new Promise((resolve) => setTimeout(resolve, 100));
                     this._rebuildTree();
+                    await new Promise((resolve) => setTimeout(resolve, 100));
                     resolve();
                 }
                 catch (err) {
                     reject(err);
                 }
+                tempWriteStream.end();
+                finalWriteStream.end();
             });
             tempWriteStream.on("error", (err) => {
+                tempWriteStream.end();
                 reject(err);
             });
             readStream.on("error", (err) => {
@@ -305,6 +315,11 @@ class DataIntegrityTree {
             files: Object.fromEntries(this.files),
         };
     }
+    static getRootOfForeignTree(treeData) {
+        const leaves = treeData.leaves.map((leaf) => Buffer.from(leaf, "hex"));
+        const tree = new merkletreejs_1.MerkleTree(leaves, crypto_js_1.SHA256, { sortPairs: true });
+        return tree.getRoot().toString("hex");
+    }
     /**
      * Deserialize a JSON object to a Merkle tree.
      * @param rootHash - The root hash of the tree.
@@ -337,7 +352,9 @@ class DataIntegrityTree {
             ? fs.readFileSync(manifestPath, "utf-8").trim().split("\n")
             : [];
         // Check if the last entry is the same as the rootHash to avoid duplicates
-        const latestRootHash = manifestContent.length > 0 ? manifestContent[manifestContent.length - 1] : null;
+        const latestRootHash = manifestContent.length > 0
+            ? manifestContent[manifestContent.length - 1]
+            : null;
         if (latestRootHash !== rootHash) {
             // Append the new rootHash if it is not the same as the last one
             fs.appendFileSync(manifestPath, `${rootHash}\n`);
@@ -413,7 +430,7 @@ class DataIntegrityTree {
      * @param rootHash - The root hash of the tree. Defaults to the latest root hash.
      * @returns The readable stream for the file.
      */
-    getValueStream(hexKey, rootHash = null) {
+    getValueStream(hexKey, rootHash = null, byteOffset = null, length = null) {
         if (!isHexString(hexKey)) {
             throw new Error("key must be a valid hex string");
         }
@@ -436,8 +453,22 @@ class DataIntegrityTree {
         if (!fs.existsSync(filePath)) {
             throw new Error(`File at path ${filePath} does not exist`);
         }
-        // Create a read stream and pipe it through a decompression stream using the same algorithm (gzip)
-        const readStream = fs.createReadStream(filePath);
+        const fileSize = fs.statSync(filePath).size;
+        // Validate offset and length
+        if (byteOffset !== null && length !== null) {
+            if (byteOffset + length > fileSize) {
+                throw new Error(`Offset (${byteOffset}) and length (${length}) exceed the file size (${fileSize}).`);
+            }
+        }
+        // Create the read stream with optional byte range
+        const options = {};
+        if (byteOffset !== null) {
+            options.start = byteOffset;
+        }
+        if (length !== null && byteOffset !== null) {
+            options.end = byteOffset + length - 1; // `end` is inclusive, hence `byteOffset + length - 1`
+        }
+        const readStream = fs.createReadStream(filePath, options);
         const decompressStream = zlib.createGunzip();
         // Return the combined stream as a generic Readable stream
         return readStream.pipe(decompressStream);
@@ -579,14 +610,13 @@ class DataIntegrityTree {
             decompressStream.on("end", () => {
                 const uncompressedSha256 = hash.digest("hex");
                 const isValid = uncompressedSha256 === sha256;
-                console.log(`SHA-256 of uncompressed file: ${uncompressedSha256}`);
                 if (!isValid) {
                     return resolve(false);
                 }
                 const tree = this.deserializeTree(rootHash);
                 const combinedHash = crypto
                     .createHash("sha256")
-                    .update(`${toHex(sha256)}/${sha256}`)
+                    .update(sha256)
                     .digest("hex");
                 const leaf = Buffer.from(combinedHash, "hex");
                 const isInTree = tree.getLeafIndex(leaf) !== -1;
@@ -608,10 +638,12 @@ class DataIntegrityTree {
      * @param sha256 - The SHA-256 hash of the file.
      * @param serializedTree - The foreign serialized Merkle tree.
      * @param expectedRootHash - The expected root hash of the Merkle tree.
+     * @param dataDir - The directory where the data is stored.
+     * @param verifiedSha256 - A boolean indicating if the SHA-256 hash has already been verified.
      * @returns A boolean indicating if the SHA-256 is present in the foreign tree and the root hash matches.
      */
-    static validateKeyIntegrityWithForeignTree(key, sha256, serializedTree, expectedRootHash) {
-        if (!isHexString(key)) {
+    static async validateKeyIntegrityWithForeignTree(hexkey, sha256, serializedTree, expectedRootHash, dataDir, verifiedSha256 = false) {
+        if (!isHexString(hexkey)) {
             throw new Error("key must be a valid hex string");
         }
         if (!isHexString(sha256)) {
@@ -620,6 +652,56 @@ class DataIntegrityTree {
         if (!isHexString(expectedRootHash)) {
             throw new Error("expectedRootHash must be a valid hex string");
         }
+        if (!verifiedSha256) {
+            // Verify the file's SHA-256 hash
+            const isFileValid = await this.verifyFileHash(sha256, dataDir);
+            if (!isFileValid) {
+                return false;
+            }
+        }
+        // Deserialize and verify the tree
+        const tree = this.deserializeAndVerifyTree(serializedTree, expectedRootHash);
+        if (!tree) {
+            return false;
+        }
+        // Check if the combined hash exists in the foreign tree's leaves
+        const isInTree = this.checkHashInTree(tree, hexkey, sha256);
+        return isInTree;
+    }
+    // Function to verify the file's SHA-256 hash
+    static async verifyFileHash(sha256, dataDir) {
+        const filePath = path.join(dataDir, sha256.match(/.{1,2}/g).join("/"));
+        // Check if the file exists
+        if (!fs.existsSync(filePath)) {
+            throw new Error(`File at path ${filePath} does not exist`);
+        }
+        const compressedReadStream = fs.createReadStream(filePath);
+        const decompressStream = zlib.createGunzip();
+        const hash = crypto.createHash("sha256");
+        // Process file decompression and hash comparison
+        return new Promise((resolve, reject) => {
+            compressedReadStream.pipe(decompressStream);
+            decompressStream.on("data", (chunk) => {
+                hash.update(chunk);
+            });
+            decompressStream.on("end", () => {
+                const uncompressedSha256 = hash.digest("hex");
+                if (uncompressedSha256 !== sha256) {
+                    console.warn(`File hash mismatch. Expected: ${sha256}, got: ${uncompressedSha256}`);
+                    return resolve(false);
+                }
+                resolve(true);
+            });
+            decompressStream.on("error", (err) => {
+                reject(err);
+            });
+            compressedReadStream.on("error", (err) => {
+                reject(err);
+            });
+        });
+    }
+    // Function to deserialize and verify the tree
+    static deserializeAndVerifyTree(serializedTree, expectedRootHash) {
         // Deserialize the foreign tree
         const leaves = serializedTree.leaves.map((leaf) => Buffer.from(leaf, "hex"));
         const tree = new merkletreejs_1.MerkleTree(leaves, crypto_js_1.SHA256, { sortPairs: true });
@@ -627,7 +709,7 @@ class DataIntegrityTree {
         const treeRootHash = tree.getRoot().toString("hex");
         if (treeRootHash !== expectedRootHash) {
             console.warn(`Expected root hash ${expectedRootHash}, but got ${treeRootHash}`);
-            return false;
+            return null;
         }
         // Rebuild the files map from the serialized tree
         // @ts-ignore
@@ -635,10 +717,13 @@ class DataIntegrityTree {
             key,
             { hash: value.hash, sha256: value.sha256 },
         ]));
-        // Check if the SHA-256 exists in the foreign tree's files
+        return tree;
+    }
+    // Function to check if the combined hash exists in the tree
+    static checkHashInTree(tree, hexkey, sha256) {
         const combinedHash = crypto
             .createHash("sha256")
-            .update(`${toHex(key)}/${sha256}`)
+            .update(`${hexkey}/${sha256}`)
             .digest("hex");
         const leaf = Buffer.from(combinedHash, "hex");
         const isInTree = tree.getLeafIndex(leaf) !== -1;

package/dist/DigNetwork/ContentServer.d.ts

@@ -3,26 +3,33 @@ import { Readable } from "stream";
 export declare class ContentServer {
     private ipAddress;
     private storeId;
+    private static certPath;
+    private static keyPath;
     private static readonly port;
     constructor(ipAddress: string, storeId: string);
     getKey(key: string, rootHash: string, challengeHex?: string): Promise<string>;
-    getPaymentAddress(): Promise<string>;
+    getKeyChunk(key: string, rootHash: string): Promise<Buffer>;
+    getPaymentAddress(): Promise<string | null>;
     getWellKnown(): Promise<any>;
     getKnownStores(): Promise<any>;
     getStoresIndex(): Promise<any>;
-    getKeysIndex(): Promise<any>;
-    headKey(key: string): Promise<{
+    getKeysIndex(rootHash?: string): Promise<any>;
+    headKey(key: string, rootHash?: string): Promise<{
         success: boolean;
         headers?: http.IncomingHttpHeaders;
     }>;
-    headStore(): Promise<{
+    headStore(options?: {
+        hasRootHash: string;
+    }): Promise<{
         success: boolean;
         headers?: http.IncomingHttpHeaders;
     }>;
-    streamKey(key: string): Promise<Readable>;
+    hasRootHash(rootHash: string): Promise<boolean>;
+    streamKey(key: string, rootHash?: string): Promise<Readable>;
     private head;
     private fetchJson;
     private fetchWithRetries;
     private fetch;
+    private fetchFirstChunk;
 }
 //# sourceMappingURL=ContentServer.d.ts.map

package/dist/DigNetwork/ContentServer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"ContentServer.d.ts","sourceRoot":"","sources":["../../src/DigNetwork/ContentServer.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;AAExB,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAGlC,qBAAa,aAAa;IACxB,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAM;gBAEtB,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM;IAMjC,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,YAAY,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAa7E,iBAAiB,IAAI,OAAO,CAAC,MAAM,CAAC;IAepC,YAAY,IAAI,OAAO,CAAC,GAAG,CAAC;IAM5B,cAAc,IAAI,OAAO,CAAC,GAAG,CAAC;IAM9B,cAAc,IAAI,OAAO,CAAC,GAAG,CAAC;IAM9B,YAAY,IAAI,OAAO,CAAC,GAAG,CAAC;IAM5B,OAAO,CAClB,GAAG,EAAE,MAAM,GACV,OAAO,CAAC;QAAE,OAAO,EAAE,OAAO,CAAC;QAAC,OAAO,CAAC,EAAE,IAAI,CAAC,mBAAmB,CAAA;KAAE,CAAC;IAMvD,SAAS,IAAI,OAAO,CAAC;QAChC,OAAO,EAAE,OAAO,CAAC;QACjB,OAAO,CAAC,EAAE,IAAI,CAAC,mBAAmB,CAAC;KACpC,CAAC;IAMK,SAAS,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,CAAC;YA0ClC,IAAI;YAuEJ,SAAS;YAMT,gBAAgB;YAiChB,KAAK;CAuDpB"}
+{"version":3,"file":"ContentServer.d.ts","sourceRoot":"","sources":["../../src/DigNetwork/ContentServer.ts"],"names":[],"mappings":"AACA,OAAO,IAAI,MAAM,MAAM,CAAC;AAExB,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAIlC,qBAAa,aAAa;IACxB,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAS;IAChC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAS;IAC/B,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAQ;gBAExB,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM;IAYjC,MAAM,CACjB,GAAG,EAAE,MAAM,EACX,QAAQ,EAAE,MAAM,EAChB,YAAY,CAAC,EAAE,MAAM,GACpB,OAAO,CAAC,MAAM,CAAC;IAaJ,WAAW,CACvB,GAAG,EAAE,MAAM,EACX,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,MAAM,CAAC;IAOL,iBAAiB,IAAI,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC;IAe3C,YAAY,IAAI,OAAO,CAAC,GAAG,CAAC;IAM5B,cAAc,IAAI,OAAO,CAAC,GAAG,CAAC;IAM9B,cAAc,IAAI,OAAO,CAAC,GAAG,CAAC;IAM9B,YAAY,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC;IAY7C,OAAO,CAClB,GAAG,EAAE,MAAM,EACX,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO,CAAC;QAAE,OAAO,EAAE,OAAO,CAAC;QAAC,OAAO,CAAC,EAAE,IAAI,CAAC,mBAAmB,CAAA;KAAE,CAAC;IAYvD,SAAS,CAAC,OAAO,CAAC,EAAE;QAAE,WAAW,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC;QACjE,OAAO,EAAE,OAAO,CAAC;QACjB,OAAO,CAAC,EAAE,IAAI,CAAC,mBAAmB,CAAC;KACpC,CAAC;IAUW,WAAW,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAUrD,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,CAAC;YAgDrD,IAAI;YA0EJ,SAAS;YAMT,gBAAgB;YAiChB,KAAK;YA+GL,eAAe;CA2G9B"}

package/dist/DigNetwork/ContentServer.js

@@ -4,23 +4,37 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.ContentServer = void 0;
+const fs_1 = __importDefault(require("fs"));
 const http_1 = __importDefault(require("http"));
 const url_1 = require("url");
+const ssl_1 = require("../utils/ssl");
+const network_1 = require("../utils/network");
 class ContentServer {
     constructor(ipAddress, storeId) {
         this.ipAddress = ipAddress;
         this.storeId = storeId;
+        if (!ContentServer.certPath || !ContentServer.keyPath) {
+            const { certPath, keyPath } = (0, ssl_1.getOrCreateSSLCerts)();
+            ContentServer.certPath = certPath;
+            ContentServer.keyPath = keyPath;
+        }
     }
     // Method to get the content of a specified key from the peer, with optional challenge query
     async getKey(key, rootHash, challengeHex) {
         // Construct the base URL
-        let url = `http://${this.ipAddress}/chia.${this.storeId}.${rootHash}/${key}`;
+        let url = `https://${(0, network_1.formatHost)(this.ipAddress)}:${ContentServer.port}/chia.${this.storeId}.${rootHash}/${key}`;
         // If a challenge is provided, append it as a query parameter
         if (challengeHex) {
             url += `?challenge=${challengeHex}`;
         }
         return this.fetchWithRetries(url);
     }
+    // New method to get only the first chunk of the content
+    async getKeyChunk(key, rootHash) {
+        // Construct the base URL
+        let url = `https://${(0, network_1.formatHost)(this.ipAddress)}:${ContentServer.port}/chia.${this.storeId}.${rootHash}/${key}`;
+        return this.fetchFirstChunk(url);
+    }
     // Method to get the payment address from the peer
     async getPaymentAddress() {
         console.log(`Fetching payment address from peer ${this.ipAddress}...`);
@@ -30,43 +44,66 @@ class ContentServer {
         }
         catch (error) {
             console.error(`Failed to fetch payment address from ${this.ipAddress}: ${error.message}`);
-            throw new Error(`Failed to fetch payment address: ${error.message}`);
+            return null;
         }
     }
     // Method to get the .well-known information
     async getWellKnown() {
-        const url = `http://${this.ipAddress}/.well-known`;
+        const url = `https://${(0, network_1.formatHost)(this.ipAddress)}:${ContentServer.port}/.well-known`;
         return this.fetchJson(url);
     }
     // Method to get the list of known stores
     async getKnownStores() {
-        const url = `http://${this.ipAddress}/.well-known/stores`;
+        const url = `https://${(0, network_1.formatHost)(this.ipAddress)}:${ContentServer.port}/.well-known/stores`;
         return this.fetchJson(url);
     }
     // Method to get the index of all stores
     async getStoresIndex() {
-        const url = `http://${this.ipAddress}/`;
+        const url = `https://${(0, network_1.formatHost)(this.ipAddress)}:${ContentServer.port}/`;
         return this.fetchJson(url);
     }
     // Method to get the index of keys in a store
-    async getKeysIndex() {
-        const url = `http://${this.ipAddress}/${this.storeId}`;
+    async getKeysIndex(rootHash) {
+        let udi = `chia.${this.storeId}`;
+        if (rootHash) {
+            udi += `.${rootHash}`;
+        }
+        const url = `https://${(0, network_1.formatHost)(this.ipAddress)}:${ContentServer.port}/${udi}`;
         return this.fetchJson(url);
     }
     // Method to check if a specific key exists (HEAD request)
-    async headKey(key) {
-        const url = `http://${this.ipAddress}/${this.storeId}/${key}`;
-        return this.head(url); // Return the object from head method
+    async headKey(key, rootHash) {
+        let udi = `chia.${this.storeId}`;
+        if (rootHash) {
+            udi += `.${rootHash}`;
+        }
+        const url = `https://${(0, network_1.formatHost)(this.ipAddress)}:${ContentServer.port}/${udi}/${key}`;
+        return this.head(url);
     }
     // Method to check if a specific store exists (HEAD request)
-    async headStore() {
-        const url = `http://${this.ipAddress}/${this.storeId}`;
-        console.log({ url });
-        return this.head(url); // Return the object from head method
+    async headStore(options) {
+        let url = `https://${(0, network_1.formatHost)(this.ipAddress)}:${ContentServer.port}/chia.${this.storeId}`;
+        if (options?.hasRootHash) {
+            url += `?hasRootHash=${options.hasRootHash}`;
+        }
+        return this.head(url);
     }
-    streamKey(key) {
+    async hasRootHash(rootHash) {
+        const { success, headers } = await this.headStore({
+            hasRootHash: rootHash,
+        });
+        if (success) {
+            return headers?.["x-has-root-hash"] === "true";
+        }
+        return false;
+    }
+    streamKey(key, rootHash) {
+        let udi = `chia.${this.storeId}`;
+        if (rootHash) {
+            udi += `.${rootHash}`;
+        }
         return new Promise((resolve, reject) => {
-            const url = `http://${this.ipAddress}/${this.storeId}/${key}`;
+            const url = `https://${(0, network_1.formatHost)(this.ipAddress)}:${ContentServer.port}/${udi}/${key}`;
             const urlObj = new url_1.URL(url);
             const requestOptions = {
                 hostname: urlObj.hostname,
@@ -93,7 +130,7 @@ class ContentServer {
                 }
             });
             request.on("error", (error) => {
-                console.error(`Request error for ${url}:`, error);
+                console.error(`GET Request error for ${url}:`, error);
                 reject(error);
             });
             request.end();
@@ -111,6 +148,9 @@ class ContentServer {
                         (urlObj.protocol === "http:" ? 80 : ContentServer.port),
                     path: urlObj.pathname + urlObj.search,
                     method: "HEAD",
+                    key: fs_1.default.readFileSync(ContentServer.keyPath),
+                    cert: fs_1.default.readFileSync(ContentServer.certPath),
+                    rejectUnauthorized: false,
                 };
                 const request = http_1.default.request(requestOptions, (response) => {
                     const { statusCode, headers } = response;
@@ -149,7 +189,7 @@ class ContentServer {
                     }
                 });
                 request.on("error", (error) => {
-                    console.error(`Request error for ${url}:`, error);
+                    console.error(`HEAD ${url}:`, error.message);
                     reject({ success: false });
                 });
                 request.end();
@@ -168,7 +208,7 @@ class ContentServer {
     // Helper method to fetch content with retries and redirection handling
     async fetchWithRetries(url) {
         let attempt = 0;
-        const maxRetries = 5;
+        const maxRetries = 1;
         const initialDelay = 2000; // 2 seconds
         const maxDelay = 10000; // 10 seconds
         const delayMultiplier = 1.5;
@@ -192,23 +232,48 @@ class ContentServer {
         }
         throw new Error(`Failed to retrieve data from ${url} after ${maxRetries} attempts.`);
     }
-    // Core method to fetch content from a URL
+    // Core method to fetch content from a URL with a 5-second inactivity timeout
     async fetch(url, maxRedirects = 5) {
         return new Promise((resolve, reject) => {
             const urlObj = new url_1.URL(url);
+            const timeoutDuration = 5000; // 5 seconds
+            let timeout = null; // Initialize timeout
             const requestOptions = {
                 hostname: urlObj.hostname,
                 port: urlObj.port || ContentServer.port,
                 path: urlObj.pathname + urlObj.search, // Include query params
                 method: "GET",
+                key: fs_1.default.readFileSync(ContentServer.keyPath),
+                cert: fs_1.default.readFileSync(ContentServer.certPath),
+                rejectUnauthorized: false,
             };
             const request = http_1.default.request(requestOptions, (response) => {
                 let data = "";
+                // Set timeout for inactivity
+                timeout = setTimeout(() => {
+                    console.error(`Request timeout: No data received for ${timeoutDuration / 1000} seconds.`);
+                    request.destroy(); // Use destroy instead of abort
+                    reject(new Error(`Request timed out after ${timeoutDuration / 1000} seconds of inactivity`));
+                }, timeoutDuration);
+                const resetTimeout = () => {
+                    if (timeout) {
+                        clearTimeout(timeout);
+                    }
+                    timeout = setTimeout(() => {
+                        console.error(`Request timeout: No data received for ${timeoutDuration / 1000} seconds.`);
+                        request.destroy(); // Use destroy instead of abort
+                        reject(new Error(`Request timed out after ${timeoutDuration / 1000} seconds of inactivity`));
+                    }, timeoutDuration);
+                };
                 if (response.statusCode === 200) {
                     response.on("data", (chunk) => {
                         data += chunk;
+                        resetTimeout(); // Reset the timeout every time data is received
                     });
                     response.on("end", () => {
+                        if (timeout) {
+                            clearTimeout(timeout);
+                        }
                         resolve(data);
                     });
                 }
@@ -217,8 +282,9 @@ class ContentServer {
                     // Handle redirects
                     if (maxRedirects > 0) {
                         const redirectUrl = new url_1.URL(response.headers.location, url); // Resolve relative URLs based on the original URL
-                        // Recursively follow the redirect, passing the same query params
-                        // console.log(`Redirecting to: ${redirectUrl.toString()}`);
+                        if (timeout) {
+                            clearTimeout(timeout);
+                        }
                         this.fetch(redirectUrl.toString(), maxRedirects - 1)
                             .then(resolve)
                             .catch(reject);
@@ -228,11 +294,93 @@ class ContentServer {
                     }
                 }
                 else {
+                    if (timeout) {
+                        clearTimeout(timeout);
+                    }
                     reject(new Error(`Failed to retrieve data from ${url}. Status code: ${response.statusCode}`));
                 }
             });
             request.on("error", (error) => {
-                console.error(`Request error for ${url}:`, error);
+                if (timeout) {
+                    clearTimeout(timeout);
+                }
+                console.error(`GET ${url}:`, error.message);
+                reject(error);
+            });
+            request.end();
+        });
+    }
+    // New core method to fetch only the first chunk without retries
+    async fetchFirstChunk(url) {
+        return new Promise((resolve, reject) => {
+            const urlObj = new url_1.URL(url);
+            const timeoutDuration = 5000; // 5 seconds
+            let timeout = null;
+            const requestOptions = {
+                hostname: urlObj.hostname,
+                port: urlObj.port || ContentServer.port,
+                path: urlObj.pathname + urlObj.search, // Include query params
+                method: "GET",
+                key: fs_1.default.readFileSync(ContentServer.keyPath),
+                cert: fs_1.default.readFileSync(ContentServer.certPath),
+                rejectUnauthorized: false,
+            };
+            const request = http_1.default.request(requestOptions, (response) => {
+                // Set timeout for inactivity
+                timeout = setTimeout(() => {
+                    console.error(`Request timeout: No data received for ${timeoutDuration / 1000} seconds.`);
+                    request.destroy(); // Use destroy instead of abort
+                    reject(new Error(`Request timed out after ${timeoutDuration / 1000} seconds of inactivity`));
+                }, timeoutDuration);
+                const resetTimeout = () => {
+                    if (timeout) {
+                        clearTimeout(timeout);
+                    }
+                    timeout = setTimeout(() => {
+                        console.error(`Request timeout: No data received for ${timeoutDuration / 1000} seconds.`);
+                        request.destroy(); // Use destroy instead of abort
+                        reject(new Error(`Request timed out after ${timeoutDuration / 1000} seconds of inactivity`));
+                    }, timeoutDuration);
+                };
+                if (response.statusCode === 200) {
+                    response.once("data", (chunk) => {
+                        if (timeout) {
+                            clearTimeout(timeout);
+                        }
+                        response.destroy(); // Close the connection after receiving the first chunk
+                        resolve(chunk);
+                    });
+                    response.on("end", () => {
+                        if (timeout) {
+                            clearTimeout(timeout);
+                        }
+                        // In case the response ends before any data is received
+                        reject(new Error("No data received"));
+                    });
+                }
+                else if ((response.statusCode === 301 || response.statusCode === 302) &&
+                    response.headers.location) {
+                    // Handle redirects
+                    const redirectUrl = new url_1.URL(response.headers.location, url).toString(); // Resolve relative URLs
+                    if (timeout) {
+                        clearTimeout(timeout);
+                    }
+                    this.fetchFirstChunk(redirectUrl)
+                        .then(resolve)
+                        .catch(reject);
+                }
+                else {
+                    if (timeout) {
+                        clearTimeout(timeout);
+                    }
+                    reject(new Error(`Failed to retrieve data from ${url}. Status code: ${response.statusCode}`));
+                }
+            });
+            request.on("error", (error) => {
+                if (timeout) {
+                    clearTimeout(timeout);
+                }
+                console.error(`GET ${url}:`, error.message);
                 reject(error);
             });
             request.end();
@@ -240,4 +388,4 @@ class ContentServer {
     }
 }
 exports.ContentServer = ContentServer;
-ContentServer.port = 80;
+ContentServer.port = 4161;

package/dist/DigNetwork/DigNetwork.d.ts

@@ -5,16 +5,10 @@ export declare class DigNetwork {
     private storeDir;
     private peerBlacklist;
     constructor(storeId: string);
-    private uploadPreflight;
-    uploadStoreHead(digPeer: DigPeer): Promise<void>;
-    uploadStore(digPeer: DigPeer): Promise<void>;
     static subscribeToStore(storeId: string): Promise<void>;
+    static findPeerWithStoreKey(storeId: string, rootHash: string, key?: string, intialBlackList?: string[]): Promise<DigPeer | null>;
     static unsubscribeFromStore(storeId: string): void;
-    downloadFiles(forceDownload?: boolean, renderProgressBar?: boolean, skipData?: boolean): Promise<void>;
-    private fetchAvailablePeers;
-    private downloadHeightFile;
-    private downloadManifestFile;
-    private downloadFileFromPeers;
-    private runProgressBar;
+    syncStoreFromPeers(maxRootsToProcess?: number): Promise<void>;
+    fetchAvailablePeers(): Promise<DigPeer[]>;
 }
 //# sourceMappingURL=DigNetwork.d.ts.map