@dignetwork/dig-sdk 0.0.1-alpha.17 → 0.0.1-alpha.171

package/dist/utils/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/utils/index.ts"],"names":[],"mappings":"AAAA,cAAc,UAAU,CAAC;AACzB,cAAc,mBAAmB,CAAC;AAClC,cAAc,oBAAoB,CAAC;AACnC,cAAc,cAAc,CAAC;AAC7B,cAAc,kBAAkB,CAAC;AACjC,cAAc,cAAc,CAAC;AAC7B,cAAc,aAAa,CAAC;AAC5B,cAAc,aAAa,CAAC;AAC5B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,WAAW,CAAC;AAC1B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,OAAO,CAAC;AACtB,cAAc,mBAAmB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/utils/index.ts"],"names":[],"mappings":"AAAA,cAAc,UAAU,CAAC;AACzB,cAAc,mBAAmB,CAAC;AAClC,cAAc,oBAAoB,CAAC;AACnC,cAAc,kBAAkB,CAAC;AACjC,cAAc,cAAc,CAAC;AAC7B,cAAc,aAAa,CAAC;AAC5B,cAAc,aAAa,CAAC;AAC5B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,WAAW,CAAC;AAC1B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,OAAO,CAAC;AACtB,cAAc,mBAAmB,CAAC;AAClC,cAAc,eAAe,CAAC;AAC9B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,cAAc,CAAC;AAC7B,cAAc,YAAY,CAAC"}

package/dist/utils/index.js

@@ -17,7 +17,6 @@ Object.defineProperty(exports, "__esModule", { value: true });
 __exportStar(require("./config"), exports);
 __exportStar(require("./connectionUtils"), exports);
 __exportStar(require("./credentialsUtils"), exports);
-__exportStar(require("./deltaUtils"), exports);
 __exportStar(require("./directoryUtils"), exports);
 __exportStar(require("./encryption"), exports);
 __exportStar(require("./FileCache"), exports);
@@ -27,3 +26,7 @@ __exportStar(require("./network"), exports);
 __exportStar(require("./spinnerUtils"), exports);
 __exportStar(require("./ssl"), exports);
 __exportStar(require("./validationUtils"), exports);
+__exportStar(require("./Environment"), exports);
+__exportStar(require("./promiseUtils"), exports);
+__exportStar(require("./PeerRanker"), exports);
+__exportStar(require("./DigCache"), exports);

package/dist/utils/merkle.d.ts

@@ -0,0 +1,2 @@
+export declare function merkleIntegrityCheck(treePath: string, tmpDir: string, dataPath: string, roothash: string): Promise<boolean>;
+//# sourceMappingURL=merkle.d.ts.map

package/dist/utils/merkle.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"merkle.d.ts","sourceRoot":"","sources":["../../src/utils/merkle.ts"],"names":[],"mappings":"AAIA,wBAAsB,oBAAoB,CACtC,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,OAAO,CAAC,CA6BlB"}

package/dist/utils/merkle.js

@@ -0,0 +1,28 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.merkleIntegrityCheck = merkleIntegrityCheck;
+const DataIntegrityTree_1 = require("../DataIntegrityTree");
+const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
+async function merkleIntegrityCheck(treePath, tmpDir, dataPath, roothash) {
+    const rootHashContent = fs_1.default.readFileSync(treePath, "utf-8");
+    const tree = JSON.parse(rootHashContent);
+    // Extract expected sha256 from dataPath
+    const expectedSha256 = dataPath.replace("data", "").replace(/\//g, "");
+    console.log("expectedSha256", expectedSha256);
+    // Find the hexKey in the tree based on matching sha256
+    const hexKey = Object.keys(tree.files).find((key) => {
+        const fileData = tree.files[key]; // Inline type definition
+        return fileData.sha256 === expectedSha256;
+    });
+    if (!hexKey) {
+        throw new Error(`No matching file found with sha256: ${expectedSha256}`);
+    }
+    // Validate the integrity with the foreign tree
+    const integrity = await DataIntegrityTree_1.DataIntegrityTree.validateKeyIntegrityWithForeignTree(hexKey, expectedSha256, tree, roothash, path_1.default.join(tmpDir, "data"));
+    console.log("Integrity check result:", integrity);
+    return integrity;
+}

package/dist/utils/network.d.ts

@@ -1,2 +1,3 @@
-export declare const getPublicIpAddress: () => Promise<string | undefined>;
+export declare const getPublicHost: () => Promise<string | undefined>;
+export declare const formatHost: (host: string) => string;
 //# sourceMappingURL=network.d.ts.map

package/dist/utils/network.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"network.d.ts","sourceRoot":"","sources":["../../src/utils/network.ts"],"names":[],"mappings":"AAKA,eAAO,MAAM,kBAAkB,QAAa,OAAO,CAAC,MAAM,GAAG,SAAS,CA8BrE,CAAC"}
+{"version":3,"file":"network.d.ts","sourceRoot":"","sources":["../../src/utils/network.ts"],"names":[],"mappings":"AAoBA,eAAO,MAAM,aAAa,QAAa,OAAO,CAAC,MAAM,GAAG,SAAS,CA+ChE,CAAC;AAIF,eAAO,MAAM,UAAU,SAAU,MAAM,KAAG,MAUzC,CAAC"}

package/dist/utils/network.js

@@ -3,36 +3,61 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getPublicIpAddress = void 0;
+exports.formatHost = exports.getPublicHost = void 0;
 const superagent_1 = __importDefault(require("superagent"));
 const MAX_RETRIES = 5;
 const RETRY_DELAY = 2000; // in milliseconds
-const getPublicIpAddress = async () => {
-    const publicIp = process.env.PUBLIC_IP;
-    if (publicIp) {
-        console.log('Public IP address from env:', publicIp);
-        return publicIp;
+// Regular expression for validating both IPv4 and IPv6 addresses
+const ipv4Regex = /^(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$/;
+const ipv6Regex = /^(([0-9a-fA-F]{1,4}:){7}([0-9a-fA-F]{1,4}|:)|(([0-9a-fA-F]{1,7}|:):){1,7}([0-9a-fA-F]{1,4}|:))$/;
+// Regular expression for validating hostnames
+const hostnameRegex = /^(([a-zA-Z0-9](-*[a-zA-Z0-9])*)\.)*[a-zA-Z]{2,}$/;
+// Helper function to validate the IP address or hostname
+const isValidHost = (host) => {
+    return ipv4Regex.test(host) || ipv6Regex.test(host) || hostnameRegex.test(host);
+};
+const getPublicHost = async () => {
+    const publicHost = process.env.PUBLIC_IP;
+    if (publicHost) {
+        console.log("Public IP/Hostname from env:", publicHost);
+        if (isValidHost(publicHost)) {
+            return publicHost;
+        }
+        console.error("Invalid public IP/Hostname in environment variable");
+        return undefined;
     }
     let attempt = 0;
     while (attempt < MAX_RETRIES) {
         try {
-            const response = await superagent_1.default.get('https://api.datalayer.storage/user/v1/get_user_ip');
+            const response = await superagent_1.default.get("https://api.datalayer.storage/user/v1/get_user_ip");
             if (response.body && response.body.success) {
-                console.log('Public IP address:', response.body);
-                return response.body.ip_address;
-            }
-            else {
-                throw new Error('Failed to retrieve public IP address');
+                const ipAddress = response.body.ip_address;
+                if (isValidHost(ipAddress)) {
+                    return ipAddress;
+                }
+                throw new Error("Invalid IP address or hostname format received");
             }
+            throw new Error("Failed to retrieve public host");
         }
         catch (error) {
             attempt++;
-            console.error(`Error fetching public IP address (Attempt ${attempt}):`, error.message);
+            console.error(`Error fetching public host (Attempt ${attempt}):`, error.message);
             if (attempt >= MAX_RETRIES) {
-                throw new Error('Could not retrieve public IP address after several attempts');
+                throw new Error("Could not retrieve public host after several attempts");
             }
             await new Promise((resolve) => setTimeout(resolve, RETRY_DELAY));
         }
     }
 };
-exports.getPublicIpAddress = getPublicIpAddress;
+exports.getPublicHost = getPublicHost;
+// Helper function to wrap IPv6 addresses in brackets
+const formatHost = (host) => {
+    const ipv6Pattern = /^[a-fA-F0-9:]+$/; // Simple regex to match raw IPv6 addresses (without brackets)
+    const hasBrackets = /^\[.*\]$/; // Regex to check if the address already has brackets
+    // If it's an IPv6 address without brackets, add them
+    if (ipv6Pattern.test(host) && !hasBrackets.test(host)) {
+        return `[${host}]`;
+    }
+    return host; // Return the host as is (IPv4, hostname, or already bracketed IPv6)
+};
+exports.formatHost = formatHost;

package/dist/utils/promiseUtils.d.ts

@@ -0,0 +1,27 @@
+/**
+ * Processes items in sequential batches with a concurrency limit.
+ * Adds a cooldown between batches.
+ * @param {number} limit - The maximum number of concurrent executions per batch.
+ * @param {Array<T>} items - The array of items to process.
+ * @param {(item: T) => Promise<R>} iteratorFn - The async function to apply to each item.
+ * @param {number} cooldownMs - The cooldown duration between batches in milliseconds.
+ * @returns {Promise<Array<R>>} - A promise that resolves when all items have been processed.
+ */
+export declare function asyncPool<T, R>(limit: number, items: T[], iteratorFn: (item: T) => Promise<R>, cooldownMs?: number): Promise<R[]>;
+/**
+* Helper function to add a timeout to a promise.
+* @param promise The original promise.
+* @param ms Timeout in milliseconds.
+* @param timeoutMessage The error message when the timeout is reached.
+* @returns Promise that resolves before the timeout or rejects with an error.
+*/
+export declare const withTimeout: <T>(promise: Promise<T>, ms: number, timeoutMessage: string) => Promise<T>;
+/**
+ * Wraps a promise to execute a callback every 30 seconds while the promise is pending.
+ *
+ * @param {Promise<T>} promise - The promise to wrap.
+ * @param {() => void} callback - The callback function to call every 30 seconds.
+ * @returns {Promise<T>} - A new promise that resolves or rejects with the original promise's result.
+ */
+export declare const withIntervalCallback: <T>(promise: Promise<T>, callback: () => void) => Promise<T>;
+//# sourceMappingURL=promiseUtils.d.ts.map

package/dist/utils/promiseUtils.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"promiseUtils.d.ts","sourceRoot":"","sources":["../../src/utils/promiseUtils.ts"],"names":[],"mappings":"AACA;;;;;;;;GAQG;AACH,wBAAsB,SAAS,CAAC,CAAC,EAAE,CAAC,EAClC,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,CAAC,EAAE,EACV,UAAU,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,OAAO,CAAC,CAAC,CAAC,EACnC,UAAU,GAAE,MAAY,GACvB,OAAO,CAAC,CAAC,EAAE,CAAC,CAkBd;AACC;;;;;;EAMC;AACH,eAAO,MAAM,WAAW,GAAI,CAAC,WAClB,OAAO,CAAC,CAAC,CAAC,MACf,MAAM,kBACM,MAAM,KACrB,OAAO,CAAC,CAAC,CAOX,CAAC;AAEF;;;;;;GAMG;AACH,eAAO,MAAM,oBAAoB,GAAI,CAAC,WAC3B,OAAO,CAAC,CAAC,CAAC,YACT,MAAM,IAAI,KACnB,OAAO,CAAC,CAAC,CAoBX,CAAC"}

package/dist/utils/promiseUtils.js

@@ -0,0 +1,69 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.withIntervalCallback = exports.withTimeout = void 0;
+exports.asyncPool = asyncPool;
+/**
+ * Processes items in sequential batches with a concurrency limit.
+ * Adds a cooldown between batches.
+ * @param {number} limit - The maximum number of concurrent executions per batch.
+ * @param {Array<T>} items - The array of items to process.
+ * @param {(item: T) => Promise<R>} iteratorFn - The async function to apply to each item.
+ * @param {number} cooldownMs - The cooldown duration between batches in milliseconds.
+ * @returns {Promise<Array<R>>} - A promise that resolves when all items have been processed.
+ */
+async function asyncPool(limit, items, iteratorFn, cooldownMs = 500 // Default cooldown of 500ms
+) {
+    const ret = [];
+    for (let i = 0; i < items.length; i += limit) {
+        const batchItems = items.slice(i, i + limit);
+        const batchPromises = batchItems.map((item) => iteratorFn(item));
+        // Wait for the current batch to complete before starting the next one
+        const batchResults = await Promise.all(batchPromises);
+        ret.push(...batchResults);
+        // Add a cooldown between batches, except after the last batch
+        if (i + limit < items.length) {
+            await new Promise((resolve) => setTimeout(resolve, cooldownMs));
+        }
+    }
+    return ret;
+}
+/**
+* Helper function to add a timeout to a promise.
+* @param promise The original promise.
+* @param ms Timeout in milliseconds.
+* @param timeoutMessage The error message when the timeout is reached.
+* @returns Promise that resolves before the timeout or rejects with an error.
+*/
+const withTimeout = (promise, ms, timeoutMessage) => {
+    return Promise.race([
+        promise,
+        new Promise((_, reject) => setTimeout(() => reject(new Error(timeoutMessage)), ms)),
+    ]);
+};
+exports.withTimeout = withTimeout;
+/**
+ * Wraps a promise to execute a callback every 30 seconds while the promise is pending.
+ *
+ * @param {Promise<T>} promise - The promise to wrap.
+ * @param {() => void} callback - The callback function to call every 30 seconds.
+ * @returns {Promise<T>} - A new promise that resolves or rejects with the original promise's result.
+ */
+const withIntervalCallback = (promise, callback) => {
+    const intervalTime = 30000; // 30 seconds in milliseconds
+    let intervalId;
+    // Start the interval that calls the callback every 30 seconds
+    intervalId = setInterval(() => {
+        callback();
+    }, intervalTime);
+    // Return a new promise that clears the interval when the original promise settles
+    return promise
+        .then((result) => {
+        clearInterval(intervalId);
+        return result;
+    })
+        .catch((error) => {
+        clearInterval(intervalId);
+        throw error;
+    });
+};
+exports.withIntervalCallback = withIntervalCallback;

package/dist/utils/ssl.js

@@ -26,7 +26,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.getOrCreateSSLCerts = void 0;
 const fs = __importStar(require("fs"));
 const path = __importStar(require("path"));
-const chia_server_coin_1 = require("chia-server-coin");
+const datalayer_driver_1 = require("@dignetwork/datalayer-driver");
 const config_1 = require("./config");
 const getOrCreateSSLCerts = () => {
     const sslDir = path.join(config_1.USER_DIR_PATH, "ssl");
@@ -38,7 +38,7 @@ const getOrCreateSSLCerts = () => {
     }
     // Check if the certificate and key exist, if not, generate them
     if (!fs.existsSync(certPath) || !fs.existsSync(keyPath)) {
-        new chia_server_coin_1.Tls(certPath, keyPath);
+        new datalayer_driver_1.Tls(certPath, keyPath);
         console.log("Client certificate and key generated successfully.");
     }
     // Return the paths to the cert and key files

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dignetwork/dig-sdk",
-  "version": "0.0.1-alpha.17",
+  "version": "0.0.1-alpha.171",
   "description": "",
   "type": "commonjs",
   "main": "./dist/index.js",
@@ -25,15 +25,20 @@
     "LICENSE"
   ],
   "dependencies": {
+    "@dignetwork/datalayer-driver": "^0.1.29",
+    "@dignetwork/dig-sdk": "^0.0.1-alpha.158",
+    "archiver": "^7.0.1",
+    "axios": "^1.7.7",
     "bip39": "^3.1.0",
+    "bottleneck": "^2.19.5",
     "chia-bls": "^1.0.2",
     "chia-config-loader": "^1.0.1",
     "chia-root-resolver": "^1.0.0",
-    "chia-server-coin": "^0.0.5",
     "chia-wallet": "^1.0.18",
     "cli-progress": "^3.12.0",
+    "colorette": "^2.0.20",
     "crypto-js": "^4.2.0",
-    "datalayer-driver": "^0.1.21",
+    "figures": "^6.1.0",
     "fs-extra": "^11.2.0",
     "ignore": "^5.3.2",
     "inquirer": "^10.1.8",
@@ -41,10 +46,16 @@
     "merkletreejs": "^0.4.0",
     "nanospinner": "^1.1.0",
     "nconf": "^0.12.1",
-    "p-limit": "^6.1.0",
-    "superagent": "^10.0.0"
+    "node-cache": "^5.1.2",
+    "node-stun": "^0.1.2",
+    "progress-stream": "^2.0.0",
+    "proper-lockfile": "^4.1.2",
+    "redis": "^4.7.0",
+    "superagent": "^10.0.0",
+    "unzipper": "^0.12.3"
   },
   "devDependencies": {
+    "@types/archiver": "^6.0.2",
     "@types/chai": "^4.3.17",
     "@types/cli-progress": "^3.11.6",
     "@types/crypto-js": "^4.2.2",
@@ -54,7 +65,10 @@
     "@types/mocha": "^10.0.7",
     "@types/nconf": "^0.10.7",
     "@types/node": "^22.1.0",
+    "@types/progress-stream": "^2.0.5",
+    "@types/proper-lockfile": "^4.1.4",
     "@types/superagent": "^8.1.8",
+    "@types/unzipper": "^0.10.10",
     "chai": "^5.1.1",
     "copyfiles": "^2.4.1",
     "mocha": "^10.7.0",

package/dist/utils/deltaUtils.d.ts

@@ -1,2 +0,0 @@
-export declare const getDeltaFiles: (storeId: string, generationIndex: number | undefined, directoryPath: string) => Promise<string[]>;
-//# sourceMappingURL=deltaUtils.d.ts.map

package/dist/utils/deltaUtils.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"deltaUtils.d.ts","sourceRoot":"","sources":["../../src/utils/deltaUtils.ts"],"names":[],"mappings":"AAIA,eAAO,MAAM,aAAa,YACf,MAAM,mBACE,MAAM,6BACR,MAAM,KACpB,OAAO,CAAC,MAAM,EAAE,CAwElB,CAAC"}

package/dist/utils/deltaUtils.js

@@ -1,83 +0,0 @@
-"use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || function (mod) {
-    if (mod && mod.__esModule) return mod;
-    var result = {};
-    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
-    __setModuleDefault(result, mod);
-    return result;
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.getDeltaFiles = void 0;
-const fs = __importStar(require("fs"));
-const path = __importStar(require("path"));
-const hashUtils_1 = require("./hashUtils");
-const getDeltaFiles = async (storeId, generationIndex = 0, directoryPath) => {
-    if (isNaN(generationIndex)) {
-        generationIndex = 0;
-    }
-    // Load manifest file
-    const manifestFilePath = path.join(directoryPath, storeId, "manifest.dat");
-    if (!fs.existsSync(manifestFilePath)) {
-        console.error("Manifest file not found", manifestFilePath);
-        return [];
-    }
-    const manifestHashes = fs
-        .readFileSync(manifestFilePath, "utf-8")
-        .split("\n")
-        .filter(Boolean);
-    console.log("");
-    console.log(`Uploading delta from generation ${generationIndex}`);
-    const filesInvolved = [];
-    // Include the height.dat file at the top of the directory
-    const heightDatFilePath = path.join(directoryPath, storeId, "height.json");
-    if (fs.existsSync(heightDatFilePath)) {
-        filesInvolved.push(heightDatFilePath);
-    }
-    // Collect files starting from generationIndex + 1
-    for (let i = generationIndex; i < manifestHashes.length; i++) {
-        const rootHash = manifestHashes[i];
-        const datFilePath = path.join(directoryPath, storeId, `${rootHash}.dat`);
-        if (!fs.existsSync(datFilePath)) {
-            console.error(`Data file for root hash ${rootHash} not found`);
-            return [];
-        }
-        const datFileContent = JSON.parse(fs.readFileSync(datFilePath, "utf-8"));
-        if (datFileContent.root !== rootHash) {
-            console.error(`Root hash in data file does not match: ${datFileContent.root} !== ${rootHash}`);
-            return [];
-        }
-        // Add the .dat file itself to the list of files involved
-        filesInvolved.push(datFilePath);
-        // Collect all files involved, ensuring correct paths
-        for (const file of Object.keys(datFileContent.files)) {
-            const filePath = (0, hashUtils_1.getFilePathFromSha256)(datFileContent.files[file].sha256, path.join(directoryPath, storeId, "data"));
-            filesInvolved.push(filePath);
-        }
-    }
-    if (process.env.DIG_DEBUG === "1") {
-        console.log("Files involved in the delta:");
-        console.table(filesInvolved);
-    }
-    // list the manifest file last, this actually
-    // helps with upload because by overriding the manifest file last, 
-    // the store can still be considered valid even when the upload is interrupted
-    filesInvolved.push(manifestFilePath);
-    return filesInvolved;
-};
-exports.getDeltaFiles = getDeltaFiles;