@dignetwork/dig-sdk 0.0.1-alpha.9 → 0.0.1-alpha.90

package/dist/utils/StoreArchiveManager.d.ts

@@ -0,0 +1,45 @@
+/**
+ * Class for managing file archiving, extraction, and working with the 'data' folder.
+ */
+declare class StoreArchiveManager {
+    private archivePath;
+    private outputStream;
+    /**
+     * Constructor that initializes the FileArchiveManager with a given path.
+     * @param archivePath - The path where the archive will be created.
+     */
+    constructor(archivePath: string);
+    /**
+     * Creates the archive at the specified path.
+     * @returns Promise<void>
+     */
+    createArchive(): Promise<void>;
+    /**
+     * Streams a file into the archive.
+     * @param filePath - The file to add to the archive.
+     * @param fileNameInArchive - The name the file should have inside the archive.
+     * @returns Promise<void>
+     */
+    addFileToArchive(filePath: string, fileNameInArchive: string): Promise<void>;
+    /**
+     * Extracts a file from the archive to a specified directory.
+     * @param extractToPath - The path where the file should be extracted.
+     * @returns Promise<void>
+     */
+    extractArchive(extractToPath: string): Promise<void>;
+    /**
+     * Merges another archive into the current one.
+     * Extracts files from the given archive and adds them into the current archive.
+     * @param archiveToMergePath - The path of the archive to be merged.
+     * @returns Promise<void>
+     */
+    mergeArchive(archiveToMergePath: string): Promise<void>;
+    /**
+     * Adds extracted files to the current archive.
+     * @param extractedFolderPath - The path of the folder containing extracted files.
+     * @returns Promise<void>
+     */
+    private addExtractedFilesToArchive;
+}
+export default StoreArchiveManager;
+//# sourceMappingURL=StoreArchiveManager.d.ts.map

package/dist/utils/StoreArchiveManager.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"StoreArchiveManager.d.ts","sourceRoot":"","sources":["../../src/utils/StoreArchiveManager.ts"],"names":[],"mappings":"AAKA;;GAEG;AACH,cAAM,mBAAmB;IACrB,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,YAAY,CAAwB;IAE5C;;;OAGG;gBACS,WAAW,EAAE,MAAM;IAK/B;;;OAGG;IACU,aAAa,IAAI,OAAO,CAAC,IAAI,CAAC;IAsB3C;;;;;OAKG;IACU,gBAAgB,CAAC,QAAQ,EAAE,MAAM,EAAE,iBAAiB,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAiBzF;;;;OAIG;IACU,cAAc,CAAC,aAAa,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IASjE;;;;;OAKG;IACU,YAAY,CAAC,kBAAkB,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAiCpE;;;;OAIG;YACW,0BAA0B;CAkB3C;AAED,eAAe,mBAAmB,CAAC"}

package/dist/utils/StoreArchiveManager.js

@@ -0,0 +1,153 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+const archiver = __importStar(require("archiver"));
+const unzipper = __importStar(require("unzipper"));
+/**
+ * Class for managing file archiving, extraction, and working with the 'data' folder.
+ */
+class StoreArchiveManager {
+    /**
+     * Constructor that initializes the FileArchiveManager with a given path.
+     * @param archivePath - The path where the archive will be created.
+     */
+    constructor(archivePath) {
+        this.archivePath = archivePath;
+        this.outputStream = null;
+    }
+    /**
+     * Creates the archive at the specified path.
+     * @returns Promise<void>
+     */
+    async createArchive() {
+        return new Promise((resolve, reject) => {
+            this.outputStream = fs.createWriteStream(this.archivePath);
+            // @ts-ignore
+            const archive = archiver('zip', {
+                zlib: { level: 9 }
+            });
+            this.outputStream.on('close', () => {
+                console.log(`${archive.pointer()} total bytes`);
+                console.log('Archive has been finalized and output file descriptor has closed.');
+                resolve();
+            });
+            archive.on('error', (err) => {
+                reject(err);
+            });
+            archive.pipe(this.outputStream);
+        });
+    }
+    /**
+     * Streams a file into the archive.
+     * @param filePath - The file to add to the archive.
+     * @param fileNameInArchive - The name the file should have inside the archive.
+     * @returns Promise<void>
+     */
+    async addFileToArchive(filePath, fileNameInArchive) {
+        if (!this.outputStream) {
+            throw new Error('Archive not yet created.');
+        }
+        return new Promise((resolve, reject) => {
+            // @ts-ignore
+            const archive = archiver('zip', { zlib: { level: 9 } });
+            const fileStream = fs.createReadStream(filePath);
+            archive.append(fileStream, { name: fileNameInArchive });
+            archive.finalize();
+            archive.on('finish', resolve);
+            archive.on('error', reject);
+        });
+    }
+    /**
+     * Extracts a file from the archive to a specified directory.
+     * @param extractToPath - The path where the file should be extracted.
+     * @returns Promise<void>
+     */
+    async extractArchive(extractToPath) {
+        return new Promise((resolve, reject) => {
+            fs.createReadStream(this.archivePath)
+                .pipe(unzipper.Extract({ path: extractToPath }))
+                .on('close', resolve)
+                .on('error', reject);
+        });
+    }
+    /**
+     * Merges another archive into the current one.
+     * Extracts files from the given archive and adds them into the current archive.
+     * @param archiveToMergePath - The path of the archive to be merged.
+     * @returns Promise<void>
+     */
+    async mergeArchive(archiveToMergePath) {
+        if (!this.outputStream) {
+            throw new Error('Archive not yet created.');
+        }
+        return new Promise((resolve, reject) => {
+            const tempExtractPath = path.join(__dirname, 'tempExtract');
+            // Ensure the temp directory exists
+            if (!fs.existsSync(tempExtractPath)) {
+                fs.mkdirSync(tempExtractPath);
+            }
+            // Step 1: Extract the contents of the archive to be merged
+            fs.createReadStream(archiveToMergePath)
+                .pipe(unzipper.Extract({ path: tempExtractPath }))
+                .on('close', async () => {
+                try {
+                    // Step 2: Read the extracted files and add them to the current archive
+                    await this.addExtractedFilesToArchive(tempExtractPath);
+                    // Clean up: Delete the temporary directory
+                    fs.rmSync(tempExtractPath, { recursive: true, force: true });
+                    resolve();
+                }
+                catch (error) {
+                    reject(error);
+                }
+            })
+                .on('error', reject);
+        });
+    }
+    /**
+     * Adds extracted files to the current archive.
+     * @param extractedFolderPath - The path of the folder containing extracted files.
+     * @returns Promise<void>
+     */
+    async addExtractedFilesToArchive(extractedFolderPath) {
+        const files = fs.readdirSync(extractedFolderPath);
+        for (const file of files) {
+            const fullFilePath = path.join(extractedFolderPath, file);
+            const stat = fs.statSync(fullFilePath);
+            if (stat.isFile()) {
+                const relativePath = path.relative(extractedFolderPath, fullFilePath);
+                // Add the file into the current archive
+                await this.addFileToArchive(fullFilePath, relativePath);
+            }
+            else if (stat.isDirectory()) {
+                // Recursively add files from nested directories
+                await this.addExtractedFilesToArchive(fullFilePath);
+            }
+        }
+    }
+}
+exports.default = StoreArchiveManager;

package/dist/utils/asyncPool.d.ts

@@ -0,0 +1,9 @@
+/**
+ * Processes items in sequential batches with a concurrency limit.
+ * @param {number} limit - The maximum number of concurrent executions per batch.
+ * @param {Array<T>} items - The array of items to process.
+ * @param {(item: T) => Promise<R>} iteratorFn - The async function to apply to each item.
+ * @returns {Promise<Array<R>>} - A promise that resolves when all items have been processed.
+ */
+export declare function asyncPool<T, R>(limit: number, items: T[], iteratorFn: (item: T) => Promise<R>): Promise<R[]>;
+//# sourceMappingURL=asyncPool.d.ts.map

package/dist/utils/asyncPool.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"asyncPool.d.ts","sourceRoot":"","sources":["../../src/utils/asyncPool.ts"],"names":[],"mappings":"AACA;;;;;;GAMG;AACH,wBAAsB,SAAS,CAAC,CAAC,EAAE,CAAC,EAChC,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,CAAC,EAAE,EACV,UAAU,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,OAAO,CAAC,CAAC,CAAC,GAClC,OAAO,CAAC,CAAC,EAAE,CAAC,CAgBd"}

package/dist/utils/asyncPool.js

@@ -0,0 +1,23 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.asyncPool = asyncPool;
+/**
+ * Processes items in sequential batches with a concurrency limit.
+ * @param {number} limit - The maximum number of concurrent executions per batch.
+ * @param {Array<T>} items - The array of items to process.
+ * @param {(item: T) => Promise<R>} iteratorFn - The async function to apply to each item.
+ * @returns {Promise<Array<R>>} - A promise that resolves when all items have been processed.
+ */
+async function asyncPool(limit, items, iteratorFn) {
+    const ret = [];
+    for (let i = 0; i < items.length; i += limit) {
+        const batchItems = items.slice(i, i + limit);
+        const batchPromises = batchItems.map((item) => iteratorFn(item));
+        // Wait for the current batch to complete before starting the next one
+        const batchResults = await Promise.all(batchPromises);
+        ret.push(...batchResults);
+        // Optional: add a cooldown between batches
+        // await new Promise((resolve) => setTimeout(resolve, 500));
+    }
+    return ret;
+}

package/dist/utils/config.d.ts

@@ -1,12 +1,12 @@
 import { DigConfig } from "../types";
 export declare const NETWORK_AGG_SIG_DATA = "ccd5bb71183532bff220ba46c268991a3ff07eb358e8255a65c30a2dce0e5fbb";
+export declare const MAIN_NET_GENISES_CHALLENGE = "ccd5bb71183532bff220ba46c268991a3ff07eb358e8255a65c30a2dce0e5fbb";
 export declare const MIN_HEIGHT = 5777842;
 export declare const MIN_HEIGHT_HEADER_HASH = "b29a4daac2434fd17a36e15ba1aac5d65012d4a66f99bed0bf2b5342e92e562c";
 export declare const DIG_FOLDER_PATH: string;
 export declare const STORE_PATH: string;
 export declare const USER_DIR_PATH: string;
 export declare const CONFIG_FILE_PATH: string;
-export declare const getManifestFilePath: (storeId: string) => string;
 export declare const getHeightFilePath: (storeId: string) => string;
 export declare const createInitialConfig: () => void;
 export declare const setRemote: (remote: string) => void;
@@ -20,9 +20,10 @@ export declare const getCoinState: (storeId: string) => {
     };
 };
 /**
- * Retrieves a list of valid store folders (64-character hexadecimal names) in the DIG folder.
+ * Retrieves the list of valid store folders.
+ * If the STORE_PATH directory does not exist, it is created.
  *
- * @returns {string[]} An array of valid folder names.
+ * @returns {string[]} An array of valid store folder names.
  */
 export declare const getStoresList: () => string[];
 /**

package/dist/utils/config.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../src/utils/config.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AAKrC,eAAO,MAAM,oBAAoB,qEACmC,CAAC;AAErE,eAAO,MAAM,UAAU,UAAU,CAAC;AAClC,eAAO,MAAM,sBAAsB,qEACiC,CAAC;AAErE,eAAO,MAAM,eAAe,QACqC,CAAC;AAElE,eAAO,MAAM,UAAU,QAAuC,CAAC;AAE/D,eAAO,MAAM,aAAa,QAAkC,CAAC;AAC7D,eAAO,MAAM,gBAAgB,QAAgD,CAAC;AAE9E,eAAO,MAAM,mBAAmB,YAAa,MAAM,KAAG,MACN,CAAC;AAEjD,eAAO,MAAM,iBAAiB,YAAa,MAAM,KAAG,MACL,CAAC;AAEhD,eAAO,MAAM,mBAAmB,QAAO,IAItC,CAAC;AAEF,eAAO,MAAM,SAAS,WAAY,MAAM,KAAG,IAQ1C,CAAC;AAEF,eAAO,MAAM,cAAc,YAAa,MAAM,KAAG,IAQhD,CAAC;AAqBF,eAAO,MAAM,YAAY,YACd,MAAM,KACd;IACD,QAAQ,EAAE;QACR,QAAQ,EAAE,MAAM,CAAC;QACjB,KAAK,EAAE,MAAM,CAAC;QACd,KAAK,EAAE,MAAM,CAAC;QACd,WAAW,EAAE,MAAM,CAAC;KACrB,CAAC;CAYH,CAAC;AAEF;;;;GAIG;AACH,eAAO,MAAM,aAAa,QAAO,MAAM,EAOtC,CAAC;AAEF;;;;;GAKG;AACH,eAAO,MAAM,gBAAgB,QAAa,OAAO,CAAC,MAAM,GAAG,IAAI,CAkC9D,CAAC;AAEF;;;;;;GAMG;AACH,eAAO,MAAM,aAAa,YAAa,MAAM,KAAG,SAS/C,CAAC;AAEF;;;;;;GAMG;AACH,eAAO,MAAM,eAAe,YAAa,MAAM,KAAG,SAoBjD,CAAC;AAEF;;;;;;;GAOG;AACH,eAAO,MAAM,eAAe,YACjB,MAAM,OACV,MAAM,SACJ,GAAG,KACT,IAMF,CAAC;AAEF;;;;;GAKG;AACH,eAAO,MAAM,kBAAkB,YAAa,MAAM,OAAO,MAAM,KAAG,IAcjE,CAAC"}
+{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../src/utils/config.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AAKrC,eAAO,MAAM,oBAAoB,qEACmC,CAAC;AAErE,eAAO,MAAM,0BAA0B,qEAAuB,CAAC;AAE/D,eAAO,MAAM,UAAU,UAAU,CAAC;AAClC,eAAO,MAAM,sBAAsB,qEACiC,CAAC;AAErE,eAAO,MAAM,eAAe,QACqC,CAAC;AAElE,eAAO,MAAM,UAAU,QAAuC,CAAC;AAE/D,eAAO,MAAM,aAAa,QAAkC,CAAC;AAC7D,eAAO,MAAM,gBAAgB,QAAgD,CAAC;AAI9E,eAAO,MAAM,iBAAiB,YAAa,MAAM,KAAG,MACL,CAAC;AAEhD,eAAO,MAAM,mBAAmB,QAAO,IAItC,CAAC;AAEF,eAAO,MAAM,SAAS,WAAY,MAAM,KAAG,IAQ1C,CAAC;AAEF,eAAO,MAAM,cAAc,YAAa,MAAM,KAAG,IAQhD,CAAC;AAqBF,eAAO,MAAM,YAAY,YACd,MAAM,KACd;IACD,QAAQ,EAAE;QACR,QAAQ,EAAE,MAAM,CAAC;QACjB,KAAK,EAAE,MAAM,CAAC;QACd,KAAK,EAAE,MAAM,CAAC;QACd,WAAW,EAAE,MAAM,CAAC;KACrB,CAAC;CAYH,CAAC;AAEF;;;;;GAKG;AACH,eAAO,MAAM,aAAa,QAAO,MAAM,EAYtC,CAAC;AAEF;;;;;GAKG;AACH,eAAO,MAAM,gBAAgB,QAAa,OAAO,CAAC,MAAM,GAAG,IAAI,CAkC9D,CAAC;AAEF;;;;;;GAMG;AACH,eAAO,MAAM,aAAa,YAAa,MAAM,KAAG,SAS/C,CAAC;AAEF;;;;;;GAMG;AACH,eAAO,MAAM,eAAe,YAAa,MAAM,KAAG,SAoBjD,CAAC;AAEF;;;;;;;GAOG;AACH,eAAO,MAAM,eAAe,YACjB,MAAM,OACV,MAAM,SACJ,GAAG,KACT,IAMF,CAAC;AAEF;;;;;GAKG;AACH,eAAO,MAAM,kBAAkB,YAAa,MAAM,OAAO,MAAM,KAAG,IAcjE,CAAC"}

package/dist/utils/config.js

@@ -26,20 +26,19 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.deleteDigConfigKey = exports.setDigConfigKey = exports.ensureDigConfig = exports.loadDigConfig = exports.getActiveStoreId = exports.getStoresList = exports.getCoinState = exports.setActiveStore = exports.setRemote = exports.createInitialConfig = exports.getHeightFilePath = exports.getManifestFilePath = exports.CONFIG_FILE_PATH = exports.USER_DIR_PATH = exports.STORE_PATH = exports.DIG_FOLDER_PATH = exports.MIN_HEIGHT_HEADER_HASH = exports.MIN_HEIGHT = exports.NETWORK_AGG_SIG_DATA = void 0;
+exports.deleteDigConfigKey = exports.setDigConfigKey = exports.ensureDigConfig = exports.loadDigConfig = exports.getActiveStoreId = exports.getStoresList = exports.getCoinState = exports.setActiveStore = exports.setRemote = exports.createInitialConfig = exports.getHeightFilePath = exports.CONFIG_FILE_PATH = exports.USER_DIR_PATH = exports.STORE_PATH = exports.DIG_FOLDER_PATH = exports.MIN_HEIGHT_HEADER_HASH = exports.MIN_HEIGHT = exports.MAIN_NET_GENISES_CHALLENGE = exports.NETWORK_AGG_SIG_DATA = void 0;
 const fs = __importStar(require("fs"));
 const path = __importStar(require("path"));
 const inquirer_1 = __importDefault(require("inquirer"));
 const os_1 = __importDefault(require("os"));
 exports.NETWORK_AGG_SIG_DATA = "ccd5bb71183532bff220ba46c268991a3ff07eb358e8255a65c30a2dce0e5fbb";
+exports.MAIN_NET_GENISES_CHALLENGE = exports.NETWORK_AGG_SIG_DATA;
 exports.MIN_HEIGHT = 5777842;
 exports.MIN_HEIGHT_HEADER_HASH = "b29a4daac2434fd17a36e15ba1aac5d65012d4a66f99bed0bf2b5342e92e562c";
 exports.DIG_FOLDER_PATH = process.env.DIG_FOLDER_PATH || path.join(process.cwd(), ".dig");
 exports.STORE_PATH = path.join(exports.DIG_FOLDER_PATH, "stores");
 exports.USER_DIR_PATH = path.join(os_1.default.homedir(), ".dig");
 exports.CONFIG_FILE_PATH = path.join(exports.DIG_FOLDER_PATH, "dig.config.json");
-const getManifestFilePath = (storeId) => path.join(exports.STORE_PATH, storeId, "manifest.dat");
-exports.getManifestFilePath = getManifestFilePath;
 const getHeightFilePath = (storeId) => path.join(exports.STORE_PATH, storeId, "height.json");
 exports.getHeightFilePath = getHeightFilePath;
 const createInitialConfig = () => {
@@ -94,11 +93,16 @@ const getCoinState = (storeId) => {
 };
 exports.getCoinState = getCoinState;
 /**
- * Retrieves a list of valid store folders (64-character hexadecimal names) in the DIG folder.
+ * Retrieves the list of valid store folders.
+ * If the STORE_PATH directory does not exist, it is created.
  *
- * @returns {string[]} An array of valid folder names.
+ * @returns {string[]} An array of valid store folder names.
  */
 const getStoresList = () => {
+    // Check if the STORE_PATH exists, create it if not
+    if (!fs.existsSync(exports.STORE_PATH)) {
+        fs.mkdirSync(exports.STORE_PATH, { recursive: true });
+    }
     const folders = fs.readdirSync(exports.STORE_PATH);
     return folders.filter((folder) => /^[a-f0-9]{64}$/.test(folder) &&
         fs.lstatSync(path.join(exports.STORE_PATH, folder)).isDirectory());

package/dist/utils/directoryUtils.d.ts

@@ -1,10 +1,4 @@
 import { DataIntegrityTree } from "../DataIntegrityTree";
-/**
- * Recursively add all files in a directory to the Merkle tree, skipping the .dig, .git folders, and files in .gitignore.
- * @param datalayer - The DataStoreManager instance.
- * @param dirPath - The directory path.
- * @param baseDir - The base directory for relative paths.
- */
 export declare const addDirectory: (datalayer: DataIntegrityTree, dirPath: string, baseDir?: string) => Promise<void>;
 /**
  * Calculate the total size of the DIG_FOLDER_PATH

package/dist/utils/directoryUtils.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"directoryUtils.d.ts","sourceRoot":"","sources":["../../src/utils/directoryUtils.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AAEzD;;;;;GAKG;AACH,eAAO,MAAM,YAAY,cACZ,iBAAiB,WACnB,MAAM,YACN,MAAM,KACd,OAAO,CAAC,IAAI,CAmCd,CAAC;AAEF;;;;GAIG;AACH,eAAO,MAAM,mBAAmB,eAAgB,MAAM,KAAG,MAiBxD,CAAC"}
+{"version":3,"file":"directoryUtils.d.ts","sourceRoot":"","sources":["../../src/utils/directoryUtils.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AA+BzD,eAAO,MAAM,YAAY,cACZ,iBAAiB,WACnB,MAAM,YACN,MAAM,KACd,OAAO,CAAC,IAAI,CA4Cd,CAAC;AAEF;;;;GAIG;AACH,eAAO,MAAM,mBAAmB,eAAgB,MAAM,KAAG,MAiBxD,CAAC"}

package/dist/utils/directoryUtils.js

@@ -27,15 +27,29 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.calculateFolderSize = exports.addDirectory = void 0;
-const fs = __importStar(require("fs"));
 const path = __importStar(require("path"));
+const fs = __importStar(require("fs"));
 const ignore_1 = __importDefault(require("ignore"));
-/**
- * Recursively add all files in a directory to the Merkle tree, skipping the .dig, .git folders, and files in .gitignore.
- * @param datalayer - The DataStoreManager instance.
- * @param dirPath - The directory path.
- * @param baseDir - The base directory for relative paths.
- */
+// Custom concurrency handler
+const limitConcurrency = async (concurrencyLimit, tasks) => {
+    const executing = [];
+    for (const task of tasks) {
+        const p = task();
+        // Add the new task to the executing array
+        executing.push(p);
+        // When a task completes, remove it from the executing array
+        const cleanup = p.finally(() => {
+            executing.splice(executing.indexOf(cleanup), 1);
+        });
+        // Once the limit is reached, wait for one to complete
+        if (executing.length >= concurrencyLimit) {
+            await new Promise((resolve) => setTimeout(resolve, 100));
+            await Promise.race(executing);
+        }
+    }
+    // Wait for all remaining tasks to complete
+    return Promise.all(executing);
+};
 const addDirectory = async (datalayer, dirPath, baseDir = dirPath) => {
     const ig = (0, ignore_1.default)();
     const gitignorePath = path.join(baseDir, ".gitignore");
@@ -45,6 +59,7 @@ const addDirectory = async (datalayer, dirPath, baseDir = dirPath) => {
         ig.add(gitignoreContent);
     }
     const files = fs.readdirSync(dirPath);
+    const tasks = [];
     for (const file of files) {
         const filePath = path.join(dirPath, file);
         const relativePath = path.relative(baseDir, filePath).replace(/\\/g, "/");
@@ -54,18 +69,22 @@ const addDirectory = async (datalayer, dirPath, baseDir = dirPath) => {
         }
         const stat = fs.statSync(filePath);
         if (stat.isDirectory()) {
-            await (0, exports.addDirectory)(datalayer, filePath, baseDir);
+            // Recursively process the directory
+            tasks.push(() => (0, exports.addDirectory)(datalayer, filePath, baseDir));
         }
         else {
-            await new Promise((resolve, reject) => {
+            // Add a task for each file to be processed
+            tasks.push(() => new Promise((resolve, reject) => {
                 const stream = fs.createReadStream(filePath);
                 datalayer
                     .upsertKey(stream, Buffer.from(relativePath).toString("hex"))
-                    .then(resolve)
+                    .then(() => resolve())
                     .catch(reject);
-            });
+            }));
         }
     }
+    // Run tasks with limited concurrency (set the concurrency limit as needed)
+    await limitConcurrency(10, tasks); // Adjust 10 based on your system limits
 };
 exports.addDirectory = addDirectory;
 /**

package/dist/utils/index.d.ts

@@ -1,7 +1,6 @@
 export * from './config';
 export * from './connectionUtils';
 export * from './credentialsUtils';
-export * from './deltaUtils';
 export * from './directoryUtils';
 export * from './encryption';
 export * from './FileCache';

package/dist/utils/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/utils/index.ts"],"names":[],"mappings":"AAAA,cAAc,UAAU,CAAC;AACzB,cAAc,mBAAmB,CAAC;AAClC,cAAc,oBAAoB,CAAC;AACnC,cAAc,cAAc,CAAC;AAC7B,cAAc,kBAAkB,CAAC;AACjC,cAAc,cAAc,CAAC;AAC7B,cAAc,aAAa,CAAC;AAC5B,cAAc,aAAa,CAAC;AAC5B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,WAAW,CAAC;AAC1B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,OAAO,CAAC;AACtB,cAAc,mBAAmB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/utils/index.ts"],"names":[],"mappings":"AAAA,cAAc,UAAU,CAAC;AACzB,cAAc,mBAAmB,CAAC;AAClC,cAAc,oBAAoB,CAAC;AACnC,cAAc,kBAAkB,CAAC;AACjC,cAAc,cAAc,CAAC;AAC7B,cAAc,aAAa,CAAC;AAC5B,cAAc,aAAa,CAAC;AAC5B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,WAAW,CAAC;AAC1B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,OAAO,CAAC;AACtB,cAAc,mBAAmB,CAAC"}

package/dist/utils/index.js

@@ -17,7 +17,6 @@ Object.defineProperty(exports, "__esModule", { value: true });
 __exportStar(require("./config"), exports);
 __exportStar(require("./connectionUtils"), exports);
 __exportStar(require("./credentialsUtils"), exports);
-__exportStar(require("./deltaUtils"), exports);
 __exportStar(require("./directoryUtils"), exports);
 __exportStar(require("./encryption"), exports);
 __exportStar(require("./FileCache"), exports);

package/dist/utils/merkle.d.ts

@@ -0,0 +1,2 @@
+export declare function merkleIntegrityCheck(treePath: string, tmpDir: string, dataPath: string, roothash: string): Promise<boolean>;
+//# sourceMappingURL=merkle.d.ts.map

package/dist/utils/merkle.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"merkle.d.ts","sourceRoot":"","sources":["../../src/utils/merkle.ts"],"names":[],"mappings":"AAIA,wBAAsB,oBAAoB,CACtC,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,OAAO,CAAC,CA6BlB"}

package/dist/utils/merkle.js

@@ -0,0 +1,28 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.merkleIntegrityCheck = merkleIntegrityCheck;
+const DataIntegrityTree_1 = require("../DataIntegrityTree");
+const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
+async function merkleIntegrityCheck(treePath, tmpDir, dataPath, roothash) {
+    const rootHashContent = fs_1.default.readFileSync(treePath, "utf-8");
+    const tree = JSON.parse(rootHashContent);
+    // Extract expected sha256 from dataPath
+    const expectedSha256 = dataPath.replace("data", "").replace(/\//g, "");
+    console.log("expectedSha256", expectedSha256);
+    // Find the hexKey in the tree based on matching sha256
+    const hexKey = Object.keys(tree.files).find((key) => {
+        const fileData = tree.files[key]; // Inline type definition
+        return fileData.sha256 === expectedSha256;
+    });
+    if (!hexKey) {
+        throw new Error(`No matching file found with sha256: ${expectedSha256}`);
+    }
+    // Validate the integrity with the foreign tree
+    const integrity = await DataIntegrityTree_1.DataIntegrityTree.validateKeyIntegrityWithForeignTree(hexKey, expectedSha256, tree, roothash, path_1.default.join(tmpDir, "data"));
+    console.log("Integrity check result:", integrity);
+    return integrity;
+}

package/dist/utils/network.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"network.d.ts","sourceRoot":"","sources":["../../src/utils/network.ts"],"names":[],"mappings":"AAKA,eAAO,MAAM,kBAAkB,QAAa,OAAO,CAAC,MAAM,GAAG,SAAS,CA8BrE,CAAC"}
+{"version":3,"file":"network.d.ts","sourceRoot":"","sources":["../../src/utils/network.ts"],"names":[],"mappings":"AAmBA,eAAO,MAAM,kBAAkB,QAAa,OAAO,CAAC,MAAM,GAAG,SAAS,CA6CrE,CAAC"}

package/dist/utils/network.js

@@ -4,32 +4,47 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.getPublicIpAddress = void 0;
+/*
+ * Stopgap until better solution for finding public IPS found
+ */
 const superagent_1 = __importDefault(require("superagent"));
 const MAX_RETRIES = 5;
 const RETRY_DELAY = 2000; // in milliseconds
+// Regular expression for validating both IPv4 and IPv6 addresses
+const ipv4Regex = /^(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$/;
+const ipv6Regex = /^(([0-9a-fA-F]{1,4}:){7}([0-9a-fA-F]{1,4}|:)|(([0-9a-fA-F]{1,4}:){1,7}|:):(([0-9a-fA-F]{1,4}:){1,6}|:):([0-9a-fA-F]{1,4}|:):([0-9a-fA-F]{1,4}|:)|::)$/;
+// Helper function to validate the IP address
+const isValidIp = (ip) => {
+    return ipv4Regex.test(ip) || ipv6Regex.test(ip);
+};
 const getPublicIpAddress = async () => {
     const publicIp = process.env.PUBLIC_IP;
     if (publicIp) {
-        console.log('Public IP address from env:', publicIp);
-        return publicIp;
+        console.log("Public IP address from env:", publicIp);
+        if (isValidIp(publicIp)) {
+            return publicIp;
+        }
+        console.error("Invalid public IP address in environment variable");
+        return undefined;
     }
     let attempt = 0;
     while (attempt < MAX_RETRIES) {
         try {
-            const response = await superagent_1.default.get('https://api.datalayer.storage/user/v1/get_user_ip');
+            const response = await superagent_1.default.get("https://api.datalayer.storage/user/v1/get_user_ip");
             if (response.body && response.body.success) {
-                console.log('Public IP address:', response.body);
-                return response.body.ip_address;
-            }
-            else {
-                throw new Error('Failed to retrieve public IP address');
+                const ipAddress = response.body.ip_address;
+                if (isValidIp(ipAddress)) {
+                    return ipAddress;
+                }
+                throw new Error("Invalid IP address format received");
             }
+            throw new Error("Failed to retrieve public IP address");
         }
         catch (error) {
             attempt++;
             console.error(`Error fetching public IP address (Attempt ${attempt}):`, error.message);
             if (attempt >= MAX_RETRIES) {
-                throw new Error('Could not retrieve public IP address after several attempts');
+                throw new Error("Could not retrieve public IP address after several attempts");
             }
             await new Promise((resolve) => setTimeout(resolve, RETRY_DELAY));
         }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dignetwork/dig-sdk",
-  "version": "0.0.1-alpha.9",
+  "version": "0.0.1-alpha.90",
   "description": "",
   "type": "commonjs",
   "main": "./dist/index.js",
@@ -25,6 +25,9 @@
     "LICENSE"
   ],
   "dependencies": {
+    "@dignetwork/datalayer-driver": "^0.1.25",
+    "archiver": "^7.0.1",
+    "axios": "^1.7.7",
     "bip39": "^3.1.0",
     "chia-bls": "^1.0.2",
     "chia-config-loader": "^1.0.1",
@@ -32,8 +35,9 @@
     "chia-server-coin": "^0.0.5",
     "chia-wallet": "^1.0.18",
     "cli-progress": "^3.12.0",
+    "colorette": "^2.0.20",
     "crypto-js": "^4.2.0",
-    "datalayer-driver": "^0.1.21",
+    "figures": "^6.1.0",
     "fs-extra": "^11.2.0",
     "ignore": "^5.3.2",
     "inquirer": "^10.1.8",
@@ -41,9 +45,13 @@
     "merkletreejs": "^0.4.0",
     "nanospinner": "^1.1.0",
     "nconf": "^0.12.1",
-    "superagent": "^10.0.0"
+    "node-cache": "^5.1.2",
+    "progress-stream": "^2.0.0",
+    "superagent": "^10.0.0",
+    "unzipper": "^0.12.3"
   },
   "devDependencies": {
+    "@types/archiver": "^6.0.2",
     "@types/chai": "^4.3.17",
     "@types/cli-progress": "^3.11.6",
     "@types/crypto-js": "^4.2.2",
@@ -53,7 +61,9 @@
     "@types/mocha": "^10.0.7",
     "@types/nconf": "^0.10.7",
     "@types/node": "^22.1.0",
+    "@types/progress-stream": "^2.0.5",
     "@types/superagent": "^8.1.8",
+    "@types/unzipper": "^0.10.10",
     "chai": "^5.1.1",
     "copyfiles": "^2.4.1",
     "mocha": "^10.7.0",

package/dist/utils/deltaUtils.d.ts

@@ -1,2 +0,0 @@
-export declare const getDeltaFiles: (storeId: string, generationIndex: number | undefined, directoryPath: string) => Promise<string[]>;
-//# sourceMappingURL=deltaUtils.d.ts.map

package/dist/utils/deltaUtils.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"deltaUtils.d.ts","sourceRoot":"","sources":["../../src/utils/deltaUtils.ts"],"names":[],"mappings":"AAIA,eAAO,MAAM,aAAa,YACf,MAAM,mBACE,MAAM,6BACR,MAAM,KACpB,OAAO,CAAC,MAAM,EAAE,CAwElB,CAAC"}