@dignetwork/dig-sdk 0.0.1-alpha.7 → 0.0.1-alpha.70
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/DataIntegrityTree/DataIntegrityTree.d.ts +5 -2
- package/dist/DataIntegrityTree/DataIntegrityTree.d.ts.map +1 -1
- package/dist/DataIntegrityTree/DataIntegrityTree.js +94 -34
- package/dist/DigNetwork/ContentServer.d.ts +9 -4
- package/dist/DigNetwork/ContentServer.d.ts.map +1 -1
- package/dist/DigNetwork/ContentServer.js +55 -19
- package/dist/DigNetwork/DigNetwork.d.ts +3 -8
- package/dist/DigNetwork/DigNetwork.d.ts.map +1 -1
- package/dist/DigNetwork/DigNetwork.js +115 -171
- package/dist/DigNetwork/DigPeer.d.ts +9 -3
- package/dist/DigNetwork/DigPeer.d.ts.map +1 -1
- package/dist/DigNetwork/DigPeer.js +68 -114
- package/dist/DigNetwork/PropagationServer.d.ts +91 -33
- package/dist/DigNetwork/PropagationServer.d.ts.map +1 -1
- package/dist/DigNetwork/PropagationServer.js +414 -372
- package/dist/blockchain/DataStore.d.ts +13 -6
- package/dist/blockchain/DataStore.d.ts.map +1 -1
- package/dist/blockchain/DataStore.js +64 -91
- package/dist/blockchain/DataStoreSerializer.d.ts +1 -1
- package/dist/blockchain/DataStoreSerializer.d.ts.map +1 -1
- package/dist/blockchain/FullNodePeer.d.ts +9 -1
- package/dist/blockchain/FullNodePeer.d.ts.map +1 -1
- package/dist/blockchain/FullNodePeer.js +38 -10
- package/dist/blockchain/ServerCoin.d.ts +11 -3
- package/dist/blockchain/ServerCoin.d.ts.map +1 -1
- package/dist/blockchain/ServerCoin.js +45 -17
- package/dist/blockchain/Wallet.d.ts +2 -2
- package/dist/blockchain/Wallet.d.ts.map +1 -1
- package/dist/blockchain/Wallet.js +2 -2
- package/dist/blockchain/coins.d.ts +1 -1
- package/dist/blockchain/coins.d.ts.map +1 -1
- package/dist/blockchain/coins.js +1 -1
- package/dist/types.d.ts +1 -0
- package/dist/types.d.ts.map +1 -1
- package/dist/utils/ContentScanner.d.ts +63 -0
- package/dist/utils/ContentScanner.d.ts.map +1 -0
- package/dist/utils/ContentScanner.js +175 -0
- package/dist/utils/FileTransfer.d.ts +47 -0
- package/dist/utils/FileTransfer.d.ts.map +1 -0
- package/dist/utils/FileTransfer.js +209 -0
- package/dist/utils/StoreArchiveManager.d.ts +45 -0
- package/dist/utils/StoreArchiveManager.d.ts.map +1 -0
- package/dist/utils/StoreArchiveManager.js +153 -0
- package/dist/utils/config.d.ts +0 -1
- package/dist/utils/config.d.ts.map +1 -1
- package/dist/utils/config.js +2 -3
- package/dist/utils/directoryUtils.d.ts +0 -6
- package/dist/utils/directoryUtils.d.ts.map +1 -1
- package/dist/utils/directoryUtils.js +30 -11
- package/dist/utils/index.d.ts +0 -1
- package/dist/utils/index.d.ts.map +1 -1
- package/dist/utils/index.js +0 -1
- package/package.json +12 -3
- package/dist/utils/deltaUtils.d.ts +0 -2
- package/dist/utils/deltaUtils.d.ts.map +0 -1
- package/dist/utils/deltaUtils.js +0 -83
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || function (mod) {
|
|
19
|
+
if (mod && mod.__esModule) return mod;
|
|
20
|
+
var result = {};
|
|
21
|
+
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
22
|
+
__setModuleDefault(result, mod);
|
|
23
|
+
return result;
|
|
24
|
+
};
|
|
25
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
26
|
+
const fs = __importStar(require("fs"));
|
|
27
|
+
const path = __importStar(require("path"));
|
|
28
|
+
const archiver = __importStar(require("archiver"));
|
|
29
|
+
const unzipper = __importStar(require("unzipper"));
|
|
30
|
+
/**
|
|
31
|
+
* Class for managing file archiving, extraction, and working with the 'data' folder.
|
|
32
|
+
*/
|
|
33
|
+
class StoreArchiveManager {
|
|
34
|
+
/**
|
|
35
|
+
* Constructor that initializes the FileArchiveManager with a given path.
|
|
36
|
+
* @param archivePath - The path where the archive will be created.
|
|
37
|
+
*/
|
|
38
|
+
constructor(archivePath) {
|
|
39
|
+
this.archivePath = archivePath;
|
|
40
|
+
this.outputStream = null;
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* Creates the archive at the specified path.
|
|
44
|
+
* @returns Promise<void>
|
|
45
|
+
*/
|
|
46
|
+
async createArchive() {
|
|
47
|
+
return new Promise((resolve, reject) => {
|
|
48
|
+
this.outputStream = fs.createWriteStream(this.archivePath);
|
|
49
|
+
// @ts-ignore
|
|
50
|
+
const archive = archiver('zip', {
|
|
51
|
+
zlib: { level: 9 }
|
|
52
|
+
});
|
|
53
|
+
this.outputStream.on('close', () => {
|
|
54
|
+
console.log(`${archive.pointer()} total bytes`);
|
|
55
|
+
console.log('Archive has been finalized and output file descriptor has closed.');
|
|
56
|
+
resolve();
|
|
57
|
+
});
|
|
58
|
+
archive.on('error', (err) => {
|
|
59
|
+
reject(err);
|
|
60
|
+
});
|
|
61
|
+
archive.pipe(this.outputStream);
|
|
62
|
+
});
|
|
63
|
+
}
|
|
64
|
+
/**
|
|
65
|
+
* Streams a file into the archive.
|
|
66
|
+
* @param filePath - The file to add to the archive.
|
|
67
|
+
* @param fileNameInArchive - The name the file should have inside the archive.
|
|
68
|
+
* @returns Promise<void>
|
|
69
|
+
*/
|
|
70
|
+
async addFileToArchive(filePath, fileNameInArchive) {
|
|
71
|
+
if (!this.outputStream) {
|
|
72
|
+
throw new Error('Archive not yet created.');
|
|
73
|
+
}
|
|
74
|
+
return new Promise((resolve, reject) => {
|
|
75
|
+
// @ts-ignore
|
|
76
|
+
const archive = archiver('zip', { zlib: { level: 9 } });
|
|
77
|
+
const fileStream = fs.createReadStream(filePath);
|
|
78
|
+
archive.append(fileStream, { name: fileNameInArchive });
|
|
79
|
+
archive.finalize();
|
|
80
|
+
archive.on('finish', resolve);
|
|
81
|
+
archive.on('error', reject);
|
|
82
|
+
});
|
|
83
|
+
}
|
|
84
|
+
/**
|
|
85
|
+
* Extracts a file from the archive to a specified directory.
|
|
86
|
+
* @param extractToPath - The path where the file should be extracted.
|
|
87
|
+
* @returns Promise<void>
|
|
88
|
+
*/
|
|
89
|
+
async extractArchive(extractToPath) {
|
|
90
|
+
return new Promise((resolve, reject) => {
|
|
91
|
+
fs.createReadStream(this.archivePath)
|
|
92
|
+
.pipe(unzipper.Extract({ path: extractToPath }))
|
|
93
|
+
.on('close', resolve)
|
|
94
|
+
.on('error', reject);
|
|
95
|
+
});
|
|
96
|
+
}
|
|
97
|
+
/**
|
|
98
|
+
* Merges another archive into the current one.
|
|
99
|
+
* Extracts files from the given archive and adds them into the current archive.
|
|
100
|
+
* @param archiveToMergePath - The path of the archive to be merged.
|
|
101
|
+
* @returns Promise<void>
|
|
102
|
+
*/
|
|
103
|
+
async mergeArchive(archiveToMergePath) {
|
|
104
|
+
if (!this.outputStream) {
|
|
105
|
+
throw new Error('Archive not yet created.');
|
|
106
|
+
}
|
|
107
|
+
return new Promise((resolve, reject) => {
|
|
108
|
+
const tempExtractPath = path.join(__dirname, 'tempExtract');
|
|
109
|
+
// Ensure the temp directory exists
|
|
110
|
+
if (!fs.existsSync(tempExtractPath)) {
|
|
111
|
+
fs.mkdirSync(tempExtractPath);
|
|
112
|
+
}
|
|
113
|
+
// Step 1: Extract the contents of the archive to be merged
|
|
114
|
+
fs.createReadStream(archiveToMergePath)
|
|
115
|
+
.pipe(unzipper.Extract({ path: tempExtractPath }))
|
|
116
|
+
.on('close', async () => {
|
|
117
|
+
try {
|
|
118
|
+
// Step 2: Read the extracted files and add them to the current archive
|
|
119
|
+
await this.addExtractedFilesToArchive(tempExtractPath);
|
|
120
|
+
// Clean up: Delete the temporary directory
|
|
121
|
+
fs.rmSync(tempExtractPath, { recursive: true, force: true });
|
|
122
|
+
resolve();
|
|
123
|
+
}
|
|
124
|
+
catch (error) {
|
|
125
|
+
reject(error);
|
|
126
|
+
}
|
|
127
|
+
})
|
|
128
|
+
.on('error', reject);
|
|
129
|
+
});
|
|
130
|
+
}
|
|
131
|
+
/**
|
|
132
|
+
* Adds extracted files to the current archive.
|
|
133
|
+
* @param extractedFolderPath - The path of the folder containing extracted files.
|
|
134
|
+
* @returns Promise<void>
|
|
135
|
+
*/
|
|
136
|
+
async addExtractedFilesToArchive(extractedFolderPath) {
|
|
137
|
+
const files = fs.readdirSync(extractedFolderPath);
|
|
138
|
+
for (const file of files) {
|
|
139
|
+
const fullFilePath = path.join(extractedFolderPath, file);
|
|
140
|
+
const stat = fs.statSync(fullFilePath);
|
|
141
|
+
if (stat.isFile()) {
|
|
142
|
+
const relativePath = path.relative(extractedFolderPath, fullFilePath);
|
|
143
|
+
// Add the file into the current archive
|
|
144
|
+
await this.addFileToArchive(fullFilePath, relativePath);
|
|
145
|
+
}
|
|
146
|
+
else if (stat.isDirectory()) {
|
|
147
|
+
// Recursively add files from nested directories
|
|
148
|
+
await this.addExtractedFilesToArchive(fullFilePath);
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
exports.default = StoreArchiveManager;
|
package/dist/utils/config.d.ts
CHANGED
|
@@ -6,7 +6,6 @@ export declare const DIG_FOLDER_PATH: string;
|
|
|
6
6
|
export declare const STORE_PATH: string;
|
|
7
7
|
export declare const USER_DIR_PATH: string;
|
|
8
8
|
export declare const CONFIG_FILE_PATH: string;
|
|
9
|
-
export declare const getManifestFilePath: (storeId: string) => string;
|
|
10
9
|
export declare const getHeightFilePath: (storeId: string) => string;
|
|
11
10
|
export declare const createInitialConfig: () => void;
|
|
12
11
|
export declare const setRemote: (remote: string) => void;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../src/utils/config.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AAKrC,eAAO,MAAM,oBAAoB,qEACmC,CAAC;AAErE,eAAO,MAAM,UAAU,UAAU,CAAC;AAClC,eAAO,MAAM,sBAAsB,qEACiC,CAAC;AAErE,eAAO,MAAM,eAAe,QACqC,CAAC;AAElE,eAAO,MAAM,UAAU,QAAuC,CAAC;AAE/D,eAAO,MAAM,aAAa,QAAkC,CAAC;AAC7D,eAAO,MAAM,gBAAgB,QAAgD,CAAC;
|
|
1
|
+
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../src/utils/config.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AAKrC,eAAO,MAAM,oBAAoB,qEACmC,CAAC;AAErE,eAAO,MAAM,UAAU,UAAU,CAAC;AAClC,eAAO,MAAM,sBAAsB,qEACiC,CAAC;AAErE,eAAO,MAAM,eAAe,QACqC,CAAC;AAElE,eAAO,MAAM,UAAU,QAAuC,CAAC;AAE/D,eAAO,MAAM,aAAa,QAAkC,CAAC;AAC7D,eAAO,MAAM,gBAAgB,QAAgD,CAAC;AAI9E,eAAO,MAAM,iBAAiB,YAAa,MAAM,KAAG,MACL,CAAC;AAEhD,eAAO,MAAM,mBAAmB,QAAO,IAItC,CAAC;AAEF,eAAO,MAAM,SAAS,WAAY,MAAM,KAAG,IAQ1C,CAAC;AAEF,eAAO,MAAM,cAAc,YAAa,MAAM,KAAG,IAQhD,CAAC;AAqBF,eAAO,MAAM,YAAY,YACd,MAAM,KACd;IACD,QAAQ,EAAE;QACR,QAAQ,EAAE,MAAM,CAAC;QACjB,KAAK,EAAE,MAAM,CAAC;QACd,KAAK,EAAE,MAAM,CAAC;QACd,WAAW,EAAE,MAAM,CAAC;KACrB,CAAC;CAYH,CAAC;AAEF;;;;GAIG;AACH,eAAO,MAAM,aAAa,QAAO,MAAM,EAOtC,CAAC;AAEF;;;;;GAKG;AACH,eAAO,MAAM,gBAAgB,QAAa,OAAO,CAAC,MAAM,GAAG,IAAI,CAkC9D,CAAC;AAEF;;;;;;GAMG;AACH,eAAO,MAAM,aAAa,YAAa,MAAM,KAAG,SAS/C,CAAC;AAEF;;;;;;GAMG;AACH,eAAO,MAAM,eAAe,YAAa,MAAM,KAAG,SAoBjD,CAAC;AAEF;;;;;;;GAOG;AACH,eAAO,MAAM,eAAe,YACjB,MAAM,OACV,MAAM,SACJ,GAAG,KACT,IAMF,CAAC;AAEF;;;;;GAKG;AACH,eAAO,MAAM,kBAAkB,YAAa,MAAM,OAAO,MAAM,KAAG,IAcjE,CAAC"}
|
package/dist/utils/config.js
CHANGED
|
@@ -26,7 +26,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
26
26
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
27
27
|
};
|
|
28
28
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
29
|
-
exports.deleteDigConfigKey = exports.setDigConfigKey = exports.ensureDigConfig = exports.loadDigConfig = exports.getActiveStoreId = exports.getStoresList = exports.getCoinState = exports.setActiveStore = exports.setRemote = exports.createInitialConfig = exports.getHeightFilePath = exports.
|
|
29
|
+
exports.deleteDigConfigKey = exports.setDigConfigKey = exports.ensureDigConfig = exports.loadDigConfig = exports.getActiveStoreId = exports.getStoresList = exports.getCoinState = exports.setActiveStore = exports.setRemote = exports.createInitialConfig = exports.getHeightFilePath = exports.CONFIG_FILE_PATH = exports.USER_DIR_PATH = exports.STORE_PATH = exports.DIG_FOLDER_PATH = exports.MIN_HEIGHT_HEADER_HASH = exports.MIN_HEIGHT = exports.NETWORK_AGG_SIG_DATA = void 0;
|
|
30
30
|
const fs = __importStar(require("fs"));
|
|
31
31
|
const path = __importStar(require("path"));
|
|
32
32
|
const inquirer_1 = __importDefault(require("inquirer"));
|
|
@@ -38,8 +38,6 @@ exports.DIG_FOLDER_PATH = process.env.DIG_FOLDER_PATH || path.join(process.cwd()
|
|
|
38
38
|
exports.STORE_PATH = path.join(exports.DIG_FOLDER_PATH, "stores");
|
|
39
39
|
exports.USER_DIR_PATH = path.join(os_1.default.homedir(), ".dig");
|
|
40
40
|
exports.CONFIG_FILE_PATH = path.join(exports.DIG_FOLDER_PATH, "dig.config.json");
|
|
41
|
-
const getManifestFilePath = (storeId) => path.join(exports.STORE_PATH, storeId, "manifest.dat");
|
|
42
|
-
exports.getManifestFilePath = getManifestFilePath;
|
|
43
41
|
const getHeightFilePath = (storeId) => path.join(exports.STORE_PATH, storeId, "height.json");
|
|
44
42
|
exports.getHeightFilePath = getHeightFilePath;
|
|
45
43
|
const createInitialConfig = () => {
|
|
@@ -167,6 +165,7 @@ exports.loadDigConfig = loadDigConfig;
|
|
|
167
165
|
*/
|
|
168
166
|
const ensureDigConfig = (baseDir) => {
|
|
169
167
|
if (!fs.existsSync(exports.CONFIG_FILE_PATH)) {
|
|
168
|
+
fs.mkdirSync(exports.DIG_FOLDER_PATH, { recursive: true });
|
|
170
169
|
const defaultConfig = { deploy_dir: "./dist" };
|
|
171
170
|
fs.writeFileSync(exports.CONFIG_FILE_PATH, JSON.stringify(defaultConfig, null, 4), "utf-8");
|
|
172
171
|
console.log(`Created new dig.config.json at ${exports.CONFIG_FILE_PATH}`);
|
|
@@ -1,10 +1,4 @@
|
|
|
1
1
|
import { DataIntegrityTree } from "../DataIntegrityTree";
|
|
2
|
-
/**
|
|
3
|
-
* Recursively add all files in a directory to the Merkle tree, skipping the .dig, .git folders, and files in .gitignore.
|
|
4
|
-
* @param datalayer - The DataStoreManager instance.
|
|
5
|
-
* @param dirPath - The directory path.
|
|
6
|
-
* @param baseDir - The base directory for relative paths.
|
|
7
|
-
*/
|
|
8
2
|
export declare const addDirectory: (datalayer: DataIntegrityTree, dirPath: string, baseDir?: string) => Promise<void>;
|
|
9
3
|
/**
|
|
10
4
|
* Calculate the total size of the DIG_FOLDER_PATH
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"directoryUtils.d.ts","sourceRoot":"","sources":["../../src/utils/directoryUtils.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;
|
|
1
|
+
{"version":3,"file":"directoryUtils.d.ts","sourceRoot":"","sources":["../../src/utils/directoryUtils.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AA+BzD,eAAO,MAAM,YAAY,cACZ,iBAAiB,WACnB,MAAM,YACN,MAAM,KACd,OAAO,CAAC,IAAI,CA4Cd,CAAC;AAEF;;;;GAIG;AACH,eAAO,MAAM,mBAAmB,eAAgB,MAAM,KAAG,MAiBxD,CAAC"}
|
|
@@ -27,15 +27,29 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
27
27
|
};
|
|
28
28
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
29
29
|
exports.calculateFolderSize = exports.addDirectory = void 0;
|
|
30
|
-
const fs = __importStar(require("fs"));
|
|
31
30
|
const path = __importStar(require("path"));
|
|
31
|
+
const fs = __importStar(require("fs"));
|
|
32
32
|
const ignore_1 = __importDefault(require("ignore"));
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
33
|
+
// Custom concurrency handler
|
|
34
|
+
const limitConcurrency = async (concurrencyLimit, tasks) => {
|
|
35
|
+
const executing = [];
|
|
36
|
+
for (const task of tasks) {
|
|
37
|
+
const p = task();
|
|
38
|
+
// Add the new task to the executing array
|
|
39
|
+
executing.push(p);
|
|
40
|
+
// When a task completes, remove it from the executing array
|
|
41
|
+
const cleanup = p.finally(() => {
|
|
42
|
+
executing.splice(executing.indexOf(cleanup), 1);
|
|
43
|
+
});
|
|
44
|
+
// Once the limit is reached, wait for one to complete
|
|
45
|
+
if (executing.length >= concurrencyLimit) {
|
|
46
|
+
await new Promise((resolve) => setTimeout(resolve, 100));
|
|
47
|
+
await Promise.race(executing);
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
// Wait for all remaining tasks to complete
|
|
51
|
+
return Promise.all(executing);
|
|
52
|
+
};
|
|
39
53
|
const addDirectory = async (datalayer, dirPath, baseDir = dirPath) => {
|
|
40
54
|
const ig = (0, ignore_1.default)();
|
|
41
55
|
const gitignorePath = path.join(baseDir, ".gitignore");
|
|
@@ -45,6 +59,7 @@ const addDirectory = async (datalayer, dirPath, baseDir = dirPath) => {
|
|
|
45
59
|
ig.add(gitignoreContent);
|
|
46
60
|
}
|
|
47
61
|
const files = fs.readdirSync(dirPath);
|
|
62
|
+
const tasks = [];
|
|
48
63
|
for (const file of files) {
|
|
49
64
|
const filePath = path.join(dirPath, file);
|
|
50
65
|
const relativePath = path.relative(baseDir, filePath).replace(/\\/g, "/");
|
|
@@ -54,18 +69,22 @@ const addDirectory = async (datalayer, dirPath, baseDir = dirPath) => {
|
|
|
54
69
|
}
|
|
55
70
|
const stat = fs.statSync(filePath);
|
|
56
71
|
if (stat.isDirectory()) {
|
|
57
|
-
|
|
72
|
+
// Recursively process the directory
|
|
73
|
+
tasks.push(() => (0, exports.addDirectory)(datalayer, filePath, baseDir));
|
|
58
74
|
}
|
|
59
75
|
else {
|
|
60
|
-
|
|
76
|
+
// Add a task for each file to be processed
|
|
77
|
+
tasks.push(() => new Promise((resolve, reject) => {
|
|
61
78
|
const stream = fs.createReadStream(filePath);
|
|
62
79
|
datalayer
|
|
63
80
|
.upsertKey(stream, Buffer.from(relativePath).toString("hex"))
|
|
64
|
-
.then(resolve)
|
|
81
|
+
.then(() => resolve())
|
|
65
82
|
.catch(reject);
|
|
66
|
-
});
|
|
83
|
+
}));
|
|
67
84
|
}
|
|
68
85
|
}
|
|
86
|
+
// Run tasks with limited concurrency (set the concurrency limit as needed)
|
|
87
|
+
await limitConcurrency(10, tasks); // Adjust 10 based on your system limits
|
|
69
88
|
};
|
|
70
89
|
exports.addDirectory = addDirectory;
|
|
71
90
|
/**
|
package/dist/utils/index.d.ts
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/utils/index.ts"],"names":[],"mappings":"AAAA,cAAc,UAAU,CAAC;AACzB,cAAc,mBAAmB,CAAC;AAClC,cAAc,oBAAoB,CAAC;AACnC,cAAc,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/utils/index.ts"],"names":[],"mappings":"AAAA,cAAc,UAAU,CAAC;AACzB,cAAc,mBAAmB,CAAC;AAClC,cAAc,oBAAoB,CAAC;AACnC,cAAc,kBAAkB,CAAC;AACjC,cAAc,cAAc,CAAC;AAC7B,cAAc,aAAa,CAAC;AAC5B,cAAc,aAAa,CAAC;AAC5B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,WAAW,CAAC;AAC1B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,OAAO,CAAC;AACtB,cAAc,mBAAmB,CAAC"}
|
package/dist/utils/index.js
CHANGED
|
@@ -17,7 +17,6 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
17
17
|
__exportStar(require("./config"), exports);
|
|
18
18
|
__exportStar(require("./connectionUtils"), exports);
|
|
19
19
|
__exportStar(require("./credentialsUtils"), exports);
|
|
20
|
-
__exportStar(require("./deltaUtils"), exports);
|
|
21
20
|
__exportStar(require("./directoryUtils"), exports);
|
|
22
21
|
__exportStar(require("./encryption"), exports);
|
|
23
22
|
__exportStar(require("./FileCache"), exports);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@dignetwork/dig-sdk",
|
|
3
|
-
"version": "0.0.1-alpha.
|
|
3
|
+
"version": "0.0.1-alpha.70",
|
|
4
4
|
"description": "",
|
|
5
5
|
"type": "commonjs",
|
|
6
6
|
"main": "./dist/index.js",
|
|
@@ -25,6 +25,9 @@
|
|
|
25
25
|
"LICENSE"
|
|
26
26
|
],
|
|
27
27
|
"dependencies": {
|
|
28
|
+
"@dignetwork/datalayer-driver": "^0.1.24",
|
|
29
|
+
"archiver": "^7.0.1",
|
|
30
|
+
"axios": "^1.7.7",
|
|
28
31
|
"bip39": "^3.1.0",
|
|
29
32
|
"chia-bls": "^1.0.2",
|
|
30
33
|
"chia-config-loader": "^1.0.1",
|
|
@@ -32,8 +35,9 @@
|
|
|
32
35
|
"chia-server-coin": "^0.0.5",
|
|
33
36
|
"chia-wallet": "^1.0.18",
|
|
34
37
|
"cli-progress": "^3.12.0",
|
|
38
|
+
"colorette": "^2.0.20",
|
|
35
39
|
"crypto-js": "^4.2.0",
|
|
36
|
-
"
|
|
40
|
+
"figures": "^6.1.0",
|
|
37
41
|
"fs-extra": "^11.2.0",
|
|
38
42
|
"ignore": "^5.3.2",
|
|
39
43
|
"inquirer": "^10.1.8",
|
|
@@ -41,9 +45,13 @@
|
|
|
41
45
|
"merkletreejs": "^0.4.0",
|
|
42
46
|
"nanospinner": "^1.1.0",
|
|
43
47
|
"nconf": "^0.12.1",
|
|
44
|
-
"
|
|
48
|
+
"node-cache": "^5.1.2",
|
|
49
|
+
"p-limit": "^6.1.0",
|
|
50
|
+
"superagent": "^10.0.0",
|
|
51
|
+
"unzipper": "^0.12.3"
|
|
45
52
|
},
|
|
46
53
|
"devDependencies": {
|
|
54
|
+
"@types/archiver": "^6.0.2",
|
|
47
55
|
"@types/chai": "^4.3.17",
|
|
48
56
|
"@types/cli-progress": "^3.11.6",
|
|
49
57
|
"@types/crypto-js": "^4.2.2",
|
|
@@ -54,6 +62,7 @@
|
|
|
54
62
|
"@types/nconf": "^0.10.7",
|
|
55
63
|
"@types/node": "^22.1.0",
|
|
56
64
|
"@types/superagent": "^8.1.8",
|
|
65
|
+
"@types/unzipper": "^0.10.10",
|
|
57
66
|
"chai": "^5.1.1",
|
|
58
67
|
"copyfiles": "^2.4.1",
|
|
59
68
|
"mocha": "^10.7.0",
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"deltaUtils.d.ts","sourceRoot":"","sources":["../../src/utils/deltaUtils.ts"],"names":[],"mappings":"AAIA,eAAO,MAAM,aAAa,YACf,MAAM,mBACE,MAAM,6BACR,MAAM,KACpB,OAAO,CAAC,MAAM,EAAE,CAwElB,CAAC"}
|
package/dist/utils/deltaUtils.js
DELETED
|
@@ -1,83 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
-
if (k2 === undefined) k2 = k;
|
|
4
|
-
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
-
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
-
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
-
}
|
|
8
|
-
Object.defineProperty(o, k2, desc);
|
|
9
|
-
}) : (function(o, m, k, k2) {
|
|
10
|
-
if (k2 === undefined) k2 = k;
|
|
11
|
-
o[k2] = m[k];
|
|
12
|
-
}));
|
|
13
|
-
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
-
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
-
}) : function(o, v) {
|
|
16
|
-
o["default"] = v;
|
|
17
|
-
});
|
|
18
|
-
var __importStar = (this && this.__importStar) || function (mod) {
|
|
19
|
-
if (mod && mod.__esModule) return mod;
|
|
20
|
-
var result = {};
|
|
21
|
-
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
22
|
-
__setModuleDefault(result, mod);
|
|
23
|
-
return result;
|
|
24
|
-
};
|
|
25
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
26
|
-
exports.getDeltaFiles = void 0;
|
|
27
|
-
const fs = __importStar(require("fs"));
|
|
28
|
-
const path = __importStar(require("path"));
|
|
29
|
-
const hashUtils_1 = require("./hashUtils");
|
|
30
|
-
const getDeltaFiles = async (storeId, generationIndex = 0, directoryPath) => {
|
|
31
|
-
if (isNaN(generationIndex)) {
|
|
32
|
-
generationIndex = 0;
|
|
33
|
-
}
|
|
34
|
-
// Load manifest file
|
|
35
|
-
const manifestFilePath = path.join(directoryPath, storeId, "manifest.dat");
|
|
36
|
-
if (!fs.existsSync(manifestFilePath)) {
|
|
37
|
-
console.error("Manifest file not found", manifestFilePath);
|
|
38
|
-
return [];
|
|
39
|
-
}
|
|
40
|
-
const manifestHashes = fs
|
|
41
|
-
.readFileSync(manifestFilePath, "utf-8")
|
|
42
|
-
.split("\n")
|
|
43
|
-
.filter(Boolean);
|
|
44
|
-
console.log("");
|
|
45
|
-
console.log(`Uploading delta from generation ${generationIndex}`);
|
|
46
|
-
const filesInvolved = [];
|
|
47
|
-
// Include the height.dat file at the top of the directory
|
|
48
|
-
const heightDatFilePath = path.join(directoryPath, storeId, "height.json");
|
|
49
|
-
if (fs.existsSync(heightDatFilePath)) {
|
|
50
|
-
filesInvolved.push(heightDatFilePath);
|
|
51
|
-
}
|
|
52
|
-
// Collect files starting from generationIndex + 1
|
|
53
|
-
for (let i = generationIndex; i < manifestHashes.length; i++) {
|
|
54
|
-
const rootHash = manifestHashes[i];
|
|
55
|
-
const datFilePath = path.join(directoryPath, storeId, `${rootHash}.dat`);
|
|
56
|
-
if (!fs.existsSync(datFilePath)) {
|
|
57
|
-
console.error(`Data file for root hash ${rootHash} not found`);
|
|
58
|
-
return [];
|
|
59
|
-
}
|
|
60
|
-
const datFileContent = JSON.parse(fs.readFileSync(datFilePath, "utf-8"));
|
|
61
|
-
if (datFileContent.root !== rootHash) {
|
|
62
|
-
console.error(`Root hash in data file does not match: ${datFileContent.root} !== ${rootHash}`);
|
|
63
|
-
return [];
|
|
64
|
-
}
|
|
65
|
-
// Add the .dat file itself to the list of files involved
|
|
66
|
-
filesInvolved.push(datFilePath);
|
|
67
|
-
// Collect all files involved, ensuring correct paths
|
|
68
|
-
for (const file of Object.keys(datFileContent.files)) {
|
|
69
|
-
const filePath = (0, hashUtils_1.getFilePathFromSha256)(datFileContent.files[file].sha256, path.join(directoryPath, storeId, "data"));
|
|
70
|
-
filesInvolved.push(filePath);
|
|
71
|
-
}
|
|
72
|
-
}
|
|
73
|
-
if (process.env.DIG_DEBUG === "1") {
|
|
74
|
-
console.log("Files involved in the delta:");
|
|
75
|
-
console.table(filesInvolved);
|
|
76
|
-
}
|
|
77
|
-
// list the manifest file last, this actually
|
|
78
|
-
// helps with upload because by overriding the manifest file last,
|
|
79
|
-
// the store can still be considered valid even when the upload is interrupted
|
|
80
|
-
filesInvolved.push(manifestFilePath);
|
|
81
|
-
return filesInvolved;
|
|
82
|
-
};
|
|
83
|
-
exports.getDeltaFiles = getDeltaFiles;
|