@dignetwork/dig-sdk 0.0.1-alpha.98 → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (162) hide show
  1. package/LICENSE +21 -7
  2. package/README.md +267 -0
  3. package/dist/adapters.cjs +315 -0
  4. package/dist/adapters.cjs.map +1 -0
  5. package/dist/adapters.d.cts +20 -0
  6. package/dist/adapters.d.ts +20 -0
  7. package/dist/adapters.js +306 -0
  8. package/dist/adapters.js.map +1 -0
  9. package/dist/dev-shim-Bgw8X2E9.d.cts +118 -0
  10. package/dist/dev-shim-Bgw8X2E9.d.ts +118 -0
  11. package/dist/dig-client-entry-BIzuZ7Rs.d.cts +263 -0
  12. package/dist/dig-client-entry-BIzuZ7Rs.d.ts +263 -0
  13. package/dist/dig-client.cjs +330 -0
  14. package/dist/dig-client.cjs.map +1 -0
  15. package/dist/dig-client.d.cts +1 -0
  16. package/dist/dig-client.d.ts +1 -0
  17. package/dist/dig-client.js +319 -0
  18. package/dist/dig-client.js.map +1 -0
  19. package/dist/index.cjs +754 -0
  20. package/dist/index.cjs.map +1 -0
  21. package/dist/index.d.cts +192 -0
  22. package/dist/index.d.ts +192 -5
  23. package/dist/index.js +731 -19
  24. package/dist/index.js.map +1 -0
  25. package/dist/next.cjs +326 -0
  26. package/dist/next.cjs.map +1 -0
  27. package/dist/next.d.cts +23 -0
  28. package/dist/next.d.ts +23 -0
  29. package/dist/next.js +319 -0
  30. package/dist/next.js.map +1 -0
  31. package/dist/spend.cjs +14 -0
  32. package/dist/spend.cjs.map +1 -0
  33. package/dist/spend.d.cts +1 -0
  34. package/dist/spend.d.ts +1 -0
  35. package/dist/spend.js +3 -0
  36. package/dist/spend.js.map +1 -0
  37. package/dist/vite.cjs +338 -0
  38. package/dist/vite.cjs.map +1 -0
  39. package/dist/vite.d.cts +35 -0
  40. package/dist/vite.d.ts +35 -0
  41. package/dist/vite.js +332 -0
  42. package/dist/vite.js.map +1 -0
  43. package/package.json +87 -59
  44. package/vendor/PROVENANCE.md +102 -0
  45. package/vendor/dig_client.d.ts +150 -0
  46. package/vendor/dig_client.mjs +805 -0
  47. package/vendor/dig_client_bg.wasm +0 -0
  48. package/dist/DataIntegrityTree/DataIntegrityTree.d.ts +0 -163
  49. package/dist/DataIntegrityTree/DataIntegrityTree.d.ts.map +0 -1
  50. package/dist/DataIntegrityTree/DataIntegrityTree.js +0 -707
  51. package/dist/DataIntegrityTree/DataLayerError.d.ts +0 -6
  52. package/dist/DataIntegrityTree/DataLayerError.d.ts.map +0 -1
  53. package/dist/DataIntegrityTree/DataLayerError.js +0 -17
  54. package/dist/DataIntegrityTree/index.d.ts +0 -2
  55. package/dist/DataIntegrityTree/index.d.ts.map +0 -1
  56. package/dist/DataIntegrityTree/index.js +0 -17
  57. package/dist/DigNetwork/ContentServer.d.ts +0 -33
  58. package/dist/DigNetwork/ContentServer.d.ts.map +0 -1
  59. package/dist/DigNetwork/ContentServer.js +0 -308
  60. package/dist/DigNetwork/DigChallenge.d.ts +0 -82
  61. package/dist/DigNetwork/DigChallenge.d.ts.map +0 -1
  62. package/dist/DigNetwork/DigChallenge.js +0 -144
  63. package/dist/DigNetwork/DigNetwork.d.ts +0 -14
  64. package/dist/DigNetwork/DigNetwork.d.ts.map +0 -1
  65. package/dist/DigNetwork/DigNetwork.js +0 -169
  66. package/dist/DigNetwork/DigPeer.d.ts +0 -24
  67. package/dist/DigNetwork/DigPeer.d.ts.map +0 -1
  68. package/dist/DigNetwork/DigPeer.js +0 -118
  69. package/dist/DigNetwork/IncentiveServer.d.ts +0 -13
  70. package/dist/DigNetwork/IncentiveServer.d.ts.map +0 -1
  71. package/dist/DigNetwork/IncentiveServer.js +0 -76
  72. package/dist/DigNetwork/PropagationServer.d.ts +0 -75
  73. package/dist/DigNetwork/PropagationServer.d.ts.map +0 -1
  74. package/dist/DigNetwork/PropagationServer.js +0 -536
  75. package/dist/DigNetwork/index.d.ts +0 -4
  76. package/dist/DigNetwork/index.d.ts.map +0 -1
  77. package/dist/DigNetwork/index.js +0 -19
  78. package/dist/blockchain/DataStore.d.ts +0 -67
  79. package/dist/blockchain/DataStore.d.ts.map +0 -1
  80. package/dist/blockchain/DataStore.js +0 -413
  81. package/dist/blockchain/DataStoreSerializer.d.ts +0 -52
  82. package/dist/blockchain/DataStoreSerializer.d.ts.map +0 -1
  83. package/dist/blockchain/DataStoreSerializer.js +0 -109
  84. package/dist/blockchain/FullNodePeer.d.ts +0 -31
  85. package/dist/blockchain/FullNodePeer.d.ts.map +0 -1
  86. package/dist/blockchain/FullNodePeer.js +0 -295
  87. package/dist/blockchain/ServerCoin.d.ts +0 -29
  88. package/dist/blockchain/ServerCoin.d.ts.map +0 -1
  89. package/dist/blockchain/ServerCoin.js +0 -321
  90. package/dist/blockchain/Wallet.d.ts +0 -28
  91. package/dist/blockchain/Wallet.d.ts.map +0 -1
  92. package/dist/blockchain/Wallet.js +0 -253
  93. package/dist/blockchain/coins.d.ts +0 -6
  94. package/dist/blockchain/coins.d.ts.map +0 -1
  95. package/dist/blockchain/coins.js +0 -102
  96. package/dist/blockchain/index.d.ts +0 -8
  97. package/dist/blockchain/index.d.ts.map +0 -1
  98. package/dist/blockchain/index.js +0 -36
  99. package/dist/index.d.ts.map +0 -1
  100. package/dist/prompts.d.ts +0 -21
  101. package/dist/prompts.d.ts.map +0 -1
  102. package/dist/prompts.js +0 -149
  103. package/dist/types.d.ts +0 -64
  104. package/dist/types.d.ts.map +0 -1
  105. package/dist/types.js +0 -2
  106. package/dist/utils/ContentScanner.d.ts +0 -63
  107. package/dist/utils/ContentScanner.d.ts.map +0 -1
  108. package/dist/utils/ContentScanner.js +0 -175
  109. package/dist/utils/Environment.d.ts +0 -18
  110. package/dist/utils/Environment.d.ts.map +0 -1
  111. package/dist/utils/Environment.js +0 -85
  112. package/dist/utils/FileCache.d.ts +0 -11
  113. package/dist/utils/FileCache.d.ts.map +0 -1
  114. package/dist/utils/FileCache.js +0 -58
  115. package/dist/utils/FileTransfer.d.ts +0 -47
  116. package/dist/utils/FileTransfer.d.ts.map +0 -1
  117. package/dist/utils/FileTransfer.js +0 -209
  118. package/dist/utils/NconfManager.d.ts +0 -11
  119. package/dist/utils/NconfManager.d.ts.map +0 -1
  120. package/dist/utils/NconfManager.js +0 -56
  121. package/dist/utils/StoreArchiveManager.d.ts +0 -45
  122. package/dist/utils/StoreArchiveManager.d.ts.map +0 -1
  123. package/dist/utils/StoreArchiveManager.js +0 -153
  124. package/dist/utils/asyncPool.d.ts +0 -9
  125. package/dist/utils/asyncPool.d.ts.map +0 -1
  126. package/dist/utils/asyncPool.js +0 -23
  127. package/dist/utils/config.d.ts +0 -68
  128. package/dist/utils/config.d.ts.map +0 -1
  129. package/dist/utils/config.js +0 -223
  130. package/dist/utils/connectionUtils.d.ts +0 -10
  131. package/dist/utils/connectionUtils.d.ts.map +0 -1
  132. package/dist/utils/connectionUtils.js +0 -18
  133. package/dist/utils/credentialsUtils.d.ts +0 -8
  134. package/dist/utils/credentialsUtils.d.ts.map +0 -1
  135. package/dist/utils/credentialsUtils.js +0 -122
  136. package/dist/utils/directoryUtils.d.ts +0 -9
  137. package/dist/utils/directoryUtils.d.ts.map +0 -1
  138. package/dist/utils/directoryUtils.js +0 -110
  139. package/dist/utils/encryption.d.ts +0 -8
  140. package/dist/utils/encryption.d.ts.map +0 -1
  141. package/dist/utils/encryption.js +0 -55
  142. package/dist/utils/hashUtils.d.ts +0 -8
  143. package/dist/utils/hashUtils.d.ts.map +0 -1
  144. package/dist/utils/hashUtils.js +0 -44
  145. package/dist/utils/index.d.ts +0 -14
  146. package/dist/utils/index.d.ts.map +0 -1
  147. package/dist/utils/index.js +0 -29
  148. package/dist/utils/merkle.d.ts +0 -2
  149. package/dist/utils/merkle.d.ts.map +0 -1
  150. package/dist/utils/merkle.js +0 -28
  151. package/dist/utils/network.d.ts +0 -2
  152. package/dist/utils/network.d.ts.map +0 -1
  153. package/dist/utils/network.js +0 -54
  154. package/dist/utils/spinnerUtils.d.ts +0 -2
  155. package/dist/utils/spinnerUtils.d.ts.map +0 -1
  156. package/dist/utils/spinnerUtils.js +0 -47
  157. package/dist/utils/ssl.d.ts +0 -5
  158. package/dist/utils/ssl.d.ts.map +0 -1
  159. package/dist/utils/ssl.js +0 -50
  160. package/dist/utils/validationUtils.d.ts +0 -9
  161. package/dist/utils/validationUtils.d.ts.map +0 -1
  162. package/dist/utils/validationUtils.js +0 -55
@@ -1,536 +0,0 @@
1
- "use strict";
2
- var __importDefault = (this && this.__importDefault) || function (mod) {
3
- return (mod && mod.__esModule) ? mod : { "default": mod };
4
- };
5
- Object.defineProperty(exports, "__esModule", { value: true });
6
- exports.PropagationServer = void 0;
7
- // Import necessary modules and dependencies
8
- const axios_1 = __importDefault(require("axios"));
9
- const fs_1 = __importDefault(require("fs"));
10
- const path_1 = __importDefault(require("path"));
11
- const form_data_1 = __importDefault(require("form-data"));
12
- const blockchain_1 = require("../blockchain");
13
- const ssl_1 = require("../utils/ssl");
14
- const credentialsUtils_1 = require("../utils/credentialsUtils");
15
- const https_1 = __importDefault(require("https"));
16
- const cli_progress_1 = __importDefault(require("cli-progress"));
17
- const colorette_1 = require("colorette");
18
- const config_1 = require("../utils/config");
19
- const hashUtils_1 = require("../utils/hashUtils");
20
- const nanospinner_1 = require("nanospinner");
21
- const progress_stream_1 = __importDefault(require("progress-stream"));
22
- const stream_1 = require("stream");
23
- const asyncPool_1 = require("../utils/asyncPool");
24
- const fs_extra_1 = __importDefault(require("fs-extra"));
25
- const os_1 = __importDefault(require("os"));
26
- const merkle_1 = require("../utils/merkle");
27
- // Helper function to trim long filenames with ellipsis and ensure consistent padding
28
- function formatFilename(filename, maxLength = 30) {
29
- if (!filename) {
30
- return "Unknown File".padEnd(maxLength, " ");
31
- }
32
- if (filename.length > maxLength) {
33
- return `...${filename.slice(-(maxLength - 3))}`.padEnd(maxLength, " ");
34
- }
35
- return filename.padEnd(maxLength, " ");
36
- }
37
- class PropagationServer {
38
- constructor(ipAddress, storeId) {
39
- this.storeId = storeId;
40
- this.sessionId = "";
41
- this.publicKey = "";
42
- this.ipAddress = ipAddress;
43
- // Get or create SSL certificates
44
- const { certPath, keyPath } = (0, ssl_1.getOrCreateSSLCerts)();
45
- this.certPath = certPath;
46
- this.keyPath = keyPath;
47
- }
48
- /**
49
- * Initialize the Wallet instance.
50
- */
51
- async initializeWallet() {
52
- this.wallet = await blockchain_1.Wallet.load("default");
53
- this.publicKey = (await this.wallet.getPublicSyntheticKey()).toString("hex");
54
- }
55
- /**
56
- * Create an Axios HTTPS Agent with self-signed certificate allowance.
57
- */
58
- createHttpsAgent() {
59
- return new https_1.default.Agent({
60
- cert: fs_1.default.readFileSync(this.certPath),
61
- key: fs_1.default.readFileSync(this.keyPath),
62
- rejectUnauthorized: false,
63
- });
64
- }
65
- /**
66
- * Check if the store and optional root hash exist by making a HEAD request.
67
- */
68
- async checkStoreExists(rootHash) {
69
- const spinner = (0, nanospinner_1.createSpinner)(`Checking if store ${this.storeId} exists...`).start();
70
- try {
71
- const config = {
72
- httpsAgent: this.createHttpsAgent(),
73
- };
74
- let url = `https://${this.ipAddress}:${PropagationServer.port}/${this.storeId}`;
75
- if (rootHash) {
76
- url += `?hasRootHash=${rootHash}`;
77
- }
78
- const response = await axios_1.default.head(url, config);
79
- // Extract store existence and root hash existence from headers
80
- const storeExists = response.headers["x-store-exists"] === "true";
81
- const rootHashExists = response.headers["x-has-root-hash"] === "true";
82
- if (storeExists) {
83
- spinner.success({
84
- text: (0, colorette_1.green)(`Store ${this.storeId} exists on peer: ${this.ipAddress}`),
85
- });
86
- }
87
- else {
88
- spinner.error({
89
- text: (0, colorette_1.yellow)(`Store ${this.storeId} does not exist on ${this.ipAddress}. Credentials will be required to push.`),
90
- });
91
- }
92
- return { storeExists, rootHashExists };
93
- }
94
- catch (error) {
95
- spinner.error({ text: (0, colorette_1.red)("Error checking if store exists:") });
96
- console.error((0, colorette_1.red)(error.message));
97
- throw error;
98
- }
99
- }
100
- /**
101
- * Start an upload session by sending a POST request with the rootHash.dat file.
102
- */
103
- async startUploadSession(rootHash) {
104
- const spinner = (0, nanospinner_1.createSpinner)(`Starting upload session for store ${this.storeId}...`).start();
105
- try {
106
- const formData = new form_data_1.default();
107
- const datFilePath = path_1.default.join(config_1.STORE_PATH, this.storeId, `${rootHash}.dat`);
108
- // Ensure the rootHash.dat file exists
109
- if (!fs_1.default.existsSync(datFilePath)) {
110
- throw new Error(`File not found: ${datFilePath}`);
111
- }
112
- formData.append("file", fs_1.default.createReadStream(datFilePath), {
113
- filename: `${rootHash}.dat`,
114
- });
115
- const config = {
116
- httpsAgent: this.createHttpsAgent(),
117
- headers: {
118
- ...formData.getHeaders(),
119
- },
120
- };
121
- // Add Basic Auth if username and password are present
122
- if (this.username && this.password) {
123
- config.auth = {
124
- username: this.username,
125
- password: this.password,
126
- };
127
- }
128
- const url = `https://${this.ipAddress}:${PropagationServer.port}/upload/${this.storeId}?roothash=${rootHash}`;
129
- const response = await axios_1.default.post(url, formData, config);
130
- this.sessionId = response.data.sessionId;
131
- spinner.success({
132
- text: (0, colorette_1.green)(`Upload session started for Root Hash: ${(0, colorette_1.cyan)(rootHash)} with session ID ${this.sessionId}`),
133
- });
134
- }
135
- catch (error) {
136
- spinner.error({ text: (0, colorette_1.red)("Error starting upload session:") });
137
- console.error((0, colorette_1.red)(error.message));
138
- throw error;
139
- }
140
- }
141
- /**
142
- * Request a nonce for a file by sending a HEAD request to the server.
143
- */
144
- async getFileNonce(filename) {
145
- try {
146
- const config = {
147
- httpsAgent: this.createHttpsAgent(),
148
- };
149
- const url = `https://${this.ipAddress}:${PropagationServer.port}/upload/${this.storeId}/${this.sessionId}/${filename}`;
150
- const response = await axios_1.default.head(url, config);
151
- // Check for 'x-file-exists' header
152
- const fileExists = response.headers["x-file-exists"] === "true";
153
- // If file exists, no need to generate a nonce
154
- const nonce = response.headers["x-nonce"];
155
- return { nonce, fileExists };
156
- }
157
- catch (error) {
158
- console.error((0, colorette_1.red)(`Error generating nonce for file ${filename}:`), error.message);
159
- throw error;
160
- }
161
- }
162
- /**
163
- * Upload a file to the server by sending a PUT request.
164
- * Logs progress using a local cli-progress bar.
165
- */
166
- async uploadFile(label, dataPath) {
167
- const filePath = path_1.default.join(config_1.STORE_PATH, this.storeId, dataPath);
168
- const { nonce, fileExists } = await this.getFileNonce(dataPath);
169
- if (fileExists) {
170
- console.log((0, colorette_1.blue)(`File ${label} already exists. Skipping upload.`));
171
- return;
172
- }
173
- const wallet = await blockchain_1.Wallet.load("default");
174
- const keyOwnershipSig = await wallet.createKeyOwnershipSignature(nonce);
175
- const publicKey = await wallet.getPublicSyntheticKey();
176
- // Get the file size
177
- const fileSize = fs_1.default.statSync(filePath).size;
178
- let progressBar;
179
- try {
180
- // Create a new progress bar for the file
181
- progressBar = new cli_progress_1.default.SingleBar({
182
- format: `${(0, colorette_1.blue)("[{bar}]")} | ${(0, colorette_1.yellow)("{filename}")} | {percentage}% | {value}/{total} bytes`,
183
- hideCursor: true,
184
- barsize: 30,
185
- align: "left",
186
- autopadding: true,
187
- noTTYOutput: false,
188
- stopOnComplete: true,
189
- clearOnComplete: false,
190
- }, cli_progress_1.default.Presets.legacy);
191
- progressBar.start(fileSize, 0, {
192
- filename: formatFilename(label),
193
- });
194
- // Create the read stream
195
- const fileReadStream = fs_1.default.createReadStream(filePath);
196
- // Create a progress stream
197
- const progressStream = (0, progress_stream_1.default)({
198
- length: fileSize,
199
- time: 500, // Adjust as needed
200
- });
201
- progressStream.on("progress", (progress) => {
202
- progressBar.update(progress.transferred);
203
- });
204
- // Create a PassThrough stream
205
- const passThroughStream = new stream_1.PassThrough();
206
- // Pipe the read stream through the progress stream into the PassThrough stream
207
- fileReadStream.pipe(progressStream).pipe(passThroughStream);
208
- // Use form-data to construct the request body
209
- const formData = new form_data_1.default();
210
- formData.append("file", passThroughStream);
211
- const headers = {
212
- ...formData.getHeaders(),
213
- "x-nonce": nonce,
214
- "x-public-key": publicKey.toString("hex"),
215
- "x-key-ownership-sig": keyOwnershipSig,
216
- };
217
- const config = {
218
- headers: headers,
219
- httpsAgent: this.createHttpsAgent(),
220
- maxContentLength: Infinity,
221
- maxBodyLength: Infinity,
222
- };
223
- const url = `https://${this.ipAddress}:${PropagationServer.port}/upload/${this.storeId}/${this.sessionId}/${dataPath}`;
224
- // Create a promise that resolves when the progress stream ends
225
- const progressPromise = new Promise((resolve, reject) => {
226
- progressStream.on("end", resolve);
227
- progressStream.on("error", reject);
228
- });
229
- // Start the upload request
230
- const uploadPromise = axios_1.default.put(url, formData, config);
231
- // Wait for both the upload and the progress stream to finish
232
- await Promise.all([uploadPromise, progressPromise]);
233
- }
234
- catch (error) {
235
- throw error;
236
- }
237
- finally {
238
- if (progressBar) {
239
- progressBar.stop();
240
- }
241
- }
242
- }
243
- /**
244
- * Commit the upload session by sending a POST request to the server.
245
- */
246
- async commitUploadSession() {
247
- const spinner = (0, nanospinner_1.createSpinner)(`Committing upload session for store ${this.storeId}...`).start();
248
- try {
249
- const config = {
250
- httpsAgent: this.createHttpsAgent(),
251
- auth: this.username && this.password
252
- ? {
253
- username: this.username,
254
- password: this.password,
255
- }
256
- : undefined,
257
- };
258
- const url = `https://${this.ipAddress}:${PropagationServer.port}/commit/${this.storeId}/${this.sessionId}`;
259
- const response = await axios_1.default.post(url, {}, config);
260
- spinner.success({
261
- text: (0, colorette_1.green)(`Upload session ${this.sessionId} successfully committed.`),
262
- });
263
- return response.data;
264
- }
265
- catch (error) {
266
- spinner.error({ text: (0, colorette_1.red)("Error committing upload session:") });
267
- console.error((0, colorette_1.red)(error.message));
268
- throw error;
269
- }
270
- }
271
- /**
272
- * Static function to handle the entire upload process for multiple files based on rootHash.
273
- */
274
- static async uploadStore(storeId, rootHash, ipAddress) {
275
- const propagationServer = new PropagationServer(ipAddress, storeId);
276
- // Initialize wallet
277
- await propagationServer.initializeWallet();
278
- // Check if the store exists
279
- const { storeExists, rootHashExists } = await propagationServer.checkStoreExists(rootHash);
280
- // If the store does not exist, prompt for credentials
281
- if (!storeExists) {
282
- console.log((0, colorette_1.red)(`Store ${storeId} does not exist. Prompting for credentials...`));
283
- const credentials = await (0, credentialsUtils_1.promptCredentials)(propagationServer.ipAddress);
284
- propagationServer.username = credentials.username;
285
- propagationServer.password = credentials.password;
286
- }
287
- if (rootHashExists) {
288
- console.log((0, colorette_1.blue)(`Root hash ${rootHash} already exists in the store. Skipping upload.`));
289
- return;
290
- }
291
- // Start the upload session
292
- await propagationServer.startUploadSession(rootHash);
293
- const dataStore = blockchain_1.DataStore.from(storeId);
294
- const files = await dataStore.getFileSetForRootHash(rootHash);
295
- // Prepare upload tasks
296
- const uploadTasks = files.map((file) => ({
297
- label: file.name,
298
- dataPath: file.path,
299
- }));
300
- // Limit the number of concurrent uploads
301
- const concurrencyLimit = 10; // Adjust this number as needed
302
- await (0, asyncPool_1.asyncPool)(concurrencyLimit, uploadTasks, async (task) => {
303
- await propagationServer.uploadFile(task.label, task.dataPath);
304
- });
305
- // Commit the session after all files have been uploaded
306
- await propagationServer.commitUploadSession();
307
- console.log((0, colorette_1.green)(`✔ All files have been uploaded to for Root Hash: ${(0, colorette_1.cyan)(rootHash)}.`));
308
- }
309
- /**
310
- * Fetch a file from the server by sending a GET request and return its content in memory.
311
- * Logs progress using a local cli-progress bar.
312
- */
313
- async fetchFile(dataPath) {
314
- const url = `https://${this.ipAddress}:${PropagationServer.port}/fetch/${this.storeId}/${dataPath}`;
315
- const config = {
316
- responseType: "stream",
317
- httpsAgent: this.createHttpsAgent(),
318
- };
319
- let progressBar;
320
- try {
321
- const response = await axios_1.default.get(url, config);
322
- const totalLengthHeader = response.headers["content-length"];
323
- const totalLength = totalLengthHeader
324
- ? parseInt(totalLengthHeader, 10)
325
- : null;
326
- if (!totalLength) {
327
- throw new Error("Content-Length header is missing");
328
- }
329
- // Create a new progress bar for the file
330
- progressBar = new cli_progress_1.default.SingleBar({
331
- format: `${(0, colorette_1.blue)("[{bar}]")} | ${(0, colorette_1.yellow)("{filename}")} | {percentage}% | {value}/{total} bytes`,
332
- hideCursor: true,
333
- barsize: 30,
334
- align: "left",
335
- autopadding: true,
336
- noTTYOutput: false,
337
- stopOnComplete: true,
338
- clearOnComplete: false,
339
- }, cli_progress_1.default.Presets.legacy);
340
- progressBar.start(totalLength, 0, {
341
- filename: formatFilename(dataPath),
342
- });
343
- let dataBuffers = [];
344
- const progressStream = (0, progress_stream_1.default)({
345
- length: totalLength,
346
- time: 500, // Adjust as needed
347
- });
348
- progressStream.on("progress", (progress) => {
349
- progressBar.update(progress.transferred);
350
- });
351
- response.data.pipe(progressStream);
352
- progressStream.on("data", (chunk) => {
353
- dataBuffers.push(chunk);
354
- });
355
- // Wait for the progress stream to finish
356
- await new Promise((resolve, reject) => {
357
- progressStream.on("end", resolve);
358
- progressStream.on("error", reject);
359
- });
360
- return Buffer.concat(dataBuffers);
361
- }
362
- catch (error) {
363
- throw error;
364
- }
365
- finally {
366
- if (progressBar) {
367
- progressBar.stop();
368
- }
369
- }
370
- }
371
- /**
372
- * Get details of a file, including whether it exists and its size.
373
- */
374
- async getFileDetails(dataPath, rootHash) {
375
- try {
376
- const config = {
377
- httpsAgent: this.createHttpsAgent(),
378
- };
379
- const url = `https://${this.ipAddress}:${PropagationServer.port}/store/${this.storeId}/${rootHash}/${dataPath}`;
380
- const response = await axios_1.default.head(url, config);
381
- // Check the headers for file existence and size
382
- const fileExists = response.headers["x-file-exists"] === "true";
383
- const fileSize = parseInt(response.headers["x-file-size"], 10);
384
- return {
385
- exists: fileExists,
386
- size: fileExists ? fileSize : 0,
387
- };
388
- }
389
- catch (error) {
390
- console.error((0, colorette_1.red)(`✖ Error checking file details for ${dataPath}:`), error.message);
391
- throw error;
392
- }
393
- }
394
- /**
395
- * Download a file from the server by sending a GET request.
396
- * Logs progress using a local cli-progress bar.
397
- */
398
- async downloadFile(label, dataPath, rootHash, baseDir) {
399
- const url = `https://${this.ipAddress}:${PropagationServer.port}/fetch/${this.storeId}/${dataPath}`;
400
- let downloadPath = path_1.default.join(baseDir, dataPath);
401
- // Ensure that the directory for the file exists
402
- fs_1.default.mkdirSync(path_1.default.dirname(downloadPath), { recursive: true });
403
- const config = {
404
- responseType: "stream",
405
- httpsAgent: this.createHttpsAgent(),
406
- };
407
- let progressBar;
408
- try {
409
- const response = await axios_1.default.get(url, config);
410
- const totalLengthHeader = response.headers["content-length"];
411
- const totalLength = totalLengthHeader
412
- ? parseInt(totalLengthHeader, 10)
413
- : null;
414
- if (!totalLength) {
415
- throw new Error("Content-Length header is missing");
416
- }
417
- // Create a new progress bar for the file
418
- progressBar = new cli_progress_1.default.SingleBar({
419
- format: `${(0, colorette_1.blue)("[{bar}]")} | ${(0, colorette_1.yellow)("{filename}")} | {percentage}% | {value}/{total} bytes`,
420
- hideCursor: true,
421
- barsize: 30,
422
- align: "left",
423
- autopadding: true,
424
- noTTYOutput: false,
425
- stopOnComplete: true,
426
- clearOnComplete: false,
427
- }, cli_progress_1.default.Presets.legacy);
428
- progressBar.start(totalLength, 0, {
429
- filename: formatFilename(label),
430
- });
431
- const fileWriteStream = fs_1.default.createWriteStream(downloadPath);
432
- const progressStream = (0, progress_stream_1.default)({
433
- length: totalLength,
434
- time: 500, // Adjust as needed
435
- });
436
- progressStream.on("progress", (progress) => {
437
- progressBar.update(progress.transferred);
438
- });
439
- response.data.pipe(progressStream).pipe(fileWriteStream);
440
- // Wait for both the file write stream and the progress stream to finish
441
- await Promise.all([
442
- new Promise((resolve, reject) => {
443
- fileWriteStream.on("finish", resolve);
444
- fileWriteStream.on("error", reject);
445
- }),
446
- new Promise((resolve, reject) => {
447
- progressStream.on("end", resolve);
448
- progressStream.on("error", reject);
449
- }),
450
- ]);
451
- if (dataPath.includes("/data")) {
452
- const integrity = await (0, merkle_1.merkleIntegrityCheck)(path_1.default.join(baseDir, `${rootHash}.dat`), baseDir, dataPath, rootHash);
453
- if (!integrity) {
454
- throw new Error(`Integrity check failed for file: ${dataPath}`);
455
- }
456
- console.log("integrity check");
457
- }
458
- }
459
- catch (error) {
460
- throw error;
461
- }
462
- finally {
463
- if (progressBar) {
464
- progressBar.stop();
465
- }
466
- }
467
- }
468
- /**
469
- * Static function to handle downloading multiple files from a DataStore based on file paths.
470
- */
471
- static async downloadStore(storeId, rootHash, ipAddress) {
472
- const propagationServer = new PropagationServer(ipAddress, storeId);
473
- // Initialize wallet
474
- await propagationServer.initializeWallet();
475
- // Check if the store exists
476
- const { storeExists, rootHashExists } = await propagationServer.checkStoreExists(rootHash);
477
- if (!storeExists || !rootHashExists) {
478
- throw new Error(`Store ${storeId} does not exist.`);
479
- }
480
- // Fetch the rootHash.dat file
481
- const datFileContent = await propagationServer.fetchFile(`${rootHash}.dat`);
482
- const root = JSON.parse(datFileContent.toString());
483
- // Prepare download tasks
484
- const downloadTasks = [];
485
- for (const [fileKey, fileData] of Object.entries(root.files)) {
486
- const dataPath = (0, hashUtils_1.getFilePathFromSha256)(root.files[fileKey].sha256, "data");
487
- const label = Buffer.from(fileKey, "hex").toString("utf-8");
488
- downloadTasks.push({ label, dataPath });
489
- }
490
- // Limit the number of concurrent downloads
491
- const concurrencyLimit = 10; // Adjust this number as needed
492
- // Create a temporary directory
493
- const tempDir = fs_1.default.mkdtempSync(path_1.default.join(os_1.default.tmpdir(), "downloadStore-"));
494
- try {
495
- // Download files to the temporary directory
496
- await (0, asyncPool_1.asyncPool)(concurrencyLimit, downloadTasks, async (task) => {
497
- await propagationServer.downloadFile(task.label, task.dataPath, rootHash, tempDir);
498
- });
499
- // Save the rootHash.dat file to the temporary directory
500
- if (!fs_1.default.existsSync(tempDir)) {
501
- fs_1.default.mkdirSync(tempDir, { recursive: true });
502
- }
503
- fs_1.default.writeFileSync(path_1.default.join(tempDir, `${rootHash}.dat`), datFileContent);
504
- // Integrity check for the downloaded files was done during the download
505
- // Here we want to make sure we got all the files or we reject the download session
506
- for (const [fileKey, fileData] of Object.entries(root.files)) {
507
- const dataPath = (0, hashUtils_1.getFilePathFromSha256)(root.files[fileKey].sha256, "data");
508
- if (!fs_1.default.existsSync(path_1.default.join(tempDir, dataPath))) {
509
- if (!fs_1.default.existsSync(path_1.default.join(config_1.STORE_PATH, storeId, dataPath))) {
510
- throw new Error(`Missing file: ${Buffer.from(fileKey, "hex")}, aborting session.`);
511
- }
512
- }
513
- }
514
- // After all downloads are complete, copy from temp directory to the main directory
515
- const destinationDir = path_1.default.join(config_1.STORE_PATH, storeId);
516
- fs_extra_1.default.copySync(tempDir, destinationDir, {
517
- overwrite: false, // Prevents overwriting existing files
518
- errorOnExist: false, // No error if file already exists
519
- });
520
- // Generate the manifest file in the main directory
521
- const dataStore = blockchain_1.DataStore.from(storeId);
522
- await dataStore.cacheStoreCreationHeight();
523
- await dataStore.generateManifestFile();
524
- console.log((0, colorette_1.green)(`✔ All files have been downloaded to ${storeId}.`));
525
- }
526
- catch (error) {
527
- console.log((0, colorette_1.red)("✖ Error downloading files:"), error);
528
- }
529
- finally {
530
- // Clean up the temporary directory
531
- fs_extra_1.default.removeSync(tempDir);
532
- }
533
- }
534
- }
535
- exports.PropagationServer = PropagationServer;
536
- PropagationServer.port = 4159; // Static port used for all requests
@@ -1,4 +0,0 @@
1
- export * from './DigPeer';
2
- export * from './DigNetwork';
3
- export * from './DigChallenge';
4
- //# sourceMappingURL=index.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/DigNetwork/index.ts"],"names":[],"mappings":"AAAA,cAAc,WAAW,CAAC;AAC1B,cAAc,cAAc,CAAC;AAC7B,cAAc,gBAAgB,CAAC"}
@@ -1,19 +0,0 @@
1
- "use strict";
2
- var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
- if (k2 === undefined) k2 = k;
4
- var desc = Object.getOwnPropertyDescriptor(m, k);
5
- if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
- desc = { enumerable: true, get: function() { return m[k]; } };
7
- }
8
- Object.defineProperty(o, k2, desc);
9
- }) : (function(o, m, k, k2) {
10
- if (k2 === undefined) k2 = k;
11
- o[k2] = m[k];
12
- }));
13
- var __exportStar = (this && this.__exportStar) || function(m, exports) {
14
- for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
15
- };
16
- Object.defineProperty(exports, "__esModule", { value: true });
17
- __exportStar(require("./DigPeer"), exports);
18
- __exportStar(require("./DigNetwork"), exports);
19
- __exportStar(require("./DigChallenge"), exports);
@@ -1,67 +0,0 @@
1
- import { DataStore as DataStoreDriver, DataStoreMetadata } from "@dignetwork/datalayer-driver";
2
- import { RootHistoryItem } from "../types";
3
- import { DataIntegrityTree, DataIntegrityTreeOptions } from "../DataIntegrityTree";
4
- import { CreateStoreUserInputs } from "../types";
5
- export declare class DataStore {
6
- private storeId;
7
- private tree;
8
- constructor(storeId: string, options?: DataIntegrityTreeOptions);
9
- get StoreId(): string;
10
- get Tree(): DataIntegrityTree;
11
- toBuffer(): Buffer;
12
- toString(): string;
13
- serialize(): string;
14
- static getActiveStore(): Promise<DataStore | undefined>;
15
- static deserialize(serialized: string): DataStore;
16
- static from(storeId: string | Buffer, rootHash?: string): DataStore;
17
- static create(inputs?: CreateStoreUserInputs): Promise<DataStore>;
18
- private static mint;
19
- /**
20
- * Instance method to calculate the disk space used by the store/storeId folder.
21
- * @returns {Promise<number>} - The total disk space used by the store in bytes.
22
- */
23
- getDiskSpace(): Promise<bigint>;
24
- /**
25
- * Static method to calculate the total disk space used by all stores in the store folder.
26
- * @returns {Promise<number>} - The total disk space used by the store folder in bytes.
27
- */
28
- static getTotalDiskSpace(): Promise<bigint>;
29
- /**
30
- * Helper method to calculate the size of a folder and its subfolders recursively.
31
- * @param folderPath - The path of the folder to calculate the size of.
32
- * @returns {Promise<number>} - The total size of the folder in bytes.
33
- */
34
- private static calculateFolderSize;
35
- static getAllStores(): DataStore[];
36
- fetchCoinInfo(): Promise<{
37
- latestStore: DataStoreDriver;
38
- latestHeight: number;
39
- latestHash: Buffer;
40
- }>;
41
- cacheStoreCreationHeight(): Promise<{
42
- createdAtHeight: number;
43
- createdAtHash: Buffer;
44
- }>;
45
- getCreationHeight(): Promise<{
46
- createdAtHeight: number;
47
- createdAtHash: Buffer;
48
- }>;
49
- private setCreationHeight;
50
- getRootHistory(bustCache?: boolean): Promise<RootHistoryItem[]>;
51
- generateManifestFile(folderPath?: string): Promise<void>;
52
- getMetaData(): Promise<DataStoreMetadata>;
53
- isSynced(): Promise<boolean>;
54
- hasMetaWritePermissions(publicSyntheticKey?: Buffer): Promise<boolean>;
55
- updateMetadata(metadata: DataStoreMetadata): Promise<DataStoreDriver>;
56
- /**
57
- * Retrieve the file set for a given root hash and validate file integrity.
58
- *
59
- * @param {string} rootHash - The root hash to fetch the file set.
60
- * @returns {Promise<{ fileName: string, file: string }[]>} - An array of unique file objects.
61
- */
62
- getFileSetForRootHash(rootHash: string): Promise<{
63
- name: string;
64
- path: string;
65
- }[]>;
66
- }
67
- //# sourceMappingURL=DataStore.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"DataStore.d.ts","sourceRoot":"","sources":["../../src/blockchain/DataStore.ts"],"names":[],"mappings":"AAEA,OAAO,EAOL,SAAS,IAAI,eAAe,EAE5B,iBAAiB,EAIlB,MAAM,8BAA8B,CAAC;AAWtC,OAAO,EAAE,eAAe,EAAW,MAAM,UAAU,CAAC;AAGpD,OAAO,EACL,iBAAiB,EACjB,wBAAwB,EACzB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,qBAAqB,EAAE,MAAM,UAAU,CAAC;AAajD,qBAAa,SAAS;IACpB,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,IAAI,CAAoB;gBAEpB,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,wBAAwB;IAiB/D,IAAW,OAAO,IAAI,MAAM,CAE3B;IAED,IAAW,IAAI,IAAI,iBAAiB,CAEnC;IAEM,QAAQ,IAAI,MAAM;IAIlB,QAAQ,IAAI,MAAM;IAIlB,SAAS,IAAI,MAAM;WAMN,cAAc,IAAI,OAAO,CAAC,SAAS,GAAG,SAAS,CAAC;WAOtD,WAAW,CAAC,UAAU,EAAE,MAAM,GAAG,SAAS;WAK1C,IAAI,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS;WAiBtD,MAAM,CACxB,MAAM,GAAE,qBAA0B,GACjC,OAAO,CAAC,SAAS,CAAC;mBAyCA,IAAI;IA4FzB;;;OAGG;IACU,YAAY,IAAI,OAAO,CAAC,MAAM,CAAC;IAK5C;;;OAGG;WACiB,iBAAiB,IAAI,OAAO,CAAC,MAAM,CAAC;IAIxD;;;;OAIG;mBACkB,mBAAmB;WAqB1B,YAAY,IAAI,SAAS,EAAE;IAW5B,aAAa,IAAI,OAAO,CAAC;QACpC,WAAW,EAAE,eAAe,CAAC;QAC7B,YAAY,EAAE,MAAM,CAAC;QACrB,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC;IAyGW,wBAAwB,IAAI,OAAO,CAAC;QAC/C,eAAe,EAAE,MAAM,CAAC;QACxB,aAAa,EAAE,MAAM,CAAC;KACvB,CAAC;IAiBW,iBAAiB,IAAI,OAAO,CAAC;QACxC,eAAe,EAAE,MAAM,CAAC;QACxB,aAAa,EAAE,MAAM,CAAC;KACvB,CAAC;YA0BY,iBAAiB;IAYlB,cAAc,CAAC,SAAS,CAAC,EAAE,OAAO,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;IA4C/D,oBAAoB,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAexD,WAAW,IAAI,OAAO,CAAC,iBAAiB,CAAC;IAKzC,QAAQ,IAAI,OAAO,CAAC,OAAO,CAAC;IAM5B,uBAAuB,CAClC,kBAAkB,CAAC,EAAE,MAAM,GAC1B,OAAO,CAAC,OAAO,CAAC;IAkBN,cAAc,CACzB,QAAQ,EAAE,iBAAiB,GAC1B,OAAO,CAAC,eAAe,CAAC;IA+C3B;;;;;OAKG;IACU,qBAAqB,CAChC,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;CAyC7C"}