@dignetwork/dig-sdk 0.0.1-alpha.99 → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (162) hide show
  1. package/LICENSE +21 -7
  2. package/README.md +267 -0
  3. package/dist/adapters.cjs +315 -0
  4. package/dist/adapters.cjs.map +1 -0
  5. package/dist/adapters.d.cts +20 -0
  6. package/dist/adapters.d.ts +20 -0
  7. package/dist/adapters.js +306 -0
  8. package/dist/adapters.js.map +1 -0
  9. package/dist/dev-shim-Bgw8X2E9.d.cts +118 -0
  10. package/dist/dev-shim-Bgw8X2E9.d.ts +118 -0
  11. package/dist/dig-client-entry-BIzuZ7Rs.d.cts +263 -0
  12. package/dist/dig-client-entry-BIzuZ7Rs.d.ts +263 -0
  13. package/dist/dig-client.cjs +330 -0
  14. package/dist/dig-client.cjs.map +1 -0
  15. package/dist/dig-client.d.cts +1 -0
  16. package/dist/dig-client.d.ts +1 -0
  17. package/dist/dig-client.js +319 -0
  18. package/dist/dig-client.js.map +1 -0
  19. package/dist/index.cjs +754 -0
  20. package/dist/index.cjs.map +1 -0
  21. package/dist/index.d.cts +192 -0
  22. package/dist/index.d.ts +192 -5
  23. package/dist/index.js +731 -19
  24. package/dist/index.js.map +1 -0
  25. package/dist/next.cjs +326 -0
  26. package/dist/next.cjs.map +1 -0
  27. package/dist/next.d.cts +23 -0
  28. package/dist/next.d.ts +23 -0
  29. package/dist/next.js +319 -0
  30. package/dist/next.js.map +1 -0
  31. package/dist/spend.cjs +14 -0
  32. package/dist/spend.cjs.map +1 -0
  33. package/dist/spend.d.cts +1 -0
  34. package/dist/spend.d.ts +1 -0
  35. package/dist/spend.js +3 -0
  36. package/dist/spend.js.map +1 -0
  37. package/dist/vite.cjs +338 -0
  38. package/dist/vite.cjs.map +1 -0
  39. package/dist/vite.d.cts +35 -0
  40. package/dist/vite.d.ts +35 -0
  41. package/dist/vite.js +332 -0
  42. package/dist/vite.js.map +1 -0
  43. package/package.json +87 -59
  44. package/vendor/PROVENANCE.md +102 -0
  45. package/vendor/dig_client.d.ts +150 -0
  46. package/vendor/dig_client.mjs +805 -0
  47. package/vendor/dig_client_bg.wasm +0 -0
  48. package/dist/DataIntegrityTree/DataIntegrityTree.d.ts +0 -163
  49. package/dist/DataIntegrityTree/DataIntegrityTree.d.ts.map +0 -1
  50. package/dist/DataIntegrityTree/DataIntegrityTree.js +0 -707
  51. package/dist/DataIntegrityTree/DataLayerError.d.ts +0 -6
  52. package/dist/DataIntegrityTree/DataLayerError.d.ts.map +0 -1
  53. package/dist/DataIntegrityTree/DataLayerError.js +0 -17
  54. package/dist/DataIntegrityTree/index.d.ts +0 -2
  55. package/dist/DataIntegrityTree/index.d.ts.map +0 -1
  56. package/dist/DataIntegrityTree/index.js +0 -17
  57. package/dist/DigNetwork/ContentServer.d.ts +0 -31
  58. package/dist/DigNetwork/ContentServer.d.ts.map +0 -1
  59. package/dist/DigNetwork/ContentServer.js +0 -176
  60. package/dist/DigNetwork/DigChallenge.d.ts +0 -82
  61. package/dist/DigNetwork/DigChallenge.d.ts.map +0 -1
  62. package/dist/DigNetwork/DigChallenge.js +0 -144
  63. package/dist/DigNetwork/DigNetwork.d.ts +0 -14
  64. package/dist/DigNetwork/DigNetwork.d.ts.map +0 -1
  65. package/dist/DigNetwork/DigNetwork.js +0 -169
  66. package/dist/DigNetwork/DigPeer.d.ts +0 -24
  67. package/dist/DigNetwork/DigPeer.d.ts.map +0 -1
  68. package/dist/DigNetwork/DigPeer.js +0 -118
  69. package/dist/DigNetwork/IncentiveServer.d.ts +0 -13
  70. package/dist/DigNetwork/IncentiveServer.d.ts.map +0 -1
  71. package/dist/DigNetwork/IncentiveServer.js +0 -76
  72. package/dist/DigNetwork/PropagationServer.d.ts +0 -81
  73. package/dist/DigNetwork/PropagationServer.d.ts.map +0 -1
  74. package/dist/DigNetwork/PropagationServer.js +0 -563
  75. package/dist/DigNetwork/index.d.ts +0 -4
  76. package/dist/DigNetwork/index.d.ts.map +0 -1
  77. package/dist/DigNetwork/index.js +0 -19
  78. package/dist/blockchain/DataStore.d.ts +0 -67
  79. package/dist/blockchain/DataStore.d.ts.map +0 -1
  80. package/dist/blockchain/DataStore.js +0 -413
  81. package/dist/blockchain/DataStoreSerializer.d.ts +0 -52
  82. package/dist/blockchain/DataStoreSerializer.d.ts.map +0 -1
  83. package/dist/blockchain/DataStoreSerializer.js +0 -109
  84. package/dist/blockchain/FullNodePeer.d.ts +0 -31
  85. package/dist/blockchain/FullNodePeer.d.ts.map +0 -1
  86. package/dist/blockchain/FullNodePeer.js +0 -295
  87. package/dist/blockchain/ServerCoin.d.ts +0 -29
  88. package/dist/blockchain/ServerCoin.d.ts.map +0 -1
  89. package/dist/blockchain/ServerCoin.js +0 -324
  90. package/dist/blockchain/Wallet.d.ts +0 -28
  91. package/dist/blockchain/Wallet.d.ts.map +0 -1
  92. package/dist/blockchain/Wallet.js +0 -253
  93. package/dist/blockchain/coins.d.ts +0 -6
  94. package/dist/blockchain/coins.d.ts.map +0 -1
  95. package/dist/blockchain/coins.js +0 -102
  96. package/dist/blockchain/index.d.ts +0 -8
  97. package/dist/blockchain/index.d.ts.map +0 -1
  98. package/dist/blockchain/index.js +0 -36
  99. package/dist/index.d.ts.map +0 -1
  100. package/dist/prompts.d.ts +0 -21
  101. package/dist/prompts.d.ts.map +0 -1
  102. package/dist/prompts.js +0 -149
  103. package/dist/types.d.ts +0 -64
  104. package/dist/types.d.ts.map +0 -1
  105. package/dist/types.js +0 -2
  106. package/dist/utils/ContentScanner.d.ts +0 -63
  107. package/dist/utils/ContentScanner.d.ts.map +0 -1
  108. package/dist/utils/ContentScanner.js +0 -175
  109. package/dist/utils/Environment.d.ts +0 -18
  110. package/dist/utils/Environment.d.ts.map +0 -1
  111. package/dist/utils/Environment.js +0 -85
  112. package/dist/utils/FileCache.d.ts +0 -11
  113. package/dist/utils/FileCache.d.ts.map +0 -1
  114. package/dist/utils/FileCache.js +0 -58
  115. package/dist/utils/FileTransfer.d.ts +0 -47
  116. package/dist/utils/FileTransfer.d.ts.map +0 -1
  117. package/dist/utils/FileTransfer.js +0 -209
  118. package/dist/utils/NconfManager.d.ts +0 -11
  119. package/dist/utils/NconfManager.d.ts.map +0 -1
  120. package/dist/utils/NconfManager.js +0 -56
  121. package/dist/utils/StoreArchiveManager.d.ts +0 -45
  122. package/dist/utils/StoreArchiveManager.d.ts.map +0 -1
  123. package/dist/utils/StoreArchiveManager.js +0 -153
  124. package/dist/utils/asyncPool.d.ts +0 -9
  125. package/dist/utils/asyncPool.d.ts.map +0 -1
  126. package/dist/utils/asyncPool.js +0 -23
  127. package/dist/utils/config.d.ts +0 -68
  128. package/dist/utils/config.d.ts.map +0 -1
  129. package/dist/utils/config.js +0 -223
  130. package/dist/utils/connectionUtils.d.ts +0 -10
  131. package/dist/utils/connectionUtils.d.ts.map +0 -1
  132. package/dist/utils/connectionUtils.js +0 -18
  133. package/dist/utils/credentialsUtils.d.ts +0 -8
  134. package/dist/utils/credentialsUtils.d.ts.map +0 -1
  135. package/dist/utils/credentialsUtils.js +0 -122
  136. package/dist/utils/directoryUtils.d.ts +0 -9
  137. package/dist/utils/directoryUtils.d.ts.map +0 -1
  138. package/dist/utils/directoryUtils.js +0 -110
  139. package/dist/utils/encryption.d.ts +0 -8
  140. package/dist/utils/encryption.d.ts.map +0 -1
  141. package/dist/utils/encryption.js +0 -55
  142. package/dist/utils/hashUtils.d.ts +0 -8
  143. package/dist/utils/hashUtils.d.ts.map +0 -1
  144. package/dist/utils/hashUtils.js +0 -44
  145. package/dist/utils/index.d.ts +0 -14
  146. package/dist/utils/index.d.ts.map +0 -1
  147. package/dist/utils/index.js +0 -29
  148. package/dist/utils/merkle.d.ts +0 -2
  149. package/dist/utils/merkle.d.ts.map +0 -1
  150. package/dist/utils/merkle.js +0 -28
  151. package/dist/utils/network.d.ts +0 -2
  152. package/dist/utils/network.d.ts.map +0 -1
  153. package/dist/utils/network.js +0 -54
  154. package/dist/utils/spinnerUtils.d.ts +0 -2
  155. package/dist/utils/spinnerUtils.d.ts.map +0 -1
  156. package/dist/utils/spinnerUtils.js +0 -47
  157. package/dist/utils/ssl.d.ts +0 -5
  158. package/dist/utils/ssl.d.ts.map +0 -1
  159. package/dist/utils/ssl.js +0 -50
  160. package/dist/utils/validationUtils.d.ts +0 -9
  161. package/dist/utils/validationUtils.d.ts.map +0 -1
  162. package/dist/utils/validationUtils.js +0 -55
@@ -1,563 +0,0 @@
1
- "use strict";
2
- var __importDefault = (this && this.__importDefault) || function (mod) {
3
- return (mod && mod.__esModule) ? mod : { "default": mod };
4
- };
5
- Object.defineProperty(exports, "__esModule", { value: true });
6
- exports.PropagationServer = void 0;
7
- const axios_1 = __importDefault(require("axios"));
8
- const cli_progress_1 = __importDefault(require("cli-progress"));
9
- const form_data_1 = __importDefault(require("form-data"));
10
- const fs_1 = __importDefault(require("fs"));
11
- const fs_extra_1 = __importDefault(require("fs-extra"));
12
- const https_1 = __importDefault(require("https"));
13
- const os_1 = __importDefault(require("os"));
14
- const path_1 = __importDefault(require("path"));
15
- const progress_stream_1 = __importDefault(require("progress-stream"));
16
- const asyncPool_1 = require("../utils/asyncPool");
17
- const nanospinner_1 = require("nanospinner");
18
- const hashUtils_1 = require("../utils/hashUtils");
19
- const ssl_1 = require("../utils/ssl");
20
- const colorette_1 = require("colorette");
21
- const merkle_1 = require("../utils/merkle");
22
- const stream_1 = require("stream");
23
- const credentialsUtils_1 = require("../utils/credentialsUtils");
24
- const config_1 = require("../utils/config");
25
- const blockchain_1 = require("../blockchain");
26
- // Helper function to trim long filenames with ellipsis and ensure consistent padding
27
- function formatFilename(filename, maxLength = 30) {
28
- if (!filename) {
29
- return "Unknown File".padEnd(maxLength, " ");
30
- }
31
- if (filename.length > maxLength) {
32
- return `...${filename.slice(-(maxLength - 3))}`.padEnd(maxLength, " ");
33
- }
34
- return filename.padEnd(maxLength, " ");
35
- }
36
- class PropagationServer {
37
- constructor(ipAddress, storeId) {
38
- this.storeId = storeId;
39
- this.sessionId = "";
40
- this.publicKey = "";
41
- this.ipAddress = ipAddress;
42
- // Get or create SSL certificates
43
- const { certPath, keyPath } = (0, ssl_1.getOrCreateSSLCerts)();
44
- this.certPath = certPath;
45
- this.keyPath = keyPath;
46
- }
47
- /**
48
- * Initialize the Wallet instance.
49
- */
50
- async initializeWallet() {
51
- this.wallet = await blockchain_1.Wallet.load("default");
52
- this.publicKey = (await this.wallet.getPublicSyntheticKey()).toString("hex");
53
- }
54
- /**
55
- * Create an Axios HTTPS Agent with self-signed certificate allowance.
56
- */
57
- createHttpsAgent() {
58
- return new https_1.default.Agent({
59
- cert: fs_1.default.readFileSync(this.certPath),
60
- key: fs_1.default.readFileSync(this.keyPath),
61
- rejectUnauthorized: false,
62
- });
63
- }
64
- /**
65
- * Adds a custom inactivity timeout for large file transfers.
66
- */
67
- addInactivityTimeout(stream, timeoutMs) {
68
- let timeoutId;
69
- const resetTimeout = () => {
70
- clearTimeout(timeoutId);
71
- timeoutId = setTimeout(() => {
72
- stream.destroy(new Error(`Inactivity timeout after ${timeoutMs / 1000} seconds`));
73
- }, timeoutMs);
74
- };
75
- // Reset timeout every time data is received
76
- stream.on("data", resetTimeout);
77
- // Set the initial timeout
78
- resetTimeout();
79
- // Clear the timeout when the stream ends
80
- stream.on("end", () => clearTimeout(timeoutId));
81
- stream.on("error", () => clearTimeout(timeoutId)); // Handle errors
82
- return stream;
83
- }
84
- /**
85
- * Check if the store and optional root hash exist by making a HEAD request.
86
- */
87
- async checkStoreExists(rootHash) {
88
- const spinner = (0, nanospinner_1.createSpinner)(`Checking if store ${this.storeId} exists...`).start();
89
- try {
90
- const config = {
91
- httpsAgent: this.createHttpsAgent(),
92
- };
93
- let url = `https://${this.ipAddress}:${PropagationServer.port}/${this.storeId}`;
94
- if (rootHash) {
95
- url += `?hasRootHash=${rootHash}`;
96
- }
97
- const response = await axios_1.default.head(url, config);
98
- // Extract store existence and root hash existence from headers
99
- const storeExists = response.headers["x-store-exists"] === "true";
100
- const rootHashExists = response.headers["x-has-root-hash"] === "true";
101
- if (storeExists) {
102
- spinner.success({
103
- text: (0, colorette_1.green)(`Store ${this.storeId} exists on peer: ${this.ipAddress}`),
104
- });
105
- }
106
- else {
107
- spinner.error({
108
- text: (0, colorette_1.yellow)(`Store ${this.storeId} does not exist on ${this.ipAddress}. Credentials will be required to push.`),
109
- });
110
- }
111
- return { storeExists, rootHashExists };
112
- }
113
- catch (error) {
114
- spinner.error({ text: (0, colorette_1.red)("Error checking if store exists:") });
115
- console.error((0, colorette_1.red)(error.message));
116
- throw error;
117
- }
118
- }
119
- /**
120
- * Start an upload session by sending a POST request with the rootHash.dat file.
121
- */
122
- async startUploadSession(rootHash) {
123
- const spinner = (0, nanospinner_1.createSpinner)(`Starting upload session for store ${this.storeId}...`).start();
124
- try {
125
- const formData = new form_data_1.default();
126
- const datFilePath = path_1.default.join(config_1.STORE_PATH, this.storeId, `${rootHash}.dat`);
127
- // Ensure the rootHash.dat file exists
128
- if (!fs_1.default.existsSync(datFilePath)) {
129
- throw new Error(`File not found: ${datFilePath}`);
130
- }
131
- formData.append("file", fs_1.default.createReadStream(datFilePath), {
132
- filename: `${rootHash}.dat`,
133
- });
134
- const config = {
135
- httpsAgent: this.createHttpsAgent(),
136
- headers: {
137
- ...formData.getHeaders(),
138
- },
139
- };
140
- // Add Basic Auth if username and password are present
141
- if (this.username && this.password) {
142
- config.auth = {
143
- username: this.username,
144
- password: this.password,
145
- };
146
- }
147
- const url = `https://${this.ipAddress}:${PropagationServer.port}/upload/${this.storeId}?roothash=${rootHash}`;
148
- const response = await axios_1.default.post(url, formData, config);
149
- this.sessionId = response.data.sessionId;
150
- spinner.success({
151
- text: (0, colorette_1.green)(`Upload session started for Root Hash: ${(0, colorette_1.cyan)(rootHash)} with session ID ${this.sessionId}`),
152
- });
153
- }
154
- catch (error) {
155
- spinner.error({ text: (0, colorette_1.red)("Error starting upload session:") });
156
- console.error((0, colorette_1.red)(error.message));
157
- throw error;
158
- }
159
- }
160
- /**
161
- * Request a nonce for a file by sending a HEAD request to the server.
162
- */
163
- async getFileNonce(filename) {
164
- try {
165
- const config = {
166
- httpsAgent: this.createHttpsAgent(),
167
- };
168
- const url = `https://${this.ipAddress}:${PropagationServer.port}/upload/${this.storeId}/${this.sessionId}/${filename}`;
169
- const response = await axios_1.default.head(url, config);
170
- // Check for 'x-file-exists' header
171
- const fileExists = response.headers["x-file-exists"] === "true";
172
- // If file exists, no need to generate a nonce
173
- const nonce = response.headers["x-nonce"];
174
- return { nonce, fileExists };
175
- }
176
- catch (error) {
177
- console.error((0, colorette_1.red)(`Error generating nonce for file ${filename}:`), error.message);
178
- throw error;
179
- }
180
- }
181
- /**
182
- * Upload a file to the server by sending a PUT request.
183
- * Logs progress using a local cli-progress bar.
184
- */
185
- async uploadFile(label, dataPath) {
186
- const filePath = path_1.default.join(config_1.STORE_PATH, this.storeId, dataPath);
187
- const { nonce, fileExists } = await this.getFileNonce(dataPath);
188
- if (fileExists) {
189
- console.log((0, colorette_1.blue)(`File ${label} already exists. Skipping upload.`));
190
- return;
191
- }
192
- const wallet = await blockchain_1.Wallet.load("default");
193
- const keyOwnershipSig = await wallet.createKeyOwnershipSignature(nonce);
194
- const publicKey = await wallet.getPublicSyntheticKey();
195
- // Get the file size
196
- const fileSize = fs_1.default.statSync(filePath).size;
197
- let progressBar;
198
- try {
199
- // Create a new progress bar for the file
200
- progressBar = new cli_progress_1.default.SingleBar({
201
- format: `${(0, colorette_1.blue)("[{bar}]")} | ${(0, colorette_1.yellow)("{filename}")} | {percentage}% | {value}/{total} bytes`,
202
- hideCursor: true,
203
- barsize: 30,
204
- align: "left",
205
- autopadding: true,
206
- noTTYOutput: false,
207
- stopOnComplete: true,
208
- clearOnComplete: false,
209
- }, cli_progress_1.default.Presets.legacy);
210
- progressBar.start(fileSize, 0, {
211
- filename: formatFilename(label),
212
- });
213
- // Create the read stream
214
- const fileReadStream = fs_1.default.createReadStream(filePath);
215
- // Create a progress stream
216
- const progressStream = (0, progress_stream_1.default)({
217
- length: fileSize,
218
- time: 500, // Adjust as needed
219
- });
220
- progressStream.on("progress", (progress) => {
221
- progressBar.update(progress.transferred);
222
- });
223
- // Add inactivity timeout to the progress stream
224
- const passThroughStream = this.addInactivityTimeout(new stream_1.PassThrough(), PropagationServer.inactivityTimeout);
225
- // Pipe the read stream through the progress stream into the PassThrough stream
226
- fileReadStream.pipe(progressStream).pipe(passThroughStream);
227
- // Use form-data to construct the request body
228
- const formData = new form_data_1.default();
229
- formData.append("file", passThroughStream);
230
- const headers = {
231
- ...formData.getHeaders(),
232
- "x-nonce": nonce,
233
- "x-public-key": publicKey.toString("hex"),
234
- "x-key-ownership-sig": keyOwnershipSig,
235
- };
236
- const config = {
237
- headers: headers,
238
- httpsAgent: this.createHttpsAgent(),
239
- maxContentLength: Infinity,
240
- maxBodyLength: Infinity,
241
- };
242
- const url = `https://${this.ipAddress}:${PropagationServer.port}/upload/${this.storeId}/${this.sessionId}/${dataPath}`;
243
- // Create a promise that resolves when the progress stream ends
244
- const progressPromise = new Promise((resolve, reject) => {
245
- progressStream.on("end", resolve);
246
- progressStream.on("error", reject);
247
- });
248
- // Start the upload request
249
- const uploadPromise = axios_1.default.put(url, formData, config);
250
- // Wait for both the upload and the progress stream to finish
251
- await Promise.all([uploadPromise, progressPromise]);
252
- }
253
- catch (error) {
254
- throw error;
255
- }
256
- finally {
257
- if (progressBar) {
258
- progressBar.stop();
259
- }
260
- }
261
- }
262
- /**
263
- * Commit the upload session by sending a POST request to the server.
264
- */
265
- async commitUploadSession() {
266
- const spinner = (0, nanospinner_1.createSpinner)(`Committing upload session for store ${this.storeId}...`).start();
267
- try {
268
- const config = {
269
- httpsAgent: this.createHttpsAgent(),
270
- auth: this.username && this.password
271
- ? {
272
- username: this.username,
273
- password: this.password,
274
- }
275
- : undefined,
276
- };
277
- const url = `https://${this.ipAddress}:${PropagationServer.port}/commit/${this.storeId}/${this.sessionId}`;
278
- const response = await axios_1.default.post(url, {}, config);
279
- spinner.success({
280
- text: (0, colorette_1.green)(`Upload session ${this.sessionId} successfully committed.`),
281
- });
282
- return response.data;
283
- }
284
- catch (error) {
285
- spinner.error({ text: (0, colorette_1.red)("Error committing upload session:") });
286
- console.error((0, colorette_1.red)(error.message));
287
- throw error;
288
- }
289
- }
290
- /**
291
- * Static function to handle the entire upload process for multiple files based on rootHash.
292
- */
293
- static async uploadStore(storeId, rootHash, ipAddress) {
294
- const propagationServer = new PropagationServer(ipAddress, storeId);
295
- // Initialize wallet
296
- await propagationServer.initializeWallet();
297
- // Check if the store exists
298
- const { storeExists, rootHashExists } = await propagationServer.checkStoreExists(rootHash);
299
- // If the store does not exist, prompt for credentials
300
- if (!storeExists) {
301
- console.log((0, colorette_1.red)(`Store ${storeId} does not exist. Prompting for credentials...`));
302
- const credentials = await (0, credentialsUtils_1.promptCredentials)(propagationServer.ipAddress);
303
- propagationServer.username = credentials.username;
304
- propagationServer.password = credentials.password;
305
- }
306
- if (rootHashExists) {
307
- console.log((0, colorette_1.blue)(`Root hash ${rootHash} already exists in the store. Skipping upload.`));
308
- return;
309
- }
310
- // Start the upload session
311
- await propagationServer.startUploadSession(rootHash);
312
- const dataStore = blockchain_1.DataStore.from(storeId);
313
- const files = await dataStore.getFileSetForRootHash(rootHash);
314
- // Prepare upload tasks
315
- const uploadTasks = files.map((file) => ({
316
- label: file.name,
317
- dataPath: file.path,
318
- }));
319
- // Limit the number of concurrent uploads
320
- const concurrencyLimit = 10; // Adjust this number as needed
321
- await (0, asyncPool_1.asyncPool)(concurrencyLimit, uploadTasks, async (task) => {
322
- await propagationServer.uploadFile(task.label, task.dataPath);
323
- });
324
- // Commit the session after all files have been uploaded
325
- await propagationServer.commitUploadSession();
326
- console.log((0, colorette_1.green)(`✔ All files have been uploaded to for Root Hash: ${(0, colorette_1.cyan)(rootHash)}.`));
327
- }
328
- /**
329
- * Fetch a file from the server by sending a GET request and return its content in memory.
330
- * Logs progress using a local cli-progress bar.
331
- */
332
- async fetchFile(dataPath) {
333
- const url = `https://${this.ipAddress}:${PropagationServer.port}/fetch/${this.storeId}/${dataPath}`;
334
- const config = {
335
- responseType: "stream",
336
- httpsAgent: this.createHttpsAgent(),
337
- };
338
- let progressBar;
339
- try {
340
- const response = await axios_1.default.get(url, config);
341
- const totalLengthHeader = response.headers["content-length"];
342
- const totalLength = totalLengthHeader
343
- ? parseInt(totalLengthHeader, 10)
344
- : null;
345
- if (!totalLength) {
346
- throw new Error("Content-Length header is missing");
347
- }
348
- // Create a new progress bar for the file
349
- progressBar = new cli_progress_1.default.SingleBar({
350
- format: `${(0, colorette_1.blue)("[{bar}]")} | ${(0, colorette_1.yellow)("{filename}")} | {percentage}% | {value}/{total} bytes`,
351
- hideCursor: true,
352
- barsize: 30,
353
- align: "left",
354
- autopadding: true,
355
- noTTYOutput: false,
356
- stopOnComplete: true,
357
- clearOnComplete: false,
358
- }, cli_progress_1.default.Presets.legacy);
359
- progressBar.start(totalLength, 0, {
360
- filename: formatFilename(dataPath),
361
- });
362
- let dataBuffers = [];
363
- const progressStream = (0, progress_stream_1.default)({
364
- length: totalLength,
365
- time: 500, // Adjust as needed
366
- });
367
- progressStream.on("progress", (progress) => {
368
- progressBar.update(progress.transferred);
369
- });
370
- // Add inactivity timeout to the progress stream
371
- const passThroughStream = this.addInactivityTimeout(new stream_1.PassThrough(), PropagationServer.inactivityTimeout);
372
- response.data.pipe(progressStream).pipe(passThroughStream);
373
- progressStream.on("data", (chunk) => {
374
- dataBuffers.push(chunk);
375
- });
376
- // Wait for the progress stream to finish
377
- await new Promise((resolve, reject) => {
378
- progressStream.on("end", resolve);
379
- progressStream.on("error", reject);
380
- });
381
- return Buffer.concat(dataBuffers);
382
- }
383
- catch (error) {
384
- throw error;
385
- }
386
- finally {
387
- if (progressBar) {
388
- progressBar.stop();
389
- }
390
- }
391
- }
392
- /**
393
- * Get details of a file, including whether it exists and its size.
394
- */
395
- async getFileDetails(dataPath, rootHash) {
396
- try {
397
- const config = {
398
- httpsAgent: this.createHttpsAgent(),
399
- };
400
- const url = `https://${this.ipAddress}:${PropagationServer.port}/store/${this.storeId}/${rootHash}/${dataPath}`;
401
- const response = await axios_1.default.head(url, config);
402
- // Check the headers for file existence and size
403
- const fileExists = response.headers["x-file-exists"] === "true";
404
- const fileSize = parseInt(response.headers["x-file-size"], 10);
405
- return {
406
- exists: fileExists,
407
- size: fileExists ? fileSize : 0,
408
- };
409
- }
410
- catch (error) {
411
- console.error((0, colorette_1.red)(`✖ Error checking file details for ${dataPath}:`), error.message);
412
- throw error;
413
- }
414
- }
415
- /**
416
- * Download a file from the server by sending a GET request.
417
- * Logs progress using a local cli-progress bar.
418
- */
419
- async downloadFile(label, dataPath, rootHash, baseDir) {
420
- const url = `https://${this.ipAddress}:${PropagationServer.port}/fetch/${this.storeId}/${dataPath}`;
421
- let downloadPath = path_1.default.join(baseDir, dataPath);
422
- // Ensure that the directory for the file exists
423
- fs_1.default.mkdirSync(path_1.default.dirname(downloadPath), { recursive: true });
424
- const config = {
425
- responseType: "stream",
426
- httpsAgent: this.createHttpsAgent(),
427
- };
428
- let progressBar;
429
- try {
430
- const response = await axios_1.default.get(url, config);
431
- const totalLengthHeader = response.headers["content-length"];
432
- const totalLength = totalLengthHeader
433
- ? parseInt(totalLengthHeader, 10)
434
- : null;
435
- if (!totalLength) {
436
- throw new Error("Content-Length header is missing");
437
- }
438
- // Create a new progress bar for the file
439
- progressBar = new cli_progress_1.default.SingleBar({
440
- format: `${(0, colorette_1.blue)("[{bar}]")} | ${(0, colorette_1.yellow)("{filename}")} | {percentage}% | {value}/{total} bytes`,
441
- hideCursor: true,
442
- barsize: 30,
443
- align: "left",
444
- autopadding: true,
445
- noTTYOutput: false,
446
- stopOnComplete: true,
447
- clearOnComplete: false,
448
- }, cli_progress_1.default.Presets.legacy);
449
- progressBar.start(totalLength, 0, {
450
- filename: formatFilename(label),
451
- });
452
- const fileWriteStream = fs_1.default.createWriteStream(downloadPath);
453
- const progressStream = (0, progress_stream_1.default)({
454
- length: totalLength,
455
- time: 500, // Adjust as needed
456
- });
457
- progressStream.on("progress", (progress) => {
458
- progressBar.update(progress.transferred);
459
- });
460
- // Add inactivity timeout to the progress stream
461
- const passThroughStream = this.addInactivityTimeout(new stream_1.PassThrough(), PropagationServer.inactivityTimeout);
462
- response.data
463
- .pipe(progressStream)
464
- .pipe(passThroughStream)
465
- .pipe(fileWriteStream);
466
- // Wait for both the file write stream and the progress stream to finish
467
- await Promise.all([
468
- new Promise((resolve, reject) => {
469
- fileWriteStream.on("finish", resolve);
470
- fileWriteStream.on("error", reject);
471
- }),
472
- new Promise((resolve, reject) => {
473
- progressStream.on("end", resolve);
474
- progressStream.on("error", reject);
475
- }),
476
- ]);
477
- if (dataPath.includes("/data")) {
478
- const integrity = await (0, merkle_1.merkleIntegrityCheck)(path_1.default.join(baseDir, `${rootHash}.dat`), baseDir, dataPath, rootHash);
479
- if (!integrity) {
480
- throw new Error(`Integrity check failed for file: ${dataPath}`);
481
- }
482
- console.log("integrity check");
483
- }
484
- }
485
- catch (error) {
486
- throw error;
487
- }
488
- finally {
489
- if (progressBar) {
490
- progressBar.stop();
491
- }
492
- }
493
- }
494
- /**
495
- * Static function to handle downloading multiple files from a DataStore based on file paths.
496
- */
497
- static async downloadStore(storeId, rootHash, ipAddress) {
498
- const propagationServer = new PropagationServer(ipAddress, storeId);
499
- // Initialize wallet
500
- await propagationServer.initializeWallet();
501
- // Check if the store exists
502
- const { storeExists, rootHashExists } = await propagationServer.checkStoreExists(rootHash);
503
- if (!storeExists || !rootHashExists) {
504
- throw new Error(`Store ${storeId} does not exist.`);
505
- }
506
- // Fetch the rootHash.dat file
507
- const datFileContent = await propagationServer.fetchFile(`${rootHash}.dat`);
508
- const root = JSON.parse(datFileContent.toString());
509
- // Prepare download tasks
510
- const downloadTasks = [];
511
- for (const [fileKey, fileData] of Object.entries(root.files)) {
512
- const dataPath = (0, hashUtils_1.getFilePathFromSha256)(root.files[fileKey].sha256, "data");
513
- const label = Buffer.from(fileKey, "hex").toString("utf-8");
514
- downloadTasks.push({ label, dataPath });
515
- }
516
- // Limit the number of concurrent downloads
517
- const concurrencyLimit = 10; // Adjust this number as needed
518
- // Create a temporary directory
519
- const tempDir = fs_1.default.mkdtempSync(path_1.default.join(os_1.default.tmpdir(), "downloadStore-"));
520
- try {
521
- // Download files to the temporary directory
522
- await (0, asyncPool_1.asyncPool)(concurrencyLimit, downloadTasks, async (task) => {
523
- await propagationServer.downloadFile(task.label, task.dataPath, rootHash, tempDir);
524
- });
525
- // Save the rootHash.dat file to the temporary directory
526
- if (!fs_1.default.existsSync(tempDir)) {
527
- fs_1.default.mkdirSync(tempDir, { recursive: true });
528
- }
529
- fs_1.default.writeFileSync(path_1.default.join(tempDir, `${rootHash}.dat`), datFileContent);
530
- // Integrity check for the downloaded files was done during the download
531
- // Here we want to make sure we got all the files or we reject the download session
532
- for (const [fileKey, fileData] of Object.entries(root.files)) {
533
- const dataPath = (0, hashUtils_1.getFilePathFromSha256)(root.files[fileKey].sha256, "data");
534
- if (!fs_1.default.existsSync(path_1.default.join(tempDir, dataPath))) {
535
- if (!fs_1.default.existsSync(path_1.default.join(config_1.STORE_PATH, storeId, dataPath))) {
536
- throw new Error(`Missing file: ${Buffer.from(fileKey, "hex")}, aborting session.`);
537
- }
538
- }
539
- }
540
- // After all downloads are complete, copy from temp directory to the main directory
541
- const destinationDir = path_1.default.join(config_1.STORE_PATH, storeId);
542
- fs_extra_1.default.copySync(tempDir, destinationDir, {
543
- overwrite: false, // Prevents overwriting existing files
544
- errorOnExist: false, // No error if file already exists
545
- });
546
- // Generate the manifest file in the main directory
547
- const dataStore = blockchain_1.DataStore.from(storeId);
548
- await dataStore.cacheStoreCreationHeight();
549
- await dataStore.generateManifestFile();
550
- console.log((0, colorette_1.green)(`✔ All files have been downloaded to ${storeId}.`));
551
- }
552
- catch (error) {
553
- console.log((0, colorette_1.red)("✖ Error downloading files:"), error);
554
- }
555
- finally {
556
- // Clean up the temporary directory
557
- fs_extra_1.default.removeSync(tempDir);
558
- }
559
- }
560
- }
561
- exports.PropagationServer = PropagationServer;
562
- PropagationServer.port = 4159; // Static port used for all requests
563
- PropagationServer.inactivityTimeout = 5000; // Inactivity timeout in milliseconds (5 seconds)
@@ -1,4 +0,0 @@
1
- export * from './DigPeer';
2
- export * from './DigNetwork';
3
- export * from './DigChallenge';
4
- //# sourceMappingURL=index.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/DigNetwork/index.ts"],"names":[],"mappings":"AAAA,cAAc,WAAW,CAAC;AAC1B,cAAc,cAAc,CAAC;AAC7B,cAAc,gBAAgB,CAAC"}
@@ -1,19 +0,0 @@
1
- "use strict";
2
- var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
- if (k2 === undefined) k2 = k;
4
- var desc = Object.getOwnPropertyDescriptor(m, k);
5
- if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
- desc = { enumerable: true, get: function() { return m[k]; } };
7
- }
8
- Object.defineProperty(o, k2, desc);
9
- }) : (function(o, m, k, k2) {
10
- if (k2 === undefined) k2 = k;
11
- o[k2] = m[k];
12
- }));
13
- var __exportStar = (this && this.__exportStar) || function(m, exports) {
14
- for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
15
- };
16
- Object.defineProperty(exports, "__esModule", { value: true });
17
- __exportStar(require("./DigPeer"), exports);
18
- __exportStar(require("./DigNetwork"), exports);
19
- __exportStar(require("./DigChallenge"), exports);
@@ -1,67 +0,0 @@
1
- import { DataStore as DataStoreDriver, DataStoreMetadata } from "@dignetwork/datalayer-driver";
2
- import { RootHistoryItem } from "../types";
3
- import { DataIntegrityTree, DataIntegrityTreeOptions } from "../DataIntegrityTree";
4
- import { CreateStoreUserInputs } from "../types";
5
- export declare class DataStore {
6
- private storeId;
7
- private tree;
8
- constructor(storeId: string, options?: DataIntegrityTreeOptions);
9
- get StoreId(): string;
10
- get Tree(): DataIntegrityTree;
11
- toBuffer(): Buffer;
12
- toString(): string;
13
- serialize(): string;
14
- static getActiveStore(): Promise<DataStore | undefined>;
15
- static deserialize(serialized: string): DataStore;
16
- static from(storeId: string | Buffer, rootHash?: string): DataStore;
17
- static create(inputs?: CreateStoreUserInputs): Promise<DataStore>;
18
- private static mint;
19
- /**
20
- * Instance method to calculate the disk space used by the store/storeId folder.
21
- * @returns {Promise<number>} - The total disk space used by the store in bytes.
22
- */
23
- getDiskSpace(): Promise<bigint>;
24
- /**
25
- * Static method to calculate the total disk space used by all stores in the store folder.
26
- * @returns {Promise<number>} - The total disk space used by the store folder in bytes.
27
- */
28
- static getTotalDiskSpace(): Promise<bigint>;
29
- /**
30
- * Helper method to calculate the size of a folder and its subfolders recursively.
31
- * @param folderPath - The path of the folder to calculate the size of.
32
- * @returns {Promise<number>} - The total size of the folder in bytes.
33
- */
34
- private static calculateFolderSize;
35
- static getAllStores(): DataStore[];
36
- fetchCoinInfo(): Promise<{
37
- latestStore: DataStoreDriver;
38
- latestHeight: number;
39
- latestHash: Buffer;
40
- }>;
41
- cacheStoreCreationHeight(): Promise<{
42
- createdAtHeight: number;
43
- createdAtHash: Buffer;
44
- }>;
45
- getCreationHeight(): Promise<{
46
- createdAtHeight: number;
47
- createdAtHash: Buffer;
48
- }>;
49
- private setCreationHeight;
50
- getRootHistory(bustCache?: boolean): Promise<RootHistoryItem[]>;
51
- generateManifestFile(folderPath?: string): Promise<void>;
52
- getMetaData(): Promise<DataStoreMetadata>;
53
- isSynced(): Promise<boolean>;
54
- hasMetaWritePermissions(publicSyntheticKey?: Buffer): Promise<boolean>;
55
- updateMetadata(metadata: DataStoreMetadata): Promise<DataStoreDriver>;
56
- /**
57
- * Retrieve the file set for a given root hash and validate file integrity.
58
- *
59
- * @param {string} rootHash - The root hash to fetch the file set.
60
- * @returns {Promise<{ fileName: string, file: string }[]>} - An array of unique file objects.
61
- */
62
- getFileSetForRootHash(rootHash: string): Promise<{
63
- name: string;
64
- path: string;
65
- }[]>;
66
- }
67
- //# sourceMappingURL=DataStore.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"DataStore.d.ts","sourceRoot":"","sources":["../../src/blockchain/DataStore.ts"],"names":[],"mappings":"AAEA,OAAO,EAOL,SAAS,IAAI,eAAe,EAE5B,iBAAiB,EAIlB,MAAM,8BAA8B,CAAC;AAWtC,OAAO,EAAE,eAAe,EAAW,MAAM,UAAU,CAAC;AAGpD,OAAO,EACL,iBAAiB,EACjB,wBAAwB,EACzB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,qBAAqB,EAAE,MAAM,UAAU,CAAC;AAajD,qBAAa,SAAS;IACpB,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,IAAI,CAAoB;gBAEpB,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,wBAAwB;IAiB/D,IAAW,OAAO,IAAI,MAAM,CAE3B;IAED,IAAW,IAAI,IAAI,iBAAiB,CAEnC;IAEM,QAAQ,IAAI,MAAM;IAIlB,QAAQ,IAAI,MAAM;IAIlB,SAAS,IAAI,MAAM;WAMN,cAAc,IAAI,OAAO,CAAC,SAAS,GAAG,SAAS,CAAC;WAOtD,WAAW,CAAC,UAAU,EAAE,MAAM,GAAG,SAAS;WAK1C,IAAI,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS;WAiBtD,MAAM,CACxB,MAAM,GAAE,qBAA0B,GACjC,OAAO,CAAC,SAAS,CAAC;mBAyCA,IAAI;IA4FzB;;;OAGG;IACU,YAAY,IAAI,OAAO,CAAC,MAAM,CAAC;IAK5C;;;OAGG;WACiB,iBAAiB,IAAI,OAAO,CAAC,MAAM,CAAC;IAIxD;;;;OAIG;mBACkB,mBAAmB;WAqB1B,YAAY,IAAI,SAAS,EAAE;IAW5B,aAAa,IAAI,OAAO,CAAC;QACpC,WAAW,EAAE,eAAe,CAAC;QAC7B,YAAY,EAAE,MAAM,CAAC;QACrB,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC;IAyGW,wBAAwB,IAAI,OAAO,CAAC;QAC/C,eAAe,EAAE,MAAM,CAAC;QACxB,aAAa,EAAE,MAAM,CAAC;KACvB,CAAC;IAiBW,iBAAiB,IAAI,OAAO,CAAC;QACxC,eAAe,EAAE,MAAM,CAAC;QACxB,aAAa,EAAE,MAAM,CAAC;KACvB,CAAC;YA0BY,iBAAiB;IAYlB,cAAc,CAAC,SAAS,CAAC,EAAE,OAAO,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;IA4C/D,oBAAoB,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAexD,WAAW,IAAI,OAAO,CAAC,iBAAiB,CAAC;IAKzC,QAAQ,IAAI,OAAO,CAAC,OAAO,CAAC;IAM5B,uBAAuB,CAClC,kBAAkB,CAAC,EAAE,MAAM,GAC1B,OAAO,CAAC,OAAO,CAAC;IAkBN,cAAc,CACzB,QAAQ,EAAE,iBAAiB,GAC1B,OAAO,CAAC,eAAe,CAAC;IA+C3B;;;;;OAKG;IACU,qBAAqB,CAChC,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;CAyC7C"}