@dignetwork/dig-sdk 0.0.1-alpha.14 → 0.0.1-alpha.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"DataIntegrityTree.d.ts","sourceRoot":"","sources":["../../src/DataIntegrityTree/DataIntegrityTree.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAC1C,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAuClC,MAAM,WAAW,wBAAwB;IACvC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAClC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAGlB,iBAAiB,CAAC,EAAE,OAAO,CAAC;CAC7B;AAED;;GAEG;AACH,cAAM,iBAAiB;IACrB,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,YAAY,CAAS;IAC7B,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,OAAO,CAAS;IACjB,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC5D,OAAO,CAAC,IAAI,CAAa;gBAEb,OAAO,EAAE,MAAM,EAAE,OAAO,GAAE,wBAA6B;WAiDrD,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,wBAAwB,GAAG,iBAAiB;IAIzF;;;OAGG;IACH,OAAO,CAAC,aAAa;IAQrB;;;OAGG;IACH,OAAO,CAAC,eAAe;IAUvB;;;;OAIG;IACH,OAAO,CAAC,kBAAkB;IAa1B;;;;OAIG;YACW,WAAW;IAazB;;;;;OAKG;IACG,SAAS,CAAC,UAAU,EAAE,QAAQ,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;
|
|
1
|
+
{"version":3,"file":"DataIntegrityTree.d.ts","sourceRoot":"","sources":["../../src/DataIntegrityTree/DataIntegrityTree.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAC1C,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAuClC,MAAM,WAAW,wBAAwB;IACvC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAClC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAGlB,iBAAiB,CAAC,EAAE,OAAO,CAAC;CAC7B;AAED;;GAEG;AACH,cAAM,iBAAiB;IACrB,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,YAAY,CAAS;IAC7B,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,OAAO,CAAS;IACjB,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC5D,OAAO,CAAC,IAAI,CAAa;gBAEb,OAAO,EAAE,MAAM,EAAE,OAAO,GAAE,wBAA6B;WAiDrD,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,wBAAwB,GAAG,iBAAiB;IAIzF;;;OAGG;IACH,OAAO,CAAC,aAAa;IAQrB;;;OAGG;IACH,OAAO,CAAC,eAAe;IAUvB;;;;OAIG;IACH,OAAO,CAAC,kBAAkB;IAa1B;;;;OAIG;YACW,WAAW;IAazB;;;;;OAKG;IACG,SAAS,CAAC,UAAU,EAAE,QAAQ,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAmHjE;;;OAGG;IACH,SAAS,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI;IAW5B;;;;OAIG;IACH,QAAQ,CAAC,QAAQ,GAAE,MAAM,GAAG,IAAW,GAAG,MAAM,EAAE;IAalD;;;OAGG;IACH,OAAO,CAAC,YAAY;IAOpB;;;;OAIG;IACH,OAAO,IAAI,MAAM;IAIjB;;;;OAIG;IACH,SAAS,CAAC,QAAQ,GAAE,MAAM,GAAG,IAAW,GAAG,MAAM;IAqBjD;;;;OAIG;IACH,eAAe,CAAC,QAAQ,EAAE,MAAM,GAAG,UAAU;IAwB7C,OAAO,CAAC,wBAAwB;IAkBhC;;OAEG;IACH,MAAM,IAAI,MAAM,GAAG,SAAS;IAoC5B;;OAEG;IACH,gBAAgB,IAAI,IAAI;IAIxB;;;;;OAKG;IACH,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,GAAE,MAAM,GAAG,IAAW,GAAG,OAAO;IA+B/D;;;;;OAKG;IACH,cAAc,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,GAAE,MAAM,GAAG,IAAW,GAAG,QAAQ;IAuCxE;;OAEG;IACH,eAAe,IAAI,IAAI;IAKvB,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS;IAUhE;;;;;;OAMG;IACH,QAAQ,CACN,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,MAAM,EACd,QAAQ,GAAE,MAAM,GAAG,IAAW,GAC7B,MAAM;IAsCT;;;;;OAKG;IACH,WAAW,CAAC,cAAc,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO;IA4B5D;;;;;OAKG;IACH,WAAW,CACT,SAAS,EAAE,MAAM,EACjB,SAAS,EAAE,MAAM,GAChB;QAAE,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAAC,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;KAAE;IA+B/D;;;;;OAKG;IACG,kBAAkB,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IA0D5E;;;;;;;;;OASG;IACH,MAAM,CAAC,mCAAmC,CACxC,GAAG,EAAE,MAAM,EACX,MAAM,EAAE,MAAM,EACd,cAAc,EAAE,MAAM,EACtB,gBAAgB,EAAE,MAAM,GACvB,OAAO;CAgDX;AAED,OAAO,EAAE,iBAAiB,EAAE,CAAC"}
|
|
@@ -188,33 +188,40 @@ class DataIntegrityTree {
|
|
|
188
188
|
const tempFilePath = path.join(tempDir, `${crypto.randomUUID()}.gz`);
|
|
189
189
|
return new Promise((resolve, reject) => {
|
|
190
190
|
const tempWriteStream = fs.createWriteStream(tempFilePath);
|
|
191
|
+
// Update the hash with the original data
|
|
191
192
|
readStream.on("data", (chunk) => {
|
|
192
193
|
uncompressedHash.update(chunk);
|
|
193
194
|
});
|
|
195
|
+
// Pipe the read stream through gzip into the temporary write stream
|
|
194
196
|
readStream.pipe(gzip).pipe(tempWriteStream);
|
|
195
197
|
tempWriteStream.on("finish", async () => {
|
|
196
|
-
|
|
197
|
-
const finalWriteStream = this._createWriteStream(sha256);
|
|
198
|
-
const finalPath = finalWriteStream.path;
|
|
199
|
-
// Ensure the directory exists before copying the file
|
|
200
|
-
const finalDir = path.dirname(finalPath);
|
|
201
|
-
if (!fs.existsSync(finalDir)) {
|
|
202
|
-
fs.mkdirSync(finalDir, { recursive: true });
|
|
203
|
-
}
|
|
198
|
+
let finalWriteStream;
|
|
204
199
|
try {
|
|
200
|
+
sha256 = uncompressedHash.digest("hex");
|
|
201
|
+
finalWriteStream = this._createWriteStream(sha256);
|
|
202
|
+
const finalPath = finalWriteStream.path;
|
|
203
|
+
// Ensure the directory exists
|
|
204
|
+
const finalDir = path.dirname(finalPath);
|
|
205
|
+
if (!fs.existsSync(finalDir)) {
|
|
206
|
+
fs.mkdirSync(finalDir, { recursive: true });
|
|
207
|
+
}
|
|
208
|
+
// Copy the temporary gzipped file to the final destination
|
|
205
209
|
await this._streamFile(tempFilePath, finalPath);
|
|
206
|
-
await unlink(tempFilePath);
|
|
210
|
+
await unlink(tempFilePath); // Clean up the temporary file
|
|
207
211
|
const combinedHash = crypto
|
|
208
212
|
.createHash("sha256")
|
|
209
213
|
.update(`${key}/${sha256}`)
|
|
210
214
|
.digest("hex");
|
|
215
|
+
// Check if the key already exists with the same hash
|
|
211
216
|
if (Array.from(this.files.values()).some((file) => file.hash === combinedHash)) {
|
|
212
217
|
console.log(`No changes detected for key: ${key}`);
|
|
213
218
|
return resolve();
|
|
214
219
|
}
|
|
220
|
+
// Delete existing key if present
|
|
215
221
|
if (this.files.has(key)) {
|
|
216
222
|
this.deleteKey(key);
|
|
217
223
|
}
|
|
224
|
+
// Insert the new key with the hash
|
|
218
225
|
console.log(`Inserted key: ${key}`);
|
|
219
226
|
this.files.set(key, {
|
|
220
227
|
hash: combinedHash,
|
|
@@ -224,13 +231,38 @@ class DataIntegrityTree {
|
|
|
224
231
|
resolve();
|
|
225
232
|
}
|
|
226
233
|
catch (err) {
|
|
234
|
+
// On error, cleanup the temporary file and reject
|
|
235
|
+
await unlink(tempFilePath).catch(() => { });
|
|
227
236
|
reject(err);
|
|
228
237
|
}
|
|
238
|
+
finally {
|
|
239
|
+
// Always close the final write stream if it exists
|
|
240
|
+
if (finalWriteStream) {
|
|
241
|
+
finalWriteStream.end();
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
});
|
|
245
|
+
tempWriteStream.on("error", async (err) => {
|
|
246
|
+
// Close streams and clean up in case of error
|
|
247
|
+
tempWriteStream.destroy();
|
|
248
|
+
gzip.destroy();
|
|
249
|
+
readStream.destroy();
|
|
250
|
+
await unlink(tempFilePath).catch(() => { }); // Clean up the temp file
|
|
251
|
+
reject(err);
|
|
229
252
|
});
|
|
230
|
-
|
|
253
|
+
readStream.on("error", async (err) => {
|
|
254
|
+
// Close streams and clean up in case of error
|
|
255
|
+
tempWriteStream.destroy();
|
|
256
|
+
gzip.destroy();
|
|
257
|
+
readStream.destroy();
|
|
258
|
+
await unlink(tempFilePath).catch(() => { }); // Clean up the temp file
|
|
231
259
|
reject(err);
|
|
232
260
|
});
|
|
233
|
-
|
|
261
|
+
gzip.on("error", (err) => {
|
|
262
|
+
// Handle errors in the gzip stream
|
|
263
|
+
tempWriteStream.destroy();
|
|
264
|
+
gzip.destroy();
|
|
265
|
+
readStream.destroy();
|
|
234
266
|
reject(err);
|
|
235
267
|
});
|
|
236
268
|
});
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"directoryUtils.d.ts","sourceRoot":"","sources":["../../src/utils/directoryUtils.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;
|
|
1
|
+
{"version":3,"file":"directoryUtils.d.ts","sourceRoot":"","sources":["../../src/utils/directoryUtils.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AA8BzD,eAAO,MAAM,YAAY,cACZ,iBAAiB,WACnB,MAAM,YACN,MAAM,KACd,OAAO,CAAC,IAAI,CAiDd,CAAC;AAEF;;;;GAIG;AACH,eAAO,MAAM,mBAAmB,eAAgB,MAAM,KAAG,MAiBxD,CAAC"}
|
|
@@ -77,10 +77,14 @@ const addDirectory = async (datalayer, dirPath, baseDir = dirPath) => {
|
|
|
77
77
|
const stream = fs.createReadStream(filePath);
|
|
78
78
|
datalayer
|
|
79
79
|
.upsertKey(stream, Buffer.from(relativePath).toString("hex"))
|
|
80
|
-
.then(
|
|
80
|
+
.then(async () => {
|
|
81
|
+
await new Promise((resolve) => setTimeout(resolve, 100));
|
|
82
|
+
resolve();
|
|
83
|
+
})
|
|
81
84
|
.catch(reject);
|
|
82
85
|
}));
|
|
83
86
|
}
|
|
87
|
+
await new Promise((resolve) => setTimeout(resolve, 100));
|
|
84
88
|
}
|
|
85
89
|
// Run tasks with limited concurrency (set the concurrency limit as needed)
|
|
86
90
|
await limitConcurrency(10, tasks); // Adjust 10 based on your system limits
|