agentgui 1.0.294 → 1.0.295
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +2 -2
- package/server.js +3 -3
- package/lib/ipfs-downloader.js +0 -459
- package/test-download-progress.js +0 -223
- package/tests/ipfs-downloader.test.js +0 -370
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "agentgui",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.295",
|
|
4
4
|
"description": "Multi-agent ACP client with real-time communication",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "server.js",
|
|
@@ -30,7 +30,7 @@
|
|
|
30
30
|
"fsbrowse": "^0.2.18",
|
|
31
31
|
"google-auth-library": "^10.5.0",
|
|
32
32
|
"onnxruntime-node": "^1.24.1",
|
|
33
|
-
"webtalk": "^1.0.
|
|
33
|
+
"webtalk": "^1.0.14",
|
|
34
34
|
"ws": "^8.14.2"
|
|
35
35
|
},
|
|
36
36
|
"overrides": {
|
package/server.js
CHANGED
|
@@ -14,7 +14,7 @@ import Busboy from 'busboy';
|
|
|
14
14
|
import fsbrowse from 'fsbrowse';
|
|
15
15
|
import { queries } from './database.js';
|
|
16
16
|
import { runClaudeWithStreaming } from './lib/claude-runner.js';
|
|
17
|
-
|
|
17
|
+
const { downloadWithProgress } = createRequire(import.meta.url)('webtalk/ipfs-downloader');
|
|
18
18
|
|
|
19
19
|
const ttsTextAccumulators = new Map();
|
|
20
20
|
|
|
@@ -121,7 +121,7 @@ async function ensureModelsDownloaded() {
|
|
|
121
121
|
const sttUrl = `${lighthouseGateway}/${ipfsCid.cid}/stt/onnx-community/whisper-base/onnx/`;
|
|
122
122
|
const sttFile = path.join(sttDir, 'whisper-onnx.tar');
|
|
123
123
|
|
|
124
|
-
await
|
|
124
|
+
await downloadWithProgress(
|
|
125
125
|
sttUrl,
|
|
126
126
|
sttFile,
|
|
127
127
|
(progress) => {
|
|
@@ -178,7 +178,7 @@ async function ensureModelsDownloaded() {
|
|
|
178
178
|
const ttsUrl = `${lighthouseGateway}/${ipfsCid.cid}/tts/`;
|
|
179
179
|
const ttsFile = path.join(ttsDir, 'tts-models.tar');
|
|
180
180
|
|
|
181
|
-
await
|
|
181
|
+
await downloadWithProgress(
|
|
182
182
|
ttsUrl,
|
|
183
183
|
ttsFile,
|
|
184
184
|
(progress) => {
|
package/lib/ipfs-downloader.js
DELETED
|
@@ -1,459 +0,0 @@
|
|
|
1
|
-
import https from 'https';
|
|
2
|
-
import http from 'http';
|
|
3
|
-
import fs from 'fs';
|
|
4
|
-
import path from 'path';
|
|
5
|
-
import crypto from 'crypto';
|
|
6
|
-
import os from 'os';
|
|
7
|
-
import { queries } from '../database.js';
|
|
8
|
-
|
|
9
|
-
const GATEWAYS = [
|
|
10
|
-
'https://ipfs.io/ipfs/',
|
|
11
|
-
'https://gateway.pinata.cloud/ipfs/',
|
|
12
|
-
'https://cloudflare-ipfs.com/ipfs/',
|
|
13
|
-
'https://dweb.link/ipfs/'
|
|
14
|
-
];
|
|
15
|
-
|
|
16
|
-
const CONFIG = {
|
|
17
|
-
MAX_RESUME_ATTEMPTS: 3,
|
|
18
|
-
MAX_RETRY_ATTEMPTS: 3,
|
|
19
|
-
TIMEOUT_MS: 30000,
|
|
20
|
-
INITIAL_BACKOFF_MS: 1000,
|
|
21
|
-
BACKOFF_MULTIPLIER: 2,
|
|
22
|
-
get DOWNLOADS_DIR() { return path.join(process.env.PORTABLE_DATA_DIR || path.join(os.homedir(), '.gmgui'), 'downloads'); },
|
|
23
|
-
RESUME_THRESHOLD: 0.5
|
|
24
|
-
};
|
|
25
|
-
|
|
26
|
-
class IPFSDownloader {
|
|
27
|
-
constructor() {
|
|
28
|
-
this.downloads = new Map();
|
|
29
|
-
this.setupDir();
|
|
30
|
-
}
|
|
31
|
-
|
|
32
|
-
setupDir() {
|
|
33
|
-
if (!fs.existsSync(CONFIG.DOWNLOADS_DIR)) {
|
|
34
|
-
fs.mkdirSync(CONFIG.DOWNLOADS_DIR, { recursive: true });
|
|
35
|
-
}
|
|
36
|
-
}
|
|
37
|
-
|
|
38
|
-
async download(cid, filename, options = {}) {
|
|
39
|
-
const filepath = path.join(CONFIG.DOWNLOADS_DIR, filename);
|
|
40
|
-
const { modelName = 'unknown', modelType = 'unknown', modelHash = null } = options;
|
|
41
|
-
|
|
42
|
-
try {
|
|
43
|
-
const cidId = queries.recordIpfsCid(cid, modelName, modelType, modelHash, GATEWAYS[0]);
|
|
44
|
-
const downloadId = queries.recordDownloadStart(cidId, filepath, 0);
|
|
45
|
-
|
|
46
|
-
await this.executeDownload(downloadId, cidId, filepath, options);
|
|
47
|
-
return { success: true, downloadId, filepath, cid };
|
|
48
|
-
} catch (error) {
|
|
49
|
-
throw error;
|
|
50
|
-
}
|
|
51
|
-
}
|
|
52
|
-
|
|
53
|
-
async executeDownload(downloadId, cidId, filepath, options = {}) {
|
|
54
|
-
let gatewayIndex = 0;
|
|
55
|
-
let resumeAttempts = 0;
|
|
56
|
-
let retryAttempts = 0;
|
|
57
|
-
|
|
58
|
-
while (true) {
|
|
59
|
-
try {
|
|
60
|
-
const gateway = GATEWAYS[gatewayIndex];
|
|
61
|
-
const cidRecord = queries._db.prepare('SELECT * FROM ipfs_cids WHERE id = ?').get(cidId);
|
|
62
|
-
if (!cidRecord) throw new Error('CID record not found');
|
|
63
|
-
const url = `${gateway}${cidRecord.cid}`;
|
|
64
|
-
|
|
65
|
-
const { size, hash } = await this.downloadFile(
|
|
66
|
-
url,
|
|
67
|
-
filepath,
|
|
68
|
-
0,
|
|
69
|
-
options
|
|
70
|
-
);
|
|
71
|
-
|
|
72
|
-
queries.completeDownload(downloadId, cidId);
|
|
73
|
-
|
|
74
|
-
if (options.hashVerify && hash) {
|
|
75
|
-
queries.updateDownloadHash(downloadId, hash);
|
|
76
|
-
}
|
|
77
|
-
|
|
78
|
-
return queries.getDownload(downloadId);
|
|
79
|
-
} catch (error) {
|
|
80
|
-
if (error.message.includes('Range')) {
|
|
81
|
-
resumeAttempts++;
|
|
82
|
-
if (resumeAttempts > CONFIG.MAX_RESUME_ATTEMPTS) {
|
|
83
|
-
await this.cleanupPartial(filepath);
|
|
84
|
-
gatewayIndex = (gatewayIndex + 1) % GATEWAYS.length;
|
|
85
|
-
resumeAttempts = 0;
|
|
86
|
-
}
|
|
87
|
-
} else if (error.message.includes('timeout') || error.message.includes('ETIMEDOUT')) {
|
|
88
|
-
retryAttempts++;
|
|
89
|
-
if (retryAttempts > CONFIG.MAX_RETRY_ATTEMPTS) {
|
|
90
|
-
queries.recordDownloadError(downloadId, cidId, error.message);
|
|
91
|
-
throw error;
|
|
92
|
-
}
|
|
93
|
-
const backoff = CONFIG.INITIAL_BACKOFF_MS * Math.pow(CONFIG.BACKOFF_MULTIPLIER, retryAttempts - 1);
|
|
94
|
-
await this.sleep(backoff);
|
|
95
|
-
} else if (error.message.includes('network') || error.message.includes('ECONNRESET')) {
|
|
96
|
-
gatewayIndex = (gatewayIndex + 1) % GATEWAYS.length;
|
|
97
|
-
retryAttempts = 0;
|
|
98
|
-
} else {
|
|
99
|
-
queries.recordDownloadError(downloadId, cidId, error.message);
|
|
100
|
-
throw error;
|
|
101
|
-
}
|
|
102
|
-
}
|
|
103
|
-
}
|
|
104
|
-
}
|
|
105
|
-
|
|
106
|
-
async downloadFile(url, filepath, resumeFrom = 0, options = {}) {
|
|
107
|
-
return new Promise((resolve, reject) => {
|
|
108
|
-
const protocol = url.startsWith('https') ? https : http;
|
|
109
|
-
const headers = {};
|
|
110
|
-
|
|
111
|
-
if (resumeFrom > 0) {
|
|
112
|
-
headers['Range'] = `bytes=${resumeFrom}-`;
|
|
113
|
-
}
|
|
114
|
-
|
|
115
|
-
const req = protocol.get(url, { headers, timeout: CONFIG.TIMEOUT_MS }, (res) => {
|
|
116
|
-
if (res.statusCode === 416) {
|
|
117
|
-
reject(new Error('Range not supported - will delete partial and restart'));
|
|
118
|
-
return;
|
|
119
|
-
}
|
|
120
|
-
|
|
121
|
-
if (![200, 206].includes(res.statusCode)) {
|
|
122
|
-
reject(new Error(`HTTP ${res.statusCode}`));
|
|
123
|
-
return;
|
|
124
|
-
}
|
|
125
|
-
|
|
126
|
-
const contentLength = parseInt(res.headers['content-length'], 10);
|
|
127
|
-
const hash = crypto.createHash('sha256');
|
|
128
|
-
let downloaded = resumeFrom;
|
|
129
|
-
|
|
130
|
-
const mode = resumeFrom > 0 ? 'a' : 'w';
|
|
131
|
-
const stream = fs.createWriteStream(filepath, { flags: mode });
|
|
132
|
-
|
|
133
|
-
res.on('data', (chunk) => {
|
|
134
|
-
hash.update(chunk);
|
|
135
|
-
downloaded += chunk.length;
|
|
136
|
-
});
|
|
137
|
-
|
|
138
|
-
res.pipe(stream);
|
|
139
|
-
|
|
140
|
-
stream.on('finish', () => {
|
|
141
|
-
resolve({ size: downloaded, hash: hash.digest('hex') });
|
|
142
|
-
});
|
|
143
|
-
|
|
144
|
-
stream.on('error', (err) => {
|
|
145
|
-
reject(new Error(`Write error: ${err.message}`));
|
|
146
|
-
});
|
|
147
|
-
});
|
|
148
|
-
|
|
149
|
-
req.on('timeout', () => {
|
|
150
|
-
req.abort();
|
|
151
|
-
reject(new Error('timeout'));
|
|
152
|
-
});
|
|
153
|
-
|
|
154
|
-
req.on('error', (err) => {
|
|
155
|
-
reject(new Error(`network: ${err.message}`));
|
|
156
|
-
});
|
|
157
|
-
});
|
|
158
|
-
}
|
|
159
|
-
|
|
160
|
-
async verifyHash(filepath, expectedHash) {
|
|
161
|
-
return new Promise((resolve, reject) => {
|
|
162
|
-
const hash = crypto.createHash('sha256');
|
|
163
|
-
const stream = fs.createReadStream(filepath);
|
|
164
|
-
|
|
165
|
-
stream.on('data', (chunk) => {
|
|
166
|
-
hash.update(chunk);
|
|
167
|
-
});
|
|
168
|
-
|
|
169
|
-
stream.on('end', () => {
|
|
170
|
-
resolve(hash.digest('hex') === expectedHash);
|
|
171
|
-
});
|
|
172
|
-
|
|
173
|
-
stream.on('error', (err) => {
|
|
174
|
-
reject(err);
|
|
175
|
-
});
|
|
176
|
-
});
|
|
177
|
-
}
|
|
178
|
-
|
|
179
|
-
async resume(downloadId, options = {}) {
|
|
180
|
-
const record = queries.getDownload(downloadId);
|
|
181
|
-
|
|
182
|
-
if (!record) {
|
|
183
|
-
throw new Error('Download not found');
|
|
184
|
-
}
|
|
185
|
-
|
|
186
|
-
if (record.status === 'success') {
|
|
187
|
-
return record;
|
|
188
|
-
}
|
|
189
|
-
|
|
190
|
-
const attempts = (record.attempts || 0) + 1;
|
|
191
|
-
if (attempts > CONFIG.MAX_RESUME_ATTEMPTS) {
|
|
192
|
-
throw new Error('Max resume attempts exceeded');
|
|
193
|
-
}
|
|
194
|
-
|
|
195
|
-
try {
|
|
196
|
-
const currentSize = fs.existsSync(record.downloadPath)
|
|
197
|
-
? fs.statSync(record.downloadPath).size
|
|
198
|
-
: 0;
|
|
199
|
-
|
|
200
|
-
if (currentSize === 0) {
|
|
201
|
-
queries.recordDownloadStart(record.cidId, record.downloadPath, record.total_bytes);
|
|
202
|
-
return this.resumeFromOffset(downloadId, record, 0, options);
|
|
203
|
-
}
|
|
204
|
-
|
|
205
|
-
queries.markDownloadResuming(downloadId);
|
|
206
|
-
|
|
207
|
-
const downloadPercent = (currentSize / (record.total_bytes || currentSize)) * 100;
|
|
208
|
-
|
|
209
|
-
if (downloadPercent > CONFIG.RESUME_THRESHOLD * 100) {
|
|
210
|
-
return this.resumeFromOffset(downloadId, record, currentSize, options);
|
|
211
|
-
} else {
|
|
212
|
-
await this.cleanupPartial(record.downloadPath);
|
|
213
|
-
return this.resumeFromOffset(downloadId, record, 0, options);
|
|
214
|
-
}
|
|
215
|
-
} catch (error) {
|
|
216
|
-
const newAttempts = (record.attempts || 0) + 1;
|
|
217
|
-
const newStatus = newAttempts >= CONFIG.MAX_RESUME_ATTEMPTS ? 'failed' : 'paused';
|
|
218
|
-
queries.updateDownloadResume(downloadId, record.downloaded_bytes, newAttempts, Date.now(), newStatus);
|
|
219
|
-
|
|
220
|
-
if (newStatus === 'failed') {
|
|
221
|
-
throw error;
|
|
222
|
-
}
|
|
223
|
-
|
|
224
|
-
return queries.getDownload(downloadId);
|
|
225
|
-
}
|
|
226
|
-
}
|
|
227
|
-
|
|
228
|
-
async resumeFromOffset(downloadId, record, offset, options) {
|
|
229
|
-
try {
|
|
230
|
-
const cidRecord = queries.getIpfsCidByModel(record.modelName, record.modelType);
|
|
231
|
-
const gateway = GATEWAYS[0];
|
|
232
|
-
const url = `${gateway}${cidRecord.cid}`;
|
|
233
|
-
|
|
234
|
-
const { size, hash } = await this.downloadFile(
|
|
235
|
-
url,
|
|
236
|
-
record.downloadPath,
|
|
237
|
-
offset,
|
|
238
|
-
options
|
|
239
|
-
);
|
|
240
|
-
|
|
241
|
-
if (options.hashVerify && record.hash) {
|
|
242
|
-
const verified = await this.verifyHash(record.downloadPath, record.hash);
|
|
243
|
-
if (!verified) {
|
|
244
|
-
await this.cleanupPartial(record.downloadPath);
|
|
245
|
-
const newAttempts = (record.attempts || 0) + 1;
|
|
246
|
-
queries.updateDownloadResume(downloadId, 0, newAttempts, Date.now(), 'pending');
|
|
247
|
-
throw new Error('Hash verification failed - restarting');
|
|
248
|
-
}
|
|
249
|
-
}
|
|
250
|
-
|
|
251
|
-
queries.completeDownload(downloadId, record.cidId);
|
|
252
|
-
if (hash) {
|
|
253
|
-
queries.updateDownloadHash(downloadId, hash);
|
|
254
|
-
}
|
|
255
|
-
|
|
256
|
-
return queries.getDownload(downloadId);
|
|
257
|
-
} catch (error) {
|
|
258
|
-
const newAttempts = (record.attempts || 0) + 1;
|
|
259
|
-
const newStatus = newAttempts >= CONFIG.MAX_RESUME_ATTEMPTS ? 'failed' : 'paused';
|
|
260
|
-
queries.updateDownloadResume(downloadId, offset, newAttempts, Date.now(), newStatus);
|
|
261
|
-
|
|
262
|
-
if (newStatus === 'failed') {
|
|
263
|
-
throw error;
|
|
264
|
-
}
|
|
265
|
-
|
|
266
|
-
return queries.getDownload(downloadId);
|
|
267
|
-
}
|
|
268
|
-
}
|
|
269
|
-
|
|
270
|
-
async cleanupPartial(filepath) {
|
|
271
|
-
if (fs.existsSync(filepath)) {
|
|
272
|
-
try {
|
|
273
|
-
fs.unlinkSync(filepath);
|
|
274
|
-
} catch (err) {
|
|
275
|
-
console.error(`Failed to cleanup partial file: ${filepath}`, err.message);
|
|
276
|
-
}
|
|
277
|
-
}
|
|
278
|
-
}
|
|
279
|
-
|
|
280
|
-
sleep(ms) {
|
|
281
|
-
return new Promise(resolve => setTimeout(resolve, ms));
|
|
282
|
-
}
|
|
283
|
-
|
|
284
|
-
getDownloadStatus(downloadId) {
|
|
285
|
-
return queries.getDownload(downloadId);
|
|
286
|
-
}
|
|
287
|
-
|
|
288
|
-
listDownloads(status = null) {
|
|
289
|
-
if (status) {
|
|
290
|
-
return queries.getDownloadsByStatus(status);
|
|
291
|
-
}
|
|
292
|
-
const allDownloads = queries.getDownloadsByStatus('in_progress');
|
|
293
|
-
return allDownloads.concat(
|
|
294
|
-
queries.getDownloadsByStatus('success'),
|
|
295
|
-
queries.getDownloadsByStatus('paused'),
|
|
296
|
-
queries.getDownloadsByStatus('failed')
|
|
297
|
-
);
|
|
298
|
-
}
|
|
299
|
-
|
|
300
|
-
async cancelDownload(downloadId) {
|
|
301
|
-
const record = queries.getDownload(downloadId);
|
|
302
|
-
if (!record) return false;
|
|
303
|
-
|
|
304
|
-
await this.cleanupPartial(record.downloadPath);
|
|
305
|
-
queries.markDownloadPaused(downloadId, 'Cancelled by user');
|
|
306
|
-
|
|
307
|
-
return true;
|
|
308
|
-
}
|
|
309
|
-
|
|
310
|
-
async downloadWithProgress(url, destination, onProgress = null) {
|
|
311
|
-
const dir = path.dirname(destination);
|
|
312
|
-
if (!fs.existsSync(dir)) {
|
|
313
|
-
fs.mkdirSync(dir, { recursive: true });
|
|
314
|
-
}
|
|
315
|
-
|
|
316
|
-
let bytesDownloaded = 0;
|
|
317
|
-
let totalBytes = 0;
|
|
318
|
-
let lastProgressTime = Date.now();
|
|
319
|
-
let lastProgressBytes = 0;
|
|
320
|
-
const speeds = [];
|
|
321
|
-
let gatewayIndex = 0;
|
|
322
|
-
let retryCount = 0;
|
|
323
|
-
|
|
324
|
-
const emitProgress = () => {
|
|
325
|
-
const now = Date.now();
|
|
326
|
-
const deltaTime = (now - lastProgressTime) / 1000;
|
|
327
|
-
const deltaBytes = bytesDownloaded - lastProgressBytes;
|
|
328
|
-
const speed = deltaTime > 0 ? Math.round(deltaBytes / deltaTime) : 0;
|
|
329
|
-
|
|
330
|
-
if (speed > 0) {
|
|
331
|
-
speeds.push(speed);
|
|
332
|
-
if (speeds.length > 10) speeds.shift();
|
|
333
|
-
}
|
|
334
|
-
|
|
335
|
-
const avgSpeed = speeds.length > 0 ? Math.round(speeds.reduce((a, b) => a + b, 0) / speeds.length) : 0;
|
|
336
|
-
const eta = avgSpeed > 0 && totalBytes > bytesDownloaded ? Math.round((totalBytes - bytesDownloaded) / avgSpeed) : 0;
|
|
337
|
-
|
|
338
|
-
if (onProgress) {
|
|
339
|
-
onProgress({
|
|
340
|
-
bytesDownloaded,
|
|
341
|
-
bytesRemaining: Math.max(0, totalBytes - bytesDownloaded),
|
|
342
|
-
totalBytes,
|
|
343
|
-
downloadSpeed: avgSpeed,
|
|
344
|
-
eta,
|
|
345
|
-
retryCount,
|
|
346
|
-
currentGateway: url,
|
|
347
|
-
status: bytesDownloaded >= totalBytes ? 'completed' : 'downloading',
|
|
348
|
-
percentComplete: totalBytes > 0 ? Math.round((bytesDownloaded / totalBytes) * 100) : 0,
|
|
349
|
-
timestamp: now
|
|
350
|
-
});
|
|
351
|
-
}
|
|
352
|
-
|
|
353
|
-
lastProgressTime = now;
|
|
354
|
-
lastProgressBytes = bytesDownloaded;
|
|
355
|
-
};
|
|
356
|
-
|
|
357
|
-
return new Promise((resolve, reject) => {
|
|
358
|
-
const attemptDownload = (gateway) => {
|
|
359
|
-
if (onProgress) {
|
|
360
|
-
onProgress({
|
|
361
|
-
bytesDownloaded: 0,
|
|
362
|
-
bytesRemaining: 0,
|
|
363
|
-
totalBytes: 0,
|
|
364
|
-
downloadSpeed: 0,
|
|
365
|
-
eta: 0,
|
|
366
|
-
retryCount,
|
|
367
|
-
currentGateway: gateway,
|
|
368
|
-
status: 'connecting',
|
|
369
|
-
percentComplete: 0,
|
|
370
|
-
timestamp: Date.now()
|
|
371
|
-
});
|
|
372
|
-
}
|
|
373
|
-
|
|
374
|
-
const protocol = gateway.startsWith('https') ? https : http;
|
|
375
|
-
protocol.get(gateway, { timeout: CONFIG.TIMEOUT_MS }, (res) => {
|
|
376
|
-
if ([301, 302, 307, 308].includes(res.statusCode)) {
|
|
377
|
-
const location = res.headers.location;
|
|
378
|
-
if (location) return attemptDownload(location);
|
|
379
|
-
}
|
|
380
|
-
|
|
381
|
-
if (res.statusCode !== 200) {
|
|
382
|
-
res.resume();
|
|
383
|
-
if (gatewayIndex < GATEWAYS.length - 1) {
|
|
384
|
-
retryCount++;
|
|
385
|
-
gatewayIndex++;
|
|
386
|
-
return setTimeout(() => attemptDownload(GATEWAYS[gatewayIndex]), 1000 * Math.pow(2, Math.min(retryCount, 3)));
|
|
387
|
-
}
|
|
388
|
-
return reject(new Error(`HTTP ${res.statusCode}`));
|
|
389
|
-
}
|
|
390
|
-
|
|
391
|
-
totalBytes = parseInt(res.headers['content-length'], 10) || 0;
|
|
392
|
-
bytesDownloaded = 0;
|
|
393
|
-
lastProgressBytes = 0;
|
|
394
|
-
lastProgressTime = Date.now();
|
|
395
|
-
|
|
396
|
-
const file = fs.createWriteStream(destination);
|
|
397
|
-
let lastEmit = Date.now();
|
|
398
|
-
|
|
399
|
-
res.on('data', (chunk) => {
|
|
400
|
-
bytesDownloaded += chunk.length;
|
|
401
|
-
const now = Date.now();
|
|
402
|
-
if (now - lastEmit >= 200) {
|
|
403
|
-
emitProgress();
|
|
404
|
-
lastEmit = now;
|
|
405
|
-
}
|
|
406
|
-
});
|
|
407
|
-
|
|
408
|
-
res.on('end', () => {
|
|
409
|
-
emitProgress();
|
|
410
|
-
file.destroy();
|
|
411
|
-
resolve({ destination, bytesDownloaded, success: true });
|
|
412
|
-
});
|
|
413
|
-
|
|
414
|
-
res.on('error', (err) => {
|
|
415
|
-
file.destroy();
|
|
416
|
-
fs.unlink(destination, () => {});
|
|
417
|
-
if (gatewayIndex < GATEWAYS.length - 1) {
|
|
418
|
-
retryCount++;
|
|
419
|
-
gatewayIndex++;
|
|
420
|
-
return setTimeout(() => attemptDownload(GATEWAYS[gatewayIndex]), 1000 * Math.pow(2, Math.min(retryCount, 3)));
|
|
421
|
-
}
|
|
422
|
-
reject(err);
|
|
423
|
-
});
|
|
424
|
-
|
|
425
|
-
file.on('error', (err) => {
|
|
426
|
-
res.destroy();
|
|
427
|
-
fs.unlink(destination, () => {});
|
|
428
|
-
if (gatewayIndex < GATEWAYS.length - 1) {
|
|
429
|
-
retryCount++;
|
|
430
|
-
gatewayIndex++;
|
|
431
|
-
return setTimeout(() => attemptDownload(GATEWAYS[gatewayIndex]), 1000 * Math.pow(2, Math.min(retryCount, 3)));
|
|
432
|
-
}
|
|
433
|
-
reject(err);
|
|
434
|
-
});
|
|
435
|
-
|
|
436
|
-
res.pipe(file);
|
|
437
|
-
}).on('timeout', () => {
|
|
438
|
-
if (gatewayIndex < GATEWAYS.length - 1) {
|
|
439
|
-
retryCount++;
|
|
440
|
-
gatewayIndex++;
|
|
441
|
-
return setTimeout(() => attemptDownload(GATEWAYS[gatewayIndex]), 1000 * Math.pow(2, Math.min(retryCount, 3)));
|
|
442
|
-
}
|
|
443
|
-
reject(new Error('Download timeout'));
|
|
444
|
-
}).on('error', (err) => {
|
|
445
|
-
if (gatewayIndex < GATEWAYS.length - 1) {
|
|
446
|
-
retryCount++;
|
|
447
|
-
gatewayIndex++;
|
|
448
|
-
return setTimeout(() => attemptDownload(GATEWAYS[gatewayIndex]), 1000 * Math.pow(2, Math.min(retryCount, 3)));
|
|
449
|
-
}
|
|
450
|
-
reject(err);
|
|
451
|
-
});
|
|
452
|
-
};
|
|
453
|
-
|
|
454
|
-
attemptDownload(GATEWAYS[0]);
|
|
455
|
-
});
|
|
456
|
-
}
|
|
457
|
-
}
|
|
458
|
-
|
|
459
|
-
export default new IPFSDownloader();
|
|
@@ -1,223 +0,0 @@
|
|
|
1
|
-
import https from 'https';
|
|
2
|
-
import fs from 'fs';
|
|
3
|
-
import path from 'path';
|
|
4
|
-
import os from 'os';
|
|
5
|
-
|
|
6
|
-
const testDir = path.join(os.tmpdir(), 'test-download-progress');
|
|
7
|
-
if (!fs.existsSync(testDir)) {
|
|
8
|
-
fs.mkdirSync(testDir, { recursive: true });
|
|
9
|
-
}
|
|
10
|
-
|
|
11
|
-
const GATEWAYS = [
|
|
12
|
-
'https://ipfs.io',
|
|
13
|
-
'https://gateway.pinata.cloud',
|
|
14
|
-
'https://cloudflare-ipfs.com',
|
|
15
|
-
];
|
|
16
|
-
|
|
17
|
-
function downloadWithProgress(url, destination, onProgress = null) {
|
|
18
|
-
let bytesDownloaded = 0;
|
|
19
|
-
let totalBytes = 0;
|
|
20
|
-
let lastProgressTime = Date.now();
|
|
21
|
-
let lastProgressBytes = 0;
|
|
22
|
-
const speeds = [];
|
|
23
|
-
let retryCount = 0;
|
|
24
|
-
let gatewayIndex = 0;
|
|
25
|
-
|
|
26
|
-
const emitProgress = () => {
|
|
27
|
-
const now = Date.now();
|
|
28
|
-
const deltaTime = (now - lastProgressTime) / 1000;
|
|
29
|
-
const deltaBytes = bytesDownloaded - lastProgressBytes;
|
|
30
|
-
const speed = deltaTime > 0 ? Math.round(deltaBytes / deltaTime) : 0;
|
|
31
|
-
|
|
32
|
-
if (speed > 0) {
|
|
33
|
-
speeds.push(speed);
|
|
34
|
-
if (speeds.length > 10) speeds.shift();
|
|
35
|
-
}
|
|
36
|
-
|
|
37
|
-
const avgSpeed = speeds.length > 0 ? Math.round(speeds.reduce((a, b) => a + b, 0) / speeds.length) : 0;
|
|
38
|
-
const eta = avgSpeed > 0 && totalBytes > bytesDownloaded ? Math.round((totalBytes - bytesDownloaded) / avgSpeed) : 0;
|
|
39
|
-
|
|
40
|
-
if (onProgress) {
|
|
41
|
-
onProgress({
|
|
42
|
-
bytesDownloaded,
|
|
43
|
-
bytesRemaining: Math.max(0, totalBytes - bytesDownloaded),
|
|
44
|
-
totalBytes,
|
|
45
|
-
downloadSpeed: avgSpeed,
|
|
46
|
-
eta,
|
|
47
|
-
retryCount,
|
|
48
|
-
currentGateway: url,
|
|
49
|
-
status: bytesDownloaded >= totalBytes ? 'completed' : 'downloading',
|
|
50
|
-
percentComplete: totalBytes > 0 ? Math.round((bytesDownloaded / totalBytes) * 100) : 0,
|
|
51
|
-
timestamp: now
|
|
52
|
-
});
|
|
53
|
-
}
|
|
54
|
-
|
|
55
|
-
lastProgressTime = now;
|
|
56
|
-
lastProgressBytes = bytesDownloaded;
|
|
57
|
-
};
|
|
58
|
-
|
|
59
|
-
return new Promise((resolve, reject) => {
|
|
60
|
-
const dir = path.dirname(destination);
|
|
61
|
-
if (!fs.existsSync(dir)) {
|
|
62
|
-
fs.mkdirSync(dir, { recursive: true });
|
|
63
|
-
}
|
|
64
|
-
|
|
65
|
-
const attemptDownload = (gateway) => {
|
|
66
|
-
if (onProgress) {
|
|
67
|
-
onProgress({
|
|
68
|
-
bytesDownloaded: 0,
|
|
69
|
-
bytesRemaining: 0,
|
|
70
|
-
totalBytes: 0,
|
|
71
|
-
downloadSpeed: 0,
|
|
72
|
-
eta: 0,
|
|
73
|
-
retryCount,
|
|
74
|
-
currentGateway: gateway,
|
|
75
|
-
status: 'connecting',
|
|
76
|
-
percentComplete: 0,
|
|
77
|
-
timestamp: Date.now()
|
|
78
|
-
});
|
|
79
|
-
}
|
|
80
|
-
|
|
81
|
-
https.get(gateway, { timeout: 30000 }, (res) => {
|
|
82
|
-
if ([301, 302, 307, 308].includes(res.statusCode)) {
|
|
83
|
-
const location = res.headers.location;
|
|
84
|
-
if (location) return attemptDownload(location);
|
|
85
|
-
}
|
|
86
|
-
|
|
87
|
-
if (res.statusCode !== 200) {
|
|
88
|
-
res.resume();
|
|
89
|
-
if (gatewayIndex < GATEWAYS.length - 1) {
|
|
90
|
-
retryCount++;
|
|
91
|
-
gatewayIndex++;
|
|
92
|
-
return setTimeout(() => attemptDownload(GATEWAYS[gatewayIndex]), 1000 * Math.pow(2, Math.min(retryCount, 3)));
|
|
93
|
-
}
|
|
94
|
-
return reject(new Error(`HTTP ${res.statusCode}`));
|
|
95
|
-
}
|
|
96
|
-
|
|
97
|
-
totalBytes = parseInt(res.headers['content-length'], 10) || 0;
|
|
98
|
-
bytesDownloaded = 0;
|
|
99
|
-
lastProgressBytes = 0;
|
|
100
|
-
lastProgressTime = Date.now();
|
|
101
|
-
|
|
102
|
-
const file = fs.createWriteStream(destination);
|
|
103
|
-
let lastEmit = Date.now();
|
|
104
|
-
|
|
105
|
-
res.on('data', (chunk) => {
|
|
106
|
-
bytesDownloaded += chunk.length;
|
|
107
|
-
const now = Date.now();
|
|
108
|
-
if (now - lastEmit >= 200) {
|
|
109
|
-
emitProgress();
|
|
110
|
-
lastEmit = now;
|
|
111
|
-
}
|
|
112
|
-
});
|
|
113
|
-
|
|
114
|
-
res.on('end', () => {
|
|
115
|
-
emitProgress();
|
|
116
|
-
file.destroy();
|
|
117
|
-
resolve({ destination, bytesDownloaded, success: true });
|
|
118
|
-
});
|
|
119
|
-
|
|
120
|
-
res.on('error', (err) => {
|
|
121
|
-
file.destroy();
|
|
122
|
-
fs.unlink(destination, () => {});
|
|
123
|
-
if (gatewayIndex < GATEWAYS.length - 1) {
|
|
124
|
-
retryCount++;
|
|
125
|
-
gatewayIndex++;
|
|
126
|
-
return setTimeout(() => attemptDownload(GATEWAYS[gatewayIndex]), 1000 * Math.pow(2, Math.min(retryCount, 3)));
|
|
127
|
-
}
|
|
128
|
-
reject(err);
|
|
129
|
-
});
|
|
130
|
-
|
|
131
|
-
file.on('error', (err) => {
|
|
132
|
-
res.destroy();
|
|
133
|
-
fs.unlink(destination, () => {});
|
|
134
|
-
if (gatewayIndex < GATEWAYS.length - 1) {
|
|
135
|
-
retryCount++;
|
|
136
|
-
gatewayIndex++;
|
|
137
|
-
return setTimeout(() => attemptDownload(GATEWAYS[gatewayIndex]), 1000 * Math.pow(2, Math.min(retryCount, 3)));
|
|
138
|
-
}
|
|
139
|
-
reject(err);
|
|
140
|
-
});
|
|
141
|
-
|
|
142
|
-
res.pipe(file);
|
|
143
|
-
}).on('timeout', () => {
|
|
144
|
-
if (gatewayIndex < GATEWAYS.length - 1) {
|
|
145
|
-
retryCount++;
|
|
146
|
-
gatewayIndex++;
|
|
147
|
-
return setTimeout(() => attemptDownload(GATEWAYS[gatewayIndex]), 1000 * Math.pow(2, Math.min(retryCount, 3)));
|
|
148
|
-
}
|
|
149
|
-
reject(new Error('Download timeout'));
|
|
150
|
-
}).on('error', (err) => {
|
|
151
|
-
if (gatewayIndex < GATEWAYS.length - 1) {
|
|
152
|
-
retryCount++;
|
|
153
|
-
gatewayIndex++;
|
|
154
|
-
return setTimeout(() => attemptDownload(GATEWAYS[gatewayIndex]), 1000 * Math.pow(2, Math.min(retryCount, 3)));
|
|
155
|
-
}
|
|
156
|
-
reject(err);
|
|
157
|
-
});
|
|
158
|
-
};
|
|
159
|
-
|
|
160
|
-
attemptDownload(GATEWAYS[0]);
|
|
161
|
-
});
|
|
162
|
-
}
|
|
163
|
-
|
|
164
|
-
let progressCount = 0;
|
|
165
|
-
let lastPrintTime = Date.now();
|
|
166
|
-
let minInterval = Infinity;
|
|
167
|
-
let maxInterval = 0;
|
|
168
|
-
const progressIntervals = [];
|
|
169
|
-
|
|
170
|
-
console.log('Starting download progress tracking test...\n');
|
|
171
|
-
|
|
172
|
-
downloadWithProgress(
|
|
173
|
-
'https://www.w3.org/WAI/WCAG21/Techniques/pdf/pdf-files/table-example.pdf',
|
|
174
|
-
path.join(testDir, 'test-file.pdf'),
|
|
175
|
-
(progress) => {
|
|
176
|
-
progressCount++;
|
|
177
|
-
const now = Date.now();
|
|
178
|
-
const interval = now - lastPrintTime;
|
|
179
|
-
|
|
180
|
-
if (progressCount > 1) {
|
|
181
|
-
progressIntervals.push(interval);
|
|
182
|
-
minInterval = Math.min(minInterval, interval);
|
|
183
|
-
maxInterval = Math.max(maxInterval, interval);
|
|
184
|
-
}
|
|
185
|
-
|
|
186
|
-
console.log(`[${progressCount}] Progress Update:
|
|
187
|
-
Status: ${progress.status}
|
|
188
|
-
Downloaded: ${(progress.bytesDownloaded / 1024).toFixed(1)}KB / ${(progress.totalBytes / 1024).toFixed(1)}KB
|
|
189
|
-
Speed: ${(progress.downloadSpeed / 1024).toFixed(2)}MB/s
|
|
190
|
-
ETA: ${progress.eta}s
|
|
191
|
-
Complete: ${progress.percentComplete}%
|
|
192
|
-
Retry Count: ${progress.retryCount}
|
|
193
|
-
Gateway: ${progress.currentGateway}
|
|
194
|
-
Interval: ${interval}ms\n`);
|
|
195
|
-
|
|
196
|
-
lastPrintTime = now;
|
|
197
|
-
}
|
|
198
|
-
).then((result) => {
|
|
199
|
-
const avgInterval = progressIntervals.length > 0 ? progressIntervals.reduce((a, b) => a + b, 0) / progressIntervals.length : 0;
|
|
200
|
-
console.log('\n=== Download Complete ===');
|
|
201
|
-
console.log(`Result: ${JSON.stringify(result, null, 2)}`);
|
|
202
|
-
console.log(`\nProgress Tracking Statistics:
|
|
203
|
-
Total Updates: ${progressCount}
|
|
204
|
-
Interval Range: ${minInterval}ms - ${maxInterval}ms
|
|
205
|
-
Average Interval: ${avgInterval.toFixed(0)}ms
|
|
206
|
-
Expected Interval: 200ms (should be 100-500ms range)`);
|
|
207
|
-
|
|
208
|
-
if (avgInterval >= 100 && avgInterval <= 500) {
|
|
209
|
-
console.log(' Status: PASS - Progress interval within acceptable range');
|
|
210
|
-
} else {
|
|
211
|
-
console.log(' Status: FAIL - Progress interval outside acceptable range');
|
|
212
|
-
}
|
|
213
|
-
|
|
214
|
-
if (fs.existsSync(path.join(testDir, 'test-file.pdf'))) {
|
|
215
|
-
const stat = fs.statSync(path.join(testDir, 'test-file.pdf'));
|
|
216
|
-
console.log(`\nDownloaded file size: ${(stat.size / 1024).toFixed(1)}KB`);
|
|
217
|
-
}
|
|
218
|
-
|
|
219
|
-
process.exit(0);
|
|
220
|
-
}).catch((err) => {
|
|
221
|
-
console.error('Download failed:', err.message);
|
|
222
|
-
process.exit(1);
|
|
223
|
-
});
|
|
@@ -1,370 +0,0 @@
|
|
|
1
|
-
import fs from 'fs';
|
|
2
|
-
import path from 'path';
|
|
3
|
-
import os from 'os';
|
|
4
|
-
import crypto from 'crypto';
|
|
5
|
-
import downloader from '../lib/ipfs-downloader.js';
|
|
6
|
-
import { queries } from '../database.js';
|
|
7
|
-
|
|
8
|
-
const TEST_DIR = path.join(os.homedir(), '.gmgui', 'test-downloads');
|
|
9
|
-
const TEST_TIMEOUT = 60000;
|
|
10
|
-
|
|
11
|
-
class TestRunner {
|
|
12
|
-
constructor() {
|
|
13
|
-
this.passed = 0;
|
|
14
|
-
this.failed = 0;
|
|
15
|
-
this.results = [];
|
|
16
|
-
}
|
|
17
|
-
|
|
18
|
-
async test(name, fn) {
|
|
19
|
-
try {
|
|
20
|
-
await fn();
|
|
21
|
-
this.passed++;
|
|
22
|
-
this.results.push({ name, status: 'PASS' });
|
|
23
|
-
console.log(`[PASS] ${name}`);
|
|
24
|
-
} catch (err) {
|
|
25
|
-
this.failed++;
|
|
26
|
-
this.results.push({ name, status: 'FAIL', error: err.message });
|
|
27
|
-
console.log(`[FAIL] ${name}: ${err.message}`);
|
|
28
|
-
}
|
|
29
|
-
}
|
|
30
|
-
|
|
31
|
-
summary() {
|
|
32
|
-
console.log(`\n=== TEST SUMMARY ===`);
|
|
33
|
-
console.log(`Passed: ${this.passed}`);
|
|
34
|
-
console.log(`Failed: ${this.failed}`);
|
|
35
|
-
console.log(`Total: ${this.passed + this.failed}`);
|
|
36
|
-
return this.failed === 0;
|
|
37
|
-
}
|
|
38
|
-
}
|
|
39
|
-
|
|
40
|
-
async function setupTestEnv() {
|
|
41
|
-
if (!fs.existsSync(TEST_DIR)) {
|
|
42
|
-
fs.mkdirSync(TEST_DIR, { recursive: true });
|
|
43
|
-
}
|
|
44
|
-
}
|
|
45
|
-
|
|
46
|
-
function cleanupTestEnv() {
|
|
47
|
-
if (fs.existsSync(TEST_DIR)) {
|
|
48
|
-
const files = fs.readdirSync(TEST_DIR);
|
|
49
|
-
for (const file of files) {
|
|
50
|
-
fs.unlinkSync(path.join(TEST_DIR, file));
|
|
51
|
-
}
|
|
52
|
-
fs.rmdirSync(TEST_DIR);
|
|
53
|
-
}
|
|
54
|
-
}
|
|
55
|
-
|
|
56
|
-
async function createMockFile(size, corrupted = false) {
|
|
57
|
-
const buffer = Buffer.alloc(size);
|
|
58
|
-
for (let i = 0; i < size; i++) {
|
|
59
|
-
buffer[i] = Math.floor(Math.random() * 256);
|
|
60
|
-
}
|
|
61
|
-
|
|
62
|
-
if (corrupted) {
|
|
63
|
-
buffer[0] = 0xFF;
|
|
64
|
-
buffer[1] = 0xFF;
|
|
65
|
-
}
|
|
66
|
-
|
|
67
|
-
const filename = `mock-${Date.now()}.bin`;
|
|
68
|
-
const filepath = path.join(TEST_DIR, filename);
|
|
69
|
-
fs.writeFileSync(filepath, buffer);
|
|
70
|
-
return { filepath, size, hash: crypto.createHash('sha256').update(buffer).digest('hex') };
|
|
71
|
-
}
|
|
72
|
-
|
|
73
|
-
async function simulatePartialDownload(filepath, targetSize) {
|
|
74
|
-
if (!fs.existsSync(filepath)) {
|
|
75
|
-
const buffer = Buffer.alloc(targetSize);
|
|
76
|
-
crypto.randomFillSync(buffer);
|
|
77
|
-
fs.writeFileSync(filepath, buffer);
|
|
78
|
-
}
|
|
79
|
-
return fs.statSync(filepath).size;
|
|
80
|
-
}
|
|
81
|
-
|
|
82
|
-
const runner = new TestRunner();
|
|
83
|
-
|
|
84
|
-
console.log('=== IPFS DOWNLOADER RESUME TESTS ===\n');
|
|
85
|
-
|
|
86
|
-
await setupTestEnv();
|
|
87
|
-
|
|
88
|
-
await runner.test('1. Detect partial download by size comparison', async () => {
|
|
89
|
-
const testFile = path.join(TEST_DIR, 'partial.bin');
|
|
90
|
-
const fullSize = 1000;
|
|
91
|
-
const partialSize = 250;
|
|
92
|
-
|
|
93
|
-
await simulatePartialDownload(testFile, partialSize);
|
|
94
|
-
const actualSize = fs.statSync(testFile).size;
|
|
95
|
-
|
|
96
|
-
if (actualSize !== partialSize) {
|
|
97
|
-
throw new Error(`Size mismatch: expected ${partialSize}, got ${actualSize}`);
|
|
98
|
-
}
|
|
99
|
-
|
|
100
|
-
if (actualSize < fullSize) {
|
|
101
|
-
console.log(` Partial download detected: ${actualSize}/${fullSize} bytes`);
|
|
102
|
-
}
|
|
103
|
-
});
|
|
104
|
-
|
|
105
|
-
await runner.test('2. Resume from offset (25% partial)', async () => {
|
|
106
|
-
const testFile = path.join(TEST_DIR, 'resume-25.bin');
|
|
107
|
-
const fullSize = 1000;
|
|
108
|
-
const partial25 = Math.floor(fullSize * 0.25);
|
|
109
|
-
|
|
110
|
-
const buffer = Buffer.alloc(partial25);
|
|
111
|
-
crypto.randomFillSync(buffer);
|
|
112
|
-
fs.writeFileSync(testFile, buffer);
|
|
113
|
-
|
|
114
|
-
const currentSize = fs.statSync(testFile).size;
|
|
115
|
-
const resumed = await simulatePartialDownload(testFile, fullSize);
|
|
116
|
-
|
|
117
|
-
if (currentSize < fullSize && resumed >= currentSize) {
|
|
118
|
-
console.log(` Successfully resumed from ${currentSize} bytes`);
|
|
119
|
-
} else {
|
|
120
|
-
throw new Error('Resume from 25% failed');
|
|
121
|
-
}
|
|
122
|
-
});
|
|
123
|
-
|
|
124
|
-
await runner.test('3. Resume from offset (50% partial)', async () => {
|
|
125
|
-
const testFile = path.join(TEST_DIR, 'resume-50.bin');
|
|
126
|
-
const fullSize = 1000;
|
|
127
|
-
const partial50 = Math.floor(fullSize * 0.5);
|
|
128
|
-
|
|
129
|
-
const buffer = Buffer.alloc(partial50);
|
|
130
|
-
crypto.randomFillSync(buffer);
|
|
131
|
-
fs.writeFileSync(testFile, buffer);
|
|
132
|
-
|
|
133
|
-
const currentSize = fs.statSync(testFile).size;
|
|
134
|
-
const resumed = await simulatePartialDownload(testFile, fullSize);
|
|
135
|
-
|
|
136
|
-
if (currentSize < fullSize && resumed >= currentSize) {
|
|
137
|
-
console.log(` Successfully resumed from ${currentSize} bytes (50%)`);
|
|
138
|
-
} else {
|
|
139
|
-
throw new Error('Resume from 50% failed');
|
|
140
|
-
}
|
|
141
|
-
});
|
|
142
|
-
|
|
143
|
-
await runner.test('4. Resume from offset (75% partial)', async () => {
|
|
144
|
-
const testFile = path.join(TEST_DIR, 'resume-75.bin');
|
|
145
|
-
const fullSize = 1000;
|
|
146
|
-
const partial75 = Math.floor(fullSize * 0.75);
|
|
147
|
-
|
|
148
|
-
const buffer = Buffer.alloc(partial75);
|
|
149
|
-
crypto.randomFillSync(buffer);
|
|
150
|
-
fs.writeFileSync(testFile, buffer);
|
|
151
|
-
|
|
152
|
-
const currentSize = fs.statSync(testFile).size;
|
|
153
|
-
const resumed = await simulatePartialDownload(testFile, fullSize);
|
|
154
|
-
|
|
155
|
-
if (currentSize < fullSize && resumed >= currentSize) {
|
|
156
|
-
console.log(` Successfully resumed from ${currentSize} bytes (75%)`);
|
|
157
|
-
} else {
|
|
158
|
-
throw new Error('Resume from 75% failed');
|
|
159
|
-
}
|
|
160
|
-
});
|
|
161
|
-
|
|
162
|
-
await runner.test('5. Hash verification after resume', async () => {
|
|
163
|
-
const mockData = await createMockFile(1000);
|
|
164
|
-
const verified = await downloader.verifyHash(mockData.filepath, mockData.hash);
|
|
165
|
-
|
|
166
|
-
if (!verified) {
|
|
167
|
-
throw new Error('Hash verification failed for valid file');
|
|
168
|
-
}
|
|
169
|
-
|
|
170
|
-
console.log(` Hash verified: ${mockData.hash.substring(0, 16)}...`);
|
|
171
|
-
});
|
|
172
|
-
|
|
173
|
-
await runner.test('6. Detect corrupted file during resume', async () => {
|
|
174
|
-
const mockData = await createMockFile(1000, true);
|
|
175
|
-
const corruptedHash = crypto.createHash('sha256').update(Buffer.alloc(1000, 0xFF)).digest('hex');
|
|
176
|
-
|
|
177
|
-
const verified = await downloader.verifyHash(mockData.filepath, corruptedHash);
|
|
178
|
-
|
|
179
|
-
if (verified) {
|
|
180
|
-
throw new Error('Hash mismatch should have been detected');
|
|
181
|
-
}
|
|
182
|
-
|
|
183
|
-
console.log(` Corruption detected correctly`);
|
|
184
|
-
});
|
|
185
|
-
|
|
186
|
-
await runner.test('7. Cleanup partial file on corruption', async () => {
|
|
187
|
-
const testFile = path.join(TEST_DIR, 'corrupted.bin');
|
|
188
|
-
const buffer = Buffer.alloc(500);
|
|
189
|
-
crypto.randomFillSync(buffer);
|
|
190
|
-
fs.writeFileSync(testFile, buffer);
|
|
191
|
-
|
|
192
|
-
if (!fs.existsSync(testFile)) {
|
|
193
|
-
throw new Error('Test file not created');
|
|
194
|
-
}
|
|
195
|
-
|
|
196
|
-
await downloader.cleanupPartial(testFile);
|
|
197
|
-
|
|
198
|
-
if (fs.existsSync(testFile)) {
|
|
199
|
-
throw new Error('Partial file was not cleaned up');
|
|
200
|
-
}
|
|
201
|
-
|
|
202
|
-
console.log(` Partial file cleaned up successfully`);
|
|
203
|
-
});
|
|
204
|
-
|
|
205
|
-
await runner.test('8. Track resume attempts in database', async () => {
|
|
206
|
-
const cidRecord = queries.recordIpfsCid(
|
|
207
|
-
'QmTest12345',
|
|
208
|
-
'test-model',
|
|
209
|
-
'test-type',
|
|
210
|
-
'test-hash',
|
|
211
|
-
'https://ipfs.io/ipfs/'
|
|
212
|
-
);
|
|
213
|
-
|
|
214
|
-
const downloadId = queries.recordDownloadStart(cidRecord, '/tmp/test.bin', 1000);
|
|
215
|
-
const initial = queries.getDownload(downloadId);
|
|
216
|
-
|
|
217
|
-
if (!initial || initial.attempts === undefined) {
|
|
218
|
-
throw new Error('Download tracking failed');
|
|
219
|
-
}
|
|
220
|
-
|
|
221
|
-
queries.updateDownloadResume(downloadId, 250, 1, Date.now(), 'resuming');
|
|
222
|
-
const updated = queries.getDownload(downloadId);
|
|
223
|
-
|
|
224
|
-
if (updated.attempts !== 1) {
|
|
225
|
-
throw new Error(`Attempt tracking failed: expected 1, got ${updated.attempts}`);
|
|
226
|
-
}
|
|
227
|
-
|
|
228
|
-
console.log(` Attempts tracked: ${updated.attempts}`);
|
|
229
|
-
});
|
|
230
|
-
|
|
231
|
-
await runner.test('9. Gateway fallback on unavailability', async () => {
|
|
232
|
-
const gateways = [
|
|
233
|
-
'https://ipfs.io/ipfs/',
|
|
234
|
-
'https://gateway.pinata.cloud/ipfs/',
|
|
235
|
-
'https://cloudflare-ipfs.com/ipfs/'
|
|
236
|
-
];
|
|
237
|
-
|
|
238
|
-
let fallbackIndex = 0;
|
|
239
|
-
const simulateFallback = (currentIndex) => {
|
|
240
|
-
fallbackIndex = (currentIndex + 1) % gateways.length;
|
|
241
|
-
return gateways[fallbackIndex];
|
|
242
|
-
};
|
|
243
|
-
|
|
244
|
-
const initialGateway = gateways[0];
|
|
245
|
-
const nextGateway = simulateFallback(0);
|
|
246
|
-
|
|
247
|
-
if (nextGateway === initialGateway) {
|
|
248
|
-
throw new Error('Gateway fallback failed');
|
|
249
|
-
}
|
|
250
|
-
|
|
251
|
-
console.log(` Fallback: ${initialGateway} -> ${nextGateway}`);
|
|
252
|
-
});
|
|
253
|
-
|
|
254
|
-
await runner.test('10. Exponential backoff for timeouts', async () => {
|
|
255
|
-
const INITIAL_BACKOFF = 1000;
|
|
256
|
-
const MULTIPLIER = 2;
|
|
257
|
-
|
|
258
|
-
const backoffs = [];
|
|
259
|
-
for (let attempt = 1; attempt <= 3; attempt++) {
|
|
260
|
-
const backoff = INITIAL_BACKOFF * Math.pow(MULTIPLIER, attempt - 1);
|
|
261
|
-
backoffs.push(backoff);
|
|
262
|
-
}
|
|
263
|
-
|
|
264
|
-
if (backoffs[0] !== 1000 || backoffs[1] !== 2000 || backoffs[2] !== 4000) {
|
|
265
|
-
throw new Error(`Backoff calculation wrong: ${backoffs}`);
|
|
266
|
-
}
|
|
267
|
-
|
|
268
|
-
console.log(` Backoff delays: ${backoffs.join('ms, ')}ms`);
|
|
269
|
-
});
|
|
270
|
-
|
|
271
|
-
await runner.test('11. Max resume attempts enforcement', async () => {
|
|
272
|
-
const cidRecord = queries.recordIpfsCid(
|
|
273
|
-
'QmTest67890',
|
|
274
|
-
'test-model-2',
|
|
275
|
-
'test-type-2',
|
|
276
|
-
'test-hash-2',
|
|
277
|
-
'https://ipfs.io/ipfs/'
|
|
278
|
-
);
|
|
279
|
-
|
|
280
|
-
const downloadId = queries.recordDownloadStart(cidRecord, '/tmp/test2.bin', 1000);
|
|
281
|
-
|
|
282
|
-
let attempts = 0;
|
|
283
|
-
while (attempts < 4) {
|
|
284
|
-
attempts++;
|
|
285
|
-
if (attempts > 3) {
|
|
286
|
-
const download = queries.getDownload(downloadId);
|
|
287
|
-
if (download.attempts >= 3) {
|
|
288
|
-
console.log(` Max attempts enforced at ${download.attempts}`);
|
|
289
|
-
break;
|
|
290
|
-
}
|
|
291
|
-
}
|
|
292
|
-
queries.updateDownloadResume(downloadId, 250 * attempts, attempts, Date.now(), 'resuming');
|
|
293
|
-
}
|
|
294
|
-
|
|
295
|
-
const final = queries.getDownload(downloadId);
|
|
296
|
-
if (final.attempts > 3) {
|
|
297
|
-
console.log(` Attempts capped at ${final.attempts}`);
|
|
298
|
-
}
|
|
299
|
-
});
|
|
300
|
-
|
|
301
|
-
await runner.test('12. Range header support detection', async () => {
|
|
302
|
-
const headers = { 'Accept-Ranges': 'bytes' };
|
|
303
|
-
const supportsRange = headers['Accept-Ranges'] === 'bytes';
|
|
304
|
-
|
|
305
|
-
if (!supportsRange) {
|
|
306
|
-
throw new Error('Range header detection failed');
|
|
307
|
-
}
|
|
308
|
-
|
|
309
|
-
console.log(` Server supports Range requests`);
|
|
310
|
-
});
|
|
311
|
-
|
|
312
|
-
await runner.test('13. Stream reset recovery strategy', async () => {
|
|
313
|
-
const fullSize = 1000;
|
|
314
|
-
const downloadedSize = 600;
|
|
315
|
-
const threshold = 0.5;
|
|
316
|
-
|
|
317
|
-
const downloadPercent = (downloadedSize / fullSize) * 100;
|
|
318
|
-
const shouldResume = downloadPercent > (threshold * 100);
|
|
319
|
-
|
|
320
|
-
if (!shouldResume) {
|
|
321
|
-
throw new Error('Stream reset strategy failed');
|
|
322
|
-
}
|
|
323
|
-
|
|
324
|
-
console.log(` Stream reset detected at ${downloadPercent}%, resuming (>50%)`);
|
|
325
|
-
});
|
|
326
|
-
|
|
327
|
-
await runner.test('14. Disk space handling during resume', async () => {
|
|
328
|
-
const testFile = path.join(TEST_DIR, 'diskspace.bin');
|
|
329
|
-
const testSize = 100;
|
|
330
|
-
|
|
331
|
-
const buffer = Buffer.alloc(testSize);
|
|
332
|
-
crypto.randomFillSync(buffer);
|
|
333
|
-
fs.writeFileSync(testFile, buffer);
|
|
334
|
-
|
|
335
|
-
const stats = fs.statSync(testFile);
|
|
336
|
-
if (stats.size !== testSize) {
|
|
337
|
-
throw new Error('File write verification failed');
|
|
338
|
-
}
|
|
339
|
-
|
|
340
|
-
console.log(` File write verified: ${stats.size} bytes`);
|
|
341
|
-
});
|
|
342
|
-
|
|
343
|
-
await runner.test('15. Download status lifecycle', async () => {
|
|
344
|
-
const cidRecord = queries.recordIpfsCid(
|
|
345
|
-
'QmTestStatus',
|
|
346
|
-
'test-model-status',
|
|
347
|
-
'test-type-status',
|
|
348
|
-
'test-hash-status',
|
|
349
|
-
'https://ipfs.io/ipfs/'
|
|
350
|
-
);
|
|
351
|
-
|
|
352
|
-
const downloadId = queries.recordDownloadStart(cidRecord, '/tmp/status.bin', 1000);
|
|
353
|
-
let download = queries.getDownload(downloadId);
|
|
354
|
-
|
|
355
|
-
const statuses = ['in_progress', 'resuming', 'paused', 'success'];
|
|
356
|
-
for (const status of statuses.slice(0, 3)) {
|
|
357
|
-
queries.updateDownloadResume(downloadId, 100, 1, Date.now(), status);
|
|
358
|
-
download = queries.getDownload(downloadId);
|
|
359
|
-
if (download.status !== status) {
|
|
360
|
-
throw new Error(`Status transition failed: ${download.status} !== ${status}`);
|
|
361
|
-
}
|
|
362
|
-
}
|
|
363
|
-
|
|
364
|
-
console.log(` Status lifecycle: ${statuses.slice(0, 3).join(' -> ')}`);
|
|
365
|
-
});
|
|
366
|
-
|
|
367
|
-
cleanupTestEnv();
|
|
368
|
-
|
|
369
|
-
const allPassed = runner.summary();
|
|
370
|
-
process.exit(allPassed ? 0 : 1);
|