agentgui 1.0.261 → 1.0.262
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +139 -0
- package/IPFS_DOWNLOADER.md +277 -0
- package/database.js +153 -0
- package/lib/ipfs-downloader.js +311 -0
- package/package.json +1 -1
- package/server.js +33 -5
- package/static/js/client.js +29 -16
- package/test-download-progress.js +223 -0
- package/test-websocket-broadcast.js +147 -0
- package/tests/ipfs-downloader.test.js +370 -0
|
@@ -0,0 +1,311 @@
|
|
|
1
|
+
import https from 'https';
|
|
2
|
+
import http from 'http';
|
|
3
|
+
import fs from 'fs';
|
|
4
|
+
import path from 'path';
|
|
5
|
+
import crypto from 'crypto';
|
|
6
|
+
import os from 'os';
|
|
7
|
+
import { queries } from '../database.js';
|
|
8
|
+
|
|
9
|
+
const GATEWAYS = [
|
|
10
|
+
'https://ipfs.io/ipfs/',
|
|
11
|
+
'https://gateway.pinata.cloud/ipfs/',
|
|
12
|
+
'https://cloudflare-ipfs.com/ipfs/',
|
|
13
|
+
'https://dweb.link/ipfs/'
|
|
14
|
+
];
|
|
15
|
+
|
|
16
|
+
const CONFIG = {
|
|
17
|
+
MAX_RESUME_ATTEMPTS: 3,
|
|
18
|
+
MAX_RETRY_ATTEMPTS: 3,
|
|
19
|
+
TIMEOUT_MS: 30000,
|
|
20
|
+
INITIAL_BACKOFF_MS: 1000,
|
|
21
|
+
BACKOFF_MULTIPLIER: 2,
|
|
22
|
+
DOWNLOADS_DIR: path.join(os.homedir(), '.gmgui', 'downloads'),
|
|
23
|
+
RESUME_THRESHOLD: 0.5
|
|
24
|
+
};
|
|
25
|
+
|
|
26
|
+
class IPFSDownloader {
|
|
27
|
+
constructor() {
|
|
28
|
+
this.downloads = new Map();
|
|
29
|
+
this.setupDir();
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
setupDir() {
|
|
33
|
+
if (!fs.existsSync(CONFIG.DOWNLOADS_DIR)) {
|
|
34
|
+
fs.mkdirSync(CONFIG.DOWNLOADS_DIR, { recursive: true });
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
async download(cid, filename, options = {}) {
|
|
39
|
+
const filepath = path.join(CONFIG.DOWNLOADS_DIR, filename);
|
|
40
|
+
const { modelName = 'unknown', modelType = 'unknown', modelHash = null } = options;
|
|
41
|
+
|
|
42
|
+
try {
|
|
43
|
+
const cidId = queries.recordIpfsCid(cid, modelName, modelType, modelHash, GATEWAYS[0]);
|
|
44
|
+
const downloadId = queries.recordDownloadStart(cidId, filepath, 0);
|
|
45
|
+
|
|
46
|
+
await this.executeDownload(downloadId, cidId, filepath, options);
|
|
47
|
+
return { success: true, downloadId, filepath, cid };
|
|
48
|
+
} catch (error) {
|
|
49
|
+
throw error;
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
async executeDownload(downloadId, cidId, filepath, options = {}) {
|
|
54
|
+
let gatewayIndex = 0;
|
|
55
|
+
let resumeAttempts = 0;
|
|
56
|
+
let retryAttempts = 0;
|
|
57
|
+
|
|
58
|
+
while (true) {
|
|
59
|
+
try {
|
|
60
|
+
const gateway = GATEWAYS[gatewayIndex];
|
|
61
|
+
const cidRecord = queries._db.prepare('SELECT * FROM ipfs_cids WHERE id = ?').get(cidId);
|
|
62
|
+
if (!cidRecord) throw new Error('CID record not found');
|
|
63
|
+
const url = `${gateway}${cidRecord.cid}`;
|
|
64
|
+
|
|
65
|
+
const { size, hash } = await this.downloadFile(
|
|
66
|
+
url,
|
|
67
|
+
filepath,
|
|
68
|
+
0,
|
|
69
|
+
options
|
|
70
|
+
);
|
|
71
|
+
|
|
72
|
+
queries.completeDownload(downloadId, cidId);
|
|
73
|
+
|
|
74
|
+
if (options.hashVerify && hash) {
|
|
75
|
+
queries.updateDownloadHash(downloadId, hash);
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
return queries.getDownload(downloadId);
|
|
79
|
+
} catch (error) {
|
|
80
|
+
if (error.message.includes('Range')) {
|
|
81
|
+
resumeAttempts++;
|
|
82
|
+
if (resumeAttempts > CONFIG.MAX_RESUME_ATTEMPTS) {
|
|
83
|
+
await this.cleanupPartial(filepath);
|
|
84
|
+
gatewayIndex = (gatewayIndex + 1) % GATEWAYS.length;
|
|
85
|
+
resumeAttempts = 0;
|
|
86
|
+
}
|
|
87
|
+
} else if (error.message.includes('timeout') || error.message.includes('ETIMEDOUT')) {
|
|
88
|
+
retryAttempts++;
|
|
89
|
+
if (retryAttempts > CONFIG.MAX_RETRY_ATTEMPTS) {
|
|
90
|
+
queries.recordDownloadError(downloadId, cidId, error.message);
|
|
91
|
+
throw error;
|
|
92
|
+
}
|
|
93
|
+
const backoff = CONFIG.INITIAL_BACKOFF_MS * Math.pow(CONFIG.BACKOFF_MULTIPLIER, retryAttempts - 1);
|
|
94
|
+
await this.sleep(backoff);
|
|
95
|
+
} else if (error.message.includes('network') || error.message.includes('ECONNRESET')) {
|
|
96
|
+
gatewayIndex = (gatewayIndex + 1) % GATEWAYS.length;
|
|
97
|
+
retryAttempts = 0;
|
|
98
|
+
} else {
|
|
99
|
+
queries.recordDownloadError(downloadId, cidId, error.message);
|
|
100
|
+
throw error;
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
async downloadFile(url, filepath, resumeFrom = 0, options = {}) {
|
|
107
|
+
return new Promise((resolve, reject) => {
|
|
108
|
+
const protocol = url.startsWith('https') ? https : http;
|
|
109
|
+
const headers = {};
|
|
110
|
+
|
|
111
|
+
if (resumeFrom > 0) {
|
|
112
|
+
headers['Range'] = `bytes=${resumeFrom}-`;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
const req = protocol.get(url, { headers, timeout: CONFIG.TIMEOUT_MS }, (res) => {
|
|
116
|
+
if (res.statusCode === 416) {
|
|
117
|
+
reject(new Error('Range not supported - will delete partial and restart'));
|
|
118
|
+
return;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
if (![200, 206].includes(res.statusCode)) {
|
|
122
|
+
reject(new Error(`HTTP ${res.statusCode}`));
|
|
123
|
+
return;
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
const contentLength = parseInt(res.headers['content-length'], 10);
|
|
127
|
+
const hash = crypto.createHash('sha256');
|
|
128
|
+
let downloaded = resumeFrom;
|
|
129
|
+
|
|
130
|
+
const mode = resumeFrom > 0 ? 'a' : 'w';
|
|
131
|
+
const stream = fs.createWriteStream(filepath, { flags: mode });
|
|
132
|
+
|
|
133
|
+
res.on('data', (chunk) => {
|
|
134
|
+
hash.update(chunk);
|
|
135
|
+
downloaded += chunk.length;
|
|
136
|
+
});
|
|
137
|
+
|
|
138
|
+
res.pipe(stream);
|
|
139
|
+
|
|
140
|
+
stream.on('finish', () => {
|
|
141
|
+
resolve({ size: downloaded, hash: hash.digest('hex') });
|
|
142
|
+
});
|
|
143
|
+
|
|
144
|
+
stream.on('error', (err) => {
|
|
145
|
+
reject(new Error(`Write error: ${err.message}`));
|
|
146
|
+
});
|
|
147
|
+
});
|
|
148
|
+
|
|
149
|
+
req.on('timeout', () => {
|
|
150
|
+
req.abort();
|
|
151
|
+
reject(new Error('timeout'));
|
|
152
|
+
});
|
|
153
|
+
|
|
154
|
+
req.on('error', (err) => {
|
|
155
|
+
reject(new Error(`network: ${err.message}`));
|
|
156
|
+
});
|
|
157
|
+
});
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
async verifyHash(filepath, expectedHash) {
|
|
161
|
+
return new Promise((resolve, reject) => {
|
|
162
|
+
const hash = crypto.createHash('sha256');
|
|
163
|
+
const stream = fs.createReadStream(filepath);
|
|
164
|
+
|
|
165
|
+
stream.on('data', (chunk) => {
|
|
166
|
+
hash.update(chunk);
|
|
167
|
+
});
|
|
168
|
+
|
|
169
|
+
stream.on('end', () => {
|
|
170
|
+
resolve(hash.digest('hex') === expectedHash);
|
|
171
|
+
});
|
|
172
|
+
|
|
173
|
+
stream.on('error', (err) => {
|
|
174
|
+
reject(err);
|
|
175
|
+
});
|
|
176
|
+
});
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
async resume(downloadId, options = {}) {
|
|
180
|
+
const record = queries.getDownload(downloadId);
|
|
181
|
+
|
|
182
|
+
if (!record) {
|
|
183
|
+
throw new Error('Download not found');
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
if (record.status === 'success') {
|
|
187
|
+
return record;
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
const attempts = (record.attempts || 0) + 1;
|
|
191
|
+
if (attempts > CONFIG.MAX_RESUME_ATTEMPTS) {
|
|
192
|
+
throw new Error('Max resume attempts exceeded');
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
try {
|
|
196
|
+
const currentSize = fs.existsSync(record.downloadPath)
|
|
197
|
+
? fs.statSync(record.downloadPath).size
|
|
198
|
+
: 0;
|
|
199
|
+
|
|
200
|
+
if (currentSize === 0) {
|
|
201
|
+
queries.recordDownloadStart(record.cidId, record.downloadPath, record.total_bytes);
|
|
202
|
+
return this.resumeFromOffset(downloadId, record, 0, options);
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
queries.markDownloadResuming(downloadId);
|
|
206
|
+
|
|
207
|
+
const downloadPercent = (currentSize / (record.total_bytes || currentSize)) * 100;
|
|
208
|
+
|
|
209
|
+
if (downloadPercent > CONFIG.RESUME_THRESHOLD * 100) {
|
|
210
|
+
return this.resumeFromOffset(downloadId, record, currentSize, options);
|
|
211
|
+
} else {
|
|
212
|
+
await this.cleanupPartial(record.downloadPath);
|
|
213
|
+
return this.resumeFromOffset(downloadId, record, 0, options);
|
|
214
|
+
}
|
|
215
|
+
} catch (error) {
|
|
216
|
+
const newAttempts = (record.attempts || 0) + 1;
|
|
217
|
+
const newStatus = newAttempts >= CONFIG.MAX_RESUME_ATTEMPTS ? 'failed' : 'paused';
|
|
218
|
+
queries.updateDownloadResume(downloadId, record.downloaded_bytes, newAttempts, Date.now(), newStatus);
|
|
219
|
+
|
|
220
|
+
if (newStatus === 'failed') {
|
|
221
|
+
throw error;
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
return queries.getDownload(downloadId);
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
async resumeFromOffset(downloadId, record, offset, options) {
|
|
229
|
+
try {
|
|
230
|
+
const cidRecord = queries.getIpfsCidByModel(record.modelName, record.modelType);
|
|
231
|
+
const gateway = GATEWAYS[0];
|
|
232
|
+
const url = `${gateway}${cidRecord.cid}`;
|
|
233
|
+
|
|
234
|
+
const { size, hash } = await this.downloadFile(
|
|
235
|
+
url,
|
|
236
|
+
record.downloadPath,
|
|
237
|
+
offset,
|
|
238
|
+
options
|
|
239
|
+
);
|
|
240
|
+
|
|
241
|
+
if (options.hashVerify && record.hash) {
|
|
242
|
+
const verified = await this.verifyHash(record.downloadPath, record.hash);
|
|
243
|
+
if (!verified) {
|
|
244
|
+
await this.cleanupPartial(record.downloadPath);
|
|
245
|
+
const newAttempts = (record.attempts || 0) + 1;
|
|
246
|
+
queries.updateDownloadResume(downloadId, 0, newAttempts, Date.now(), 'pending');
|
|
247
|
+
throw new Error('Hash verification failed - restarting');
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
queries.completeDownload(downloadId, record.cidId);
|
|
252
|
+
if (hash) {
|
|
253
|
+
queries.updateDownloadHash(downloadId, hash);
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
return queries.getDownload(downloadId);
|
|
257
|
+
} catch (error) {
|
|
258
|
+
const newAttempts = (record.attempts || 0) + 1;
|
|
259
|
+
const newStatus = newAttempts >= CONFIG.MAX_RESUME_ATTEMPTS ? 'failed' : 'paused';
|
|
260
|
+
queries.updateDownloadResume(downloadId, offset, newAttempts, Date.now(), newStatus);
|
|
261
|
+
|
|
262
|
+
if (newStatus === 'failed') {
|
|
263
|
+
throw error;
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
return queries.getDownload(downloadId);
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
async cleanupPartial(filepath) {
|
|
271
|
+
if (fs.existsSync(filepath)) {
|
|
272
|
+
try {
|
|
273
|
+
fs.unlinkSync(filepath);
|
|
274
|
+
} catch (err) {
|
|
275
|
+
console.error(`Failed to cleanup partial file: ${filepath}`, err.message);
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
sleep(ms) {
|
|
281
|
+
return new Promise(resolve => setTimeout(resolve, ms));
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
getDownloadStatus(downloadId) {
|
|
285
|
+
return queries.getDownload(downloadId);
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
listDownloads(status = null) {
|
|
289
|
+
if (status) {
|
|
290
|
+
return queries.getDownloadsByStatus(status);
|
|
291
|
+
}
|
|
292
|
+
const allDownloads = queries.getDownloadsByStatus('in_progress');
|
|
293
|
+
return allDownloads.concat(
|
|
294
|
+
queries.getDownloadsByStatus('success'),
|
|
295
|
+
queries.getDownloadsByStatus('paused'),
|
|
296
|
+
queries.getDownloadsByStatus('failed')
|
|
297
|
+
);
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
async cancelDownload(downloadId) {
|
|
301
|
+
const record = queries.getDownload(downloadId);
|
|
302
|
+
if (!record) return false;
|
|
303
|
+
|
|
304
|
+
await this.cleanupPartial(record.downloadPath);
|
|
305
|
+
queries.markDownloadPaused(downloadId, 'Cancelled by user');
|
|
306
|
+
|
|
307
|
+
return true;
|
|
308
|
+
}
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
export default new IPFSDownloader();
|
package/package.json
CHANGED
package/server.js
CHANGED
|
@@ -32,12 +32,31 @@ const modelDownloadState = {
|
|
|
32
32
|
downloading: false,
|
|
33
33
|
progress: null,
|
|
34
34
|
error: null,
|
|
35
|
-
complete: false
|
|
35
|
+
complete: false,
|
|
36
|
+
startTime: null,
|
|
37
|
+
downloadMetrics: new Map()
|
|
36
38
|
};
|
|
37
39
|
|
|
38
40
|
function broadcastModelProgress(progress) {
|
|
39
41
|
modelDownloadState.progress = progress;
|
|
40
|
-
|
|
42
|
+
const broadcastData = {
|
|
43
|
+
type: 'model_download_progress',
|
|
44
|
+
modelId: progress.type || 'unknown',
|
|
45
|
+
bytesDownloaded: progress.bytesDownloaded || 0,
|
|
46
|
+
bytesRemaining: progress.bytesRemaining || 0,
|
|
47
|
+
totalBytes: progress.totalBytes || 0,
|
|
48
|
+
downloadSpeed: progress.downloadSpeed || 0,
|
|
49
|
+
eta: progress.eta || 0,
|
|
50
|
+
retryCount: progress.retryCount || 0,
|
|
51
|
+
currentGateway: progress.currentGateway || '',
|
|
52
|
+
status: progress.status || (progress.done ? 'completed' : progress.downloading ? 'downloading' : 'paused'),
|
|
53
|
+
percentComplete: progress.percentComplete || 0,
|
|
54
|
+
completedFiles: progress.completedFiles || 0,
|
|
55
|
+
totalFiles: progress.totalFiles || 0,
|
|
56
|
+
timestamp: Date.now(),
|
|
57
|
+
...progress
|
|
58
|
+
};
|
|
59
|
+
broadcastSync(broadcastData);
|
|
41
60
|
}
|
|
42
61
|
|
|
43
62
|
async function ensureModelsDownloaded() {
|
|
@@ -85,14 +104,22 @@ async function ensureModelsDownloaded() {
|
|
|
85
104
|
if (!sttOk) {
|
|
86
105
|
console.log('[MODELS] Downloading STT model...');
|
|
87
106
|
broadcastModelProgress({ started: true, done: false, downloading: true, type: 'stt', completedFiles, totalFiles });
|
|
88
|
-
|
|
107
|
+
try {
|
|
108
|
+
await webtalkWhisper.ensureModel('onnx-community/whisper-base', config);
|
|
109
|
+
} catch (err) {
|
|
110
|
+
console.warn('[MODELS] STT download failed, falling back to HuggingFace:', err.message);
|
|
111
|
+
}
|
|
89
112
|
completedFiles += 10;
|
|
90
113
|
}
|
|
91
114
|
|
|
92
115
|
if (!ttsOk) {
|
|
93
116
|
console.log('[MODELS] Downloading TTS models...');
|
|
94
117
|
broadcastModelProgress({ started: true, done: false, downloading: true, type: 'tts', completedFiles, totalFiles });
|
|
95
|
-
|
|
118
|
+
try {
|
|
119
|
+
await webtalkTTS.ensureTTSModels(config);
|
|
120
|
+
} catch (err) {
|
|
121
|
+
console.warn('[MODELS] TTS download failed, falling back to HuggingFace:', err.message);
|
|
122
|
+
}
|
|
96
123
|
completedFiles += 6;
|
|
97
124
|
}
|
|
98
125
|
|
|
@@ -3274,7 +3301,8 @@ const BROADCAST_TYPES = new Set([
|
|
|
3274
3301
|
'message_created', 'conversation_created', 'conversation_updated',
|
|
3275
3302
|
'conversations_updated', 'conversation_deleted', 'queue_status', 'queue_updated',
|
|
3276
3303
|
'rate_limit_hit', 'rate_limit_clear',
|
|
3277
|
-
'script_started', 'script_stopped', 'script_output'
|
|
3304
|
+
'script_started', 'script_stopped', 'script_output',
|
|
3305
|
+
'model_download_progress'
|
|
3278
3306
|
]);
|
|
3279
3307
|
|
|
3280
3308
|
const wsBatchQueues = new Map();
|
package/static/js/client.js
CHANGED
|
@@ -494,7 +494,7 @@ class AgentGUIClient {
|
|
|
494
494
|
this.handleRateLimitClear(data);
|
|
495
495
|
break;
|
|
496
496
|
case 'model_download_progress':
|
|
497
|
-
this._handleModelDownloadProgress(data.progress);
|
|
497
|
+
this._handleModelDownloadProgress(data.progress || data);
|
|
498
498
|
break;
|
|
499
499
|
default:
|
|
500
500
|
break;
|
|
@@ -2028,16 +2028,16 @@ class AgentGUIClient {
|
|
|
2028
2028
|
|
|
2029
2029
|
_handleModelDownloadProgress(progress) {
|
|
2030
2030
|
this._modelDownloadProgress = progress;
|
|
2031
|
-
|
|
2032
|
-
if (progress.error) {
|
|
2031
|
+
|
|
2032
|
+
if (progress.status === 'failed' || progress.error) {
|
|
2033
2033
|
this._modelDownloadInProgress = false;
|
|
2034
|
-
console.error('[Models] Download error:', progress.error);
|
|
2034
|
+
console.error('[Models] Download error:', progress.error || progress.status);
|
|
2035
2035
|
this._updateConnectionIndicator(this.wsManager?.latency?.quality || 'unknown');
|
|
2036
2036
|
if (window._voiceProgressDialog) {
|
|
2037
2037
|
window._voiceProgressDialog.close();
|
|
2038
2038
|
window._voiceProgressDialog = null;
|
|
2039
2039
|
}
|
|
2040
|
-
const errorMsg = 'Failed to download voice models: ' + progress.error;
|
|
2040
|
+
const errorMsg = 'Failed to download voice models: ' + (progress.error || 'unknown error');
|
|
2041
2041
|
if (window.UIDialog) {
|
|
2042
2042
|
window.UIDialog.alert(errorMsg, 'Download Error');
|
|
2043
2043
|
} else {
|
|
@@ -2045,8 +2045,8 @@ class AgentGUIClient {
|
|
|
2045
2045
|
}
|
|
2046
2046
|
return;
|
|
2047
2047
|
}
|
|
2048
|
-
|
|
2049
|
-
if (progress.done) {
|
|
2048
|
+
|
|
2049
|
+
if (progress.done || progress.status === 'completed') {
|
|
2050
2050
|
this._modelDownloadInProgress = false;
|
|
2051
2051
|
console.log('[Models] Download complete');
|
|
2052
2052
|
this._updateConnectionIndicator(this.wsManager?.latency?.quality || 'unknown');
|
|
@@ -2067,8 +2067,8 @@ class AgentGUIClient {
|
|
|
2067
2067
|
}
|
|
2068
2068
|
return;
|
|
2069
2069
|
}
|
|
2070
|
-
|
|
2071
|
-
if (progress.started || progress.downloading) {
|
|
2070
|
+
|
|
2071
|
+
if (progress.started || progress.downloading || progress.status === 'downloading' || progress.status === 'connecting') {
|
|
2072
2072
|
this._modelDownloadInProgress = true;
|
|
2073
2073
|
this._updateConnectionIndicator(this.wsManager?.latency?.quality || 'unknown');
|
|
2074
2074
|
|
|
@@ -2076,13 +2076,26 @@ class AgentGUIClient {
|
|
|
2076
2076
|
window.__showVoiceDownloadProgress();
|
|
2077
2077
|
}
|
|
2078
2078
|
|
|
2079
|
-
if (window._voiceProgressDialog
|
|
2080
|
-
|
|
2081
|
-
|
|
2082
|
-
|
|
2083
|
-
|
|
2084
|
-
|
|
2085
|
-
|
|
2079
|
+
if (window._voiceProgressDialog) {
|
|
2080
|
+
let displayText = 'Downloading models...';
|
|
2081
|
+
|
|
2082
|
+
if (progress.status === 'connecting') {
|
|
2083
|
+
displayText = 'Connecting to ' + (progress.currentGateway || 'gateway') + '...';
|
|
2084
|
+
} else if (progress.totalBytes > 0) {
|
|
2085
|
+
const downloaded = (progress.bytesDownloaded || 0) / 1024 / 1024;
|
|
2086
|
+
const total = progress.totalBytes / 1024 / 1024;
|
|
2087
|
+
const speed = progress.downloadSpeed ? (progress.downloadSpeed / 1024 / 1024).toFixed(2) : '0';
|
|
2088
|
+
const eta = progress.eta ? Math.ceil(progress.eta) + 's' : '...';
|
|
2089
|
+
const retryInfo = progress.retryCount > 0 ? ` (retry ${progress.retryCount})` : '';
|
|
2090
|
+
|
|
2091
|
+
displayText = `Downloading ${downloaded.toFixed(1)}MB / ${total.toFixed(1)}MB @ ${speed}MB/s (ETA: ${eta})${retryInfo}`;
|
|
2092
|
+
} else if (progress.file) {
|
|
2093
|
+
displayText = 'Loading ' + progress.file + '...';
|
|
2094
|
+
} else if (progress.completedFiles && progress.totalFiles) {
|
|
2095
|
+
displayText = `Downloaded ${progress.completedFiles}/${progress.totalFiles} files`;
|
|
2096
|
+
}
|
|
2097
|
+
|
|
2098
|
+
window._voiceProgressDialog.update(progress.percentComplete || 0, displayText);
|
|
2086
2099
|
}
|
|
2087
2100
|
}
|
|
2088
2101
|
}
|
|
@@ -0,0 +1,223 @@
|
|
|
1
|
+
import https from 'https';
|
|
2
|
+
import fs from 'fs';
|
|
3
|
+
import path from 'path';
|
|
4
|
+
import os from 'os';
|
|
5
|
+
|
|
6
|
+
const testDir = path.join(os.tmpdir(), 'test-download-progress');
|
|
7
|
+
if (!fs.existsSync(testDir)) {
|
|
8
|
+
fs.mkdirSync(testDir, { recursive: true });
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
const GATEWAYS = [
|
|
12
|
+
'https://ipfs.io',
|
|
13
|
+
'https://gateway.pinata.cloud',
|
|
14
|
+
'https://cloudflare-ipfs.com',
|
|
15
|
+
];
|
|
16
|
+
|
|
17
|
+
function downloadWithProgress(url, destination, onProgress = null) {
|
|
18
|
+
let bytesDownloaded = 0;
|
|
19
|
+
let totalBytes = 0;
|
|
20
|
+
let lastProgressTime = Date.now();
|
|
21
|
+
let lastProgressBytes = 0;
|
|
22
|
+
const speeds = [];
|
|
23
|
+
let retryCount = 0;
|
|
24
|
+
let gatewayIndex = 0;
|
|
25
|
+
|
|
26
|
+
const emitProgress = () => {
|
|
27
|
+
const now = Date.now();
|
|
28
|
+
const deltaTime = (now - lastProgressTime) / 1000;
|
|
29
|
+
const deltaBytes = bytesDownloaded - lastProgressBytes;
|
|
30
|
+
const speed = deltaTime > 0 ? Math.round(deltaBytes / deltaTime) : 0;
|
|
31
|
+
|
|
32
|
+
if (speed > 0) {
|
|
33
|
+
speeds.push(speed);
|
|
34
|
+
if (speeds.length > 10) speeds.shift();
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
const avgSpeed = speeds.length > 0 ? Math.round(speeds.reduce((a, b) => a + b, 0) / speeds.length) : 0;
|
|
38
|
+
const eta = avgSpeed > 0 && totalBytes > bytesDownloaded ? Math.round((totalBytes - bytesDownloaded) / avgSpeed) : 0;
|
|
39
|
+
|
|
40
|
+
if (onProgress) {
|
|
41
|
+
onProgress({
|
|
42
|
+
bytesDownloaded,
|
|
43
|
+
bytesRemaining: Math.max(0, totalBytes - bytesDownloaded),
|
|
44
|
+
totalBytes,
|
|
45
|
+
downloadSpeed: avgSpeed,
|
|
46
|
+
eta,
|
|
47
|
+
retryCount,
|
|
48
|
+
currentGateway: url,
|
|
49
|
+
status: bytesDownloaded >= totalBytes ? 'completed' : 'downloading',
|
|
50
|
+
percentComplete: totalBytes > 0 ? Math.round((bytesDownloaded / totalBytes) * 100) : 0,
|
|
51
|
+
timestamp: now
|
|
52
|
+
});
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
lastProgressTime = now;
|
|
56
|
+
lastProgressBytes = bytesDownloaded;
|
|
57
|
+
};
|
|
58
|
+
|
|
59
|
+
return new Promise((resolve, reject) => {
|
|
60
|
+
const dir = path.dirname(destination);
|
|
61
|
+
if (!fs.existsSync(dir)) {
|
|
62
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
const attemptDownload = (gateway) => {
|
|
66
|
+
if (onProgress) {
|
|
67
|
+
onProgress({
|
|
68
|
+
bytesDownloaded: 0,
|
|
69
|
+
bytesRemaining: 0,
|
|
70
|
+
totalBytes: 0,
|
|
71
|
+
downloadSpeed: 0,
|
|
72
|
+
eta: 0,
|
|
73
|
+
retryCount,
|
|
74
|
+
currentGateway: gateway,
|
|
75
|
+
status: 'connecting',
|
|
76
|
+
percentComplete: 0,
|
|
77
|
+
timestamp: Date.now()
|
|
78
|
+
});
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
https.get(gateway, { timeout: 30000 }, (res) => {
|
|
82
|
+
if ([301, 302, 307, 308].includes(res.statusCode)) {
|
|
83
|
+
const location = res.headers.location;
|
|
84
|
+
if (location) return attemptDownload(location);
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
if (res.statusCode !== 200) {
|
|
88
|
+
res.resume();
|
|
89
|
+
if (gatewayIndex < GATEWAYS.length - 1) {
|
|
90
|
+
retryCount++;
|
|
91
|
+
gatewayIndex++;
|
|
92
|
+
return setTimeout(() => attemptDownload(GATEWAYS[gatewayIndex]), 1000 * Math.pow(2, Math.min(retryCount, 3)));
|
|
93
|
+
}
|
|
94
|
+
return reject(new Error(`HTTP ${res.statusCode}`));
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
totalBytes = parseInt(res.headers['content-length'], 10) || 0;
|
|
98
|
+
bytesDownloaded = 0;
|
|
99
|
+
lastProgressBytes = 0;
|
|
100
|
+
lastProgressTime = Date.now();
|
|
101
|
+
|
|
102
|
+
const file = fs.createWriteStream(destination);
|
|
103
|
+
let lastEmit = Date.now();
|
|
104
|
+
|
|
105
|
+
res.on('data', (chunk) => {
|
|
106
|
+
bytesDownloaded += chunk.length;
|
|
107
|
+
const now = Date.now();
|
|
108
|
+
if (now - lastEmit >= 200) {
|
|
109
|
+
emitProgress();
|
|
110
|
+
lastEmit = now;
|
|
111
|
+
}
|
|
112
|
+
});
|
|
113
|
+
|
|
114
|
+
res.on('end', () => {
|
|
115
|
+
emitProgress();
|
|
116
|
+
file.destroy();
|
|
117
|
+
resolve({ destination, bytesDownloaded, success: true });
|
|
118
|
+
});
|
|
119
|
+
|
|
120
|
+
res.on('error', (err) => {
|
|
121
|
+
file.destroy();
|
|
122
|
+
fs.unlink(destination, () => {});
|
|
123
|
+
if (gatewayIndex < GATEWAYS.length - 1) {
|
|
124
|
+
retryCount++;
|
|
125
|
+
gatewayIndex++;
|
|
126
|
+
return setTimeout(() => attemptDownload(GATEWAYS[gatewayIndex]), 1000 * Math.pow(2, Math.min(retryCount, 3)));
|
|
127
|
+
}
|
|
128
|
+
reject(err);
|
|
129
|
+
});
|
|
130
|
+
|
|
131
|
+
file.on('error', (err) => {
|
|
132
|
+
res.destroy();
|
|
133
|
+
fs.unlink(destination, () => {});
|
|
134
|
+
if (gatewayIndex < GATEWAYS.length - 1) {
|
|
135
|
+
retryCount++;
|
|
136
|
+
gatewayIndex++;
|
|
137
|
+
return setTimeout(() => attemptDownload(GATEWAYS[gatewayIndex]), 1000 * Math.pow(2, Math.min(retryCount, 3)));
|
|
138
|
+
}
|
|
139
|
+
reject(err);
|
|
140
|
+
});
|
|
141
|
+
|
|
142
|
+
res.pipe(file);
|
|
143
|
+
}).on('timeout', () => {
|
|
144
|
+
if (gatewayIndex < GATEWAYS.length - 1) {
|
|
145
|
+
retryCount++;
|
|
146
|
+
gatewayIndex++;
|
|
147
|
+
return setTimeout(() => attemptDownload(GATEWAYS[gatewayIndex]), 1000 * Math.pow(2, Math.min(retryCount, 3)));
|
|
148
|
+
}
|
|
149
|
+
reject(new Error('Download timeout'));
|
|
150
|
+
}).on('error', (err) => {
|
|
151
|
+
if (gatewayIndex < GATEWAYS.length - 1) {
|
|
152
|
+
retryCount++;
|
|
153
|
+
gatewayIndex++;
|
|
154
|
+
return setTimeout(() => attemptDownload(GATEWAYS[gatewayIndex]), 1000 * Math.pow(2, Math.min(retryCount, 3)));
|
|
155
|
+
}
|
|
156
|
+
reject(err);
|
|
157
|
+
});
|
|
158
|
+
};
|
|
159
|
+
|
|
160
|
+
attemptDownload(GATEWAYS[0]);
|
|
161
|
+
});
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
let progressCount = 0;
|
|
165
|
+
let lastPrintTime = Date.now();
|
|
166
|
+
let minInterval = Infinity;
|
|
167
|
+
let maxInterval = 0;
|
|
168
|
+
const progressIntervals = [];
|
|
169
|
+
|
|
170
|
+
console.log('Starting download progress tracking test...\n');
|
|
171
|
+
|
|
172
|
+
downloadWithProgress(
|
|
173
|
+
'https://www.w3.org/WAI/WCAG21/Techniques/pdf/pdf-files/table-example.pdf',
|
|
174
|
+
path.join(testDir, 'test-file.pdf'),
|
|
175
|
+
(progress) => {
|
|
176
|
+
progressCount++;
|
|
177
|
+
const now = Date.now();
|
|
178
|
+
const interval = now - lastPrintTime;
|
|
179
|
+
|
|
180
|
+
if (progressCount > 1) {
|
|
181
|
+
progressIntervals.push(interval);
|
|
182
|
+
minInterval = Math.min(minInterval, interval);
|
|
183
|
+
maxInterval = Math.max(maxInterval, interval);
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
console.log(`[${progressCount}] Progress Update:
|
|
187
|
+
Status: ${progress.status}
|
|
188
|
+
Downloaded: ${(progress.bytesDownloaded / 1024).toFixed(1)}KB / ${(progress.totalBytes / 1024).toFixed(1)}KB
|
|
189
|
+
Speed: ${(progress.downloadSpeed / 1024).toFixed(2)}MB/s
|
|
190
|
+
ETA: ${progress.eta}s
|
|
191
|
+
Complete: ${progress.percentComplete}%
|
|
192
|
+
Retry Count: ${progress.retryCount}
|
|
193
|
+
Gateway: ${progress.currentGateway}
|
|
194
|
+
Interval: ${interval}ms\n`);
|
|
195
|
+
|
|
196
|
+
lastPrintTime = now;
|
|
197
|
+
}
|
|
198
|
+
).then((result) => {
|
|
199
|
+
const avgInterval = progressIntervals.length > 0 ? progressIntervals.reduce((a, b) => a + b, 0) / progressIntervals.length : 0;
|
|
200
|
+
console.log('\n=== Download Complete ===');
|
|
201
|
+
console.log(`Result: ${JSON.stringify(result, null, 2)}`);
|
|
202
|
+
console.log(`\nProgress Tracking Statistics:
|
|
203
|
+
Total Updates: ${progressCount}
|
|
204
|
+
Interval Range: ${minInterval}ms - ${maxInterval}ms
|
|
205
|
+
Average Interval: ${avgInterval.toFixed(0)}ms
|
|
206
|
+
Expected Interval: 200ms (should be 100-500ms range)`);
|
|
207
|
+
|
|
208
|
+
if (avgInterval >= 100 && avgInterval <= 500) {
|
|
209
|
+
console.log(' Status: PASS - Progress interval within acceptable range');
|
|
210
|
+
} else {
|
|
211
|
+
console.log(' Status: FAIL - Progress interval outside acceptable range');
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
if (fs.existsSync(path.join(testDir, 'test-file.pdf'))) {
|
|
215
|
+
const stat = fs.statSync(path.join(testDir, 'test-file.pdf'));
|
|
216
|
+
console.log(`\nDownloaded file size: ${(stat.size / 1024).toFixed(1)}KB`);
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
process.exit(0);
|
|
220
|
+
}).catch((err) => {
|
|
221
|
+
console.error('Download failed:', err.message);
|
|
222
|
+
process.exit(1);
|
|
223
|
+
});
|