justget 0.2.4 → 0.2.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.browser.md +1 -1
- package/README.en.md +13 -11
- package/README.md +16 -13
- package/dist/browser/core/downloader.d.ts +11 -11
- package/dist/browser/core/downloader.d.ts.map +1 -1
- package/dist/browser/core/downloader.js +28 -27
- package/dist/browser/core/downloader.js.map +1 -1
- package/dist/browser/core/http.d.ts +13 -13
- package/dist/browser/core/http.d.ts.map +1 -1
- package/dist/browser/core/http.js +8 -8
- package/dist/browser/core/http.js.map +1 -1
- package/dist/browser/index.d.ts +2 -2
- package/dist/browser/index.js +2 -2
- package/dist/browser/utils/buffer.d.ts +4 -4
- package/dist/browser/utils/buffer.d.ts.map +1 -1
- package/dist/browser/utils/buffer.js +4 -4
- package/dist/browser/utils/buffer.js.map +1 -1
- package/dist/browser/utils/cache.d.ts +12 -11
- package/dist/browser/utils/cache.d.ts.map +1 -1
- package/dist/browser/utils/cache.js +14 -13
- package/dist/browser/utils/cache.js.map +1 -1
- package/dist/browser/utils/hash.d.ts +10 -11
- package/dist/browser/utils/hash.d.ts.map +1 -1
- package/dist/browser/utils/hash.js +20 -20
- package/dist/browser/utils/hash.js.map +1 -1
- package/dist/cli.d.ts +2 -9
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +66 -72
- package/dist/cli.js.map +1 -1
- package/dist/config.d.ts +2 -2
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +23 -21
- package/dist/config.js.map +1 -1
- package/dist/core/downloader.d.ts +18 -9
- package/dist/core/downloader.d.ts.map +1 -1
- package/dist/core/downloader.js +478 -258
- package/dist/core/downloader.js.map +1 -1
- package/dist/core/http.d.ts +19 -18
- package/dist/core/http.d.ts.map +1 -1
- package/dist/core/http.js +41 -39
- package/dist/core/http.js.map +1 -1
- package/dist/core/temp.d.ts +23 -19
- package/dist/core/temp.d.ts.map +1 -1
- package/dist/core/temp.js +29 -29
- package/dist/core/temp.js.map +1 -1
- package/dist/format.d.ts +16 -0
- package/dist/format.d.ts.map +1 -0
- package/dist/format.js +47 -0
- package/dist/format.js.map +1 -0
- package/dist/i18n.d.ts +21 -0
- package/dist/i18n.d.ts.map +1 -0
- package/dist/i18n.js +109 -0
- package/dist/i18n.js.map +1 -0
- package/dist/index.d.ts +5 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +10 -7
- package/dist/index.js.map +1 -1
- package/dist/render.d.ts +85 -0
- package/dist/render.d.ts.map +1 -0
- package/dist/render.js +221 -0
- package/dist/render.js.map +1 -0
- package/dist/types.d.ts +22 -5
- package/dist/types.d.ts.map +1 -1
- package/dist/utils/hash.d.ts +13 -2
- package/dist/utils/hash.d.ts.map +1 -1
- package/dist/utils/hash.js +35 -4
- package/dist/utils/hash.js.map +1 -1
- package/package.json +4 -3
- package/dist/downloader.d.ts +0 -22
- package/dist/downloader.d.ts.map +0 -1
- package/dist/downloader.js +0 -564
- package/dist/downloader.js.map +0 -1
package/dist/core/downloader.js
CHANGED
|
@@ -1,38 +1,80 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
3
|
-
* Core downloader: multi-source racing chunked download
|
|
2
|
+
* Core downloader: multi-source racing with lease-based dynamic range splitting
|
|
4
3
|
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
4
|
+
* Strategy (see docs/plans/2026-09-16-lease-download-strategy.md):
|
|
5
|
+
* - The primary issues one Range request for the whole file (a single lease [0, total))
|
|
6
|
+
* - When a mirror joins, the second half of the slowest-ETA lease's remaining range
|
|
7
|
+
* is carved out for it
|
|
8
|
+
* - When a lease completes: 1) overtake a slow right neighbor that has written
|
|
9
|
+
* little; 2) otherwise take a free lease and resume from its cursor (zero waste);
|
|
10
|
+
* 3) otherwise idle (never exit) until every lease is done
|
|
11
|
+
* - Slow-source replacement: abort -> lease goes to the free pool (partial file
|
|
12
|
+
* kept) -> the fastest source seamlessly takes over and resumes
|
|
13
|
+
* - Servers without Range support: whole-file racing (single-stream), first wins
|
|
14
|
+
*
|
|
15
|
+
* probe -> initial lease -> warmup -> decide mirrors -> split/takeover
|
|
16
|
+
* -> replace -> merge -> verify -> commit
|
|
8
17
|
*/
|
|
9
18
|
import * as fs from 'node:fs';
|
|
10
19
|
import * as path from 'node:path';
|
|
11
20
|
import { createWriteStream } from 'node:fs';
|
|
12
21
|
import { mergeConfig, validateConfig, isImmediateFailure, REPLACEMENT_CONFIG, } from '../config.js';
|
|
13
|
-
import { calculateChunkSize, calculateChunkCount, createChunkRanges, } from '../chunks/strategy.js';
|
|
14
22
|
import { SpeedMonitor } from '../speed/speed-monitor.js';
|
|
15
23
|
import { probe, openRangeStream, consumeBody, HttpError } from './http.js';
|
|
16
|
-
import { cleanupOldTempFiles,
|
|
24
|
+
import { cleanupOldTempFiles, buildLeaseTempPath, findResumableMerged, mergeChunks, commitMerged, removeAllTempFiles, removeNonMergedTempFiles, } from './temp.js';
|
|
17
25
|
import { validateFile } from '../utils/validator.js';
|
|
18
|
-
import {
|
|
26
|
+
import { getMergedTempFileName } from '../utils/hash.js';
|
|
27
|
+
import { t } from '../i18n.js';
|
|
28
|
+
const DEFAULT_MIN_SLICE = 4 * 1024 * 1024;
|
|
29
|
+
const DEFAULT_OVERTAKE_RATIO = 0.5;
|
|
19
30
|
function sleep(ms) {
|
|
20
31
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
21
32
|
}
|
|
33
|
+
/** Lease boundary shrank after a split: the owner stops the request and re-issues it */
|
|
34
|
+
class LeaseResizedError extends Error {
|
|
35
|
+
}
|
|
36
|
+
/** Grace period before taking over a released lease, so the previous writer settles */
|
|
37
|
+
const HANDOFF_QUIET_MS = 400;
|
|
38
|
+
function minSliceOf(cfg) {
|
|
39
|
+
return cfg.minSliceSize ?? DEFAULT_MIN_SLICE;
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* Average throughput including in-flight lease bytes; independent of the 1s
|
|
43
|
+
* sampler so short downloads and fresh starts still get a usable signal.
|
|
44
|
+
*/
|
|
45
|
+
function avgSpeed(w) {
|
|
46
|
+
const total = w.bytesWritten + (w.current ? w.current.downloaded - w.current.base : 0);
|
|
47
|
+
return total / Math.max(1, Date.now() - w.startedAt);
|
|
48
|
+
}
|
|
49
|
+
/** Current cursor of a lease: the file size is the safe resume point */
|
|
50
|
+
function leaseCursor(lease) {
|
|
51
|
+
const size = (() => {
|
|
52
|
+
try {
|
|
53
|
+
return fs.statSync(lease.tempPath).size;
|
|
54
|
+
}
|
|
55
|
+
catch {
|
|
56
|
+
return 0;
|
|
57
|
+
}
|
|
58
|
+
})();
|
|
59
|
+
return lease.start + Math.min(size, lease.end - lease.start + 1);
|
|
60
|
+
}
|
|
22
61
|
/**
|
|
23
|
-
*
|
|
24
|
-
* Multi-source racing
|
|
62
|
+
* Multi-source racing lease-based download
|
|
63
|
+
* Multi-source racing lease-based download
|
|
25
64
|
*/
|
|
26
65
|
export async function download(options) {
|
|
27
66
|
const startedAt = Date.now();
|
|
28
|
-
const { url, output
|
|
67
|
+
const { url, output } = options;
|
|
29
68
|
if (!url)
|
|
30
|
-
throw new Error('url is required
|
|
69
|
+
throw new Error(t('url is required'));
|
|
31
70
|
if (!output)
|
|
32
|
-
throw new Error('output is required
|
|
71
|
+
throw new Error(t('output is required'));
|
|
72
|
+
// Dedupe mirrors: drop duplicates of the primary and of each other (trim, keep order)
|
|
73
|
+
const primary = url.trim();
|
|
74
|
+
const mirrors = [...new Set((options.mirrors ?? []).map((m) => m.trim()))].filter((m) => m.length > 0 && m !== primary);
|
|
33
75
|
const cfg = mergeConfig(options.options);
|
|
34
76
|
validateConfig(cfg);
|
|
35
|
-
const result = await runDownload(
|
|
77
|
+
const result = await runDownload(primary, output, mirrors, cfg);
|
|
36
78
|
const final = { ...result, duration: Date.now() - startedAt };
|
|
37
79
|
if (cfg.onComplete)
|
|
38
80
|
cfg.onComplete(final);
|
|
@@ -43,16 +85,16 @@ async function runDownload(url, output, mirrors, cfg) {
|
|
|
43
85
|
const outPath = path.resolve(output);
|
|
44
86
|
const dir = path.dirname(outPath);
|
|
45
87
|
fs.mkdirSync(dir, { recursive: true });
|
|
46
|
-
//
|
|
88
|
+
// clean stale temp files
|
|
47
89
|
cleanupOldTempFiles(outPath);
|
|
48
|
-
//
|
|
90
|
+
// probe primary
|
|
49
91
|
const primaryProbe = await probe(url, {
|
|
50
92
|
connectTimeout: cfg.connectTimeout,
|
|
51
93
|
timeout: cfg.timeout,
|
|
52
94
|
});
|
|
53
95
|
const contentLength = primaryProbe.contentLength;
|
|
54
96
|
const supportsRange = primaryProbe.supportsRange;
|
|
55
|
-
//
|
|
97
|
+
// resume: output already complete, or a resumable merged temp file exists
|
|
56
98
|
if (contentLength !== undefined) {
|
|
57
99
|
try {
|
|
58
100
|
if (fs.statSync(outPath).size === contentLength) {
|
|
@@ -60,63 +102,71 @@ async function runDownload(url, output, mirrors, cfg) {
|
|
|
60
102
|
}
|
|
61
103
|
}
|
|
62
104
|
catch {
|
|
63
|
-
//
|
|
105
|
+
// not there yet
|
|
64
106
|
}
|
|
65
107
|
const merged = findResumableMerged(outPath, contentLength);
|
|
66
108
|
if (merged && (await tryResumeMerged(merged, outPath, cfg))) {
|
|
67
109
|
return summarize(outPath, contentLength, [], startedAt, []);
|
|
68
110
|
}
|
|
69
111
|
}
|
|
70
|
-
// 2)
|
|
71
|
-
const
|
|
112
|
+
// 2) initial lease: the primary requests the whole file (no Range -> single-stream)
|
|
113
|
+
const leases = [];
|
|
72
114
|
if (supportsRange && contentLength !== undefined && contentLength > 0) {
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
start,
|
|
82
|
-
end,
|
|
83
|
-
size: end - start + 1,
|
|
84
|
-
tempPath: tempPaths[i].tempPath,
|
|
85
|
-
status: 'pending',
|
|
86
|
-
failures: 0,
|
|
87
|
-
});
|
|
88
|
-
}
|
|
115
|
+
leases.push({
|
|
116
|
+
id: 0,
|
|
117
|
+
start: 0,
|
|
118
|
+
end: contentLength - 1,
|
|
119
|
+
tempPath: buildLeaseTempPath(outPath, 0),
|
|
120
|
+
status: 'leased',
|
|
121
|
+
owner: 'primary',
|
|
122
|
+
});
|
|
89
123
|
}
|
|
90
124
|
const ctx = {
|
|
91
125
|
output: outPath,
|
|
92
126
|
contentLength,
|
|
93
|
-
|
|
127
|
+
leases,
|
|
128
|
+
nextLeaseId: leases.length,
|
|
129
|
+
maxLeases: typeof cfg.chunks === 'number' ? Math.max(1, cfg.chunks) : Number.POSITIVE_INFINITY,
|
|
94
130
|
workers: [],
|
|
95
131
|
cfg,
|
|
96
132
|
done: false,
|
|
97
133
|
mirrorsStarted: false,
|
|
98
134
|
primaryFailedImmediately: false,
|
|
99
135
|
};
|
|
100
|
-
// 是否已原子提交到最终输出(决定失败时是否清理临时文件)
|
|
101
136
|
// whether the output was atomically committed (drives failure cleanup)
|
|
102
137
|
let committed = false;
|
|
103
138
|
const emitProgress = () => {
|
|
104
139
|
if (!cfg.onProgress)
|
|
105
140
|
return;
|
|
106
|
-
const sources = ctx.workers.map((w) =>
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
141
|
+
const sources = ctx.workers.map((w) => {
|
|
142
|
+
const completed = ctx.leases.filter((l) => l.status === 'done' && l.owner === w.id);
|
|
143
|
+
return {
|
|
144
|
+
id: w.id,
|
|
145
|
+
url: w.url,
|
|
146
|
+
downloadedBytes: w.bytesWritten,
|
|
147
|
+
speed: w.monitor.getCurrentSpeed(),
|
|
148
|
+
status: w.status,
|
|
149
|
+
completedRanges: completed.length > 0 ? completed.map((l) => ({ start: l.start, end: l.end })) : undefined,
|
|
150
|
+
activeRange: w.current
|
|
151
|
+
? { start: w.current.start, end: w.current.end, downloaded: w.current.downloaded }
|
|
152
|
+
: undefined,
|
|
153
|
+
};
|
|
154
|
+
});
|
|
155
|
+
// downloaded bytes: actual temp file sizes (smooth; counts partial writes across takeovers)
|
|
156
|
+
const downloadedBytes = ctx.leases.length > 0
|
|
157
|
+
? ctx.leases.reduce((sum, l) => {
|
|
158
|
+
let size = 0;
|
|
159
|
+
try {
|
|
160
|
+
size = fs.statSync(l.tempPath).size;
|
|
161
|
+
}
|
|
162
|
+
catch {
|
|
163
|
+
// ignore
|
|
164
|
+
}
|
|
165
|
+
return sum + Math.min(size, l.end - l.start + 1);
|
|
166
|
+
}, 0)
|
|
115
167
|
: ctx.workers.reduce((sum, w) => sum + w.bytesWritten, 0);
|
|
116
168
|
const total = ctx.contentLength ?? downloadedBytes;
|
|
117
|
-
//
|
|
118
|
-
// instant speed; falls back to average when sampling hasn't caught up (e.g.
|
|
119
|
-
// a fast download finishing before the 1s sampler ticks)
|
|
169
|
+
// instant speed; fall back to the average while the sampler lags (avoids 0 B/s)
|
|
120
170
|
const instant = ctx.workers.reduce((sum, w) => sum + w.monitor.getCurrentSpeed(), 0);
|
|
121
171
|
const elapsedMs = Math.max(1, Date.now() - startedAt);
|
|
122
172
|
const speed = instant > 0 ? instant : downloadedBytes > 0 ? (downloadedBytes / elapsedMs) * 1000 : 0;
|
|
@@ -130,18 +180,18 @@ async function runDownload(url, output, mirrors, cfg) {
|
|
|
130
180
|
};
|
|
131
181
|
cfg.onProgress(progress);
|
|
132
182
|
};
|
|
133
|
-
// 3)
|
|
183
|
+
// 3) mirror-start decision (unchanged speed-threshold logic)
|
|
134
184
|
const shouldStartMirrors = () => {
|
|
135
185
|
const primarySpeed = primaryWorker.monitor.getCurrentSpeed();
|
|
136
186
|
const size = ctx.contentLength ?? 0;
|
|
137
187
|
if (primarySpeed >= (cfg.fastPrimaryThreshold ?? 512 * 1024 * 4))
|
|
138
188
|
return false;
|
|
139
|
-
if (size < (cfg.sizeThreshold ??
|
|
189
|
+
if (size < (cfg.sizeThreshold ?? 1024 * 1024))
|
|
140
190
|
return false;
|
|
141
191
|
if (primarySpeed < (cfg.speedThreshold ?? 512 * 1024))
|
|
142
192
|
return true;
|
|
143
|
-
const
|
|
144
|
-
if (size > 0 &&
|
|
193
|
+
const doneBytes = ctx.leases.reduce((s, l) => s + (l.status === 'done' ? l.end - l.start + 1 : 0), 0);
|
|
194
|
+
if (size > 0 && doneBytes / size < 0.3 && primarySpeed < (cfg.speedThreshold ?? 512 * 1024) * 1.5)
|
|
145
195
|
return true;
|
|
146
196
|
return false;
|
|
147
197
|
};
|
|
@@ -169,7 +219,7 @@ async function runDownload(url, output, mirrors, cfg) {
|
|
|
169
219
|
const maybeStartMirrors = () => {
|
|
170
220
|
if (ctx.done || ctx.mirrorsStarted || mirrors.length === 0)
|
|
171
221
|
return;
|
|
172
|
-
//
|
|
222
|
+
// immediate primary failure → start mirrors regardless of size
|
|
173
223
|
if (ctx.primaryFailedImmediately) {
|
|
174
224
|
startMirrors();
|
|
175
225
|
return;
|
|
@@ -179,12 +229,15 @@ async function runDownload(url, output, mirrors, cfg) {
|
|
|
179
229
|
};
|
|
180
230
|
const startWorker = (w) => {
|
|
181
231
|
ctx.workers.push(w);
|
|
182
|
-
|
|
232
|
+
// Sample live lease increments minus the takeover baseline (only bytes this
|
|
233
|
+
// source added), otherwise bytesWritten is constant until a lease completes
|
|
234
|
+
// and the sampled speed stays at zero.
|
|
235
|
+
w.monitor.startSampling(() => w.bytesWritten + (w.current ? w.current.downloaded - w.current.base : 0));
|
|
183
236
|
void runWorker(w, ctx, emitProgress).catch((err) => {
|
|
184
237
|
w.error = err;
|
|
185
238
|
w.status = 'failed';
|
|
186
239
|
ctx.lastError = ctx.lastError ?? err;
|
|
187
|
-
//
|
|
240
|
+
// any HTTP 4xx/5xx or network-level failure -> start mirrors early
|
|
188
241
|
if (w.isPrimary &&
|
|
189
242
|
(err instanceof HttpError ||
|
|
190
243
|
isImmediateFailure(err, err.status))) {
|
|
@@ -193,7 +246,7 @@ async function runDownload(url, output, mirrors, cfg) {
|
|
|
193
246
|
}
|
|
194
247
|
});
|
|
195
248
|
};
|
|
196
|
-
//
|
|
249
|
+
// start primary
|
|
197
250
|
const primaryWorker = {
|
|
198
251
|
id: 'primary',
|
|
199
252
|
url,
|
|
@@ -208,88 +261,123 @@ async function runDownload(url, output, mirrors, cfg) {
|
|
|
208
261
|
status: 'active',
|
|
209
262
|
};
|
|
210
263
|
startWorker(primaryWorker);
|
|
211
|
-
//
|
|
264
|
+
// decide after warmup
|
|
212
265
|
const warmupTimer = setTimeout(() => maybeStartMirrors(), cfg.primaryWarmupTime ?? 5000);
|
|
213
|
-
// 6)
|
|
266
|
+
// 6) slow-source replacement: abort the slow source, its lease goes to the
|
|
267
|
+
// free pool and the fastest source seamlessly resumes from its cursor.
|
|
268
|
+
// Only lease holders are compared; idle sources are exempt (speed 0 does
|
|
269
|
+
// not mean slow). With a single holder plus idle waiters, hand over when
|
|
270
|
+
// the holder is below the absolute speedThreshold.
|
|
214
271
|
const replaceCheck = setInterval(() => {
|
|
215
272
|
if (ctx.done)
|
|
216
273
|
return;
|
|
217
274
|
const active = ctx.workers.filter((w) => w.status === 'active' && !w.aborted);
|
|
218
|
-
|
|
275
|
+
const hasLease = (w) => ctx.leases.some((l) => l.status === 'leased' && l.owner === w.id);
|
|
276
|
+
const holders = active.filter(hasLease);
|
|
277
|
+
const idle = active.filter((w) => !hasLease(w));
|
|
278
|
+
if (holders.length === 0)
|
|
219
279
|
return;
|
|
220
|
-
const
|
|
221
|
-
if (
|
|
280
|
+
const minReplaceTime = cfg.minReplaceTime ?? 30_000;
|
|
281
|
+
if (Date.now() - startedAt < minReplaceTime)
|
|
222
282
|
return;
|
|
223
|
-
const
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
if (
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
283
|
+
const condemn = (w) => {
|
|
284
|
+
w.aborted = true;
|
|
285
|
+
w.status = 'failed';
|
|
286
|
+
w.controller.abort();
|
|
287
|
+
const lease = ctx.leases.find((l) => l.owner === w.id && l.status === 'leased');
|
|
288
|
+
if (lease) {
|
|
289
|
+
lease.status = 'free';
|
|
290
|
+
lease.owner = undefined;
|
|
291
|
+
lease.releasedAt = Date.now(); // let the slow writer settle before takeover
|
|
292
|
+
}
|
|
293
|
+
emitProgress();
|
|
294
|
+
};
|
|
295
|
+
if (holders.length >= 2) {
|
|
296
|
+
const fastest = holders.reduce((a, b) => (avgSpeed(a) > avgSpeed(b) ? a : b));
|
|
297
|
+
const fastestSpeed = avgSpeed(fastest);
|
|
298
|
+
if (fastestSpeed <= 0)
|
|
299
|
+
return;
|
|
300
|
+
for (const w of holders) {
|
|
301
|
+
if (w === fastest)
|
|
302
|
+
continue;
|
|
303
|
+
const ratio = avgSpeed(w) / fastestSpeed;
|
|
304
|
+
if (ratio < REPLACEMENT_CONFIG.slowThresholdRatio) {
|
|
305
|
+
w.slowChecks += 1;
|
|
306
|
+
if (w.slowChecks >= REPLACEMENT_CONFIG.maxSlowChecks)
|
|
307
|
+
condemn(w);
|
|
308
|
+
}
|
|
309
|
+
else {
|
|
310
|
+
w.slowChecks = 0;
|
|
239
311
|
}
|
|
240
312
|
}
|
|
313
|
+
}
|
|
314
|
+
else if (idle.length > 0) {
|
|
315
|
+
// single holder + idle waiters: below the absolute threshold -> hand over
|
|
316
|
+
const holder = holders[0];
|
|
317
|
+
if (avgSpeed(holder) < (cfg.speedThreshold ?? 512 * 1024)) {
|
|
318
|
+
holder.slowChecks += 1;
|
|
319
|
+
if (holder.slowChecks >= REPLACEMENT_CONFIG.maxSlowChecks)
|
|
320
|
+
condemn(holder);
|
|
321
|
+
}
|
|
241
322
|
else {
|
|
242
|
-
|
|
323
|
+
holder.slowChecks = 0;
|
|
243
324
|
}
|
|
244
325
|
}
|
|
245
326
|
}, cfg.replaceCheckInterval ?? 10_000);
|
|
246
|
-
//
|
|
327
|
+
// progress tick
|
|
247
328
|
const progressTimer = setInterval(emitProgress, 1000);
|
|
248
329
|
try {
|
|
249
|
-
//
|
|
330
|
+
// wait for completion
|
|
250
331
|
await waitForCompletion(ctx, cfg);
|
|
251
332
|
if (!ctx.done) {
|
|
252
|
-
throw ctx.lastError ?? new Error('Download failed: no source could complete
|
|
333
|
+
throw ctx.lastError ?? new Error(t('Download failed: no source could complete'));
|
|
253
334
|
}
|
|
254
|
-
//
|
|
255
|
-
// workers exit without updating status; mark them completed here
|
|
335
|
+
// mark workers completed
|
|
256
336
|
for (const w of ctx.workers) {
|
|
257
337
|
if (w.status === 'active')
|
|
258
338
|
w.status = 'completed';
|
|
259
339
|
}
|
|
260
|
-
//
|
|
340
|
+
// integrity check + merge
|
|
261
341
|
const mergedPath = path.join(dir, getMergedTempFileName(outPath));
|
|
262
|
-
if (ctx.
|
|
263
|
-
const
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
342
|
+
if (ctx.leases.length > 0) {
|
|
343
|
+
const sorted = [...ctx.leases].sort((a, b) => a.start - b.start);
|
|
344
|
+
let pos = 0;
|
|
345
|
+
for (const l of sorted) {
|
|
346
|
+
if (l.status !== 'done') {
|
|
347
|
+
throw new Error(`${t('Lease not done')}: ${l.id}`);
|
|
348
|
+
}
|
|
349
|
+
if (l.start !== pos) {
|
|
350
|
+
throw new Error(`${t('Lease gap')}: expected start ${pos}, got ${l.start}`);
|
|
351
|
+
}
|
|
352
|
+
pos = l.end + 1;
|
|
353
|
+
}
|
|
354
|
+
if (ctx.contentLength !== undefined && pos !== ctx.contentLength) {
|
|
355
|
+
throw new Error(`${t('Lease coverage mismatch')}: ${pos}/${ctx.contentLength}`);
|
|
268
356
|
}
|
|
269
|
-
await mergeChunks(
|
|
357
|
+
await mergeChunks(sorted.map((l) => ({ start: l.start, end: l.end, tempPath: l.tempPath })), mergedPath);
|
|
270
358
|
}
|
|
271
359
|
else {
|
|
272
|
-
//
|
|
360
|
+
// single-stream: winner temp file is the result
|
|
273
361
|
const winner = ctx.singleWinnerPath;
|
|
274
362
|
if (!winner || !fs.existsSync(winner)) {
|
|
275
|
-
throw new Error('No download produced output
|
|
363
|
+
throw new Error(t('No download produced output'));
|
|
276
364
|
}
|
|
277
365
|
fs.renameSync(winner, mergedPath);
|
|
278
366
|
}
|
|
279
|
-
//
|
|
367
|
+
// verify checksum
|
|
280
368
|
if (cfg.checksum) {
|
|
281
369
|
const validation = await validateFile(mergedPath, cfg.checksum, cfg.checksumAlgorithm ?? 'sha256');
|
|
282
370
|
if (!validation.valid) {
|
|
283
|
-
throw new Error(
|
|
371
|
+
throw new Error(`${t('Checksum mismatch')}: expected ${cfg.checksum}, got ${validation.actual ?? 'n/a'}`);
|
|
284
372
|
}
|
|
285
373
|
}
|
|
286
|
-
//
|
|
374
|
+
// atomic commit
|
|
287
375
|
commitMerged(mergedPath, outPath);
|
|
288
376
|
committed = true;
|
|
289
|
-
//
|
|
377
|
+
// cleanup
|
|
290
378
|
removeAllTempFiles(outPath);
|
|
291
379
|
const finalSize = contentLength ?? fs.statSync(outPath).size;
|
|
292
|
-
return summarize(outPath, finalSize, ctx.workers, startedAt, ctx.
|
|
380
|
+
return summarize(outPath, finalSize, ctx.workers, startedAt, ctx.leases);
|
|
293
381
|
}
|
|
294
382
|
finally {
|
|
295
383
|
clearTimeout(warmupTimer);
|
|
@@ -297,17 +385,16 @@ async function runDownload(url, output, mirrors, cfg) {
|
|
|
297
385
|
clearInterval(progressTimer);
|
|
298
386
|
for (const w of ctx.workers) {
|
|
299
387
|
w.controller.abort();
|
|
300
|
-
w.monitor.stopSampling(); //
|
|
388
|
+
w.monitor.stopSampling(); // stop speed-sampling timers (leak fix)
|
|
301
389
|
}
|
|
302
|
-
//
|
|
303
|
-
//
|
|
304
|
-
//
|
|
305
|
-
// single-stream temp files, keep merged for resume.
|
|
390
|
+
// Not committed (download/merge/checksum failure): remove range and
|
|
391
|
+
// single-stream temp files, keep merged for resume. Success already ran
|
|
392
|
+
// removeAllTempFiles.
|
|
306
393
|
if (!committed)
|
|
307
394
|
removeNonMergedTempFiles(outPath);
|
|
308
395
|
}
|
|
309
396
|
}
|
|
310
|
-
/**
|
|
397
|
+
/** try to resume from an existing merged temp file */
|
|
311
398
|
async function tryResumeMerged(mergedPath, outPath, cfg) {
|
|
312
399
|
try {
|
|
313
400
|
if (cfg.checksum) {
|
|
@@ -322,232 +409,365 @@ async function tryResumeMerged(mergedPath, outPath, cfg) {
|
|
|
322
409
|
return false;
|
|
323
410
|
}
|
|
324
411
|
}
|
|
325
|
-
/**
|
|
412
|
+
/** wait until all leases done or all workers failed */
|
|
326
413
|
async function waitForCompletion(ctx, cfg) {
|
|
327
414
|
const deadline = Date.now() + (cfg.timeout ?? 300_000) * 3;
|
|
328
415
|
while (Date.now() < deadline) {
|
|
329
416
|
if (ctx.done)
|
|
330
417
|
return;
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
// 单流模式:等某个 worker 完成(downloadSingle 会置 ctx.done)
|
|
334
|
-
if (!active && ctx.workers.length > 0) {
|
|
335
|
-
throw ctx.lastError ?? new Error('All sources failed / 所有源均失败');
|
|
336
|
-
}
|
|
337
|
-
}
|
|
338
|
-
else {
|
|
339
|
-
if (ctx.chunks.every((c) => c.status === 'done')) {
|
|
418
|
+
if (ctx.leases.length > 0) {
|
|
419
|
+
if (ctx.leases.every((l) => l.status === 'done')) {
|
|
340
420
|
ctx.done = true;
|
|
341
421
|
return;
|
|
342
422
|
}
|
|
343
|
-
|
|
344
|
-
|
|
423
|
+
const active = ctx.workers.some((w) => w.status === 'active' && !w.aborted);
|
|
424
|
+
if (!active) {
|
|
425
|
+
throw ctx.lastError ?? new Error(t('No active workers and leases pending'));
|
|
426
|
+
}
|
|
427
|
+
}
|
|
428
|
+
else {
|
|
429
|
+
// single-stream: wait for a worker to finish (downloadSingle sets ctx.done)
|
|
430
|
+
const active = ctx.workers.some((w) => w.status === 'active' && !w.aborted);
|
|
431
|
+
if (!active && ctx.workers.length > 0) {
|
|
432
|
+
throw ctx.lastError ?? new Error(t('All sources failed'));
|
|
345
433
|
}
|
|
346
434
|
}
|
|
347
435
|
await sleep(100);
|
|
348
436
|
}
|
|
349
|
-
throw new Error(
|
|
437
|
+
throw new Error(`${t('Download timeout')} (${(cfg.timeout ?? 300_000) * 3}ms)`);
|
|
350
438
|
}
|
|
351
|
-
/**
|
|
439
|
+
/** worker loop: hold, take over, or split leases */
|
|
352
440
|
async function runWorker(w, ctx, emit) {
|
|
353
441
|
const cfg = ctx.cfg;
|
|
354
|
-
//
|
|
355
|
-
if (ctx.
|
|
442
|
+
// single-stream (no Range): whole-file racing, first to finish wins
|
|
443
|
+
if (ctx.leases.length === 0) {
|
|
356
444
|
await downloadSingle(w, ctx, emit);
|
|
357
445
|
return;
|
|
358
446
|
}
|
|
359
447
|
for (;;) {
|
|
360
448
|
if (w.aborted || ctx.done)
|
|
361
449
|
return;
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
450
|
+
// 1) lease I already hold
|
|
451
|
+
let lease = ctx.leases.find((l) => l.status === 'leased' && l.owner === w.id);
|
|
452
|
+
// 2) free pool (leftmost gap first)
|
|
453
|
+
if (!lease)
|
|
454
|
+
lease = takeFreeLease(ctx, w);
|
|
455
|
+
// 3) overtake: right neighbor much slower and barely written -> abort, take over
|
|
456
|
+
if (!lease)
|
|
457
|
+
lease = tryOvertakeRightNeighbor(ctx, w);
|
|
458
|
+
// 4) split the slowest lease's second half (bounded by maxLeases)
|
|
459
|
+
if (!lease && ctx.leases.length < ctx.maxLeases)
|
|
460
|
+
lease = splitSlowestLease(ctx, w);
|
|
461
|
+
if (!lease) {
|
|
462
|
+
// nothing to do: stay alive waiting for splits/releases; exit only when all done
|
|
463
|
+
if (ctx.leases.every((l) => l.status === 'done')) {
|
|
365
464
|
w.status = 'completed';
|
|
465
|
+
emit();
|
|
366
466
|
return;
|
|
367
467
|
}
|
|
368
468
|
await sleep(150);
|
|
369
469
|
continue;
|
|
370
470
|
}
|
|
371
|
-
|
|
372
|
-
let
|
|
471
|
+
// download (with retries)
|
|
472
|
+
let ok = false;
|
|
473
|
+
let resized = false;
|
|
474
|
+
let writtenNow = 0;
|
|
373
475
|
for (let attempt = 0; attempt <= (cfg.retries ?? 0) && !w.aborted; attempt++) {
|
|
374
476
|
if (attempt > 0)
|
|
375
477
|
await sleep(cfg.retryDelay ?? 0);
|
|
376
478
|
try {
|
|
377
|
-
await
|
|
378
|
-
|
|
479
|
+
const r = await downloadLeaseToFile(w, lease, ctx);
|
|
480
|
+
writtenNow = r.written;
|
|
481
|
+
ok = r.done;
|
|
482
|
+
resized = !r.done; // lease shrank after a split; re-request the new boundary
|
|
379
483
|
break;
|
|
380
484
|
}
|
|
381
485
|
catch (err) {
|
|
382
486
|
const e = err;
|
|
487
|
+
if (w.aborted)
|
|
488
|
+
break; // replaced or aborted
|
|
383
489
|
if (isImmediateFailure(e, e.status))
|
|
384
|
-
break; //
|
|
490
|
+
break; // immediate failure: no retry
|
|
385
491
|
if (attempt === (cfg.retries ?? 0))
|
|
386
492
|
w.error = e;
|
|
387
493
|
}
|
|
388
494
|
}
|
|
389
|
-
if (
|
|
390
|
-
|
|
391
|
-
w.
|
|
495
|
+
if (ok) {
|
|
496
|
+
lease.status = 'done';
|
|
497
|
+
w.lastLeaseEnd = lease.end;
|
|
498
|
+
w.bytesWritten += writtenNow;
|
|
392
499
|
emit();
|
|
500
|
+
continue;
|
|
393
501
|
}
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
if (w.aborted) {
|
|
399
|
-
emit();
|
|
400
|
-
return; // 被替换/中止 / replaced or aborted
|
|
401
|
-
}
|
|
402
|
-
w.status = 'failed';
|
|
403
|
-
w.aborted = true;
|
|
404
|
-
w.controller.abort();
|
|
405
|
-
emit();
|
|
406
|
-
return;
|
|
502
|
+
// resize interruption and the lease is still mine -> resume the new boundary
|
|
503
|
+
if (resized && !w.aborted && lease.owner === w.id && lease.status === 'leased') {
|
|
504
|
+
w.bytesWritten += writtenNow;
|
|
505
|
+
continue;
|
|
407
506
|
}
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
if (w.isPrimary) {
|
|
413
|
-
for (const c of ctx.chunks) {
|
|
414
|
-
if (c.status === 'pending')
|
|
415
|
-
return c;
|
|
507
|
+
// failure: release only leases still owned by me (may have been taken over)
|
|
508
|
+
if (lease.owner === w.id) {
|
|
509
|
+
lease.status = 'free';
|
|
510
|
+
lease.owner = undefined;
|
|
416
511
|
}
|
|
512
|
+
emit();
|
|
513
|
+
if (w.aborted)
|
|
514
|
+
return; // replaced or aborted
|
|
515
|
+
w.status = 'failed';
|
|
516
|
+
w.aborted = true;
|
|
517
|
+
w.controller.abort();
|
|
518
|
+
emit();
|
|
519
|
+
return;
|
|
417
520
|
}
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
521
|
+
}
|
|
522
|
+
/** Take a free lease: leftmost gap first (sequential completion) */
|
|
523
|
+
function takeFreeLease(ctx, w) {
|
|
524
|
+
const frees = ctx.leases.filter((l) => l.status === 'free').sort((a, b) => a.start - b.start);
|
|
525
|
+
const lease = frees[0];
|
|
526
|
+
if (!lease)
|
|
527
|
+
return undefined;
|
|
528
|
+
lease.status = 'leased';
|
|
529
|
+
lease.owner = w.id;
|
|
530
|
+
return lease;
|
|
531
|
+
}
|
|
532
|
+
/**
|
|
533
|
+
* Overtake: abort a much slower right neighbor that has written little, take over its lease
|
|
534
|
+
*/
|
|
535
|
+
function tryOvertakeRightNeighbor(ctx, w) {
|
|
536
|
+
if (w.lastLeaseEnd === undefined)
|
|
537
|
+
return undefined;
|
|
538
|
+
const neighbor = ctx.leases.find((l) => l.status === 'leased' && l.start === w.lastLeaseEnd + 1 && l.owner && l.owner !== w.id);
|
|
539
|
+
if (!neighbor)
|
|
540
|
+
return undefined;
|
|
541
|
+
const other = ctx.workers.find((x) => x.id === neighbor.owner);
|
|
542
|
+
if (!other || other.aborted)
|
|
543
|
+
return undefined;
|
|
544
|
+
const mySpeed = avgSpeed(w);
|
|
545
|
+
const otherSpeed = avgSpeed(other);
|
|
546
|
+
const ratio = ctx.cfg.overtakeSpeedRatio ?? DEFAULT_OVERTAKE_RATIO;
|
|
547
|
+
const otherWritten = other.current?.downloaded ?? 0;
|
|
548
|
+
if (mySpeed <= 0)
|
|
549
|
+
return undefined;
|
|
550
|
+
if (otherSpeed >= mySpeed * ratio)
|
|
551
|
+
return undefined;
|
|
552
|
+
if (otherWritten >= minSliceOf(ctx.cfg))
|
|
553
|
+
return undefined;
|
|
554
|
+
// abort the neighbor; the lease stays leased with me as owner and resumes from its cursor
|
|
555
|
+
other.aborted = true;
|
|
556
|
+
other.status = 'failed';
|
|
557
|
+
other.controller.abort();
|
|
558
|
+
neighbor.owner = w.id;
|
|
559
|
+
neighbor.releasedAt = Date.now(); // let the previous writer settle before resuming
|
|
560
|
+
return neighbor;
|
|
561
|
+
}
|
|
562
|
+
/**
|
|
563
|
+
* Split the second half of the slowest-ETA lease's remaining range for a new worker
|
|
564
|
+
* Split the second half of the slowest-ETA lease for a new worker
|
|
565
|
+
*/
|
|
566
|
+
function splitSlowestLease(ctx, w) {
|
|
567
|
+
const minSlice = minSliceOf(ctx.cfg);
|
|
568
|
+
let best;
|
|
569
|
+
let bestEta = -1;
|
|
570
|
+
for (const l of ctx.leases) {
|
|
571
|
+
if (l.status !== 'leased' || !l.owner)
|
|
572
|
+
continue;
|
|
573
|
+
const cursor = leaseCursor(l);
|
|
574
|
+
const remaining = l.end + 1 - cursor;
|
|
575
|
+
if (remaining < minSlice * 2)
|
|
576
|
+
continue; // both halves must be >= minSlice
|
|
577
|
+
const owner = ctx.workers.find((x) => x.id === l.owner);
|
|
578
|
+
const speed = Math.max(1, owner ? owner.monitor.getCurrentSpeed() : 1);
|
|
579
|
+
const eta = remaining / speed;
|
|
580
|
+
if (eta > bestEta) {
|
|
581
|
+
bestEta = eta;
|
|
582
|
+
best = l;
|
|
422
583
|
}
|
|
423
584
|
}
|
|
424
|
-
|
|
585
|
+
if (!best)
|
|
586
|
+
return undefined;
|
|
587
|
+
const cursor = leaseCursor(best);
|
|
588
|
+
const remaining = best.end + 1 - cursor;
|
|
589
|
+
const mid = cursor + Math.floor(remaining / 2);
|
|
590
|
+
const lease = {
|
|
591
|
+
id: ctx.nextLeaseId++,
|
|
592
|
+
start: mid,
|
|
593
|
+
end: best.end,
|
|
594
|
+
tempPath: buildLeaseTempPath(ctx.output, mid),
|
|
595
|
+
status: 'leased',
|
|
596
|
+
owner: w.id,
|
|
597
|
+
};
|
|
598
|
+
best.end = mid - 1; // shrink the lease; the written prefix stays in its temp file
|
|
599
|
+
ctx.leases.push(lease);
|
|
600
|
+
return lease;
|
|
425
601
|
}
|
|
426
|
-
/**
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
602
|
+
/**
|
|
603
|
+
* Download a lease: resume from the existing temp file size through end
|
|
604
|
+
*
|
|
605
|
+
* Returns {done, written}: done=false means the boundary shrank; re-request it
|
|
606
|
+
*/
|
|
607
|
+
async function downloadLeaseToFile(w, lease, ctx) {
|
|
608
|
+
// taking over a released lease: wait for the previous writer to settle
|
|
609
|
+
if (lease.releasedAt !== undefined) {
|
|
610
|
+
const quiet = HANDOFF_QUIET_MS - (Date.now() - lease.releasedAt);
|
|
611
|
+
if (quiet > 0)
|
|
612
|
+
await sleep(quiet);
|
|
613
|
+
lease.releasedAt = undefined;
|
|
614
|
+
}
|
|
615
|
+
const leaseSize = lease.end - lease.start + 1;
|
|
616
|
+
const endAtStart = lease.end;
|
|
617
|
+
let have = 0;
|
|
437
618
|
try {
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
619
|
+
have = fs.statSync(lease.tempPath).size;
|
|
620
|
+
}
|
|
621
|
+
catch {
|
|
622
|
+
// not there yet
|
|
623
|
+
}
|
|
624
|
+
if (have > leaseSize)
|
|
625
|
+
have = 0; // corrupt leftover: rewrite
|
|
626
|
+
if (have >= leaseSize)
|
|
627
|
+
return { done: true, written: 0 }; // already complete
|
|
628
|
+
const reqStart = lease.start + have;
|
|
629
|
+
const append = have > 0;
|
|
630
|
+
w.current = { start: lease.start, end: lease.end, downloaded: have, base: have };
|
|
631
|
+
try {
|
|
632
|
+
const stream = await openRangeStream(w.url, reqStart, lease.end, {
|
|
633
|
+
connectTimeout: ctx.cfg.connectTimeout,
|
|
634
|
+
timeout: ctx.cfg.timeout,
|
|
635
|
+
signal: w.controller.signal,
|
|
636
|
+
});
|
|
637
|
+
const ws = createWriteStream(lease.tempPath, { flags: append ? 'a' : 'w' });
|
|
638
|
+
let written = have;
|
|
639
|
+
let skipped = 0;
|
|
640
|
+
const skipBytes = stream.supportsRange ? 0 : reqStart;
|
|
641
|
+
try {
|
|
642
|
+
await consumeBody(stream.body, async (value) => {
|
|
643
|
+
if (w.aborted)
|
|
644
|
+
throw new Error('Aborted');
|
|
645
|
+
// boundary shrank after a split: stop; the worker re-requests the new range
|
|
646
|
+
if (lease.end !== endAtStart)
|
|
647
|
+
throw new LeaseResizedError();
|
|
648
|
+
let buffer;
|
|
649
|
+
if (skipBytes > skipped) {
|
|
650
|
+
// server ignored Range: skip leading bytes
|
|
651
|
+
const remain = skipBytes - skipped;
|
|
652
|
+
if (value.byteLength <= remain) {
|
|
653
|
+
skipped += value.byteLength;
|
|
654
|
+
return;
|
|
655
|
+
}
|
|
656
|
+
buffer = Buffer.from(value.subarray(remain));
|
|
657
|
+
skipped = skipBytes;
|
|
448
658
|
}
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
}
|
|
452
|
-
else {
|
|
453
|
-
buffer = Buffer.from(value);
|
|
454
|
-
}
|
|
455
|
-
if (buffer.length > 0) {
|
|
456
|
-
if (!ws.write(buffer)) {
|
|
457
|
-
await new Promise((resolve) => ws.once('drain', resolve));
|
|
659
|
+
else {
|
|
660
|
+
buffer = Buffer.from(value);
|
|
458
661
|
}
|
|
459
|
-
|
|
662
|
+
if (buffer.length > 0) {
|
|
663
|
+
if (!ws.write(buffer)) {
|
|
664
|
+
await new Promise((resolve) => ws.once('drain', resolve));
|
|
665
|
+
}
|
|
666
|
+
written += buffer.length;
|
|
667
|
+
if (w.current)
|
|
668
|
+
w.current.downloaded = written; // live position progress
|
|
669
|
+
}
|
|
670
|
+
}, w.controller.signal);
|
|
671
|
+
ws.end();
|
|
672
|
+
await new Promise((resolve, reject) => {
|
|
673
|
+
ws.once('finish', resolve);
|
|
674
|
+
ws.once('error', reject);
|
|
675
|
+
});
|
|
676
|
+
}
|
|
677
|
+
catch (err) {
|
|
678
|
+
if (err instanceof LeaseResizedError) {
|
|
679
|
+
// deliberate shrink: flush what we have and let the worker re-request
|
|
680
|
+
stream.cancel();
|
|
681
|
+
ws.end();
|
|
682
|
+
await new Promise((resolve) => {
|
|
683
|
+
ws.once('finish', () => resolve());
|
|
684
|
+
ws.once('error', () => resolve());
|
|
685
|
+
});
|
|
686
|
+
return { done: false, written: written - have };
|
|
460
687
|
}
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
ws.once('finish', resolve);
|
|
465
|
-
ws.once('error', reject);
|
|
466
|
-
});
|
|
467
|
-
}
|
|
468
|
-
catch (err) {
|
|
469
|
-
ws.destroy();
|
|
470
|
-
stream.cancel();
|
|
471
|
-
try {
|
|
472
|
-
fs.unlinkSync(chunk.tempPath);
|
|
688
|
+
ws.destroy();
|
|
689
|
+
stream.cancel();
|
|
690
|
+
throw err;
|
|
473
691
|
}
|
|
474
|
-
|
|
475
|
-
|
|
692
|
+
// verify lease size
|
|
693
|
+
if (written < leaseSize) {
|
|
694
|
+
throw new Error(`${t('Lease incomplete')}: ${written}/${leaseSize} bytes`);
|
|
476
695
|
}
|
|
477
|
-
|
|
696
|
+
return { done: true, written: written - have };
|
|
478
697
|
}
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
if (written < expected) {
|
|
482
|
-
try {
|
|
483
|
-
fs.unlinkSync(chunk.tempPath);
|
|
484
|
-
}
|
|
485
|
-
catch {
|
|
486
|
-
// 忽略
|
|
487
|
-
}
|
|
488
|
-
throw new Error(`Chunk ${chunk.index} incomplete: ${written}/${expected} bytes / 分块不完整`);
|
|
698
|
+
finally {
|
|
699
|
+
w.current = undefined;
|
|
489
700
|
}
|
|
490
701
|
}
|
|
491
|
-
/**
|
|
702
|
+
/** no-Range mode: full-file racing */
|
|
492
703
|
async function downloadSingle(w, ctx, emit) {
|
|
493
|
-
const tempPath = path.join(path.dirname(ctx.output),
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
timeout: ctx.cfg.timeout,
|
|
497
|
-
signal: w.controller.signal,
|
|
498
|
-
});
|
|
499
|
-
const ws = createWriteStream(tempPath, { flags: 'w' });
|
|
500
|
-
let written = 0;
|
|
704
|
+
const tempPath = path.join(path.dirname(ctx.output), getMergedTempFileName(ctx.output) + '.single-' + w.id);
|
|
705
|
+
// end<=0 = unknown length, renderer falls back
|
|
706
|
+
w.current = { start: 0, end: ctx.contentLength ?? 0, downloaded: 0, base: 0 };
|
|
501
707
|
try {
|
|
502
|
-
await
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
if (!ws.write(buffer)) {
|
|
507
|
-
await new Promise((resolve) => ws.once('drain', resolve));
|
|
508
|
-
}
|
|
509
|
-
written += buffer.length;
|
|
510
|
-
}, w.controller.signal);
|
|
511
|
-
ws.end();
|
|
512
|
-
await new Promise((resolve, reject) => {
|
|
513
|
-
ws.once('finish', resolve);
|
|
514
|
-
ws.once('error', reject);
|
|
708
|
+
const stream = await openRangeStream(w.url, undefined, undefined, {
|
|
709
|
+
connectTimeout: ctx.cfg.connectTimeout,
|
|
710
|
+
timeout: ctx.cfg.timeout,
|
|
711
|
+
signal: w.controller.signal,
|
|
515
712
|
});
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
if (
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
713
|
+
const ws = createWriteStream(tempPath, { flags: 'w' });
|
|
714
|
+
let written = 0;
|
|
715
|
+
try {
|
|
716
|
+
await consumeBody(stream.body, async (value) => {
|
|
717
|
+
if (w.aborted)
|
|
718
|
+
throw new Error('Aborted');
|
|
719
|
+
const buffer = Buffer.from(value);
|
|
720
|
+
if (!ws.write(buffer)) {
|
|
721
|
+
await new Promise((resolve) => ws.once('drain', resolve));
|
|
722
|
+
}
|
|
723
|
+
written += buffer.length;
|
|
724
|
+
if (w.current)
|
|
725
|
+
w.current.downloaded = written;
|
|
726
|
+
}, w.controller.signal);
|
|
727
|
+
ws.end();
|
|
728
|
+
await new Promise((resolve, reject) => {
|
|
729
|
+
ws.once('finish', resolve);
|
|
730
|
+
ws.once('error', reject);
|
|
731
|
+
});
|
|
732
|
+
w.bytesWritten = written;
|
|
733
|
+
w.status = 'completed';
|
|
734
|
+
if (!ctx.done) {
|
|
735
|
+
// first completed worker wins, abort others
|
|
736
|
+
ctx.done = true;
|
|
737
|
+
ctx.singleWinnerPath = tempPath;
|
|
738
|
+
for (const other of ctx.workers) {
|
|
739
|
+
if (other !== w && !other.aborted) {
|
|
740
|
+
other.aborted = true;
|
|
741
|
+
other.status = 'failed';
|
|
742
|
+
other.controller.abort();
|
|
743
|
+
}
|
|
527
744
|
}
|
|
528
745
|
}
|
|
746
|
+
emit();
|
|
747
|
+
}
|
|
748
|
+
catch (err) {
|
|
749
|
+
ws.destroy();
|
|
750
|
+
stream.cancel();
|
|
751
|
+
throw err;
|
|
529
752
|
}
|
|
530
|
-
emit();
|
|
531
753
|
}
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
stream.cancel();
|
|
535
|
-
throw err;
|
|
754
|
+
finally {
|
|
755
|
+
w.current = undefined;
|
|
536
756
|
}
|
|
537
757
|
}
|
|
538
|
-
/**
|
|
758
|
+
/** OO wrapper over download() */
|
|
539
759
|
export class Downloader {
|
|
540
760
|
options;
|
|
541
761
|
constructor(options) {
|
|
542
762
|
this.options = options;
|
|
543
763
|
}
|
|
544
|
-
/**
|
|
764
|
+
/** start download */
|
|
545
765
|
start() {
|
|
546
766
|
return download(this.options);
|
|
547
767
|
}
|
|
548
768
|
}
|
|
549
|
-
/**
|
|
550
|
-
function summarize(outPath, bytes, workers, startedAt,
|
|
769
|
+
/** summarize result */
|
|
770
|
+
function summarize(outPath, bytes, workers, startedAt, leases) {
|
|
551
771
|
const sources = workers.map((w) => ({
|
|
552
772
|
id: w.id,
|
|
553
773
|
url: w.url,
|
|
@@ -556,7 +776,7 @@ function summarize(outPath, bytes, workers, startedAt, chunks) {
|
|
|
556
776
|
averageSpeed: (w.bytesWritten / Math.max(1, Date.now() - w.startedAt)) * 1000,
|
|
557
777
|
status: w.status,
|
|
558
778
|
}));
|
|
559
|
-
const totalBytes =
|
|
779
|
+
const totalBytes = leases.reduce((s, l) => s + (l.status === 'done' ? l.end - l.start + 1 : 0), 0) || bytes;
|
|
560
780
|
const duration = Date.now() - startedAt;
|
|
561
781
|
return {
|
|
562
782
|
path: outPath,
|