kovence-cli 0.0.0-stage → 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +15 -0
- package/LICENSE +21 -0
- package/README.md +386 -2
- package/SECURITY.md +13 -0
- package/nd.js +700 -0
- package/package.json +62 -5
- package/providers.js +530 -0
- package/src/agent/history.js +157 -0
- package/src/agent/project_context.js +136 -0
- package/src/agent/resilience.js +100 -0
- package/src/agent/turn.js +782 -0
- package/src/cli/args.js +171 -0
- package/src/cli/sinks.js +326 -0
- package/src/commands/handlers.js +1280 -0
- package/src/config/memory.js +175 -0
- package/src/config/model_registry.js +123 -0
- package/src/config/paths.js +268 -0
- package/src/config/providers.js +209 -0
- package/src/config/sessions.js +201 -0
- package/src/permissions/engine.js +846 -0
- package/src/permissions/store.js +113 -0
- package/src/settings/schema.js +340 -0
- package/src/settings/store.js +136 -0
- package/src/tools/checkpoints.js +108 -0
- package/src/tools/download.js +513 -0
- package/src/tools/exec.js +155 -0
- package/src/tools/fs_walk.js +165 -0
- package/src/tools/net.js +467 -0
- package/src/tools/patch_git.js +205 -0
- package/src/tools/sudo.js +212 -0
- package/src/tools/tasks.js +127 -0
- package/src/tui/settings_panel.js +223 -0
- package/src/tui/text_width.js +218 -0
- package/src/tui/themes.js +143 -0
- package/store.js +107 -0
- package/tools.js +1267 -0
- package/tui.js +1634 -0
- package/ui.js +394 -0
|
@@ -0,0 +1,513 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
const crypto = require("node:crypto");
|
|
4
|
+
const fs = require("node:fs");
|
|
5
|
+
const http = require("node:http");
|
|
6
|
+
const https = require("node:https");
|
|
7
|
+
const path = require("node:path");
|
|
8
|
+
const { BROWSER_HEADERS, assertSafePublicUrl, formatNetworkError } = require("./net");
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* Robust resumable downloader:
|
|
12
|
+
* - writes to `<dest>.part` and atomically renames only after size/checksum verification
|
|
13
|
+
* - resumes with HTTP Range (+If-Range) after any drop, stall, 5xx/429 or reconnect
|
|
14
|
+
* - separate connect timeout and idle (stall) timeout instead of one fixed total timeout
|
|
15
|
+
* - jittered exponential backoff, IPv4 fallback, attempts only count when no progress was made
|
|
16
|
+
* - SSRF re-validation on every redirect hop, identity encoding so sizes stay exact
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
const RETRYABLE_CODES = new Set([
|
|
20
|
+
"ECONNRESET",
|
|
21
|
+
"ETIMEDOUT",
|
|
22
|
+
"ECONNREFUSED",
|
|
23
|
+
"ECONNABORTED",
|
|
24
|
+
"ENETUNREACH",
|
|
25
|
+
"EHOSTUNREACH",
|
|
26
|
+
"ENETDOWN",
|
|
27
|
+
"EAI_AGAIN",
|
|
28
|
+
"EPIPE",
|
|
29
|
+
"UND_ERR_SOCKET",
|
|
30
|
+
"ERR_STREAM_PREMATURE_CLOSE",
|
|
31
|
+
]);
|
|
32
|
+
|
|
33
|
+
const DEFAULTS = {
|
|
34
|
+
maxAttempts: 8,
|
|
35
|
+
connectTimeoutMs: 20000,
|
|
36
|
+
idleTimeoutMs: 45000,
|
|
37
|
+
retryBaseDelayMs: 1000,
|
|
38
|
+
maxRedirects: 8,
|
|
39
|
+
maxBytes: 10 * 1024 * 1024 * 1024,
|
|
40
|
+
};
|
|
41
|
+
|
|
42
|
+
class DownloadError extends Error {
|
|
43
|
+
constructor(
|
|
44
|
+
message,
|
|
45
|
+
{ retryable = false, aborted = false, resetPart = false, retryAfterMs = 0, phase = "", code = "" } = {}
|
|
46
|
+
) {
|
|
47
|
+
super(message);
|
|
48
|
+
this.name = "DownloadError";
|
|
49
|
+
this.retryable = retryable;
|
|
50
|
+
this.aborted = aborted;
|
|
51
|
+
this.resetPart = resetPart;
|
|
52
|
+
this.retryAfterMs = retryAfterMs;
|
|
53
|
+
this.phase = phase;
|
|
54
|
+
this.code = code;
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
function wrapNetworkError(err, phase = "") {
|
|
59
|
+
if (err instanceof DownloadError) return err;
|
|
60
|
+
const inner = Array.isArray(err?.errors) ? err.errors : [];
|
|
61
|
+
const codes = [err?.code, ...inner.map((e) => e?.code)].filter(Boolean);
|
|
62
|
+
const message = formatNetworkError(err);
|
|
63
|
+
const retryable =
|
|
64
|
+
codes.some((c) => RETRYABLE_CODES.has(c)) ||
|
|
65
|
+
/timed out|socket hang up|reset|closed|aborted|premature|network|EAI_AGAIN|ENETUNREACH/i.test(message);
|
|
66
|
+
return new DownloadError(message, { retryable, phase, code: codes[0] || "" });
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
function isRetryable(err) {
|
|
70
|
+
if (err instanceof DownloadError) return err.retryable && !err.aborted;
|
|
71
|
+
return /DNS resolution failed/i.test(err?.message || "");
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
function sleepAbortable(ms, signal) {
|
|
75
|
+
return new Promise((resolve, reject) => {
|
|
76
|
+
if (signal?.aborted) return reject(new DownloadError("Download aborted by user", { aborted: true }));
|
|
77
|
+
const timer = setTimeout(() => {
|
|
78
|
+
signal?.removeEventListener("abort", onAbort);
|
|
79
|
+
resolve();
|
|
80
|
+
}, ms);
|
|
81
|
+
const onAbort = () => {
|
|
82
|
+
clearTimeout(timer);
|
|
83
|
+
reject(new DownloadError("Download aborted by user", { aborted: true }));
|
|
84
|
+
};
|
|
85
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
86
|
+
});
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
function statSize(file) {
|
|
90
|
+
try {
|
|
91
|
+
return fs.statSync(file).size;
|
|
92
|
+
} catch {
|
|
93
|
+
return 0;
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
function closeFile(file) {
|
|
98
|
+
if (!file) return Promise.resolve();
|
|
99
|
+
if (file._kovenceClose) return file._kovenceClose;
|
|
100
|
+
file._kovenceClose = new Promise((resolve) => {
|
|
101
|
+
if (file.closed) return resolve();
|
|
102
|
+
file.once("close", resolve);
|
|
103
|
+
file.once("error", () => resolve());
|
|
104
|
+
try {
|
|
105
|
+
file.end();
|
|
106
|
+
} catch {
|
|
107
|
+
resolve();
|
|
108
|
+
}
|
|
109
|
+
});
|
|
110
|
+
return file._kovenceClose;
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* Issue one GET and resolve once response headers arrive (connect timeout enforced).
|
|
115
|
+
*/
|
|
116
|
+
function requestOnce(parsed, { headers, family, connectTimeoutMs, signal }) {
|
|
117
|
+
return new Promise((resolve, reject) => {
|
|
118
|
+
const client = parsed.protocol === "https:" ? https : http;
|
|
119
|
+
const req = client.request(parsed, {
|
|
120
|
+
method: "GET",
|
|
121
|
+
headers,
|
|
122
|
+
autoSelectFamily: true,
|
|
123
|
+
autoSelectFamilyAttemptTimeout: 3000,
|
|
124
|
+
...(family ? { family } : {}),
|
|
125
|
+
});
|
|
126
|
+
const connTimer = setTimeout(() => {
|
|
127
|
+
req.destroy(
|
|
128
|
+
new DownloadError(`Connection timed out after ${Math.round(connectTimeoutMs / 1000)}s`, {
|
|
129
|
+
retryable: true,
|
|
130
|
+
phase: "connect",
|
|
131
|
+
code: "ETIMEDOUT",
|
|
132
|
+
})
|
|
133
|
+
);
|
|
134
|
+
}, connectTimeoutMs);
|
|
135
|
+
|
|
136
|
+
const onAbort = () =>
|
|
137
|
+
req.destroy(signal.reason || new DownloadError("Download aborted by user", { aborted: true }));
|
|
138
|
+
if (signal?.aborted) onAbort();
|
|
139
|
+
else signal?.addEventListener("abort", onAbort, { once: true });
|
|
140
|
+
|
|
141
|
+
req.on("response", (res) => {
|
|
142
|
+
clearTimeout(connTimer);
|
|
143
|
+
resolve({ res, req });
|
|
144
|
+
});
|
|
145
|
+
req.on("error", (err) => {
|
|
146
|
+
clearTimeout(connTimer);
|
|
147
|
+
signal?.removeEventListener("abort", onAbort);
|
|
148
|
+
reject(wrapNetworkError(err, "connect"));
|
|
149
|
+
});
|
|
150
|
+
req.end();
|
|
151
|
+
});
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
/**
|
|
155
|
+
* Follow redirects manually so every hop passes SSRF validation.
|
|
156
|
+
*/
|
|
157
|
+
async function openResponse(startUrl, { headers, family, connectTimeoutMs, signal, allowPrivate, maxRedirects }) {
|
|
158
|
+
let current = startUrl;
|
|
159
|
+
let hopHeaders = headers;
|
|
160
|
+
for (let hop = 0; hop <= maxRedirects; hop++) {
|
|
161
|
+
const parsed = await assertSafePublicUrl(current, allowPrivate);
|
|
162
|
+
const { res } = await requestOnce(parsed, { headers: hopHeaders, family, connectTimeoutMs, signal });
|
|
163
|
+
if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
|
|
164
|
+
res.resume();
|
|
165
|
+
const next = new URL(res.headers.location, current).toString();
|
|
166
|
+
hopHeaders = { ...hopHeaders, Referer: current };
|
|
167
|
+
current = next;
|
|
168
|
+
continue;
|
|
169
|
+
}
|
|
170
|
+
return { res, finalUrl: current };
|
|
171
|
+
}
|
|
172
|
+
throw new DownloadError("Too many redirects", { retryable: false });
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
function checkDiskSpace(destDir, remainingBytes) {
|
|
176
|
+
if (!remainingBytes || typeof fs.statfsSync !== "function") return;
|
|
177
|
+
try {
|
|
178
|
+
const st = fs.statfsSync(destDir);
|
|
179
|
+
const free = st.bavail * st.bsize;
|
|
180
|
+
if (free < remainingBytes + 16 * 1024 * 1024) {
|
|
181
|
+
throw new DownloadError(
|
|
182
|
+
`Not enough disk space in ${destDir}: need ~${Math.ceil(remainingBytes / 1048576)} MB, free ${Math.floor(free / 1048576)} MB`,
|
|
183
|
+
{ retryable: false }
|
|
184
|
+
);
|
|
185
|
+
}
|
|
186
|
+
} catch (err) {
|
|
187
|
+
if (err instanceof DownloadError) throw err;
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
function hashFile(file, algorithm = "sha256") {
|
|
192
|
+
return new Promise((resolve, reject) => {
|
|
193
|
+
const hash = crypto.createHash(algorithm);
|
|
194
|
+
fs.createReadStream(file)
|
|
195
|
+
.on("data", (c) => hash.update(c))
|
|
196
|
+
.on("error", reject)
|
|
197
|
+
.on("end", () => resolve(hash.digest("hex")));
|
|
198
|
+
});
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
function moveFile(from, to) {
|
|
202
|
+
try {
|
|
203
|
+
fs.renameSync(from, to);
|
|
204
|
+
} catch (err) {
|
|
205
|
+
if (err.code !== "EXDEV") throw err;
|
|
206
|
+
fs.copyFileSync(from, to);
|
|
207
|
+
fs.rmSync(from, { force: true });
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
/**
|
|
212
|
+
* Download with resume, retry, stall detection, progress, and integrity verification.
|
|
213
|
+
*/
|
|
214
|
+
async function downloadResumable(url, destPath, options = {}) {
|
|
215
|
+
const opts = { ...DEFAULTS, ...options };
|
|
216
|
+
const { signal = null, allowPrivate = false, onProgress = null, sha256 = "", extraHeaders = {} } = opts;
|
|
217
|
+
const partPath = `${destPath}.part`;
|
|
218
|
+
const startedAt = Date.now();
|
|
219
|
+
const state = { family: undefined, validator: "", totalBytes: 0, samples: [], lastEmitAt: 0, attempt: 0 };
|
|
220
|
+
|
|
221
|
+
if (signal?.aborted) throw new DownloadError("Download aborted by user", { aborted: true });
|
|
222
|
+
await assertSafePublicUrl(url, allowPrivate);
|
|
223
|
+
fs.mkdirSync(path.dirname(destPath), { recursive: true });
|
|
224
|
+
|
|
225
|
+
const emit = (info, force = false) => {
|
|
226
|
+
if (!onProgress) return;
|
|
227
|
+
const now = Date.now();
|
|
228
|
+
if (!force && now - state.lastEmitAt < 100) return;
|
|
229
|
+
state.lastEmitAt = now;
|
|
230
|
+
onProgress({
|
|
231
|
+
maxAttempts: opts.maxAttempts,
|
|
232
|
+
attempt: Math.max(1, state.attempt),
|
|
233
|
+
totalBytes: state.totalBytes,
|
|
234
|
+
elapsedMs: now - startedAt,
|
|
235
|
+
...info,
|
|
236
|
+
});
|
|
237
|
+
};
|
|
238
|
+
|
|
239
|
+
const speedOf = (bytes, sessionStartBytes, sessionStartAt) => {
|
|
240
|
+
const now = Date.now();
|
|
241
|
+
state.samples.push({ t: now, b: bytes });
|
|
242
|
+
while (state.samples.length > 2 && now - state.samples[0].t > 4000) state.samples.shift();
|
|
243
|
+
const first = state.samples[0];
|
|
244
|
+
const dt = (now - first.t) / 1000;
|
|
245
|
+
if (dt >= 0.4) return Math.max(0, (bytes - first.b) / dt);
|
|
246
|
+
const sdt = Math.max(0.05, (now - sessionStartAt) / 1000);
|
|
247
|
+
return Math.max(0, (bytes - sessionStartBytes) / sdt);
|
|
248
|
+
};
|
|
249
|
+
|
|
250
|
+
const runAttempt = async () => {
|
|
251
|
+
let existing = statSize(partPath);
|
|
252
|
+
const ctl = new AbortController();
|
|
253
|
+
const onParent = () => ctl.abort(new DownloadError("Download aborted by user", { aborted: true }));
|
|
254
|
+
if (signal?.aborted) onParent();
|
|
255
|
+
else signal?.addEventListener("abort", onParent, { once: true });
|
|
256
|
+
let idleTimer = null;
|
|
257
|
+
let file = null;
|
|
258
|
+
const clearIdle = () => idleTimer && clearTimeout(idleTimer);
|
|
259
|
+
|
|
260
|
+
try {
|
|
261
|
+
emit({ status: "connecting", bytes: existing }, true);
|
|
262
|
+
const headers = {
|
|
263
|
+
...BROWSER_HEADERS,
|
|
264
|
+
Accept: "*/*",
|
|
265
|
+
"Accept-Encoding": "identity",
|
|
266
|
+
...extraHeaders,
|
|
267
|
+
};
|
|
268
|
+
if (existing > 0) {
|
|
269
|
+
headers.Range = `bytes=${existing}-`;
|
|
270
|
+
if (state.validator) headers["If-Range"] = state.validator;
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
const { res } = await openResponse(url, {
|
|
274
|
+
headers,
|
|
275
|
+
family: state.family,
|
|
276
|
+
connectTimeoutMs: opts.connectTimeoutMs,
|
|
277
|
+
signal: ctl.signal,
|
|
278
|
+
allowPrivate,
|
|
279
|
+
maxRedirects: opts.maxRedirects,
|
|
280
|
+
});
|
|
281
|
+
|
|
282
|
+
const status = res.statusCode;
|
|
283
|
+
if (status === 416) {
|
|
284
|
+
res.resume();
|
|
285
|
+
const m = /bytes \*\/(\d+)/.exec(String(res.headers["content-range"] || ""));
|
|
286
|
+
const total = m ? Number(m[1]) : 0;
|
|
287
|
+
if (existing > 0 && total && existing === total) {
|
|
288
|
+
state.totalBytes = total;
|
|
289
|
+
return { complete: true, size: existing, resumed: true };
|
|
290
|
+
}
|
|
291
|
+
throw new DownloadError("Partial file no longer matches server, restarting", {
|
|
292
|
+
retryable: true,
|
|
293
|
+
resetPart: true,
|
|
294
|
+
});
|
|
295
|
+
}
|
|
296
|
+
if (status < 200 || status >= 300) {
|
|
297
|
+
res.resume();
|
|
298
|
+
const retryAfter = Number(res.headers["retry-after"]);
|
|
299
|
+
const retryable = status === 408 || status === 425 || status === 429 || status >= 500;
|
|
300
|
+
throw new DownloadError(`HTTP ${status} ${res.statusMessage || ""}`.trim(), {
|
|
301
|
+
retryable,
|
|
302
|
+
retryAfterMs:
|
|
303
|
+
Number.isFinite(retryAfter) && retryAfter > 0 && retryAfter <= 60 ? retryAfter * 1000 : 0,
|
|
304
|
+
});
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
let resuming = false;
|
|
308
|
+
let total = 0;
|
|
309
|
+
if (status === 206) {
|
|
310
|
+
const m = /bytes (\d+)-(\d+)\/(\d+|\*)/.exec(String(res.headers["content-range"] || ""));
|
|
311
|
+
if (!m || Number(m[1]) !== existing) {
|
|
312
|
+
res.resume();
|
|
313
|
+
throw new DownloadError("Server returned unexpected byte range, restarting", {
|
|
314
|
+
retryable: true,
|
|
315
|
+
resetPart: true,
|
|
316
|
+
});
|
|
317
|
+
}
|
|
318
|
+
resuming = true;
|
|
319
|
+
total = m[3] === "*" ? 0 : Number(m[3]);
|
|
320
|
+
} else {
|
|
321
|
+
// 200: server ignored Range (or fresh download) -> start from zero
|
|
322
|
+
existing = 0;
|
|
323
|
+
total = Number(res.headers["content-length"]) || 0;
|
|
324
|
+
}
|
|
325
|
+
state.totalBytes = total || state.totalBytes;
|
|
326
|
+
state.validator = String(res.headers.etag || res.headers["last-modified"] || state.validator || "");
|
|
327
|
+
if (opts.maxBytes && total > opts.maxBytes) {
|
|
328
|
+
res.resume();
|
|
329
|
+
throw new DownloadError(
|
|
330
|
+
`File is larger than the ${Math.round(opts.maxBytes / 1048576)} MB download limit`,
|
|
331
|
+
{ retryable: false }
|
|
332
|
+
);
|
|
333
|
+
}
|
|
334
|
+
checkDiskSpace(path.dirname(destPath), total ? total - existing : 0);
|
|
335
|
+
|
|
336
|
+
file = fs.createWriteStream(partPath, { flags: resuming ? "a" : "w" });
|
|
337
|
+
let bytes = existing;
|
|
338
|
+
const sessionStartBytes = existing;
|
|
339
|
+
const sessionStartAt = Date.now();
|
|
340
|
+
state.samples = [{ t: sessionStartAt, b: bytes }];
|
|
341
|
+
const resumedFrom = resuming ? existing : 0;
|
|
342
|
+
|
|
343
|
+
const progressInfo = () => {
|
|
344
|
+
const bytesPerSec = Math.round(speedOf(bytes, sessionStartBytes, sessionStartAt));
|
|
345
|
+
const remaining = total > 0 ? Math.max(0, total - bytes) : 0;
|
|
346
|
+
return {
|
|
347
|
+
status: "downloading",
|
|
348
|
+
bytes,
|
|
349
|
+
percent: total > 0 ? Math.min(100, (bytes / total) * 100) : null,
|
|
350
|
+
bytesPerSec,
|
|
351
|
+
etaSec: total > 0 && bytesPerSec > 0 ? remaining / bytesPerSec : null,
|
|
352
|
+
resumedFrom,
|
|
353
|
+
};
|
|
354
|
+
};
|
|
355
|
+
|
|
356
|
+
await new Promise((resolve, reject) => {
|
|
357
|
+
let settled = false;
|
|
358
|
+
const fail = (err) => {
|
|
359
|
+
if (settled) return;
|
|
360
|
+
settled = true;
|
|
361
|
+
res.destroy();
|
|
362
|
+
reject(err);
|
|
363
|
+
};
|
|
364
|
+
const armIdle = () => {
|
|
365
|
+
clearIdle();
|
|
366
|
+
idleTimer = setTimeout(() => {
|
|
367
|
+
ctl.abort(
|
|
368
|
+
new DownloadError(
|
|
369
|
+
`Stalled: no data received for ${Math.round(opts.idleTimeoutMs / 1000)}s`,
|
|
370
|
+
{
|
|
371
|
+
retryable: true,
|
|
372
|
+
code: "ETIMEDOUT",
|
|
373
|
+
}
|
|
374
|
+
)
|
|
375
|
+
);
|
|
376
|
+
}, opts.idleTimeoutMs);
|
|
377
|
+
};
|
|
378
|
+
ctl.signal.addEventListener("abort", () => fail(ctl.signal.reason), { once: true });
|
|
379
|
+
if (ctl.signal.aborted) return fail(ctl.signal.reason);
|
|
380
|
+
|
|
381
|
+
armIdle();
|
|
382
|
+
res.on("data", (chunk) => {
|
|
383
|
+
bytes += chunk.length;
|
|
384
|
+
armIdle();
|
|
385
|
+
if (opts.maxBytes && bytes > opts.maxBytes) {
|
|
386
|
+
ctl.abort(
|
|
387
|
+
new DownloadError("Download exceeded the maximum allowed size", { retryable: false })
|
|
388
|
+
);
|
|
389
|
+
return;
|
|
390
|
+
}
|
|
391
|
+
emit(progressInfo(), false);
|
|
392
|
+
});
|
|
393
|
+
res.on("error", (err) => fail(ctl.signal.aborted ? ctl.signal.reason : wrapNetworkError(err)));
|
|
394
|
+
res.on("close", () => {
|
|
395
|
+
if (!res.complete) {
|
|
396
|
+
fail(
|
|
397
|
+
ctl.signal.aborted
|
|
398
|
+
? ctl.signal.reason
|
|
399
|
+
: new DownloadError("Connection closed before download completed", { retryable: true })
|
|
400
|
+
);
|
|
401
|
+
}
|
|
402
|
+
});
|
|
403
|
+
file.on("error", (err) =>
|
|
404
|
+
fail(new DownloadError(`Disk write failed: ${err.message}`, { retryable: false }))
|
|
405
|
+
);
|
|
406
|
+
file.on("finish", () => {
|
|
407
|
+
if (settled || !res.complete) return;
|
|
408
|
+
settled = true;
|
|
409
|
+
resolve();
|
|
410
|
+
});
|
|
411
|
+
res.pipe(file);
|
|
412
|
+
});
|
|
413
|
+
|
|
414
|
+
clearIdle();
|
|
415
|
+
await closeFile(file);
|
|
416
|
+
const size = statSize(partPath);
|
|
417
|
+
if (total > 0 && size !== total) {
|
|
418
|
+
throw new DownloadError(`Size mismatch: got ${size} of ${total} bytes`, {
|
|
419
|
+
retryable: true,
|
|
420
|
+
resetPart: size > total,
|
|
421
|
+
});
|
|
422
|
+
}
|
|
423
|
+
emit({ ...progressInfo(), bytes: size, percent: 100 }, true);
|
|
424
|
+
return { complete: true, size, resumed: resumedFrom > 0 };
|
|
425
|
+
} finally {
|
|
426
|
+
clearIdle();
|
|
427
|
+
signal?.removeEventListener("abort", onParent);
|
|
428
|
+
await closeFile(file);
|
|
429
|
+
}
|
|
430
|
+
};
|
|
431
|
+
|
|
432
|
+
let noProgressFailures = 0;
|
|
433
|
+
let totalTries = 0;
|
|
434
|
+
let everResumed = false;
|
|
435
|
+
let result = null;
|
|
436
|
+
|
|
437
|
+
while (true) {
|
|
438
|
+
totalTries += 1;
|
|
439
|
+
state.attempt = totalTries;
|
|
440
|
+
const sizeBefore = statSize(partPath);
|
|
441
|
+
try {
|
|
442
|
+
result = await runAttempt();
|
|
443
|
+
break;
|
|
444
|
+
} catch (err) {
|
|
445
|
+
if (signal?.aborted || err?.aborted) {
|
|
446
|
+
const saved = statSize(partPath);
|
|
447
|
+
throw new DownloadError(
|
|
448
|
+
`Download cancelled${saved ? ` (${Math.round(saved / 1024)} KB kept in ${path.basename(partPath)}; run again to resume)` : ""}`,
|
|
449
|
+
{ aborted: true }
|
|
450
|
+
);
|
|
451
|
+
}
|
|
452
|
+
const saved = statSize(partPath);
|
|
453
|
+
if (err.resetPart) fs.rmSync(partPath, { force: true });
|
|
454
|
+
const progressed = saved > sizeBefore;
|
|
455
|
+
noProgressFailures = progressed ? 0 : noProgressFailures + 1;
|
|
456
|
+
|
|
457
|
+
if (!isRetryable(err) || noProgressFailures >= opts.maxAttempts || totalTries >= opts.maxAttempts * 6) {
|
|
458
|
+
const keep = statSize(partPath);
|
|
459
|
+
const hint =
|
|
460
|
+
keep > 0
|
|
461
|
+
? ` Partial data (${(keep / 1048576).toFixed(2)} MB) kept at ${partPath}; call download_file again to resume.`
|
|
462
|
+
: "";
|
|
463
|
+
throw new DownloadError(`Download failed after ${totalTries} attempt(s): ${err.message}.${hint}`, {
|
|
464
|
+
retryable: false,
|
|
465
|
+
code: err.code,
|
|
466
|
+
});
|
|
467
|
+
}
|
|
468
|
+
if (err.phase === "connect" && state.family === undefined) state.family = 4;
|
|
469
|
+
if (saved > 0) everResumed = true;
|
|
470
|
+
|
|
471
|
+
const backoff = Math.min(15000, opts.retryBaseDelayMs * 2 ** Math.min(noProgressFailures, 4));
|
|
472
|
+
const delay =
|
|
473
|
+
(err.retryAfterMs || backoff) + Math.floor(Math.random() * Math.min(250, opts.retryBaseDelayMs));
|
|
474
|
+
emit(
|
|
475
|
+
{
|
|
476
|
+
status: "retrying",
|
|
477
|
+
bytes: statSize(partPath),
|
|
478
|
+
note: `${err.message} - retry in ${Math.ceil(delay / 1000)}s (${noProgressFailures}/${opts.maxAttempts})`,
|
|
479
|
+
bytesPerSec: 0,
|
|
480
|
+
},
|
|
481
|
+
true
|
|
482
|
+
);
|
|
483
|
+
await sleepAbortable(delay, signal);
|
|
484
|
+
}
|
|
485
|
+
}
|
|
486
|
+
|
|
487
|
+
if (sha256) {
|
|
488
|
+
const actual = await hashFile(partPath);
|
|
489
|
+
if (actual.toLowerCase() !== String(sha256).trim().toLowerCase()) {
|
|
490
|
+
fs.rmSync(partPath, { force: true });
|
|
491
|
+
throw new DownloadError(`SHA-256 mismatch: expected ${sha256}, got ${actual}. Corrupt file removed.`, {
|
|
492
|
+
retryable: false,
|
|
493
|
+
});
|
|
494
|
+
}
|
|
495
|
+
}
|
|
496
|
+
moveFile(partPath, destPath);
|
|
497
|
+
|
|
498
|
+
return {
|
|
499
|
+
bytes: result.size,
|
|
500
|
+
totalBytes: state.totalBytes || result.size,
|
|
501
|
+
elapsedMs: Date.now() - startedAt,
|
|
502
|
+
attempts: totalTries,
|
|
503
|
+
resumed: everResumed || result.resumed,
|
|
504
|
+
verified: Boolean(sha256),
|
|
505
|
+
};
|
|
506
|
+
}
|
|
507
|
+
|
|
508
|
+
module.exports = {
|
|
509
|
+
DownloadError,
|
|
510
|
+
DEFAULTS,
|
|
511
|
+
downloadResumable,
|
|
512
|
+
streamDownloadSafe: downloadResumable,
|
|
513
|
+
};
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
const { spawn } = require("node:child_process");
|
|
4
|
+
|
|
5
|
+
const SECRET_ENV_KEY_RE =
|
|
6
|
+
/(?:^|_)(API_KEY|SECRET|TOKEN|PASSWORD|PASSWD|PRIVATE_KEY|ACCESS_KEY|CREDENTIAL|AUTH)(?:$|_)/i;
|
|
7
|
+
|
|
8
|
+
const EXTRA_SCRUB_KEYS = new Set([
|
|
9
|
+
"OPENAI_API_KEY",
|
|
10
|
+
"ANTHROPIC_API_KEY",
|
|
11
|
+
"LITELLM_API_KEY",
|
|
12
|
+
"GOOGLE_API_KEY",
|
|
13
|
+
"BRAVE_API_KEY",
|
|
14
|
+
"TAVILY_API_KEY",
|
|
15
|
+
"API_KEY",
|
|
16
|
+
]);
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* Strip provider/API secrets from child process env so shell commands never expose user keys.
|
|
20
|
+
*/
|
|
21
|
+
function scrubChildEnv(baseEnv = process.env, extraKeys = []) {
|
|
22
|
+
const clean = {};
|
|
23
|
+
const extraSet = new Set(extraKeys.filter(Boolean).map((k) => String(k).trim()));
|
|
24
|
+
for (const [k, v] of Object.entries(baseEnv || {})) {
|
|
25
|
+
if (v === undefined) continue;
|
|
26
|
+
if (EXTRA_SCRUB_KEYS.has(k) || extraSet.has(k)) continue;
|
|
27
|
+
if (SECRET_ENV_KEY_RE.test(k) && !/^XDG_|^GNOME_|^DBUS_|^GPG_|^SSH_AUTH_SOCK$/i.test(k)) {
|
|
28
|
+
continue;
|
|
29
|
+
}
|
|
30
|
+
clean[k] = v;
|
|
31
|
+
}
|
|
32
|
+
return clean;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* Kill a child process group cleanly (POSIX negative PID), falling back to direct kill.
|
|
37
|
+
*/
|
|
38
|
+
function killProcessGroup(child, sig = "SIGTERM") {
|
|
39
|
+
if (!child || !child.pid) return;
|
|
40
|
+
try {
|
|
41
|
+
process.kill(-child.pid, sig);
|
|
42
|
+
} catch {
|
|
43
|
+
try {
|
|
44
|
+
child.kill(sig);
|
|
45
|
+
} catch {
|
|
46
|
+
// Process may already have exited.
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* Spawn shell command in its own process group with AbortSignal, timeout, and optional live streaming.
|
|
53
|
+
*/
|
|
54
|
+
function spawnShell(
|
|
55
|
+
command,
|
|
56
|
+
{
|
|
57
|
+
cwd = process.cwd(),
|
|
58
|
+
timeout = 15000,
|
|
59
|
+
stdinText = null,
|
|
60
|
+
signal = null,
|
|
61
|
+
env = null,
|
|
62
|
+
maxCapture = 64000,
|
|
63
|
+
onData = null,
|
|
64
|
+
} = {}
|
|
65
|
+
) {
|
|
66
|
+
return new Promise((resolve) => {
|
|
67
|
+
if (signal?.aborted) {
|
|
68
|
+
resolve({
|
|
69
|
+
code: "aborted",
|
|
70
|
+
stdout: "",
|
|
71
|
+
stderr: "Aborted before execution",
|
|
72
|
+
timedOut: false,
|
|
73
|
+
aborted: true,
|
|
74
|
+
});
|
|
75
|
+
return;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
const child = spawn("sh", ["-c", command], {
|
|
79
|
+
cwd,
|
|
80
|
+
env: env || scrubChildEnv(process.env),
|
|
81
|
+
detached: process.platform !== "win32",
|
|
82
|
+
stdio: ["pipe", "pipe", "pipe"],
|
|
83
|
+
});
|
|
84
|
+
|
|
85
|
+
let stdout = "";
|
|
86
|
+
let stderr = "";
|
|
87
|
+
let timedOut = false;
|
|
88
|
+
let aborted = false;
|
|
89
|
+
let settled = false;
|
|
90
|
+
let forceKillTimer = null;
|
|
91
|
+
|
|
92
|
+
const terminateTree = () => {
|
|
93
|
+
killProcessGroup(child, "SIGTERM");
|
|
94
|
+
forceKillTimer = setTimeout(() => {
|
|
95
|
+
killProcessGroup(child, "SIGKILL");
|
|
96
|
+
}, 300);
|
|
97
|
+
forceKillTimer.unref?.();
|
|
98
|
+
};
|
|
99
|
+
|
|
100
|
+
const onAbort = () => {
|
|
101
|
+
aborted = true;
|
|
102
|
+
terminateTree();
|
|
103
|
+
};
|
|
104
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
105
|
+
|
|
106
|
+
const timer = setTimeout(() => {
|
|
107
|
+
timedOut = true;
|
|
108
|
+
terminateTree();
|
|
109
|
+
}, timeout);
|
|
110
|
+
|
|
111
|
+
child.stdout.on("data", (chunk) => {
|
|
112
|
+
const text = chunk.toString();
|
|
113
|
+
onData?.("stdout", text);
|
|
114
|
+
if (stdout.length < maxCapture) stdout += text;
|
|
115
|
+
});
|
|
116
|
+
|
|
117
|
+
child.stderr.on("data", (chunk) => {
|
|
118
|
+
const text = chunk.toString();
|
|
119
|
+
onData?.("stderr", text);
|
|
120
|
+
if (stderr.length < maxCapture) stderr += text;
|
|
121
|
+
});
|
|
122
|
+
|
|
123
|
+
const finish = (code) => {
|
|
124
|
+
if (settled) return;
|
|
125
|
+
settled = true;
|
|
126
|
+
clearTimeout(timer);
|
|
127
|
+
if (forceKillTimer) clearTimeout(forceKillTimer);
|
|
128
|
+
signal?.removeEventListener("abort", onAbort);
|
|
129
|
+
resolve({
|
|
130
|
+
code: aborted ? "aborted" : timedOut ? "timeout" : code,
|
|
131
|
+
stdout,
|
|
132
|
+
stderr,
|
|
133
|
+
timedOut,
|
|
134
|
+
aborted,
|
|
135
|
+
});
|
|
136
|
+
};
|
|
137
|
+
|
|
138
|
+
child.on("close", finish);
|
|
139
|
+
child.on("error", (err) => {
|
|
140
|
+
stderr += (stderr ? "\n" : "") + err.message;
|
|
141
|
+
finish(1);
|
|
142
|
+
});
|
|
143
|
+
|
|
144
|
+
if (stdinText !== null && stdinText !== undefined) {
|
|
145
|
+
child.stdin.write(stdinText);
|
|
146
|
+
}
|
|
147
|
+
child.stdin.end();
|
|
148
|
+
});
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
module.exports = {
|
|
152
|
+
scrubChildEnv,
|
|
153
|
+
killProcessGroup,
|
|
154
|
+
spawnShell,
|
|
155
|
+
};
|