@having5548/dsh-downloader 0.0.0-stage → 0.5.2

package/lib/download/fetch-file.js

@@ -0,0 +1,363 @@
+/**
+ * Streaming file download with integrity, limits, and cancellation.
+ *
+ * Invariants:
+ * - the target file only ever appears complete (written to `<target>.part`,
+ *   then renamed);
+ * - **every** failure path removes the `.part` file;
+ * - the byte count is capped while streaming, so a wrong URL cannot fill a disk.
+ * @module dsh-downloader/fetch-file
+ */
+import fs from "node:fs";
+import path from "node:path";
+import { createHash } from "node:crypto";
+import { httpFlow, readBody } from "./http.js";
+import { humanSize } from "./util.js";
+import { RangeUnusableError, SEGMENT_MIN_TOTAL, downloadSegmented, supportsRanges } from "./segmented.js";
+
+/** Raised for a non-2xx response, carrying the status for callers that retry. */
+export class HttpStatusError extends Error {
+	constructor(message, status, url) {
+		super(message);
+		this.name = "HttpStatusError";
+		this.status = status;
+		this.url = url;
+	}
+}
+
+/** Stream a response body into `partPath`, hashing and capping as it goes. */
+function streamToFile(flow, partPath, { maxBytes, stallMs, signal, onProgress }) {
+	const hash = createHash("sha256");
+	let written = 0;
+	let lastTick = Date.now();
+	const sink = fs.createWriteStream(partPath);
+
+	return new Promise((resolve, reject) => {
+		let failure = null;
+		const cleanup = () => {
+			clearInterval(stallWatch);
+			signal?.removeEventListener("abort", onAbort);
+		};
+		const fail = (error) => {
+			if (failure === null) failure = error instanceof Error ? error : new Error(String(error));
+			try {
+				flow.res.destroy();
+			} catch {
+				/* already gone */
+			}
+			try {
+				sink.destroy();
+			} catch {
+				/* already gone */
+			}
+		};
+		const onAbort = () => fail(new Error("下载已取消"));
+		const stallWatch = setInterval(() => {
+			if (Date.now() - lastTick > stallMs) fail(new Error(`下载停滞：已 ${Math.round(stallMs / 1000)} 秒没有收到数据`));
+		}, 1000);
+
+		if (signal !== null && signal !== undefined) {
+			if (signal.aborted) {
+				onAbort();
+			} else {
+				signal.addEventListener("abort", onAbort, { once: true });
+			}
+		}
+
+		flow.res.on("data", (chunk) => {
+			if (failure !== null) return;
+			written += chunk.length;
+			lastTick = Date.now();
+			if (written > maxBytes) {
+				fail(new Error(`文件超过配置的大小上限（${humanSize(maxBytes)}）；确认文件大小后可提高 max_mb`));
+				return;
+			}
+			hash.update(chunk);
+			onProgress?.(written);
+			if (!sink.write(chunk)) flow.res.pause();
+		});
+		sink.on("drain", () => flow.res.resume());
+		flow.res.once("end", () => sink.end());
+		flow.res.once("error", (error) => fail(error));
+		sink.once("error", (error) => fail(error));
+		sink.once("close", () => {
+			cleanup();
+			if (failure !== null) reject(failure);
+			else resolve({ bytes: written, sha256: hash.digest("hex") });
+		});
+	});
+}
+
+/** One plain GET streamed into `partPath`; used when the probe body is gone. */
+async function attemptSingleStream({ url, partPath, proxy, insecure, maxRedirects, maxBytes, stallMs, signal, onEvent, headers = null }) {
+	const flow = await httpFlow(url, { proxy, insecure, maxRedirects, timeoutMs: 20000, signal, headers });
+	if (flow.status >= 400) {
+		let snippet = "";
+		try {
+			snippet = (await readBody(flow, 16 * 1024)).slice(0, 200).replace(/\s+/g, " ");
+		} catch {
+			/* body already consumed or torn down */
+		}
+		throw new HttpStatusError(
+			`HTTP ${flow.status}${flow.statusMessage ? ` ${flow.statusMessage}` : ""} @ ${flow.url}${snippet ? ` | ${snippet}` : ""}`,
+			flow.status,
+			flow.url
+		);
+	}
+	const declared = Number(flow.headers["content-length"]);
+	if (Number.isFinite(declared) && declared > maxBytes) {
+		flow.res.resume();
+		flow.abort();
+		throw new Error(`服务端声明大小 ${humanSize(declared)}，超过配置上限（${humanSize(maxBytes)}）；提高 max_mb 后重试`);
+	}
+	const streamed = await streamToFile(flow, partPath, {
+		maxBytes,
+		stallMs,
+		signal,
+		onProgress: (bytes) => onEvent?.({ kind: "progress", bytes })
+	});
+	return {
+		...streamed,
+		http: {
+			status: flow.status,
+			redirects: flow.redirects,
+			content_length: Number.isFinite(declared) ? declared : null,
+			content_type: flow.headers["content-type"] ?? null
+		},
+		finalUrl: flow.url
+	};
+}
+
+/**
+ * Try the multi-connection path.
+ *
+ * Returns null (never throws) whenever the segmented attempt is not applicable
+ * or turned out to be unusable, so the caller can fall back to one stream. The
+ * probe response is discarded: a range request is issued per slice instead, so
+ * a server that lies about `Accept-Ranges` costs one wasted request, not a
+ * corrupt file.
+ */
+async function trySegmented(flow, options, context) {
+	const { partPath, maxBytes, threads, signal, onEvent, stallMs, onConsume } = context;
+	const declaredLength = Number(flow.headers["content-length"]);
+	if (threads <= 1) return null;
+	if (!Number.isFinite(declaredLength) || declaredLength < SEGMENT_MIN_TOTAL) return null;
+	if (!supportsRanges(flow.status, flow.headers)) return null;
+	if (declaredLength > maxBytes) return null; // the single-stream path raises the proper error
+
+	// Only headers were needed; drop the body before opening our own requests.
+	try {
+		flow.res.resume();
+		flow.abort();
+	} catch {
+		/* already gone */
+	}
+	onConsume();
+
+	const validator = flow.headers.etag ?? flow.headers["last-modified"] ?? null;
+	const handle = await fs.promises.open(partPath, "w+");
+	try {
+		onEvent?.({ kind: "segments", threads, bytes: declaredLength });
+		return await downloadSegmented({
+			url: flow.url,
+			handle,
+			totalBytes: declaredLength,
+			threads,
+			proxy: options.proxy,
+			insecure: options.insecure,
+			maxRedirects: options.maxRedirects,
+			timeoutMs: options.timeoutMs,
+			stallMs,
+			maxBytes,
+			signal,
+			validator,
+			headers: options.headers,
+			onProgress: (bytes) => onEvent?.({ kind: "progress", bytes })
+		});
+	} finally {
+		await handle.close().catch(() => {});
+	}
+}
+
+/**
+ * Perform exactly one download attempt through an explicit proxy (or directly).
+ *
+ * @param options - target, destination, proxy, limits, and cancellation.
+ * @returns the attempt result; the caller owns route reporting.
+ * @throws {HttpStatusError} on a non-2xx response.
+ */
+export async function attemptDownload(options) {
+	const {
+		url,
+		savePath,
+		overwrite = false,
+		proxy = null,
+		maxBytes = 512 * 1024 * 1024,
+		timeoutS = 600,
+		stallS = 30,
+		threads = 1,
+		insecure = false,
+		maxRedirects = 10,
+		signal = null,
+		headers = null,
+		onEvent = null
+	} = options;
+
+	if (fs.existsSync(savePath) && !overwrite) {
+		const stats = fs.statSync(savePath);
+		if (stats.isDirectory()) throw new Error(`目标是目录，不是文件：${savePath}`);
+		throw new Error(`目标文件已存在：${savePath}（${humanSize(stats.size)}）；加 overwrite=true 覆盖，或换一个 save_path`);
+	}
+
+	fs.mkdirSync(path.dirname(savePath), { recursive: true });
+	const partPath = `${savePath}.part`;
+	const started = Date.now();
+	const controller = new AbortController();
+	const forwardAbort = () => controller.abort(new Error("下载已取消"));
+	if (signal !== null && signal !== undefined) {
+		if (signal.aborted) throw new Error("下载已取消");
+		signal.addEventListener("abort", forwardAbort, { once: true });
+	}
+	const deadline = setTimeout(
+		() => controller.abort(new Error(`下载超过 ${timeoutS} 秒预算；大文件请提高 timeout_seconds`)),
+		Math.max(1, timeoutS) * 1000
+	);
+
+	let flow = null;
+	try {
+		flow = await httpFlow(url, {
+			proxy,
+			headers,
+			timeoutMs: 20000,
+			signal: controller.signal,
+			insecure,
+			maxRedirects,
+			onHop: (hop) => onEvent?.({ kind: "redirect", ...hop })
+		});
+		if (flow.status >= 400) {
+			let snippet = "";
+			try {
+				snippet = (await readBody(flow, 16 * 1024)).slice(0, 200).replace(/\s+/g, " ");
+			} catch {
+				/* body already consumed or torn down */
+			}
+			throw new HttpStatusError(
+				`HTTP ${flow.status}${flow.statusMessage ? ` ${flow.statusMessage}` : ""} @ ${flow.url}${snippet ? ` | ${snippet}` : ""}`,
+				flow.status,
+				flow.url
+			);
+		}
+
+		const declaredLength = Number(flow.headers["content-length"]);
+		if (Number.isFinite(declaredLength) && declaredLength > maxBytes) {
+			flow.res.resume();
+			flow.abort();
+			throw new Error(
+				`服务端声明大小 ${humanSize(declaredLength)}，超过配置上限（${humanSize(maxBytes)}）；提高 max_mb 后重试`
+			);
+		}
+
+		let probeConsumed = false;
+		const streamed = await (async () => {
+			try {
+				const segmented = await trySegmented(flow, { url, proxy, insecure, maxRedirects, timeoutMs: 20000, headers }, {
+					partPath,
+					maxBytes,
+					threads,
+					signal: controller.signal,
+					onEvent,
+					stallMs: Math.max(1, stallS) * 1000,
+					onConsume: () => { probeConsumed = true; }
+				});
+				if (segmented !== null) return { ...segmented, viaThreads: true };
+			} catch (error) {
+				// A range problem must never lose the download: redo it on one stream.
+				if (!(error instanceof RangeUnusableError)) throw error;
+				onEvent?.({ kind: "fallback", reason: `分片下载不可用（${error.message}），改用单连接` });
+			}
+			if (!probeConsumed) {
+				return {
+					...(await streamToFile(flow, partPath, {
+						maxBytes,
+						stallMs: Math.max(1, stallS) * 1000,
+						signal: controller.signal,
+						onProgress: (bytes) => onEvent?.({ kind: "progress", bytes })
+					})),
+					viaThreads: false
+				};
+			}
+			// The probe body was consumed, so the single-stream retry needs its own request.
+			const retried = await attemptSingleStream({
+				url: flow.url,
+				partPath,
+				proxy,
+				insecure,
+				maxRedirects,
+				headers,
+				maxBytes,
+				stallMs: Math.max(1, stallS) * 1000,
+				signal: controller.signal,
+				onEvent
+			});
+			return { ...retried, viaThreads: false };
+		})();
+
+		if (fs.existsSync(savePath) && !overwrite) {
+			throw new Error(`下载期间目标文件出现了：${savePath}`);
+		}
+		fs.renameSync(partPath, savePath);
+		const elapsedMs = Date.now() - started;
+		return {
+			savedTo: savePath,
+			bytes: streamed.bytes,
+			sha256: streamed.sha256,
+			elapsedMs,
+			speedBps: elapsedMs > 0 ? Math.round((streamed.bytes / elapsedMs) * 1000) : null,
+			viaThreads: streamed.viaThreads,
+			segments: streamed.segments ?? null,
+			http: {
+				status: flow.status,
+				redirects: flow.redirects,
+				content_length: Number.isFinite(declaredLength) ? declaredLength : null,
+				content_type: flow.headers["content-type"] ?? null
+			},
+			finalUrl: flow.url
+		};
+	} catch (error) {
+		try {
+			fs.rmSync(partPath, { force: true });
+		} catch {
+			/* nothing to clean */
+		}
+		throw error;
+	} finally {
+		clearTimeout(deadline);
+		signal?.removeEventListener("abort", forwardAbort);
+	}
+}
+
+/**
+ * Download with the automatic direct → proxy fallback.
+ *
+ * @param options - as {@link attemptDownload} plus `decidedRoute` and `upstream`.
+ * @returns the attempt result plus `route` / `routeReason` / `fallbackUsed`.
+ */
+export async function downloadWithFallback(options) {
+	const { decidedRoute, upstream, forceRoute = "auto", onEvent = null, ...attempt } = options;
+	const canFallback = decidedRoute.route === "direct" && forceRoute !== "direct" && upstream.available === true && upstream.proxy !== null;
+
+	try {
+		const result = await attemptDownload({ ...attempt, proxy: decidedRoute.route === "proxy" ? upstream.proxy : null, onEvent });
+		return { ...result, route: decidedRoute.route, routeReason: decidedRoute.reason, fallbackUsed: false };
+	} catch (error) {
+		if (!canFallback || error instanceof HttpStatusError) throw error;
+		onEvent?.({ kind: "fallback", reason: error instanceof Error ? error.message : String(error) });
+		const result = await attemptDownload({ ...attempt, proxy: upstream.proxy, onEvent });
+		return {
+			...result,
+			route: "proxy",
+			routeReason: `直连失败（${error instanceof Error ? error.message : String(error)}），已改用代理重试`,
+			fallbackUsed: true
+		};
+	}
+}

package/lib/download/http.js

@@ -0,0 +1,299 @@
+/**
+ * Minimal HTTP/1.1 client for the download pipeline.
+ *
+ * Deliberately built on `node:http` / `node:tls` instead of undici so that the
+ * proxy hop is explicit and observable: the caller passes a `{ host, port }`
+ * HTTP-CONNECT proxy (in practice always the plugin's own loopback rule core),
+ * and nothing about the request depends on a process-wide dispatcher.
+ *
+ * Every `openTarget` resolves on response headers; the body stays streaming on
+ * the returned `res` for the caller to consume or discard.
+ * @module dsh-downloader/http
+ */
+import http from "node:http";
+import https from "node:https";
+import net from "node:net";
+import tls from "node:tls";
+import { hasSensitiveHeaders, stripSensitiveHeaders } from "./auth.js";
+
+/** User agent used for every plugin-originated request. */
+export const DEFAULT_UA = "DeepSeek-Harness/dsh-downloader (+https://github.com/having5548/dsh-downloader)";
+
+/**
+ * Parse a proxy URL into `{ socks, host, port, raw, auth }`, or null.
+ * @param value - `host:port`, `http://…`, `https://…`, `socks5://…` or null.
+ */
+export function parseProxyUrl(value) {
+	if (typeof value !== "string" || value.trim().length === 0) return null;
+	const raw = value.trim();
+	try {
+		const url = new URL(raw.includes("://") ? raw : `http://${raw}`);
+		const protocol = url.protocol.toLowerCase();
+		const socks = protocol === "socks5:" || protocol === "socks5h:" || protocol === "socks4:" || protocol === "socks:";
+		const supported = socks || protocol === "http:" || protocol === "https:";
+		if (!supported) return null;
+		const port = Number(url.port) || (socks ? 1080 : protocol === "https:" ? 443 : 80);
+		const auth = url.username.length > 0
+			? { username: decodeURIComponent(url.username), password: decodeURIComponent(url.password) }
+			: null;
+		return { socks, secure: protocol === "https:", host: url.hostname, port, raw, auth };
+	} catch {
+		return null;
+	}
+}
+
+/**
+ * Open an HTTP CONNECT tunnel through `proxy` to `host:port`.
+ * @param proxy - a parsed proxy (must not be a SOCKS proxy).
+ * @returns the raw tunnelled socket.
+ */
+export function connectTunnel(proxy, host, port, timeoutMs) {
+	return new Promise((resolve, reject) => {
+		const headers = { Host: `${host}:${port}` };
+		if (proxy.auth !== null && proxy.auth !== undefined) {
+			const token = Buffer.from(`${proxy.auth.username}:${proxy.auth.password}`).toString("base64");
+			headers["Proxy-Authorization"] = `Basic ${token}`;
+		}
+		// [dsh-downloader] The plugin's own loopback core is token-gated so that
+		// nothing outside it can borrow the listener as a proxy.
+		if (typeof proxy.token === "string" && proxy.token.length > 0) {
+			headers["Proxy-Authorization"] = `Bearer ${proxy.token}`;
+		}
+		const transport = proxy.secure === true ? https : http;
+		const request = transport.request({
+			host: proxy.host,
+			port: proxy.port,
+			method: "CONNECT",
+			path: `${host}:${port}`,
+			headers,
+			agent: false,
+			...(proxy.secure === true ? { rejectUnauthorized: true } : {})
+		});
+		const timer = setTimeout(() => request.destroy(new Error(`代理 CONNECT 超时（${timeoutMs}ms）`)), timeoutMs);
+		request.once("connect", (response, socket) => {
+			clearTimeout(timer);
+			if (response.statusCode === 200) resolve(socket);
+			else {
+				socket.destroy();
+				reject(new Error(`代理 CONNECT 被拒绝：HTTP ${response.statusCode}`));
+			}
+		});
+		request.once("error", (error) => {
+			clearTimeout(timer);
+			reject(error);
+		});
+		request.end();
+	});
+}
+
+/**
+ * Start one HTTP(S) request, optionally through an HTTP CONNECT proxy.
+ *
+ * Resolves as soon as response headers arrive. `result.abort()` tears the
+ * request down; the caller owns `result.res`.
+ *
+ * @param rawUrl - absolute http/https URL.
+ * @param options - method, headers, proxy (`{host,port,socks,auth,secure}`|null), timeoutMs, signal, insecure, body.
+ * @returns `{ status, statusMessage, headers, res, url, abort }`.
+ */
+export function openTarget(rawUrl, options = {}) {
+	const {
+		method = "GET",
+		headers = {},
+		proxy = null,
+		timeoutMs = 20000,
+		signal = null,
+		insecure = false,
+		body = null
+	} = options;
+
+	return new Promise((resolve, reject) => {
+		let url;
+		try {
+			url = new URL(rawUrl);
+		} catch {
+			reject(new Error(`无效的 URL：${rawUrl}`));
+			return;
+		}
+		const isHttps = url.protocol === "https:";
+		const port = Number(url.port) || (isHttps ? 443 : 80);
+
+		const controller = new AbortController();
+		const onOuterAbort = () => controller.abort(new Error("调用方已取消"));
+		if (signal !== null && signal !== undefined) {
+			if (signal.aborted) {
+				reject(new Error("调用方已取消"));
+				return;
+			}
+			signal.addEventListener("abort", onOuterAbort, { once: true });
+		}
+		const timer = setTimeout(() => controller.abort(new Error(`请求超时（${timeoutMs}ms）：${url.host}`)), timeoutMs);
+
+		let settled = false;
+		let request = null;
+		const finish = (error, response) => {
+			if (settled) return;
+			settled = true;
+			clearTimeout(timer);
+			signal?.removeEventListener("abort", onOuterAbort);
+			if (error !== null && error !== undefined) reject(error instanceof Error ? error : new Error(String(error)));
+			else {
+				resolve({
+					status: response.statusCode,
+					statusMessage: response.statusMessage,
+					headers: response.headers,
+					res: response,
+					url: url.href,
+					abort: () => {
+						try {
+							request?.destroy();
+						} catch {
+							/* already gone */
+						}
+					}
+				});
+			}
+		};
+
+		const baseHeaders = { "User-Agent": DEFAULT_UA, "Accept-Encoding": "identity", ...headers };
+		const onResponse = (response) => finish(null, response);
+		const onRequestError = (error) => {
+			const reason = controller.signal.reason;
+			finish(controller.signal.aborted && reason instanceof Error ? reason : error);
+		};
+
+		if (proxy !== null && proxy !== undefined && proxy.socks === true) {
+			finish(new Error("SOCKS 代理由规则内核处理，请改传内核的 HTTP 端口"));
+			return;
+		}
+
+		if (proxy !== null && proxy !== undefined && isHttps) {
+			connectTunnel(proxy, url.hostname, port, Math.min(timeoutMs, 15000))
+				.then((socket) => {
+					socket.once("error", () => {});
+					const tlsSocket = tls.connect({ socket, servername: url.hostname, rejectUnauthorized: !insecure });
+					tlsSocket.once("error", () => {});
+					request = https.request(
+						url,
+						{ method, headers: baseHeaders, signal: controller.signal, agent: false, createConnection: () => tlsSocket },
+						onResponse
+					);
+					request.once("error", onRequestError);
+					if (body !== null && body !== undefined) request.end(body);
+					else request.end();
+				})
+				.catch((error) => finish(error));
+			return;
+		}
+
+		try {
+			if (proxy !== null && proxy !== undefined) {
+				// Plaintext http through a proxy: absolute-form request line.
+				const proxyHeaders = { ...baseHeaders, Host: url.host };
+				if (typeof proxy.token === "string" && proxy.token.length > 0) {
+					proxyHeaders["Proxy-Authorization"] = `Bearer ${proxy.token}`;
+				}
+				request = http.request(
+					{
+						host: proxy.host,
+						port: proxy.port,
+						path: url.href,
+						method,
+						headers: proxyHeaders,
+						signal: controller.signal,
+						agent: false
+					},
+					onResponse
+				);
+			} else if (isHttps) {
+				request = https.request(url, { method, headers: baseHeaders, signal: controller.signal, agent: false, rejectUnauthorized: !insecure }, onResponse);
+			} else {
+				request = http.request(url, { method, headers: baseHeaders, signal: controller.signal, agent: false }, onResponse);
+			}
+		} catch (error) {
+			finish(error);
+			return;
+		}
+
+		request.once("error", onRequestError);
+		if (body !== null && body !== undefined) request.end(body);
+		else request.end();
+	});
+}
+
+/** Read a response body up to `limit` bytes and decode it as UTF-8. */
+export async function readBody(result, limit = 64 * 1024) {
+	const chunks = [];
+	let size = 0;
+	await new Promise((resolve, reject) => {
+		result.res.on("data", (chunk) => {
+			size += chunk.length;
+			if (size <= limit) chunks.push(chunk);
+			else result.abort();
+		});
+		result.res.once("end", resolve);
+		result.res.once("aborted", () => reject(new Error("响应被中断")));
+		result.res.once("error", reject);
+	});
+	return Buffer.concat(chunks).toString("utf8");
+}
+
+/**
+ * Follow redirects and return the final response.
+ *
+ * `onHop` (optional) is called once per hop with `{ status, url, location }`
+ * before the redirect is followed, so callers can log the chain.
+ *
+ * [dsh-downloader] 跨主机跳转时会**丢掉凭据类请求头**：GitHub 的
+ * `archive_download_url` 会 302 到 blob 域，把 `Authorization` 带过去既没用，
+ * 还可能被对端以 400 拒绝（也等于把 token 交给了第三方主机）。
+ */
+export async function httpFlow(rawUrl, options = {}) {
+	const { maxRedirects = 10, onHop, headers, ...rest } = options;
+	let current = rawUrl;
+	let currentHeaders = headers;
+	let redirects = 0;
+	for (;;) {
+		const result = await openTarget(current, { ...rest, headers: currentHeaders });
+		if ([301, 302, 303, 307, 308].includes(result.status)) {
+			const location = result.headers.location;
+			result.res.resume();
+			result.abort();
+			if (typeof location !== "string" || location.length === 0) throw new Error(`HTTP ${result.status} 重定向缺少 Location 头`);
+			if (++redirects > maxRedirects) throw new Error(`重定向次数超过上限（> ${maxRedirects}）`);
+			const next = new URL(location, current).href;
+			if (hasSensitiveHeaders(currentHeaders)) {
+				let sameHost = false;
+				try {
+					sameHost = new URL(next).host === new URL(current).host;
+				} catch {
+					sameHost = false;
+				}
+				if (!sameHost) currentHeaders = stripSensitiveHeaders(currentHeaders);
+			}
+			onHop?.({ status: result.status, url: current, location: next });
+			current = next;
+			continue;
+		}
+		return { ...result, redirects };
+	}
+}
+
+/** Open a plain TCP connection with a deadline. Rejects on error or timeout. */
+export function tcpConnect(host, port, timeoutMs = 3000) {
+	return new Promise((resolve, reject) => {
+		const socket = net.connect({ host, port });
+		const timer = setTimeout(() => {
+			socket.destroy();
+			reject(new Error(`TCP 连接超时（${timeoutMs}ms）`));
+		}, timeoutMs);
+		socket.once("connect", () => {
+			clearTimeout(timer);
+			resolve(socket);
+		});
+		socket.once("error", (error) => {
+			clearTimeout(timer);
+			reject(error);
+		});
+	});
+}