@having5548/dsh-downloader 0.0.0-stage → 0.5.2

package/lib/download/net-probe.js

@@ -0,0 +1,126 @@
+/**
+ * Network diagnostics used by the `dsh_geo_check` tool and by the route
+ * decider's "unknown geography" fallback.
+ *
+ * These helpers are never on the hot download path: routing prefers the
+ * offline rule engine, so a sandbox with no DNS or no ICMP still downloads fine.
+ * @module dsh-downloader/net-probe
+ */
+import dns from "node:dns/promises";
+import { spawn } from "node:child_process";
+import { openTarget, readBody, tcpConnect } from "./http.js";
+
+/** Resolve every A/AAAA record for a host. Returns [] on failure. */
+export async function resolveHost(host) {
+	try {
+		const records = await dns.lookup(host, { all: true, verbatim: true });
+		return records.map((record) => record.address);
+	} catch {
+		return [];
+	}
+}
+
+/**
+ * ICMP ping via the platform `ping` binary.
+ * @returns `{ ok, timeMs, raw }` — `ok: false` with a reason when unavailable.
+ */
+export function icmpPing(host, timeoutMs = 3000) {
+	return new Promise((resolve) => {
+		const windows = process.platform === "win32";
+		const argv = windows ? ["-n", "1", "-w", String(timeoutMs), host] : ["-c", "1", "-W", String(Math.ceil(timeoutMs / 1000)), host];
+		let stdout = "";
+		let settled = false;
+		const done = (value) => {
+			if (settled) return;
+			settled = true;
+			resolve(value);
+		};
+		let child;
+		try {
+			child = spawn(windows ? "ping.exe" : "ping", argv, { windowsHide: true, stdio: ["ignore", "pipe", "ignore"] });
+		} catch (error) {
+			done({ ok: false, timeMs: null, raw: String(error?.message ?? error) });
+			return;
+		}
+		const timer = setTimeout(() => {
+			try {
+				child.kill();
+			} catch {
+				/* already gone */
+			}
+			done({ ok: false, timeMs: null, raw: "icmp timeout" });
+		}, timeoutMs + 1500);
+		child.stdout?.on("data", (chunk) => {
+			stdout += chunk.toString("utf8");
+		});
+		child.once("error", (error) => {
+			clearTimeout(timer);
+			done({ ok: false, timeMs: null, raw: String(error?.message ?? error) });
+		});
+		child.once("close", (code) => {
+			clearTimeout(timer);
+			// Both the English and the localized Windows output carry "<n>ms" / "=1ms".
+			const match = /[=<]\s*([\d.]+)\s*ms/i.exec(stdout);
+			done({ ok: code === 0, timeMs: match === null ? null : Number(match[1]), raw: stdout.trim().split(/\r?\n/).slice(0, 4).join(" | ") });
+		});
+	});
+}
+
+/** TCP connect probe. @returns `{ ok, timeMs, error }`. */
+export async function tcpPing(host, port = 443, timeoutMs = 3000) {
+	const started = Date.now();
+	try {
+		const socket = await tcpConnect(host, port, timeoutMs);
+		const elapsed = Date.now() - started;
+		socket.destroy();
+		return { ok: true, timeMs: elapsed, error: null };
+	} catch (error) {
+		return { ok: false, timeMs: null, error: error instanceof Error ? error.message : String(error) };
+	}
+}
+
+/** GeoIP providers tried in order. Both are keyless. */
+const GEO_PROVIDERS = [
+	{
+		name: "ip-api.com",
+		url: (ip) => `http://ip-api.com/json/${encodeURIComponent(ip)}?fields=status,country,countryCode,isp,query`,
+		parse: (json) => (json.status === "success"
+			? { ip: json.query ?? null, country: json.country ?? null, countryCode: (json.countryCode ?? "").toUpperCase() || null, isp: json.isp ?? null }
+			: null)
+	},
+	{
+		name: "ipwho.is",
+		url: (ip) => `https://ipwho.is/${encodeURIComponent(ip)}`,
+		parse: (json) => (json.success === true
+			? { ip: json.ip ?? null, country: json.country ?? null, countryCode: (json.country_code ?? "").toUpperCase() || null, isp: json.connection?.isp ?? null }
+			: null)
+	}
+];
+
+/**
+ * Look up the country of an IP address.
+ * @param ip - the address to look up.
+ * @param options - optional proxy, timeout, and the provider order to try.
+ * @returns `{ country, countryCode, isp, provider }` or null.
+ */
+export async function geoLookupIP(ip, options = {}) {
+	const { proxy = null, timeoutMs = 6000, providers = GEO_PROVIDERS, signal = null } = options;
+	for (const provider of providers) {
+		try {
+			const result = await openTarget(provider.url(ip), { proxy, timeoutMs, signal });
+			if (result.status < 200 || result.status >= 300) {
+				result.res.resume();
+				result.abort();
+				continue;
+			}
+			const parsed = provider.parse(JSON.parse(await readBody(result, 64 * 1024)));
+			if (parsed !== null) return { ...parsed, provider: provider.name };
+		} catch {
+			// Try the next provider.
+		}
+	}
+	return null;
+}
+
+/** True when the host resolves to at least one non-private address. */
+export const __internals = { GEO_PROVIDERS };

package/lib/download/route.js

@@ -0,0 +1,103 @@
+/**
+ * Route decision for one download target.
+ *
+ * The decision is made **offline** from the rule engine (embedded CN domain
+ * suffixes and CN CIDR blocks + the AI-platform protection list + the
+ * subscription's own rules), so routing never depends on a reachable GeoIP
+ * service. Online lookups live in `net-probe.js` and are used only by the
+ * `dsh_geo_check` diagnostic tool.
+ *
+ * 面向模型与用户的文案一律中文；标识符、配置字段名、协议名保持原样。
+ * @module dsh-downloader/route
+ */
+import { isHttpUrl, isPrivateIp } from "./util.js";
+
+/** Raised when a target must go through a proxy but no usable exit exists. */
+export class NoUpstreamError extends Error {
+	constructor(message, hint) {
+		super(message);
+		this.name = "NoUpstreamError";
+		this.hint = hint;
+	}
+}
+
+/** Extract the hostname from a URL without throwing. */
+export function hostOf(rawUrl) {
+	try {
+		return new URL(rawUrl).hostname.toLowerCase();
+	} catch {
+		return "";
+	}
+}
+
+/**
+ * True for hosts that must never be sent to a proxy: loopback, `.local`
+ * names, and private/loopback IP literals. The rule tables cannot express
+ * "this machine", so the check lives here rather than in `RuleEngine`.
+ */
+function isLocalHost(host) {
+	if (host.length === 0) return false;
+	if (host === "localhost" || host.endsWith(".localhost") || host.endsWith(".local")) return true;
+	if (host.startsWith("[") && host.endsWith("]")) {
+		const inner = host.slice(1, -1);
+		return inner === "::1" || isPrivateIp(inner);
+	}
+	return isPrivateIp(host);
+}
+
+/**
+ * Decide whether a target is fetched directly or through the rule core.
+ *
+ * @param target - the download URL.
+ * @param upstream - the resolved upstream (`{ mode, engine, available, description }`).
+ * @param forceRoute - `auto` | `direct` | `proxy`.
+ * @returns `{ route, reason, host, warning }`.
+ * @throws {NoUpstreamError} when the target needs a proxy and none is usable.
+ */
+export function decideRoute(target, upstream, forceRoute = "auto") {
+	if (!isHttpUrl(target)) throw new Error(`不是 http/https 下载地址：${target}`);
+	const host = hostOf(target);
+	const forced = String(forceRoute ?? "auto").toLowerCase();
+
+	if (forced === "direct") {
+		return { route: "direct", reason: "调用方指定直连", host, warning: null };
+	}
+	if (forced === "proxy") {
+		if (upstream.available !== true) {
+			throw new NoUpstreamError(
+				`调用方指定走代理，但没有可用的代理出口（${upstream.reason}）`,
+				"请在「设置 → 下载代理」里填写订阅地址或 proxyUrl。"
+			);
+		}
+		if (isLocalHost(host)) {
+			return { route: "direct", reason: `${host} 是本机或私有地址，回环流量永不经过代理`, host, warning: null };
+		}
+		return { route: "proxy", reason: "调用方指定走代理", host, warning: null };
+	}
+
+	if (isLocalHost(host)) {
+		return { route: "direct", reason: `${host} 是本机或私有地址`, host, warning: null };
+	}
+
+	if (upstream.engine === null || upstream.engine === undefined) {
+		return {
+			route: "direct",
+			reason: upstream.reason ?? "未配置代理，直接下载",
+			host,
+			warning: "未配置代理，所有目标都会直接下载"
+		};
+	}
+
+	const decision = upstream.engine.decide(host);
+	if (decision === "reject") throw new Error(`当前规则拒绝访问该主机：${host}`);
+	if (decision === "direct") {
+		return { route: "direct", reason: `当前规则判定 ${host} 直连`, host, warning: null };
+	}
+	if (upstream.available !== true) {
+		throw new NoUpstreamError(
+			`当前规则要求 ${host} 走代理，但没有可用节点（${upstream.reason}）`,
+			"请在「设置 → 下载代理」检查订阅，或传 force_route=\"direct\" 绕过。"
+		);
+	}
+	return { route: "proxy", reason: `当前规则判定 ${host} 走代理`, host, warning: null };
+}

package/lib/download/segmented.js

@@ -0,0 +1,279 @@
+/**
+ * Segmented (multi-connection) download.
+ *
+ * Modelled on gopeed's `internal/protocol/http/fetcher.go`: instead of splitting
+ * the file into fixed equal parts, every connection owns a chunk and, when it
+ * finishes, **steals the tail half of the chunk with the most work left**. That
+ * is what keeps a slow server from holding up the whole transfer — a static
+ * split is only as fast as its slowest connection.
+ *
+ * Everything here is opt-in: the caller falls back to a single stream whenever
+ * the server does not advertise byte ranges, the resource is too small for
+ * threads to pay off, or a range response turns out to be unusable.
+ * @module dsh-downloader/segmented
+ */
+import fs from "node:fs";
+import { createHash } from "node:crypto";
+import { httpFlow } from "./http.js";
+import { humanSize } from "./util.js";
+
+/** Smallest slice worth a separate request (gopeed's `stealMinChunkSize`). */
+export const STEAL_MIN_BYTES = 512 * 1024;
+
+/** Below this, one connection is faster than the coordination overhead. */
+export const SEGMENT_MIN_TOTAL = 1024 * 1024;
+
+/** Raised when the segmented attempt cannot proceed and the caller must retry single-stream. */
+export class RangeUnusableError extends Error {
+	constructor(message) {
+		super(message);
+		this.name = "RangeUnusableError";
+	}
+}
+
+/** True when the response advertises byte-range support. */
+export function supportsRanges(status, headers) {
+	if (status === 206) return true;
+	const accept = String(headers?.["accept-ranges"] ?? "").toLowerCase();
+	return accept.includes("bytes");
+}
+
+/**
+ * Parse a `Content-Range: bytes start-end/total` header.
+ * @returns `{ start, end, total }`; `total` is null for `*`.
+ */
+export function parseContentRange(value) {
+	const match = /^bytes\s+(\d+)-(\d+)\/(\d+|\*)$/i.exec(String(value ?? "").trim());
+	if (match === null) return null;
+	return {
+		start: Number.parseInt(match[1], 10),
+		end: Number.parseInt(match[2], 10),
+		total: match[3] === "*" ? null : Number.parseInt(match[3], 10)
+	};
+}
+
+/** Split `total` into up to `threads` starting chunks. */
+export function planChunks(total, threads) {
+	const count = Math.max(1, Math.min(Number(threads) || 1, Math.max(1, Math.floor(total / STEAL_MIN_BYTES))));
+	const size = Math.ceil(total / count);
+	const chunks = [];
+	for (let begin = 0; begin < total; begin += size) {
+		chunks.push({ begin, end: Math.min(begin + size, total) - 1 });
+	}
+	return chunks;
+}
+
+/**
+ * Take the next slice of work.
+ *
+ * Picks the chunk with the most bytes left and cuts **half of it** off, so a
+ * worker that just finished keeps pulling its weight while a slow one still
+ * makes progress. A chunk smaller than two steal-minimums is handed over whole.
+ * @returns `{ begin, end }` or null when everything is claimed.
+ */
+export function stealSlice(chunks) {
+	let best = null;
+	for (const chunk of chunks) {
+		const remain = chunk.end - chunk.begin + 1;
+		if (remain <= 0) continue;
+		if (best === null || remain > best.end - best.begin + 1) best = chunk;
+	}
+	if (best === null) return null;
+	const remain = best.end - best.begin + 1;
+	const take = remain >= 2 * STEAL_MIN_BYTES ? Math.floor(remain / 2) : remain;
+	const slice = { begin: best.begin, end: best.begin + take - 1 };
+	best.begin += take;
+	return slice;
+}
+
+/** Bytes still unclaimed across every chunk. */
+export function remainingBytes(chunks) {
+	let total = 0;
+	for (const chunk of chunks) total += Math.max(0, chunk.end - chunk.begin + 1);
+	return total;
+}
+
+/**
+ * Write a whole buffer at an absolute offset.
+ *
+ * `fs.promises.FileHandle.write` is promise-only — passing it a callback the way
+ * `fs.write` takes one silently ignores the callback and the promise never
+ * settles (which hangs the whole download).
+ */
+async function writeAt(handle, buffer, offset) {
+	const { bytesWritten } = await handle.write(buffer, 0, buffer.length, offset);
+	if (bytesWritten !== buffer.length) {
+		throw new Error(`写入不完整：偏移 ${offset} 期望 ${buffer.length} 字节，实际 ${bytesWritten}`);
+	}
+}
+
+/**
+ * Fetch one slice into the file at its absolute offset.
+ * @throws {RangeUnusableError} when the response is not the requested 206.
+ */
+async function fetchSlice(slice, handle, context) {
+	const { url, proxy, insecure, maxRedirects, validator, timeoutMs, controller, bump, headers } = context;
+	const requestHeaders = { ...(headers ?? {}), range: `bytes=${slice.begin}-${slice.end}` };
+	if (validator !== null) requestHeaders["if-range"] = validator;
+
+	const flow = await httpFlow(url, {
+		proxy,
+		headers: requestHeaders,
+		insecure,
+		maxRedirects,
+		timeoutMs,
+		signal: controller.signal
+	});
+	if (flow.status !== 206) {
+		try {
+			flow.res.resume();
+			flow.abort();
+		} catch {
+			/* already gone */
+		}
+		throw new RangeUnusableError(`服务端对 Range 请求返回了 HTTP ${flow.status}`);
+	}
+	const range = parseContentRange(flow.headers["content-range"]);
+	if (range === null) {
+		flow.res.resume();
+		flow.abort();
+		throw new RangeUnusableError("Range 响应缺少合法的 Content-Range");
+	}
+	if (range.start !== slice.begin) {
+		flow.res.resume();
+		flow.abort();
+		throw new RangeUnusableError(`Range 响应起点不符：请求 ${slice.begin}，返回 ${range.start}`);
+	}
+
+	let offset = slice.begin;
+	let pending = Promise.resolve();
+	await new Promise((resolve, reject) => {
+		let settled = false;
+		const fail = (error) => {
+			if (settled) return;
+			settled = true;
+			try {
+				flow.res.destroy();
+			} catch {
+				/* already gone */
+			}
+			reject(error);
+		};
+		flow.res.on("data", (chunk) => {
+			if (settled) return;
+			// Positional writes let every worker share one file descriptor.
+			pending = pending.then(() => {
+				if (settled) return undefined;
+				return writeAt(handle, chunk, offset).then(() => {
+					offset += chunk.length;
+					bump(chunk.length);
+				});
+			}).catch(fail);
+		});
+		flow.res.once("end", () => {
+			pending.then(() => {
+				if (settled) return;
+				settled = true;
+				resolve();
+			}, fail);
+		});
+		flow.res.once("error", fail);
+	});
+	return offset - slice.begin;
+}
+
+/**
+ * Download `url` into `handle` using several connections.
+ *
+ * @param options - target, an open file handle, size, thread count and the same
+ * transport knobs the single-stream path takes.
+ * @returns `{ bytes, sha256, segments }`.
+ * @throws {RangeUnusableError} when the segmented path cannot be used.
+ */
+export async function downloadSegmented(options) {
+	const {
+		url,
+		handle,
+		totalBytes,
+		threads = 4,
+		proxy = null,
+		insecure = false,
+		maxRedirects = 10,
+		timeoutMs = 20000,
+		stallMs = 30000,
+		maxBytes = 512 * 1024 * 1024,
+		signal = null,
+		validator = null,
+		headers = null,
+		onProgress = null
+	} = options;
+
+	if (totalBytes > maxBytes) {
+		throw new Error(`服务端声明大小 ${humanSize(totalBytes)}，超过配置上限（${humanSize(maxBytes)}）；提高 max_mb 后重试`);
+	}
+
+	const chunks = planChunks(totalBytes, threads);
+	const controller = new AbortController();
+	const forwardAbort = () => controller.abort(new Error("下载已取消"));
+	if (signal !== null && signal !== undefined) {
+		if (signal.aborted) throw new Error("下载已取消");
+		signal.addEventListener("abort", forwardAbort, { once: true });
+	}
+
+	let written = 0;
+	let lastTick = Date.now();
+	let failure = null;
+	const context = {
+		url, proxy, insecure, maxRedirects, validator, timeoutMs, controller, headers,
+		bump: (bytes) => {
+			written += bytes;
+			lastTick = Date.now();
+			onProgress?.(written);
+		}
+	};
+
+	const stallWatch = setInterval(() => {
+		if (failure === null && Date.now() - lastTick > stallMs) {
+			failure = new Error(`下载停滞：已 ${Math.round(stallMs / 1000)} 秒没有收到数据`);
+			controller.abort(failure);
+		}
+	}, 1000);
+
+	try {
+		const workerCount = Math.max(1, Math.min(Number(threads) || 1, chunks.length));
+		const workers = [];
+		for (let index = 0; index < workerCount; index += 1) {
+			workers.push((async () => {
+				for (;;) {
+					if (failure !== null) return;
+					const slice = stealSlice(chunks);
+					if (slice === null) return;
+					await fetchSlice(slice, handle, context);
+				}
+			})());
+		}
+		await Promise.all(workers);
+		if (failure !== null) throw failure;
+
+		const shortfall = totalBytes - written;
+		if (shortfall !== 0) throw new RangeUnusableError(`分片合计 ${written} 字节，与声明大小 ${totalBytes} 不符`);
+	} catch (error) {
+		if (failure !== null) throw failure;
+		throw error;
+	} finally {
+		clearInterval(stallWatch);
+		signal?.removeEventListener("abort", forwardAbort);
+	}
+
+	// Hash after the merge: sha256 must cover the file in order, and the workers
+	// write out of order.
+	const hash = createHash("sha256");
+	await new Promise((resolve, reject) => {
+		const reader = handle.createReadStream({ autoClose: false, start: 0 });
+		reader.on("data", (chunk) => hash.update(chunk));
+		reader.once("error", reject);
+		reader.once("end", resolve);
+	});
+
+	return { bytes: written, sha256: hash.digest("hex"), segments: chunks.length };
+}

package/lib/download/util.js

@@ -0,0 +1,103 @@
+/**
+ * Filename / size / address helpers shared by the download pipeline.
+ * @module dsh-downloader/util
+ */
+import net from "node:net";
+
+/** Render a byte count as a short human string. */
+export function humanSize(n) {
+	const units = ["B", "KB", "MB", "GB", "TB"];
+	let value = Number(n);
+	let unit = 0;
+	while (value >= 1024 && unit < units.length - 1) {
+		value /= 1024;
+		unit += 1;
+	}
+	return `${unit === 0 ? value : value.toFixed(value >= 100 ? 0 : value >= 10 ? 1 : 2)} ${units[unit]}`;
+}
+
+/** True for loopback / link-local / private / CGNAT addresses (v4 and v6). */
+export function isPrivateIp(ip) {
+	const family = net.isIP(ip);
+	if (family === 4) {
+		const parts = ip.split(".").map(Number);
+		if (parts.length !== 4 || parts.some((part) => !Number.isInteger(part) || part < 0 || part > 255)) return false;
+		const [a, b] = parts;
+		if (a === 0 || a === 10 || a === 127) return true;
+		if (a === 169 && b === 254) return true;
+		if (a === 172 && b >= 16 && b <= 31) return true;
+		if (a === 192 && b === 168) return true;
+		if (a === 100 && b >= 64 && b <= 127) return true; // CGNAT 100.64/10
+		if (a >= 224) return true; // multicast + reserved
+		return false;
+	}
+	if (family === 6) {
+		const host = ip.toLowerCase().split("%")[0];
+		if (host === "::" || host === "::1") return true;
+		if (host.startsWith("fe80") || host.startsWith("fec0")) return true; // link-local
+		if (/^f[cd]/.test(host)) return true; // unique local fc00::/7
+		if (host.startsWith("::ffff:")) return isPrivateIp(host.slice(7));
+		return false;
+	}
+	return false;
+}
+
+/** Strip path separators, characters Windows rejects, and control characters. */
+export function sanitizeFilename(name) {
+	const cleaned = String(name ?? "")
+		.replace(/[\\/]+/g, "_")
+		.replace(/[<>:"|?*]+/g, "_")
+		// eslint-disable-next-line no-control-regex
+		.replace(/[\u0000-\u001f\u007f]/g, "")
+		.replace(/^\.+$/, "")
+		.trim();
+	return cleaned.length > 0 ? cleaned.slice(0, 180) : "";
+}
+
+/** Best-effort file name from a URL path, or "" when the URL has none. */
+export function filenameFromUrl(rawUrl) {
+	try {
+		const url = new URL(rawUrl);
+		const last = url.pathname.split("/").filter(Boolean).pop() ?? "";
+		return sanitizeFilename(decodeURIComponent(last));
+	} catch {
+		return "";
+	}
+}
+
+/** File name from a Content-Disposition header, or "" when absent/unusable. */
+export function filenameFromDisposition(header) {
+	if (typeof header !== "string" || header.length === 0) return "";
+	const extended = /filename\*\s*=\s*([^;]+)/i.exec(header);
+	if (extended !== null) {
+		const value = extended[1].trim().replace(/^["']|["']$/g, "");
+		const withoutCharset = value.includes("''") ? value.slice(value.indexOf("''") + 2) : value;
+		try {
+			return sanitizeFilename(decodeURIComponent(withoutCharset));
+		} catch {
+			return sanitizeFilename(withoutCharset);
+		}
+	}
+	const plain = /filename\s*=\s*("([^"]*)"|([^;]+))/i.exec(header);
+	if (plain === null) return "";
+	return sanitizeFilename((plain[2] ?? plain[3] ?? "").trim());
+}
+
+/** Decide the file name to save under: Content-Disposition first, then the URL. */
+export function pickFilename(rawUrl, headers) {
+	const fromHeader = filenameFromDisposition(headers?.["content-disposition"]);
+	if (fromHeader.length > 0) return fromHeader;
+	const fromUrl = filenameFromUrl(rawUrl);
+	if (fromUrl.length > 0) return fromUrl;
+	return "download.bin";
+}
+
+/** True when the URL scheme is one this plugin downloads. */
+export function isHttpUrl(value) {
+	try {
+		const url = new URL(value);
+		return url.protocol === "http:" || url.protocol === "https:";
+	} catch {
+		return false;
+	}
+}