cursedbelt-server 4.42.0 → 4.43.1

package/src/server/youtube/ytdlp.ts

@@ -0,0 +1,454 @@
+/**
+ * `cursedbelt-server/youtube/ytdlp` — run yt-dlp on the Mac and hand back the file it made.
+ *
+ * ── Why it is here (forge task 2349) ────────────────────────────────────────────────────
+ * Music downloads the owner's YouTube playlists as AUDIO (task 2348) and Collections downloads
+ * them as VIDEO (task 2349). Both runners were the same machine with a different `-f`: spawn
+ * yt-dlp, read its progress off a marker line, sort a failure by what fixes it, install the
+ * binary when it is missing and update it when YouTube has moved on. So that machine is here,
+ * once, and each app hands it a {@link YtDlpProfile} — the format, the size ceiling, the noun
+ * its sentences use — plus the one command that installs or updates yt-dlp from that app.
+ *
+ * What is NOT here, on purpose: where the binary lives (each app passes `binary`, normally
+ * {@link ytDlpPath} of the generation's state root), what a downloaded file becomes (Music files
+ * a track; Collections ingests a video), and any re-encode (only Collections has one, and only
+ * when no H.264 stream exists).
+ *
+ * ── The file stays on disk ──────────────────────────────────────────────────────────────
+ * A download resolves to a PATH inside a fresh temporary directory, and the caller disposes of
+ * it. A 1080p video is gigabytes, and reading it into memory to hand it over would make the
+ * agent's footprint the size of the largest video in a playlist. Music, whose files are a few
+ * MB, reads the bytes and disposes at once.
+ *
+ * ── Self-healing in the two ways that need no owner ─────────────────────────────────────
+ * A missing binary is installed before the first run, and a failure that reads as "this
+ * yt-dlp is out of date" updates it (at most once an hour) and tries again. What is left after
+ * that is a sentence that NAMES the fix — "yt-dlp is out of date … Run: cd … && bun run yt-dlp".
+ *
+ * YouTube's JS challenges need a JavaScript runtime and yt-dlp enables only Deno by default,
+ * so it is handed the Bun running this process — which it supports — rather than a second
+ * runtime installed for one tool.
+ *
+ * 🔴 Mac-only. It spawns a process and writes files; nothing a Worker mounts may import it.
+ * It imports `node:*` and nothing else, which `leafSubpathsImportNothing.spec.ts` holds it to.
+ */
+import { spawn } from "node:child_process";
+import { chmod, mkdir, mkdtemp, rename, rm, stat, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { dirname, join } from "node:path";
+
+/** Where the standalone build is published. `latest` always names the newest release. */
+const YTDLP_RELEASE_URL = "https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_macos";
+
+/** The markers this module asks yt-dlp to print, so its own chatter can never be mistaken for them. */
+const PROGRESS_MARK = "ytprogress ";
+const INFO_MARK = "ytinfo ";
+
+/**
+ * A download that failed and was tried again. yt-dlp answers an intermittent `403` on the
+ * media request (measured 2026-09-30: one in three identical runs), and a retry is the fix.
+ */
+const ATTEMPTS = 3;
+
+/** How often a fetcher may update yt-dlp by itself. A release that does not fix it will not
+ *  start fixing it on the tenth try. */
+const SELF_UPDATE_EVERY_MS = 60 * 60_000;
+
+/** `$STATE/bin/yt-dlp` — one binary for every app on the Mac. `state` is the generation's state root. */
+export function ytDlpPath(state: string): string {
+	return join(state, "bin", "yt-dlp");
+}
+
+/** yt-dlp's cache (the player JS it has already solved), kept with the generation's state. */
+export function ytDlpCacheDir(state: string): string {
+	return join(state, "cache", "yt-dlp");
+}
+
+/**
+ * What one app downloads. Music's is audio as YouTube serves it; Collections' is H.264 + AAC
+ * in MP4. Everything else about a run is the same.
+ */
+export interface YtDlpProfile {
+	/** yt-dlp's `-f` selector. */
+	format: string;
+	/** Arguments placed after the format — e.g. `--merge-output-format mp4`. */
+	extraArgs?: readonly string[];
+	/** `--max-filesize`, in yt-dlp's own notation (`200M`, `4G`). */
+	maxFilesize: string;
+	/** What a file is called in a sentence: "That video's audio …", "That video …". */
+	noun: string;
+}
+
+/** What yt-dlp says about a video — the fields an app reads. */
+export interface YtInfo {
+	id: string;
+	title?: string;
+	/** Set for YouTube Music tracks (the "Topic" channels): the song, not the video title. */
+	track?: string;
+	artist?: string;
+	creator?: string;
+	channel?: string;
+	uploader?: string;
+	album?: string;
+	release_year?: number;
+	/** `YYYYMMDD`. */
+	upload_date?: string;
+	duration?: number;
+	thumbnail?: string;
+	ext?: string;
+	filepath?: string;
+	/** The chosen video stream's codec — `avc1.640028`, `vp09…`, `none` for audio. */
+	vcodec?: string;
+	acodec?: string;
+	width?: number;
+	height?: number;
+}
+
+/** One finished download. `path` is inside `dir`; {@link dispose} removes both. */
+export interface YtDownload {
+	path: string;
+	/** `webm`, `m4a`, `mp4` — what the file IS, for its name and its MIME type. */
+	extension: string;
+	info: YtInfo;
+	/** The temporary directory the file is in — a caller may write a re-encode beside it. */
+	dir: string;
+	dispose(): Promise<void>;
+}
+
+/** Fetch one video. `onProgress` gets the fraction of the file received, 0–1. */
+export type YtDlpFetch = (videoId: string, onProgress?: (fraction: number) => void) => Promise<YtDownload>;
+
+/**
+ * Why a download failed, sorted by what the owner can do about it.
+ *
+ *  · `missing`     — no binary, and installing one failed.
+ *  · `outdated`    — YouTube changed and this yt-dlp has not caught up. Update fixes it.
+ *  · `blocked`     — YouTube is refusing this Mac as a bot. Time fixes it, usually.
+ *  · `unavailable` — the video itself: private, deleted, region-locked, members-only.
+ *  · `too-big`     — over the one-download ceiling.
+ *  · `failed`      — anything else, with yt-dlp's own last error line.
+ */
+export type YtDlpFailure = "missing" | "outdated" | "blocked" | "unavailable" | "too-big" | "failed";
+
+export class YtDlpError extends Error {
+	constructor(
+		message: string,
+		readonly kind: YtDlpFailure,
+	) {
+		super(message);
+		this.name = "YtDlpError";
+	}
+}
+
+/** A failure no retry and no update can change — the video's own fault, or the ceiling. */
+export const isFinalYtDlpFailure = (kind: YtDlpFailure): boolean => kind === "unavailable" || kind === "too-big";
+
+// ── Pure: reading what yt-dlp prints ───────────────────────────────────────────────────
+
+/**
+ * One progress line, as a fraction — or `null` for any other line.
+ *
+ * The template prints `downloaded total estimate`, and yt-dlp writes `NA` for a field it does
+ * not know. The exact total wins; the estimate is what a stream without a `content-length`
+ * has. With neither there is no honest fraction, so none is claimed.
+ */
+export function parseYtProgress(line: string): number | null {
+	if (!line.startsWith(PROGRESS_MARK)) return null;
+	const [downloaded, total, estimate] = line.slice(PROGRESS_MARK.length).trim().split(/\s+/).map(Number);
+	const whole = Number.isFinite(total) && (total as number) > 0 ? total : estimate;
+	if (!Number.isFinite(downloaded) || !Number.isFinite(whole) || (whole as number) <= 0) return null;
+	return Math.min(Math.max((downloaded as number) / (whole as number), 0), 1);
+}
+
+/** The info JSON printed after the file is in place, or `null` for any other line. */
+export function parseYtInfo(line: string): YtInfo | null {
+	if (!line.startsWith(INFO_MARK)) return null;
+	try {
+		const value = JSON.parse(line.slice(INFO_MARK.length)) as unknown;
+		return value && typeof value === "object" && typeof (value as YtInfo).id === "string" ? (value as YtInfo) : null;
+	} catch {
+		return null;
+	}
+}
+
+/** yt-dlp's last `ERROR:` line — the one that says why — or the stderr's last line. */
+function lastYtDlpError(stderr: string): string {
+	const lines = stderr
+		.split("\n")
+		.map((l) => l.trim())
+		.filter(Boolean);
+	const errors = lines.filter((l) => l.startsWith("ERROR:"));
+	return (errors.at(-1) ?? lines.at(-1) ?? "").replace(/^ERROR:\s*/, "");
+}
+
+const UNAVAILABLE =
+	/video unavailable|this video is unavailable|private video|has been removed|no longer available|account associated with this video has been terminated|not available in your country|blocked it in your country|members[- ]only|join this channel|confirm your age|age-restricted|copyright/i;
+const BLOCKED = /confirm you.?re not a bot|sign in to confirm|too many requests|http error 429/i;
+const OUTDATED =
+	/unable to extract|nsig|n challenge|signature (extraction|function)|precondition check failed|http error 403|requested format is not available|please report this issue|js challenge|player .*(response|js)|unsupported url|jsc/i;
+const TOO_BIG = /larger than max-filesize|max-filesize/i;
+
+/** Sort a failure by what can be done about it — see {@link YtDlpFailure}. */
+export function classifyYtDlpFailure(stderr: string): YtDlpFailure {
+	const line = lastYtDlpError(stderr);
+	if (TOO_BIG.test(line)) return "too-big";
+	if (UNAVAILABLE.test(line)) return "unavailable";
+	if (BLOCKED.test(line)) return "blocked";
+	if (OUTDATED.test(line)) return "outdated";
+	return "failed";
+}
+
+/** The sentence a downloads table shows. Every kind that has a remedy NAMES it. */
+function describeYtDlpFailure(
+	kind: YtDlpFailure,
+	detail: string,
+	context: { updated: boolean; fixCommand: string; profile: Pick<YtDlpProfile, "maxFilesize" | "noun"> },
+): string {
+	const fix = context.fixCommand;
+	switch (kind) {
+		case "missing":
+			return `yt-dlp is not installed on the Mac and installing it failed (${detail}). Run: ${fix}`;
+		case "outdated":
+			return (
+				`yt-dlp could not read YouTube — it is out of date for a change YouTube made (${detail}). ` +
+				(context.updated
+					? `The agent already updated it and it still fails, so the fix is not released yet; try again in a day, or run: ${fix}`
+					: `Run: ${fix}`)
+			);
+		case "blocked":
+			return `YouTube is refusing this Mac's requests as a bot (${detail}). It usually lifts within hours — try again later. If it persists, update yt-dlp: ${fix}`;
+		case "unavailable":
+			return `YouTube will not play this video to anyone signed out (${detail}), so there is nothing to download.`;
+		case "too-big":
+			return `${context.profile.noun} is over this app's ${context.profile.maxFilesize}B ceiling for one download.`;
+		default:
+			return `yt-dlp failed: ${detail || "no reason given"}. If YouTube links keep failing, update it: ${fix}`;
+	}
+}
+
+/** The arguments for one download. Exported so a test pins the flags that matter. */
+export function ytDlpArgs(
+	videoId: string,
+	outDir: string,
+	profile: Pick<YtDlpProfile, "format" | "extraArgs" | "maxFilesize">,
+	options: { cacheDir?: string; jsRuntime?: string } = {},
+): string[] {
+	return [
+		...(options.jsRuntime ? ["--js-runtimes", `bun:${options.jsRuntime}`] : []),
+		...(options.cacheDir ? ["--cache-dir", options.cacheDir] : []),
+		"--no-playlist",
+		"-f",
+		profile.format,
+		...(profile.extraArgs ?? []),
+		"--max-filesize",
+		profile.maxFilesize,
+		"--no-part",
+		"--no-mtime",
+		"--no-colors",
+		"--newline",
+		// `--print` implies `--quiet`, which would swallow the progress lines — hence `--progress`.
+		"--progress",
+		"--progress-template",
+		`download:${PROGRESS_MARK}%(progress.downloaded_bytes)s %(progress.total_bytes)s %(progress.total_bytes_estimate)s`,
+		"--print",
+		`after_move:${INFO_MARK}%()j`,
+		"-o",
+		join(outDir, "%(id)s.%(ext)s"),
+		"--",
+		`https://www.youtube.com/watch?v=${videoId}`,
+	];
+}
+
+// ── The process ────────────────────────────────────────────────────────────────────────
+
+/** Run a program, handing each stdout line over as it arrives. Injected in tests. */
+export type RunLines = (
+	binary: string,
+	args: string[],
+	onLine: (line: string) => void,
+) => Promise<{ code: number; stderr: string }>;
+
+const runLines: RunLines = (binary, args, onLine) =>
+	new Promise((resolve, reject) => {
+		const child = spawn(binary, args, { stdio: ["ignore", "pipe", "pipe"] });
+		let buffer = "";
+		let stderr = "";
+		child.stdout.setEncoding("utf8");
+		child.stdout.on("data", (chunk: string) => {
+			buffer += chunk;
+			let at = buffer.indexOf("\n");
+			while (at >= 0) {
+				onLine(buffer.slice(0, at).replace(/\r$/, ""));
+				buffer = buffer.slice(at + 1);
+				at = buffer.indexOf("\n");
+			}
+		});
+		child.stderr.setEncoding("utf8");
+		child.stderr.on("data", (chunk: string) => {
+			// The tail is the part that says why; a runaway stderr must not grow without bound.
+			stderr = (stderr + chunk).slice(-16_384);
+		});
+		child.on("error", reject);
+		child.on("close", (code) => {
+			if (buffer) onLine(buffer);
+			resolve({ code: code ?? 1, stderr });
+		});
+	});
+
+/**
+ * Put the newest standalone build at `binary`. Written beside it and renamed into place, so a
+ * download that dies halfway never leaves a truncated executable where a fetcher looks.
+ */
+export async function installYtDlp(binary: string, fetchImpl: typeof fetch = fetch): Promise<void> {
+	const res = await fetchImpl(YTDLP_RELEASE_URL, { redirect: "follow", signal: AbortSignal.timeout(120_000) });
+	if (!res.ok) throw new Error(`GitHub answered ${res.status} for ${YTDLP_RELEASE_URL}`);
+	const bytes = new Uint8Array(await res.arrayBuffer());
+	// The standalone build is a Mach-O executable; anything small is an error page.
+	if (bytes.length < 1_000_000) throw new Error(`the download was ${bytes.length} bytes, not a yt-dlp build`);
+	await mkdir(dirname(binary), { recursive: true });
+	const partial = `${binary}.partial`;
+	await writeFile(partial, bytes);
+	await chmod(partial, 0o755);
+	await rename(partial, binary);
+}
+
+/** `yt-dlp -U` — the standalone build updates itself in place. Resolves to what it printed. */
+export async function updateYtDlp(binary: string, run: RunLines = runLines): Promise<string> {
+	const lines: string[] = [];
+	const { code, stderr } = await run(binary, ["-U"], (line) => lines.push(line));
+	const said = [...lines, ...stderr.split("\n")].map((l) => l.trim()).filter(Boolean);
+	if (code !== 0) throw new Error(said.at(-1) ?? `yt-dlp -U exited ${code}`);
+	return said.at(-1) ?? "";
+}
+
+/**
+ * Install yt-dlp when it is missing, else update it — the body of each app's `bun run yt-dlp`.
+ * Resolves to the sentence to print.
+ */
+export async function installOrUpdateYtDlp(binary: string): Promise<string> {
+	if (!(await isFile(binary))) {
+		await installYtDlp(binary);
+		return `installed the latest release at ${binary}`;
+	}
+	return updateYtDlp(binary);
+}
+
+const isFile = async (path: string): Promise<boolean> => {
+	try {
+		return (await stat(path)).isFile();
+	} catch {
+		return false;
+	}
+};
+
+export interface YtDlpOptions {
+	binary: string;
+	profile: YtDlpProfile;
+	/** The ONE command, runnable from any directory, that installs or updates yt-dlp for this app. */
+	fixCommand: string;
+	/** The prefix of every log line — `[music]`, `[collections]`. */
+	label?: string;
+	cacheDir?: string;
+	/** The Bun binary yt-dlp may use for YouTube's JS challenges. Defaults to this process's. */
+	jsRuntime?: string;
+	/** Where each download's temporary directory is made. Defaults to the OS temp dir. */
+	tmpRoot?: string;
+	/** Injected in tests — the process, the installer, the updater, the clock. */
+	run?: RunLines;
+	install?: (binary: string) => Promise<void>;
+	update?: (binary: string) => Promise<string>;
+	now?: () => number;
+	log?: (line: string) => void;
+}
+
+/**
+ * The fetcher an app's download agent is handed. See the header for what it heals by itself
+ * and what it leaves as a sentence.
+ */
+export function createYtDlp(options: YtDlpOptions): YtDlpFetch {
+	const run = options.run ?? runLines;
+	const install = options.install ?? ((binary: string) => installYtDlp(binary));
+	const update = options.update ?? ((binary: string) => updateYtDlp(binary, run));
+	const now = options.now ?? Date.now;
+	const label = options.label ?? "[yt-dlp]";
+	const jsRuntime = options.jsRuntime ?? (process.versions.bun ? process.execPath : undefined);
+	let lastUpdate = Number.NEGATIVE_INFINITY;
+	const describe = (kind: YtDlpFailure, detail: string, updated: boolean): string =>
+		describeYtDlpFailure(kind, detail, { updated, fixCommand: options.fixCommand, profile: options.profile });
+
+	const ensureInstalled = async (): Promise<void> => {
+		if (await isFile(options.binary)) return;
+		options.log?.(`${label} yt-dlp: not at ${options.binary} — installing the latest release`);
+		try {
+			await install(options.binary);
+		} catch (error) {
+			const detail = error instanceof Error ? error.message : String(error);
+			throw new YtDlpError(describe("missing", detail, false), "missing");
+		}
+	};
+
+	/** One yt-dlp run into a fresh directory. Throws a classified error on failure. */
+	const attempt = async (videoId: string, onProgress?: (fraction: number) => void): Promise<YtDownload> => {
+		const dir = await mkdtemp(join(options.tmpRoot ?? tmpdir(), "ytdlp-"));
+		const dispose = () => rm(dir, { recursive: true, force: true });
+		try {
+			let info: YtInfo | null = null;
+			const { code, stderr } = await run(
+				options.binary,
+				ytDlpArgs(videoId, dir, options.profile, {
+					...(options.cacheDir ? { cacheDir: options.cacheDir } : {}),
+					...(jsRuntime ? { jsRuntime } : {}),
+				}),
+				(line) => {
+					const fraction = parseYtProgress(line);
+					if (fraction !== null) onProgress?.(fraction);
+					info = parseYtInfo(line) ?? info;
+				},
+			);
+			const found = info as YtInfo | null;
+			if (code !== 0 || !found?.filepath) {
+				const kind = code === 0 ? "failed" : classifyYtDlpFailure(stderr);
+				throw new YtDlpError(code === 0 ? "yt-dlp finished without saying where the file went" : lastYtDlpError(stderr), kind);
+			}
+			if (!(await isFile(found.filepath))) throw new YtDlpError(`yt-dlp named ${found.filepath} and it is not there`, "failed");
+			const extension = (found.ext || found.filepath.split(".").at(-1) || "bin").toLowerCase();
+			return { path: found.filepath, extension, info: found, dir, dispose };
+		} catch (error) {
+			await dispose();
+			throw error;
+		}
+	};
+
+	return async (videoId, onProgress) => {
+		await ensureInstalled();
+		let updated = false;
+		let last: YtDlpError | null = null;
+		for (let tries = 1; tries <= ATTEMPTS + 1; tries++) {
+			try {
+				return await attempt(videoId, onProgress);
+			} catch (error) {
+				last =
+					error instanceof YtDlpError ? error : new YtDlpError(error instanceof Error ? error.message : String(error), "failed");
+				if (isFinalYtDlpFailure(last.kind)) break;
+				if (tries < ATTEMPTS) continue;
+				// Out of plain retries. One more, after an update, if it reads as out of date.
+				if (last.kind === "outdated" && !updated && now() - lastUpdate >= SELF_UPDATE_EVERY_MS) {
+					lastUpdate = now();
+					try {
+						const said = await update(options.binary);
+						updated = true;
+						options.log?.(`${label} yt-dlp: updated after "${last.message}" — ${said}`);
+						continue;
+					} catch (updateError) {
+						options.log?.(
+							`${label} yt-dlp: update failed — ${updateError instanceof Error ? updateError.message : String(updateError)}`,
+						);
+					}
+				}
+				break;
+			}
+		}
+		const failure = last ?? new YtDlpError("no attempt ran", "failed");
+		throw new YtDlpError(describe(failure.kind, failure.message, updated), failure.kind);
+	};
+}