cursedbelt-server 4.42.0 → 4.43.1

package/src/server/youtube/ytdlp.spec.ts

@@ -0,0 +1,259 @@
+/**
+ * The shared yt-dlp runner (forge task 2349 — these specs came over from Music's own runner), with
+ * yt-dlp itself injected — nothing here reaches YouTube.
+ *
+ * What is worth pinning: that progress is read from the lines this file asked for and from
+ * nothing else, that a failure is sorted by what fixes it, and that the two self-healing paths
+ * (install when missing, update when out of date) run exactly when they should and say so
+ * when they did not help.
+ */
+import { afterEach, beforeEach, describe, expect, test } from "bun:test";
+import { chmod, mkdtemp, readdir, readFile, rm, stat, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import {
+	classifyYtDlpFailure,
+	createYtDlp,
+	installOrUpdateYtDlp,
+	parseYtInfo,
+	parseYtProgress,
+	type RunLines,
+	YtDlpError,
+	type YtDlpProfile,
+	ytDlpArgs,
+	ytDlpPath,
+} from "./ytdlp";
+
+/** Music's profile, as Music hands it in — audio as YouTube serves it. */
+const AUDIO: YtDlpProfile = {
+	format: "bestaudio[ext=webm][acodec=opus]/bestaudio[ext=m4a]",
+	maxFilesize: "200M",
+	noun: "That video's audio",
+};
+/** Collections' — H.264 + AAC merged into MP4. */
+const VIDEO: YtDlpProfile = {
+	format: "bv*[vcodec^=avc1][height<=1080]+ba[ext=m4a]/b[ext=mp4]",
+	extraArgs: ["--merge-output-format", "mp4"],
+	maxFilesize: "8G",
+	noun: "That video",
+};
+const FIX = "cd /somewhere && bun run yt-dlp";
+
+describe("reading what yt-dlp prints", () => {
+	test("a progress line is a fraction of the exact total", () => {
+		expect(parseYtProgress("ytprogress 2551 10202 NA")).toBeCloseTo(0.25, 3);
+	});
+
+	test("the estimate stands in when there is no exact total", () => {
+		expect(parseYtProgress("ytprogress 50 NA 200")).toBe(0.25);
+	});
+
+	test("🔴 no total and no estimate is no progress, never a made-up one", () => {
+		expect(parseYtProgress("ytprogress 50 NA NA")).toBeNull();
+	});
+
+	test("yt-dlp's own chatter is not progress", () => {
+		expect(parseYtProgress("[download]  41.1% of 9.73MiB")).toBeNull();
+		expect(parseYtInfo("[info] aqz: Downloading 1 format(s): 251")).toBeNull();
+	});
+
+	test("the info line carries the file's path and what it is", () => {
+		expect(parseYtInfo('ytinfo {"id":"abc","ext":"webm","filepath":"/t/abc.webm"}')).toMatchObject({
+			id: "abc",
+			ext: "webm",
+			filepath: "/t/abc.webm",
+		});
+		expect(parseYtInfo("ytinfo {not json")).toBeNull();
+	});
+
+	test("🔴 the profile's format and extra args are what yt-dlp is given, and the video is one video", () => {
+		const args = ytDlpArgs("abc", "/out", VIDEO, { jsRuntime: "/bin/bun" });
+		expect(args[args.indexOf("-f") + 1]).toBe(VIDEO.format);
+		expect(args.slice(args.indexOf("-f") + 2, args.indexOf("-f") + 4)).toEqual(["--merge-output-format", "mp4"]);
+		expect(args[args.indexOf("--max-filesize") + 1]).toBe("8G");
+		expect(args).toContain("--no-playlist");
+		expect(args.slice(0, 2)).toEqual(["--js-runtimes", "bun:/bin/bun"]);
+		expect(args.at(-1)).toBe("https://www.youtube.com/watch?v=abc");
+		expect(args.at(-2)).toBe("--");
+		expect(ytDlpArgs("abc", "/out", AUDIO)).not.toContain("--merge-output-format");
+	});
+
+	test("the binary is one per generation, under its state root", () => {
+		expect(ytDlpPath("/state")).toBe("/state/bin/yt-dlp");
+	});
+});
+
+describe("sorting a failure by what fixes it", () => {
+	const err = (line: string) => `WARNING: something\nERROR: ${line}\n`;
+	test.each([
+		["[youtube] abc: Video unavailable. This video is private", "unavailable"],
+		["[youtube] abc: Sign in to confirm your age", "unavailable"],
+		["[youtube] abc: Sign in to confirm you're not a bot", "blocked"],
+		["unable to download video data: HTTP Error 403: Forbidden", "outdated"],
+		["[youtube] abc: Unable to extract nsig function code", "outdated"],
+		["[youtube] abc: Requested format is not available", "outdated"],
+		["File is larger than max-filesize (300000000 bytes > 209715200 bytes). Aborting.", "too-big"],
+		["something nobody has seen before", "failed"],
+	] as const)("%s → %s", (line, kind) => {
+		expect(classifyYtDlpFailure(err(line))).toBe(kind);
+	});
+});
+
+describe("the fetcher", () => {
+	let dir: string;
+	let binary: string;
+	let runs: string[][];
+	let updates: number;
+	let installs: number;
+
+	beforeEach(async () => {
+		dir = await mkdtemp(join(tmpdir(), "ytdlp-test-"));
+		binary = join(dir, "yt-dlp");
+		await writeFile(binary, "#!/bin/sh\n");
+		runs = [];
+		updates = 0;
+		installs = 0;
+	});
+	afterEach(async () => {
+		await rm(dir, { recursive: true, force: true });
+	});
+
+	/** A yt-dlp that answers each run from the script, in order; the last answer repeats. */
+	const scripted =
+		(...answers: ("ok" | string)[]): RunLines =>
+		async (_bin, args, onLine) => {
+			runs.push(args);
+			const answer = answers[Math.min(runs.length - 1, answers.length - 1)] as string;
+			if (answer !== "ok") return { code: 1, stderr: `ERROR: ${answer}\n` };
+			const out = args[args.indexOf("-o") + 1] as string;
+			const file = out.replace("%(id)s.%(ext)s", "abc.webm");
+			await writeFile(file, new Uint8Array([0x1a, 0x45, 0xdf, 0xa3, 1, 2, 3]));
+			onLine("ytprogress 1 4 NA");
+			onLine("[download] 100%");
+			onLine(`ytinfo ${JSON.stringify({ id: "abc", title: "T", ext: "webm", filepath: file })}`);
+			return { code: 0, stderr: "" };
+		};
+
+	const make = (run: RunLines, profile: YtDlpProfile = AUDIO) =>
+		createYtDlp({
+			binary,
+			profile,
+			fixCommand: FIX,
+			tmpRoot: dir,
+			jsRuntime: "/bin/bun",
+			run,
+			install: async () => {
+				installs++;
+			},
+			update: async () => {
+				updates++;
+				return "Updated yt-dlp to stable@2099.01.01";
+			},
+		});
+
+	test("a download hands back the file, the extension and the info, and reports progress", async () => {
+		const fractions: number[] = [];
+		const audio = await make(scripted("ok"))("abc", (f) => fractions.push(f));
+		expect(audio.extension).toBe("webm");
+		expect(audio.info.title).toBe("T");
+		expect(new Uint8Array(await readFile(audio.path)).slice(0, 4)).toEqual(new Uint8Array([0x1a, 0x45, 0xdf, 0xa3]));
+		expect(fractions).toEqual([0.25]);
+		expect(installs).toBe(0);
+		// 🔴 The file is the caller's until it disposes — then it and its directory are gone.
+		await audio.dispose();
+		expect(await stat(audio.dir).catch(() => null)).toBeNull();
+	});
+
+	test("🔴 a failed run leaves no temporary directory behind", async () => {
+		await make(scripted("[youtube] abc: Private video"))("abc").catch(() => undefined);
+		expect((await readdir(dir)).filter((name) => name.startsWith("ytdlp-"))).toEqual([]);
+	});
+
+	test("the ceiling sentence uses the profile's own noun and size", async () => {
+		const failure = await make(scripted("File is larger than max-filesize (9 > 8). Aborting."), VIDEO)("abc").then(
+			() => {
+				throw new Error("expected a failure");
+			},
+			(e: unknown) => e as YtDlpError,
+		);
+		expect(failure.kind).toBe("too-big");
+		expect(failure.message).toBe("That video is over this app's 8GB ceiling for one download.");
+	});
+
+	test("an intermittent 403 is retried, and a retry that works is simply the answer", async () => {
+		const audio = await make(scripted("HTTP Error 403: Forbidden", "ok"))("abc");
+		expect(audio.info.id).toBe("abc");
+		expect(runs).toHaveLength(2);
+		expect(updates).toBe(0);
+	});
+
+	test("🔴 out of date: retried, then updated ONCE, then tried again — and the update is what fixed it", async () => {
+		const audio = await make(scripted("Unable to extract nsig", "Unable to extract nsig", "Unable to extract nsig", "ok"))("abc");
+		expect(audio.info.id).toBe("abc");
+		expect(updates).toBe(1);
+		expect(runs).toHaveLength(4);
+	});
+
+	test("🔴 out of date and the update did not help: the sentence names the command to run", async () => {
+		const failure = await make(scripted("Unable to extract nsig"))("abc").catch((e: unknown) => e);
+		expect(failure).toBeInstanceOf(YtDlpError);
+		expect((failure as YtDlpError).kind).toBe("outdated");
+		expect((failure as YtDlpError).message).toMatch(/out of date/);
+		expect((failure as YtDlpError).message).toMatch(/already updated it/);
+		expect((failure as YtDlpError).message).toMatch(/run: cd \/somewhere && bun run yt-dlp$/);
+		expect(updates).toBe(1);
+	});
+
+	test("the self-update runs at most once an hour, however many videos fail", async () => {
+		const fetchAudio = make(scripted("Unable to extract nsig"));
+		await fetchAudio("abc").catch(() => undefined);
+		const second = await fetchAudio("abc").then(() => { throw new Error("expected a failure"); }, (e: unknown) => e as YtDlpError);
+		expect(updates).toBe(1);
+		// Not updated THIS time, so the sentence tells the owner to do it rather than claiming it was done.
+		expect(second.message).toMatch(/Run: cd /);
+	});
+
+	test("a private video is not retried and not a reason to update", async () => {
+		const failure = await make(scripted("[youtube] abc: Private video"))("abc").then(() => { throw new Error("expected a failure"); }, (e: unknown) => e as YtDlpError);
+		expect(failure.kind).toBe("unavailable");
+		expect(runs).toHaveLength(1);
+		expect(updates).toBe(0);
+	});
+
+	test("🔴 a missing binary is installed before the first run", async () => {
+		await rm(binary);
+		await make(scripted("ok"))("abc");
+		expect(installs).toBe(1);
+	});
+
+	test("🔴 an install that fails says so and names the command", async () => {
+		await rm(binary);
+		const failure = await createYtDlp({
+			binary,
+			profile: AUDIO,
+			fixCommand: FIX,
+			tmpRoot: dir,
+			run: scripted("ok"),
+			install: async () => {
+				throw new Error("GitHub answered 503");
+			},
+		})("abc").then(() => { throw new Error("expected a failure"); }, (e: unknown) => e as YtDlpError);
+		expect(failure.kind).toBe("missing");
+		expect(failure.message).toMatch(/GitHub answered 503.*bun run yt-dlp/);
+		expect(runs).toHaveLength(0);
+	});
+});
+
+describe("bun run yt-dlp", () => {
+	test("🔴 an installed binary is UPDATED, never re-downloaded over itself", async () => {
+		const dir = await mkdtemp(join(tmpdir(), "ytdlp-cli-"));
+		try {
+			const binary = join(dir, "yt-dlp");
+			await writeFile(binary, "#!/bin/sh\necho 'yt-dlp is up to date (stable@2099.01.01)'\n");
+			await chmod(binary, 0o755);
+			expect(await installOrUpdateYtDlp(binary)).toBe("yt-dlp is up to date (stable@2099.01.01)");
+		} finally {
+			await rm(dir, { recursive: true, force: true });
+		}
+	});
+});