cursedbelt-server 4.41.0 → 4.43.0

package/src/server/youtube/youtubeApi.ts

@@ -0,0 +1,325 @@
+/**
+ * `cursedbelt-server/youtube` — the YouTube Data API v3, READ half only, normalized.
+ *
+ * Arrived in 4.43.0 (forge task 2349) from Music (`apps/music/src/server/tube/youtubeApi.ts`),
+ * where it was already catalog-agnostic, so Collections could list the same owner's playlists
+ * without a second copy. Worker-safe: it is `fetch` and nothing else.
+ *
+ * 🔴 **It never writes to YouTube.** The scope a consumer asks for is
+ * {@link YOUTUBE_READONLY_SCOPE}, and there is no code path here that could create, reorder or
+ * delete anything in the owner's account. The failure mode of a write-back mistake is
+ * destroying a playlist on a service no app here owns.
+ *
+ * ── Quota ────────────────────────────────────────────────────────────────────────────────
+ * Every call costs Data API quota (10,000 units/day by default; a `playlistItems` page costs 1,
+ * a `search` 100 — which is why there is no search). Playback is not the Data API, so a
+ * consumer reads once, when the owner asks, and never on a timer.
+ *
+ * Every read that fails says why AND what to do ({@link TubeApiResult}) — the same law
+ * `./google-oauth` runs on. Normalized shapes come out, so no route and no browser ever touches
+ * a Google response envelope.
+ */
+import type { FetchLike } from "../google-oauth/googleOAuth";
+
+const API_BASE = "https://www.googleapis.com/youtube/v3";
+
+/** YouTube's own still for a video — the fallback when a playlist row carries no thumbnail. */
+export const youtubeThumbnailUrl = (videoId: string): string =>
+	videoId ? `https://i.ytimg.com/vi/${videoId}/mqdefault.jpg` : "";
+
+/** Read-only, and nothing else. See the header. */
+export const YOUTUBE_READONLY_SCOPE = "https://www.googleapis.com/auth/youtube.readonly";
+
+export interface TubeApiVideo {
+	videoId: string;
+	title: string;
+	channelTitle: string;
+	thumbnail: string;
+	url: string;
+}
+
+export interface TubeApiPlaylist {
+	id: string;
+	title: string;
+	description: string;
+	thumbnail: string;
+	itemCount: number;
+}
+
+export interface TubeApiChannel {
+	id: string;
+	title: string;
+	thumbnail: string;
+}
+
+export interface TubePage<T> {
+	items: T[];
+	nextPageToken?: string;
+}
+
+/** A read that failed says why and what to do — the same law the token store runs on. */
+export type TubeApiResult<T> =
+	| { status: "ok"; value: T }
+	| { status: "error"; reason: string; fix: string };
+
+type Json = Record<string, unknown>;
+type Thumbs = Record<string, { url?: string } | undefined> | undefined;
+
+/**
+ * `medium` first, deliberately. `default` is 120×90 and looks like a thumbnail from 2009 on
+ * a retina phone; `maxres` does not exist for most videos, so preferring it would leave
+ * half a list with no picture at all.
+ */
+const pickThumb = (thumbs: Thumbs): string =>
+	thumbs?.medium?.url || thumbs?.high?.url || thumbs?.standard?.url || thumbs?.default?.url || "";
+
+const watchUrl = (videoId: string): string => `https://www.youtube.com/watch?v=${videoId}`;
+
+/**
+ * A playlist keeps a tombstone for a video the uploader deleted or made private. It has a
+ * title and a thumbnail and NOTHING that will ever play, so importing it would put dead
+ * rows in the owner's list that look exactly like live ones until pressed.
+ */
+const UNPLAYABLE_TITLES = new Set(["Private video", "Deleted video"]);
+
+async function get(
+	accessToken: string,
+	path: string,
+	params: Record<string, string>,
+	fetchImpl: FetchLike,
+): Promise<TubeApiResult<Json>> {
+	let res: Response;
+	try {
+		res = await fetchImpl(`${API_BASE}/${path}?${new URLSearchParams(params).toString()}`, {
+			headers: { authorization: `Bearer ${accessToken}` },
+			signal: AbortSignal.timeout(30_000),
+		});
+	} catch (err) {
+		return {
+			status: "error",
+			reason: `could not reach YouTube: ${err instanceof Error ? err.message : String(err)}`,
+			fix: "check this machine's internet connection and try again",
+		};
+	}
+	const body = (await res.json().catch(() => ({}))) as Json;
+	if (!res.ok) {
+		const detail = ((body.error as Json | undefined)?.message as string) || `HTTP ${res.status}`;
+		// 403 is overwhelmingly one of two things and they have different fixes, so the
+		// message says both rather than making the owner guess which.
+		if (res.status === 403) {
+			return {
+				status: "error",
+				reason: `YouTube refused the read: ${detail}`,
+				fix: "this is either the daily Data API quota (it resets at midnight Pacific) or the YouTube Data API v3 not being enabled for this Google Cloud project",
+			};
+		}
+		return {
+			status: "error",
+			reason: `YouTube refused the read: ${detail}`,
+			fix:
+				res.status === 401
+					? "the connection has lapsed — press Disconnect and Connect again"
+					: "try again; if it persists, check the YouTube API status",
+		};
+	}
+	return { status: "ok", value: body };
+}
+
+const rows = (body: Json): Json[] => (Array.isArray(body.items) ? (body.items as Json[]) : []);
+const nextToken = (body: Json): { nextPageToken?: string } =>
+	typeof body.nextPageToken === "string" ? { nextPageToken: body.nextPageToken } : {};
+
+/** `PT1H2M3S` → 3723. YouTube's ISO 8601 durations never carry years or months. */
+function parseIsoDuration(value: string): number | null {
+	const m = /^P(?:(\d+)D)?(?:T(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?)?$/.exec(value);
+	if (!m || value === "P" || value.endsWith("T")) return null;
+	const [, d, h, min, s] = m.map((x) => Number(x ?? 0));
+	return (d as number) * 86_400 + (h as number) * 3600 + (min as number) * 60 + (s as number);
+}
+
+export function createYouTubeApi(fetchImpl: FetchLike = fetch) {
+	return {
+		/** Who this grant belongs to — the name shown beside "Connected". */
+		async channel(accessToken: string): Promise<TubeApiResult<TubeApiChannel | null>> {
+			const res = await get(accessToken, "channels", { part: "snippet", mine: "true" }, fetchImpl);
+			if (res.status !== "ok") return res;
+			const first = rows(res.value)[0];
+			if (!first) return { status: "ok", value: null };
+			const snippet = (first.snippet as Json) ?? {};
+			return {
+				status: "ok",
+				value: {
+					id: String(first.id ?? ""),
+					title: String(snippet.title ?? ""),
+					thumbnail: pickThumb(snippet.thumbnails as Thumbs),
+				},
+			};
+		},
+
+		async playlists(
+			accessToken: string,
+			pageToken?: string,
+		): Promise<TubeApiResult<TubePage<TubeApiPlaylist>>> {
+			const res = await get(
+				accessToken,
+				"playlists",
+				{
+					part: "snippet,contentDetails",
+					mine: "true",
+					maxResults: "50",
+					...(pageToken ? { pageToken } : {}),
+				},
+				fetchImpl,
+			);
+			if (res.status !== "ok") return res;
+			return {
+				status: "ok",
+				value: {
+					...nextToken(res.value),
+					items: rows(res.value).map((p) => {
+						const snippet = (p.snippet as Json) ?? {};
+						const details = (p.contentDetails as Json) ?? {};
+						return {
+							id: String(p.id ?? ""),
+							title: String(snippet.title ?? ""),
+							description: String(snippet.description ?? ""),
+							thumbnail: pickThumb(snippet.thumbnails as Thumbs),
+							itemCount: typeof details.itemCount === "number" ? details.itemCount : 0,
+						};
+					}),
+				},
+			};
+		},
+
+		async playlistItems(
+			accessToken: string,
+			playlistId: string,
+			pageToken?: string,
+		): Promise<TubeApiResult<TubePage<TubeApiVideo>>> {
+			const res = await get(
+				accessToken,
+				"playlistItems",
+				{
+					part: "snippet,contentDetails",
+					playlistId,
+					maxResults: "50",
+					...(pageToken ? { pageToken } : {}),
+				},
+				fetchImpl,
+			);
+			if (res.status !== "ok") return res;
+			return {
+				status: "ok",
+				value: {
+					...nextToken(res.value),
+					items: rows(res.value)
+						.map((item): TubeApiVideo => {
+							const snippet = (item.snippet as Json) ?? {};
+							const details = (item.contentDetails as Json) ?? {};
+							const resource = snippet.resourceId as Json | undefined;
+							const videoId = String(details.videoId ?? resource?.videoId ?? "");
+							return {
+								videoId,
+								title: String(snippet.title ?? ""),
+								channelTitle: String(snippet.videoOwnerChannelTitle ?? ""),
+								thumbnail: pickThumb(snippet.thumbnails as Thumbs) || youtubeThumbnailUrl(videoId),
+								url: watchUrl(videoId),
+							};
+						})
+						.filter((v) => v.videoId && !UNPLAYABLE_TITLES.has(v.title)),
+				},
+			};
+		},
+
+		/**
+		 * How long each video is, in seconds — `playlistItems` does not say. Up to 50 ids per
+		 * call (1 quota unit each call), so a 200-video playlist costs four. A video YouTube no
+		 * longer returns (private since, deleted) is simply absent from the map.
+		 */
+		async durations(accessToken: string, videoIds: readonly string[]): Promise<TubeApiResult<Map<string, number>>> {
+			const out = new Map<string, number>();
+			for (let at = 0; at < videoIds.length; at += 50) {
+				const res = await get(
+					accessToken,
+					"videos",
+					{ part: "contentDetails", id: videoIds.slice(at, at + 50).join(","), maxResults: "50" },
+					fetchImpl,
+				);
+				if (res.status !== "ok") return res;
+				for (const v of rows(res.value)) {
+					const seconds = parseIsoDuration(String((v.contentDetails as Json)?.duration ?? ""));
+					if (seconds !== null) out.set(String(v.id ?? ""), seconds);
+				}
+			}
+			return { status: "ok", value: out };
+		},
+
+		/**
+		 * Liked videos. Not a playlist read: the "LL" pseudo-playlist is not reliably
+		 * listable through `playlistItems`, whereas `videos?myRating=like` is the documented
+		 * way to ask and costs the same.
+		 */
+		async liked(
+			accessToken: string,
+			pageToken?: string,
+		): Promise<TubeApiResult<TubePage<TubeApiVideo>>> {
+			const res = await get(
+				accessToken,
+				"videos",
+				{
+					part: "snippet",
+					myRating: "like",
+					maxResults: "50",
+					...(pageToken ? { pageToken } : {}),
+				},
+				fetchImpl,
+			);
+			if (res.status !== "ok") return res;
+			return {
+				status: "ok",
+				value: {
+					...nextToken(res.value),
+					items: rows(res.value)
+						.map((v): TubeApiVideo => {
+							const snippet = (v.snippet as Json) ?? {};
+							const videoId = String(v.id ?? "");
+							return {
+								videoId,
+								title: String(snippet.title ?? ""),
+								channelTitle: String(snippet.channelTitle ?? ""),
+								thumbnail: pickThumb(snippet.thumbnails as Thumbs) || youtubeThumbnailUrl(videoId),
+								url: watchUrl(videoId),
+							};
+						})
+						.filter((v) => v.videoId),
+				},
+			};
+		},
+	};
+}
+
+export type YouTubeApi = ReturnType<typeof createYouTubeApi>;
+
+/**
+ * Walk every page of a paginated read, stopping at `maxPages` and SAYING SO (`truncated`).
+ *
+ * A list that quietly stops at the cap is indistinguishable from one that is exactly that long,
+ * so the cap is reported. 40 pages of 50 is 2,000 videos — larger than any real playlist, and
+ * small enough that a pathological account cannot turn one button into a thousand API calls.
+ */
+export async function collectYouTubePages<T>(
+	fetchPage: (pageToken?: string) => Promise<TubeApiResult<TubePage<T>>>,
+	maxPages = 40,
+): Promise<{ status: "ok"; items: T[]; truncated: boolean } | { status: "error"; reason: string; fix: string }> {
+	const items: T[] = [];
+	let pageToken: string | undefined;
+	let pages = 0;
+	do {
+		const page = await fetchPage(pageToken);
+		if (page.status !== "ok") return page;
+		items.push(...page.value.items);
+		pageToken = page.value.nextPageToken;
+		pages += 1;
+	} while (pageToken && pages < maxPages);
+	return { status: "ok", items, truncated: Boolean(pageToken) };
+}

package/src/server/youtube/ytdlp.spec.ts

@@ -0,0 +1,259 @@
+/**
+ * The shared yt-dlp runner (forge task 2349 — these specs came over from Music's own runner), with
+ * yt-dlp itself injected — nothing here reaches YouTube.
+ *
+ * What is worth pinning: that progress is read from the lines this file asked for and from
+ * nothing else, that a failure is sorted by what fixes it, and that the two self-healing paths
+ * (install when missing, update when out of date) run exactly when they should and say so
+ * when they did not help.
+ */
+import { afterEach, beforeEach, describe, expect, test } from "bun:test";
+import { chmod, mkdtemp, readdir, readFile, rm, stat, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import {
+	classifyYtDlpFailure,
+	createYtDlp,
+	installOrUpdateYtDlp,
+	parseYtInfo,
+	parseYtProgress,
+	type RunLines,
+	YtDlpError,
+	type YtDlpProfile,
+	ytDlpArgs,
+	ytDlpPath,
+} from "./ytdlp";
+
+/** Music's profile, as Music hands it in — audio as YouTube serves it. */
+const AUDIO: YtDlpProfile = {
+	format: "bestaudio[ext=webm][acodec=opus]/bestaudio[ext=m4a]",
+	maxFilesize: "200M",
+	noun: "That video's audio",
+};
+/** Collections' — H.264 + AAC merged into MP4. */
+const VIDEO: YtDlpProfile = {
+	format: "bv*[vcodec^=avc1][height<=1080]+ba[ext=m4a]/b[ext=mp4]",
+	extraArgs: ["--merge-output-format", "mp4"],
+	maxFilesize: "8G",
+	noun: "That video",
+};
+const FIX = "cd /somewhere && bun run yt-dlp";
+
+describe("reading what yt-dlp prints", () => {
+	test("a progress line is a fraction of the exact total", () => {
+		expect(parseYtProgress("ytprogress 2551 10202 NA")).toBeCloseTo(0.25, 3);
+	});
+
+	test("the estimate stands in when there is no exact total", () => {
+		expect(parseYtProgress("ytprogress 50 NA 200")).toBe(0.25);
+	});
+
+	test("🔴 no total and no estimate is no progress, never a made-up one", () => {
+		expect(parseYtProgress("ytprogress 50 NA NA")).toBeNull();
+	});
+
+	test("yt-dlp's own chatter is not progress", () => {
+		expect(parseYtProgress("[download]  41.1% of 9.73MiB")).toBeNull();
+		expect(parseYtInfo("[info] aqz: Downloading 1 format(s): 251")).toBeNull();
+	});
+
+	test("the info line carries the file's path and what it is", () => {
+		expect(parseYtInfo('ytinfo {"id":"abc","ext":"webm","filepath":"/t/abc.webm"}')).toMatchObject({
+			id: "abc",
+			ext: "webm",
+			filepath: "/t/abc.webm",
+		});
+		expect(parseYtInfo("ytinfo {not json")).toBeNull();
+	});
+
+	test("🔴 the profile's format and extra args are what yt-dlp is given, and the video is one video", () => {
+		const args = ytDlpArgs("abc", "/out", VIDEO, { jsRuntime: "/bin/bun" });
+		expect(args[args.indexOf("-f") + 1]).toBe(VIDEO.format);
+		expect(args.slice(args.indexOf("-f") + 2, args.indexOf("-f") + 4)).toEqual(["--merge-output-format", "mp4"]);
+		expect(args[args.indexOf("--max-filesize") + 1]).toBe("8G");
+		expect(args).toContain("--no-playlist");
+		expect(args.slice(0, 2)).toEqual(["--js-runtimes", "bun:/bin/bun"]);
+		expect(args.at(-1)).toBe("https://www.youtube.com/watch?v=abc");
+		expect(args.at(-2)).toBe("--");
+		expect(ytDlpArgs("abc", "/out", AUDIO)).not.toContain("--merge-output-format");
+	});
+
+	test("the binary is one per generation, under its state root", () => {
+		expect(ytDlpPath("/state")).toBe("/state/bin/yt-dlp");
+	});
+});
+
+describe("sorting a failure by what fixes it", () => {
+	const err = (line: string) => `WARNING: something\nERROR: ${line}\n`;
+	test.each([
+		["[youtube] abc: Video unavailable. This video is private", "unavailable"],
+		["[youtube] abc: Sign in to confirm your age", "unavailable"],
+		["[youtube] abc: Sign in to confirm you're not a bot", "blocked"],
+		["unable to download video data: HTTP Error 403: Forbidden", "outdated"],
+		["[youtube] abc: Unable to extract nsig function code", "outdated"],
+		["[youtube] abc: Requested format is not available", "outdated"],
+		["File is larger than max-filesize (300000000 bytes > 209715200 bytes). Aborting.", "too-big"],
+		["something nobody has seen before", "failed"],
+	] as const)("%s → %s", (line, kind) => {
+		expect(classifyYtDlpFailure(err(line))).toBe(kind);
+	});
+});
+
+describe("the fetcher", () => {
+	let dir: string;
+	let binary: string;
+	let runs: string[][];
+	let updates: number;
+	let installs: number;
+
+	beforeEach(async () => {
+		dir = await mkdtemp(join(tmpdir(), "ytdlp-test-"));
+		binary = join(dir, "yt-dlp");
+		await writeFile(binary, "#!/bin/sh\n");
+		runs = [];
+		updates = 0;
+		installs = 0;
+	});
+	afterEach(async () => {
+		await rm(dir, { recursive: true, force: true });
+	});
+
+	/** A yt-dlp that answers each run from the script, in order; the last answer repeats. */
+	const scripted =
+		(...answers: ("ok" | string)[]): RunLines =>
+		async (_bin, args, onLine) => {
+			runs.push(args);
+			const answer = answers[Math.min(runs.length - 1, answers.length - 1)] as string;
+			if (answer !== "ok") return { code: 1, stderr: `ERROR: ${answer}\n` };
+			const out = args[args.indexOf("-o") + 1] as string;
+			const file = out.replace("%(id)s.%(ext)s", "abc.webm");
+			await writeFile(file, new Uint8Array([0x1a, 0x45, 0xdf, 0xa3, 1, 2, 3]));
+			onLine("ytprogress 1 4 NA");
+			onLine("[download] 100%");
+			onLine(`ytinfo ${JSON.stringify({ id: "abc", title: "T", ext: "webm", filepath: file })}`);
+			return { code: 0, stderr: "" };
+		};
+
+	const make = (run: RunLines, profile: YtDlpProfile = AUDIO) =>
+		createYtDlp({
+			binary,
+			profile,
+			fixCommand: FIX,
+			tmpRoot: dir,
+			jsRuntime: "/bin/bun",
+			run,
+			install: async () => {
+				installs++;
+			},
+			update: async () => {
+				updates++;
+				return "Updated yt-dlp to stable@2099.01.01";
+			},
+		});
+
+	test("a download hands back the file, the extension and the info, and reports progress", async () => {
+		const fractions: number[] = [];
+		const audio = await make(scripted("ok"))("abc", (f) => fractions.push(f));
+		expect(audio.extension).toBe("webm");
+		expect(audio.info.title).toBe("T");
+		expect(new Uint8Array(await readFile(audio.path)).slice(0, 4)).toEqual(new Uint8Array([0x1a, 0x45, 0xdf, 0xa3]));
+		expect(fractions).toEqual([0.25]);
+		expect(installs).toBe(0);
+		// 🔴 The file is the caller's until it disposes — then it and its directory are gone.
+		await audio.dispose();
+		expect(await stat(audio.dir).catch(() => null)).toBeNull();
+	});
+
+	test("🔴 a failed run leaves no temporary directory behind", async () => {
+		await make(scripted("[youtube] abc: Private video"))("abc").catch(() => undefined);
+		expect((await readdir(dir)).filter((name) => name.startsWith("ytdlp-"))).toEqual([]);
+	});
+
+	test("the ceiling sentence uses the profile's own noun and size", async () => {
+		const failure = await make(scripted("File is larger than max-filesize (9 > 8). Aborting."), VIDEO)("abc").then(
+			() => {
+				throw new Error("expected a failure");
+			},
+			(e: unknown) => e as YtDlpError,
+		);
+		expect(failure.kind).toBe("too-big");
+		expect(failure.message).toBe("That video is over this app's 8GB ceiling for one download.");
+	});
+
+	test("an intermittent 403 is retried, and a retry that works is simply the answer", async () => {
+		const audio = await make(scripted("HTTP Error 403: Forbidden", "ok"))("abc");
+		expect(audio.info.id).toBe("abc");
+		expect(runs).toHaveLength(2);
+		expect(updates).toBe(0);
+	});
+
+	test("🔴 out of date: retried, then updated ONCE, then tried again — and the update is what fixed it", async () => {
+		const audio = await make(scripted("Unable to extract nsig", "Unable to extract nsig", "Unable to extract nsig", "ok"))("abc");
+		expect(audio.info.id).toBe("abc");
+		expect(updates).toBe(1);
+		expect(runs).toHaveLength(4);
+	});
+
+	test("🔴 out of date and the update did not help: the sentence names the command to run", async () => {
+		const failure = await make(scripted("Unable to extract nsig"))("abc").catch((e: unknown) => e);
+		expect(failure).toBeInstanceOf(YtDlpError);
+		expect((failure as YtDlpError).kind).toBe("outdated");
+		expect((failure as YtDlpError).message).toMatch(/out of date/);
+		expect((failure as YtDlpError).message).toMatch(/already updated it/);
+		expect((failure as YtDlpError).message).toMatch(/run: cd \/somewhere && bun run yt-dlp$/);
+		expect(updates).toBe(1);
+	});
+
+	test("the self-update runs at most once an hour, however many videos fail", async () => {
+		const fetchAudio = make(scripted("Unable to extract nsig"));
+		await fetchAudio("abc").catch(() => undefined);
+		const second = await fetchAudio("abc").then(() => { throw new Error("expected a failure"); }, (e: unknown) => e as YtDlpError);
+		expect(updates).toBe(1);
+		// Not updated THIS time, so the sentence tells the owner to do it rather than claiming it was done.
+		expect(second.message).toMatch(/Run: cd /);
+	});
+
+	test("a private video is not retried and not a reason to update", async () => {
+		const failure = await make(scripted("[youtube] abc: Private video"))("abc").then(() => { throw new Error("expected a failure"); }, (e: unknown) => e as YtDlpError);
+		expect(failure.kind).toBe("unavailable");
+		expect(runs).toHaveLength(1);
+		expect(updates).toBe(0);
+	});
+
+	test("🔴 a missing binary is installed before the first run", async () => {
+		await rm(binary);
+		await make(scripted("ok"))("abc");
+		expect(installs).toBe(1);
+	});
+
+	test("🔴 an install that fails says so and names the command", async () => {
+		await rm(binary);
+		const failure = await createYtDlp({
+			binary,
+			profile: AUDIO,
+			fixCommand: FIX,
+			tmpRoot: dir,
+			run: scripted("ok"),
+			install: async () => {
+				throw new Error("GitHub answered 503");
+			},
+		})("abc").then(() => { throw new Error("expected a failure"); }, (e: unknown) => e as YtDlpError);
+		expect(failure.kind).toBe("missing");
+		expect(failure.message).toMatch(/GitHub answered 503.*bun run yt-dlp/);
+		expect(runs).toHaveLength(0);
+	});
+});
+
+describe("bun run yt-dlp", () => {
+	test("🔴 an installed binary is UPDATED, never re-downloaded over itself", async () => {
+		const dir = await mkdtemp(join(tmpdir(), "ytdlp-cli-"));
+		try {
+			const binary = join(dir, "yt-dlp");
+			await writeFile(binary, "#!/bin/sh\necho 'yt-dlp is up to date (stable@2099.01.01)'\n");
+			await chmod(binary, 0o755);
+			expect(await installOrUpdateYtDlp(binary)).toBe("yt-dlp is up to date (stable@2099.01.01)");
+		} finally {
+			await rm(dir, { recursive: true, force: true });
+		}
+	});
+});