@oh-my-pi/pi-coding-agent 3.24.0 → 3.30.0

package/src/core/tools/web-fetch-handlers/youtube.test.ts

@@ -0,0 +1,198 @@
+import { describe, expect, it } from "bun:test";
+import { handleYouTube } from "./youtube";
+
+const SKIP = !process.env.WEB_FETCH_INTEGRATION;
+
+describe.skipIf(SKIP)("handleYouTube", () => {
+	it("returns null for non-YouTube URLs", async () => {
+		const result = await handleYouTube("https://example.com", 10);
+		expect(result).toBeNull();
+	});
+
+	it("returns null for invalid YouTube URLs", async () => {
+		const result = await handleYouTube("https://youtube.com/invalid", 10);
+		expect(result).toBeNull();
+	});
+
+	it("handles youtube.com/watch?v= format", async () => {
+		// Use Rick Astley's "Never Gonna Give You Up" - a stable, well-known video
+		const result = await handleYouTube("https://www.youtube.com/watch?v=dQw4w9WgXcQ", 30);
+		expect(result).not.toBeNull();
+		expect(result?.method).toMatch(/^youtube/);
+		expect(result?.contentType).toBe("text/markdown");
+		expect(result?.content).toContain("Video ID");
+		expect(result?.content).toContain("dQw4w9WgXcQ");
+	}, 30000);
+
+	it("handles youtu.be/ short format", async () => {
+		const result = await handleYouTube("https://youtu.be/dQw4w9WgXcQ", 30);
+		expect(result).not.toBeNull();
+		expect(result?.method).toMatch(/^youtube/);
+		expect(result?.content).toContain("dQw4w9WgXcQ");
+	}, 30000);
+
+	it("handles youtube.com/shorts/ format", async () => {
+		// Use a stable YouTube Shorts video
+		const result = await handleYouTube("https://www.youtube.com/shorts/jNQXAC9IVRw", 30);
+		expect(result).not.toBeNull();
+		expect(result?.method).toMatch(/^youtube/);
+		expect(result?.content).toContain("jNQXAC9IVRw");
+	}, 30000);
+
+	it("handles youtube.com/embed/ format", async () => {
+		const result = await handleYouTube("https://www.youtube.com/embed/dQw4w9WgXcQ", 30);
+		expect(result).not.toBeNull();
+		expect(result?.method).toMatch(/^youtube/);
+		expect(result?.content).toContain("dQw4w9WgXcQ");
+	}, 30000);
+
+	it("handles youtube.com/v/ format", async () => {
+		const result = await handleYouTube("https://www.youtube.com/v/dQw4w9WgXcQ", 30);
+		expect(result).not.toBeNull();
+		expect(result?.method).toMatch(/^youtube/);
+		expect(result?.content).toContain("dQw4w9WgXcQ");
+	}, 30000);
+
+	it("handles m.youtube.com mobile URLs", async () => {
+		const result = await handleYouTube("https://m.youtube.com/watch?v=dQw4w9WgXcQ", 30);
+		expect(result).not.toBeNull();
+		expect(result?.method).toMatch(/^youtube/);
+		expect(result?.content).toContain("dQw4w9WgXcQ");
+	}, 30000);
+
+	it("extracts video metadata when yt-dlp is available", async () => {
+		const result = await handleYouTube("https://www.youtube.com/watch?v=dQw4w9WgXcQ", 30);
+		expect(result).not.toBeNull();
+
+		// If yt-dlp is available, should have metadata
+		if (result?.method === "youtube") {
+			expect(result.content).toContain("Video ID");
+			expect(result.content).toContain("Channel");
+			// May have duration, views, upload date, etc.
+		}
+
+		// If yt-dlp is not available, should indicate that
+		if (result?.method === "youtube-no-ytdlp") {
+			expect(result.content).toContain("yt-dlp could not be installed");
+			expect(result.notes).toContain("yt-dlp installation failed");
+		}
+	}, 30000);
+
+	it("handles videos with transcripts gracefully", async () => {
+		// This video should have captions
+		const result = await handleYouTube("https://www.youtube.com/watch?v=dQw4w9WgXcQ", 30);
+		expect(result).not.toBeNull();
+
+		if (result?.method === "youtube") {
+			// Either has transcript or explicitly notes it's not available
+			const hasTranscript = result.content.includes("Transcript");
+			const noTranscriptNote = result.content.includes("No transcript available");
+			expect(hasTranscript || noTranscriptNote).toBe(true);
+		}
+	}, 30000);
+
+	it("handles videos without transcripts gracefully", async () => {
+		// Many music videos lack captions, but this is not guaranteed
+		// Just verify the handler doesn't crash and provides some info
+		const result = await handleYouTube("https://www.youtube.com/watch?v=kJQP7kiw5Fk", 30);
+		expect(result).not.toBeNull();
+
+		if (result?.method === "youtube") {
+			// Should still have basic metadata
+			expect(result.content).toContain("Video ID");
+		}
+	}, 30000);
+
+	it("returns appropriate response when yt-dlp is not available", async () => {
+		// We can't force yt-dlp to be unavailable in tests, but we can verify
+		// the return structure matches expectations for both cases
+		const result = await handleYouTube("https://www.youtube.com/watch?v=dQw4w9WgXcQ", 30);
+		expect(result).not.toBeNull();
+
+		// Should have one of these two methods
+		expect(["youtube", "youtube-no-ytdlp"]).toContain(result!.method);
+
+		// Both should have required fields
+		expect(result?.url).toBe("https://www.youtube.com/watch?v=dQw4w9WgXcQ");
+		expect(result?.finalUrl).toContain("youtube.com");
+		expect(result?.fetchedAt).toBeTruthy();
+		expect(typeof result?.truncated).toBe("boolean");
+		expect(Array.isArray(result?.notes)).toBe(true);
+	}, 30000);
+
+	it("normalizes video URLs to canonical format", async () => {
+		// Different input formats should normalize to same canonical URL
+		const result = await handleYouTube("https://youtu.be/dQw4w9WgXcQ", 30);
+		expect(result).not.toBeNull();
+		expect(result?.finalUrl).toBe("https://www.youtube.com/watch?v=dQw4w9WgXcQ");
+	}, 30000);
+
+	it("handles playlist URLs by extracting video ID", async () => {
+		const result = await handleYouTube(
+			"https://www.youtube.com/watch?v=dQw4w9WgXcQ&list=PLrAXtmErZgOeiKm4sgNOknGvNjby9efdf",
+			30,
+		);
+		expect(result).not.toBeNull();
+		expect(result?.content).toContain("dQw4w9WgXcQ");
+	}, 30000);
+
+	it("includes subtitle source information when available", async () => {
+		const result = await handleYouTube("https://www.youtube.com/watch?v=dQw4w9WgXcQ", 30);
+
+		if (result?.method === "youtube") {
+			// If transcript is present, should note the source
+			const hasManualNote = result.notes.includes("Using manual subtitles");
+			const hasAutoNote = result.notes.includes("Using auto-generated captions");
+			const hasNoSubsNote = result.notes.includes("No subtitles/captions available");
+
+			// Should have exactly one of these
+			const noteCount = [hasManualNote, hasAutoNote, hasNoSubsNote].filter(Boolean).length;
+			expect(noteCount).toBeGreaterThanOrEqual(1);
+		}
+	}, 30000);
+
+	it("formats duration in human readable format", async () => {
+		const result = await handleYouTube("https://www.youtube.com/watch?v=dQw4w9WgXcQ", 30);
+
+		if (result?.method === "youtube" && result.content.includes("Duration")) {
+			// Should have duration in M:SS or H:MM:SS format
+			expect(result.content).toMatch(/Duration.*\d+:\d{2}/);
+		}
+	}, 30000);
+
+	it("formats view count in readable format", async () => {
+		const result = await handleYouTube("https://www.youtube.com/watch?v=dQw4w9WgXcQ", 30);
+
+		if (result?.method === "youtube" && result.content.includes("Views")) {
+			// Should have views formatted (e.g., 1.5B, 100M, 10.5K)
+			expect(result.content).toMatch(/Views.*\d+(\.\d+)?[KM]?/);
+		}
+	}, 30000);
+
+	it("includes upload date when available", async () => {
+		const result = await handleYouTube("https://www.youtube.com/watch?v=dQw4w9WgXcQ", 30);
+
+		if (result?.method === "youtube" && result.content.includes("Uploaded")) {
+			// Should have date in YYYY-MM-DD format
+			expect(result.content).toMatch(/Uploaded.*\d{4}-\d{2}-\d{2}/);
+		}
+	}, 30000);
+
+	it("truncates long descriptions", async () => {
+		const result = await handleYouTube("https://www.youtube.com/watch?v=dQw4w9WgXcQ", 30);
+
+		if (result?.method === "youtube" && result.content.includes("Description")) {
+			// Description section should exist
+			expect(result.content).toContain("## Description");
+		}
+	}, 30000);
+
+	it("handles www prefix variations", async () => {
+		const withWww = await handleYouTube("https://www.youtube.com/watch?v=dQw4w9WgXcQ", 30);
+		const withoutWww = await handleYouTube("https://youtube.com/watch?v=dQw4w9WgXcQ", 30);
+
+		expect(withWww).not.toBeNull();
+		expect(withoutWww).not.toBeNull();
+		expect(withWww?.finalUrl).toBe(withoutWww?.finalUrl);
+	}, 30000);
+});

package/src/core/tools/web-fetch-handlers/youtube.ts

@@ -0,0 +1,319 @@
+import { unlinkSync } from "node:fs";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import { ensureTool } from "../../../utils/tools-manager";
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput } from "./types";
+
+/**
+ * Execute a command and return stdout
+ */
+function exec(
+	cmd: string,
+	args: string[],
+	options?: { timeout?: number; input?: string | Buffer },
+): { stdout: string; stderr: string; ok: boolean } {
+	const result = Bun.spawnSync([cmd, ...args], {
+		stdin: options?.input ? (options.input as any) : "ignore",
+		stdout: "pipe",
+		stderr: "pipe",
+	});
+	return {
+		stdout: result.stdout?.toString() ?? "",
+		stderr: result.stderr?.toString() ?? "",
+		ok: result.exitCode === 0,
+	};
+}
+
+interface YouTubeUrl {
+	videoId: string;
+	playlistId?: string;
+}
+
+/**
+ * Parse YouTube URL into components
+ */
+function parseYouTubeUrl(url: string): YouTubeUrl | null {
+	try {
+		const parsed = new URL(url);
+		const hostname = parsed.hostname.replace(/^www\./, "");
+
+		// youtube.com/watch?v=VIDEO_ID
+		if ((hostname === "youtube.com" || hostname === "m.youtube.com") && parsed.pathname === "/watch") {
+			const videoId = parsed.searchParams.get("v");
+			const playlistId = parsed.searchParams.get("list") || undefined;
+			if (videoId) return { videoId, playlistId };
+		}
+
+		// youtube.com/v/VIDEO_ID or youtube.com/embed/VIDEO_ID
+		if (hostname === "youtube.com" || hostname === "m.youtube.com") {
+			const match = parsed.pathname.match(/^\/(v|embed)\/([a-zA-Z0-9_-]{11})/);
+			if (match) return { videoId: match[2] };
+		}
+
+		// youtu.be/VIDEO_ID
+		if (hostname === "youtu.be") {
+			const videoId = parsed.pathname.slice(1).split("/")[0];
+			if (videoId && /^[a-zA-Z0-9_-]{11}$/.test(videoId)) {
+				return { videoId };
+			}
+		}
+
+		// youtube.com/shorts/VIDEO_ID
+		if (hostname === "youtube.com" && parsed.pathname.startsWith("/shorts/")) {
+			const videoId = parsed.pathname.replace("/shorts/", "").split("/")[0];
+			if (videoId && /^[a-zA-Z0-9_-]{11}$/.test(videoId)) {
+				return { videoId };
+			}
+		}
+	} catch {}
+
+	return null;
+}
+
+/**
+ * Clean VTT subtitle content to plain text
+ */
+function cleanVttToText(vtt: string): string {
+	const lines = vtt.split("\n");
+	const textLines: string[] = [];
+	let lastLine = "";
+
+	for (const line of lines) {
+		// Skip WEBVTT header, timestamps, and metadata
+		if (
+			line.startsWith("WEBVTT") ||
+			line.startsWith("Kind:") ||
+			line.startsWith("Language:") ||
+			line.match(/^\d{2}:\d{2}/) || // Timestamp lines
+			line.match(/^[a-f0-9-]{36}$/) || // UUID cue identifiers
+			line.match(/^\d+$/) || // Numeric cue identifiers
+			line.includes("-->") ||
+			line.trim() === ""
+		) {
+			continue;
+		}
+
+		// Remove inline timestamp tags like <00:00:01.520>
+		let cleaned = line.replace(/<\d{2}:\d{2}:\d{2}\.\d{3}>/g, "");
+		// Remove other VTT tags like <c> </c>
+		cleaned = cleaned.replace(/<\/?[^>]+>/g, "");
+		cleaned = cleaned.trim();
+
+		// Skip duplicates (auto-generated captions often repeat)
+		if (cleaned && cleaned !== lastLine) {
+			textLines.push(cleaned);
+			lastLine = cleaned;
+		}
+	}
+
+	return textLines.join(" ").replace(/\s+/g, " ").trim();
+}
+
+/**
+ * Format duration from seconds to human readable
+ */
+function formatDuration(seconds: number): string {
+	const h = Math.floor(seconds / 3600);
+	const m = Math.floor((seconds % 3600) / 60);
+	const s = Math.floor(seconds % 60);
+	if (h > 0) return `${h}:${m.toString().padStart(2, "0")}:${s.toString().padStart(2, "0")}`;
+	return `${m}:${s.toString().padStart(2, "0")}`;
+}
+
+/**
+ * Handle YouTube URLs - fetch metadata and transcript
+ */
+export const handleYouTube: SpecialHandler = async (url: string, timeout: number): Promise<RenderResult | null> => {
+	const yt = parseYouTubeUrl(url);
+	if (!yt) return null;
+
+	// Ensure yt-dlp is available (auto-download if missing)
+	const ytdlp = await ensureTool("yt-dlp", true);
+	if (!ytdlp) {
+		return {
+			url,
+			finalUrl: url,
+			contentType: "text/plain",
+			method: "youtube-no-ytdlp",
+			content: "YouTube video detected but yt-dlp could not be installed.",
+			fetchedAt: new Date().toISOString(),
+			truncated: false,
+			notes: ["yt-dlp installation failed"],
+		};
+	}
+
+	const fetchedAt = new Date().toISOString();
+	const notes: string[] = [];
+	const videoUrl = `https://www.youtube.com/watch?v=${yt.videoId}`;
+
+	// Fetch video metadata
+	const metaResult = exec(ytdlp, ["--dump-json", "--no-warnings", "--no-playlist", "--skip-download", videoUrl], {
+		timeout: timeout * 1000,
+	});
+
+	let title = "YouTube Video";
+	let channel = "";
+	let description = "";
+	let duration = 0;
+	let uploadDate = "";
+	let viewCount = 0;
+
+	if (metaResult.ok && metaResult.stdout.trim()) {
+		try {
+			const meta = JSON.parse(metaResult.stdout) as {
+				title?: string;
+				channel?: string;
+				uploader?: string;
+				description?: string;
+				duration?: number;
+				upload_date?: string;
+				view_count?: number;
+			};
+			title = meta.title || title;
+			channel = meta.channel || meta.uploader || "";
+			description = meta.description || "";
+			duration = meta.duration || 0;
+			uploadDate = meta.upload_date || "";
+			viewCount = meta.view_count || 0;
+		} catch {}
+	}
+
+	// Format upload date
+	let formattedDate = "";
+	if (uploadDate && uploadDate.length === 8) {
+		formattedDate = `${uploadDate.slice(0, 4)}-${uploadDate.slice(4, 6)}-${uploadDate.slice(6, 8)}`;
+	}
+
+	// Try to fetch subtitles
+	let transcript = "";
+	let transcriptSource = "";
+
+	// First, list available subtitles
+	const listResult = exec(ytdlp, ["--list-subs", "--no-warnings", "--no-playlist", "--skip-download", videoUrl], {
+		timeout: timeout * 1000,
+	});
+
+	const hasManualSubs = listResult.stdout.includes("[info] Available subtitles");
+	const hasAutoSubs = listResult.stdout.includes("[info] Available automatic captions");
+
+	// Create temp directory for subtitle download
+	const tmpDir = tmpdir();
+	const tmpBase = path.join(tmpDir, `yt-${yt.videoId}-${Date.now()}`);
+
+	try {
+		// Try manual subtitles first (English preferred)
+		if (hasManualSubs) {
+			const subResult = exec(
+				ytdlp,
+				[
+					"--write-sub",
+					"--sub-lang",
+					"en,en-US,en-GB",
+					"--sub-format",
+					"vtt",
+					"--skip-download",
+					"--no-warnings",
+					"--no-playlist",
+					"-o",
+					tmpBase,
+					videoUrl,
+				],
+				{ timeout: timeout * 1000 },
+			);
+
+			if (subResult.ok) {
+				// Find the downloaded subtitle file using glob
+				const subFiles = await Array.fromAsync(new Bun.Glob(`${tmpBase}*.vtt`).scan({ absolute: true }));
+				if (subFiles.length > 0) {
+					const vttContent = await Bun.file(subFiles[0]).text();
+					transcript = cleanVttToText(vttContent);
+					transcriptSource = "manual";
+					notes.push("Using manual subtitles");
+				}
+			}
+		}
+
+		// Fall back to auto-generated captions
+		if (!transcript && hasAutoSubs) {
+			const autoResult = exec(
+				ytdlp,
+				[
+					"--write-auto-sub",
+					"--sub-lang",
+					"en,en-US,en-GB",
+					"--sub-format",
+					"vtt",
+					"--skip-download",
+					"--no-warnings",
+					"--no-playlist",
+					"-o",
+					tmpBase,
+					videoUrl,
+				],
+				{ timeout: timeout * 1000 },
+			);
+
+			if (autoResult.ok) {
+				const subFiles = await Array.fromAsync(new Bun.Glob(`${tmpBase}*.vtt`).scan({ absolute: true }));
+				if (subFiles.length > 0) {
+					const vttContent = await Bun.file(subFiles[0]).text();
+					transcript = cleanVttToText(vttContent);
+					transcriptSource = "auto-generated";
+					notes.push("Using auto-generated captions");
+				}
+			}
+		}
+	} finally {
+		// Cleanup temp files using sync unlink to avoid leaving handles open
+		try {
+			const tmpFiles = await Array.fromAsync(new Bun.Glob(`${tmpBase}*`).scan({ absolute: true }));
+			for (const f of tmpFiles) {
+				try {
+					unlinkSync(f);
+				} catch {}
+			}
+		} catch {}
+	}
+
+	// Build markdown output
+	let md = `# ${title}\n\n`;
+	if (channel) md += `**Channel:** ${channel}\n`;
+	if (formattedDate) md += `**Uploaded:** ${formattedDate}\n`;
+	if (duration > 0) md += `**Duration:** ${formatDuration(duration)}\n`;
+	if (viewCount > 0) {
+		const formatted =
+			viewCount >= 1_000_000
+				? `${(viewCount / 1_000_000).toFixed(1)}M`
+				: viewCount >= 1_000
+					? `${(viewCount / 1_000).toFixed(1)}K`
+					: String(viewCount);
+		md += `**Views:** ${formatted}\n`;
+	}
+	md += `**Video ID:** ${yt.videoId}\n\n`;
+
+	if (description) {
+		// Truncate long descriptions
+		const descPreview = description.length > 1000 ? `${description.slice(0, 1000)}...` : description;
+		md += `---\n\n## Description\n\n${descPreview}\n\n`;
+	}
+
+	if (transcript) {
+		md += `---\n\n## Transcript (${transcriptSource})\n\n${transcript}\n`;
+	} else {
+		notes.push("No subtitles/captions available");
+		md += `---\n\n*No transcript available for this video.*\n`;
+	}
+
+	const output = finalizeOutput(md);
+	return {
+		url,
+		finalUrl: videoUrl,
+		contentType: "text/markdown",
+		method: "youtube",
+		content: output.content,
+		fetchedAt,
+		truncated: output.truncated,
+		notes,
+	};
+};