@oh-my-pi/pi-coding-agent 3.25.0 → 3.31.0

package/src/core/tools/web-scrapers/wikipedia.test.ts

@@ -0,0 +1,73 @@
+import { describe, expect, it } from "bun:test";
+import { handleWikipedia } from "./wikipedia";
+
+const SKIP = !process.env.WEB_FETCH_INTEGRATION;
+
+describe.skipIf(SKIP)("handleWikipedia", () => {
+	it("returns null for non-Wikipedia URLs", async () => {
+		const result = await handleWikipedia("https://example.com", 10);
+		expect(result).toBeNull();
+	});
+
+	it("returns null for Wikipedia URLs without /wiki/ path", async () => {
+		const result = await handleWikipedia("https://en.wikipedia.org/", 10);
+		expect(result).toBeNull();
+	});
+
+	it("fetches a known article with full metadata", async () => {
+		// "Computer" is a stable, well-established article
+		const result = await handleWikipedia("https://en.wikipedia.org/wiki/Computer", 20);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("wikipedia");
+		expect(result?.contentType).toBe("text/markdown");
+		expect(result?.content).toContain("Computer");
+		expect(result?.url).toBe("https://en.wikipedia.org/wiki/Computer");
+		expect(result?.finalUrl).toBe("https://en.wikipedia.org/wiki/Computer");
+		expect(result?.truncated).toBe(false);
+		expect(result?.notes).toContain("Fetched via Wikipedia API");
+		expect(result?.fetchedAt).toBeDefined();
+		// Should be a valid ISO timestamp
+		expect(() => new Date(result?.fetchedAt ?? "")).not.toThrow();
+		// The handler should filter out References and External links sections
+		const content = result?.content ?? "";
+		const hasReferencesHeading = /^## References$/m.test(content);
+		const hasExternalLinksHeading = /^## External links$/m.test(content);
+		// At least one of these should be filtered out
+		expect(hasReferencesHeading || hasExternalLinksHeading).toBe(false);
+	});
+
+	it("handles different language wikis", async () => {
+		// German Wikipedia article for "Computer"
+		const result = await handleWikipedia("https://de.wikipedia.org/wiki/Computer", 20);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("wikipedia");
+		expect(result?.contentType).toBe("text/markdown");
+		expect(result?.content).toContain("Computer");
+	});
+
+	it("handles article with special characters in title", async () => {
+		// Article with special characters: "C++"
+		const result = await handleWikipedia("https://en.wikipedia.org/wiki/C%2B%2B", 20);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("wikipedia");
+		expect(result?.contentType).toBe("text/markdown");
+		expect(result?.content).toMatch(/C\+\+/);
+	});
+
+	it("handles article with spaces and parentheses in title", async () => {
+		// Artificial intelligence uses underscores for spaces
+		const result = await handleWikipedia("https://en.wikipedia.org/wiki/Artificial_intelligence", 20);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("wikipedia");
+		expect(result?.contentType).toBe("text/markdown");
+		expect(result?.content).toMatch(/[Aa]rtificial intelligence/);
+	});
+
+	it("handles non-existent articles gracefully", async () => {
+		const result = await handleWikipedia(
+			"https://en.wikipedia.org/wiki/ThisArticleDefinitelyDoesNotExist123456789",
+			20,
+		);
+		expect(result).toBeNull();
+	});
+});

package/src/core/tools/web-scrapers/wikipedia.ts

@@ -0,0 +1,95 @@
+import { parse as parseHtml } from "node-html-parser";
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, loadPage } from "./types";
+
+/**
+ * Handle Wikipedia URLs via Wikipedia API
+ */
+export const handleWikipedia: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		// Match *.wikipedia.org
+		const wikiMatch = parsed.hostname.match(/^(\w+)\.wikipedia\.org$/);
+		if (!wikiMatch) return null;
+
+		const lang = wikiMatch[1];
+		const titleMatch = parsed.pathname.match(/\/wiki\/(.+)/);
+		if (!titleMatch) return null;
+
+		const title = decodeURIComponent(titleMatch[1]);
+		const fetchedAt = new Date().toISOString();
+
+		// Use Wikipedia API to get plain text extract
+		const apiUrl = `https://${lang}.wikipedia.org/api/rest_v1/page/summary/${encodeURIComponent(title)}`;
+		const summaryResult = await loadPage(apiUrl, { timeout, signal });
+
+		let md = "";
+
+		if (summaryResult.ok) {
+			const summary = JSON.parse(summaryResult.content) as {
+				title: string;
+				description?: string;
+				extract: string;
+			};
+			md = `# ${summary.title}\n\n`;
+			if (summary.description) md += `*${summary.description}*\n\n`;
+			md += `${summary.extract}\n\n---\n\n`;
+		}
+
+		// Get full article content via mobile-html or parse API
+		const contentUrl = `https://${lang}.wikipedia.org/api/rest_v1/page/mobile-html/${encodeURIComponent(title)}`;
+		const contentResult = await loadPage(contentUrl, { timeout, signal });
+
+		if (contentResult.ok) {
+			const doc = parseHtml(contentResult.content);
+
+			// Extract main content sections
+			const sections = doc.querySelectorAll("section");
+			for (const section of sections) {
+				const heading = section.querySelector("h2, h3, h4");
+				const headingText = heading?.text?.trim();
+
+				// Skip certain sections
+				if (
+					headingText &&
+					["References", "External links", "See also", "Notes", "Further reading"].includes(headingText)
+				) {
+					continue;
+				}
+
+				if (headingText) {
+					const level = heading?.tagName === "H2" ? "##" : "###";
+					md += `${level} ${headingText}\n\n`;
+				}
+
+				const paragraphs = section.querySelectorAll("p");
+				for (const p of paragraphs) {
+					const text = p.text?.trim();
+					if (text && text.length > 20) {
+						md += `${text}\n\n`;
+					}
+				}
+			}
+		}
+
+		if (!md) return null;
+
+		const output = finalizeOutput(md);
+		return {
+			url,
+			finalUrl: url,
+			contentType: "text/markdown",
+			method: "wikipedia",
+			content: output.content,
+			fetchedAt,
+			truncated: output.truncated,
+			notes: ["Fetched via Wikipedia API"],
+		};
+	} catch {}
+
+	return null;
+};

package/src/core/tools/web-scrapers/youtube.test.ts

@@ -0,0 +1,198 @@
+import { describe, expect, it } from "bun:test";
+import { handleYouTube } from "./youtube";
+
+const SKIP = !process.env.WEB_FETCH_INTEGRATION;
+
+describe.skipIf(SKIP)("handleYouTube", () => {
+	it("returns null for non-YouTube URLs", async () => {
+		const result = await handleYouTube("https://example.com", 10);
+		expect(result).toBeNull();
+	});
+
+	it("returns null for invalid YouTube URLs", async () => {
+		const result = await handleYouTube("https://youtube.com/invalid", 10);
+		expect(result).toBeNull();
+	});
+
+	it("handles youtube.com/watch?v= format", async () => {
+		// Use Rick Astley's "Never Gonna Give You Up" - a stable, well-known video
+		const result = await handleYouTube("https://www.youtube.com/watch?v=dQw4w9WgXcQ", 30);
+		expect(result).not.toBeNull();
+		expect(result?.method).toMatch(/^youtube/);
+		expect(result?.contentType).toBe("text/markdown");
+		expect(result?.content).toContain("Video ID");
+		expect(result?.content).toContain("dQw4w9WgXcQ");
+	}, 30000);
+
+	it("handles youtu.be/ short format", async () => {
+		const result = await handleYouTube("https://youtu.be/dQw4w9WgXcQ", 30);
+		expect(result).not.toBeNull();
+		expect(result?.method).toMatch(/^youtube/);
+		expect(result?.content).toContain("dQw4w9WgXcQ");
+	}, 30000);
+
+	it("handles youtube.com/shorts/ format", async () => {
+		// Use a stable YouTube Shorts video
+		const result = await handleYouTube("https://www.youtube.com/shorts/jNQXAC9IVRw", 30);
+		expect(result).not.toBeNull();
+		expect(result?.method).toMatch(/^youtube/);
+		expect(result?.content).toContain("jNQXAC9IVRw");
+	}, 30000);
+
+	it("handles youtube.com/embed/ format", async () => {
+		const result = await handleYouTube("https://www.youtube.com/embed/dQw4w9WgXcQ", 30);
+		expect(result).not.toBeNull();
+		expect(result?.method).toMatch(/^youtube/);
+		expect(result?.content).toContain("dQw4w9WgXcQ");
+	}, 30000);
+
+	it("handles youtube.com/v/ format", async () => {
+		const result = await handleYouTube("https://www.youtube.com/v/dQw4w9WgXcQ", 30);
+		expect(result).not.toBeNull();
+		expect(result?.method).toMatch(/^youtube/);
+		expect(result?.content).toContain("dQw4w9WgXcQ");
+	}, 30000);
+
+	it("handles m.youtube.com mobile URLs", async () => {
+		const result = await handleYouTube("https://m.youtube.com/watch?v=dQw4w9WgXcQ", 30);
+		expect(result).not.toBeNull();
+		expect(result?.method).toMatch(/^youtube/);
+		expect(result?.content).toContain("dQw4w9WgXcQ");
+	}, 30000);
+
+	it("extracts video metadata when yt-dlp is available", async () => {
+		const result = await handleYouTube("https://www.youtube.com/watch?v=dQw4w9WgXcQ", 30);
+		expect(result).not.toBeNull();
+
+		// If yt-dlp is available, should have metadata
+		if (result?.method === "youtube") {
+			expect(result.content).toContain("Video ID");
+			expect(result.content).toContain("Channel");
+			// May have duration, views, upload date, etc.
+		}
+
+		// If yt-dlp is not available, should indicate that
+		if (result?.method === "youtube-no-ytdlp") {
+			expect(result.content).toContain("yt-dlp could not be installed");
+			expect(result.notes).toContain("yt-dlp installation failed");
+		}
+	}, 30000);
+
+	it("handles videos with transcripts gracefully", async () => {
+		// This video should have captions
+		const result = await handleYouTube("https://www.youtube.com/watch?v=dQw4w9WgXcQ", 30);
+		expect(result).not.toBeNull();
+
+		if (result?.method === "youtube") {
+			// Either has transcript or explicitly notes it's not available
+			const hasTranscript = result.content.includes("Transcript");
+			const noTranscriptNote = result.content.includes("No transcript available");
+			expect(hasTranscript || noTranscriptNote).toBe(true);
+		}
+	}, 30000);
+
+	it("handles videos without transcripts gracefully", async () => {
+		// Many music videos lack captions, but this is not guaranteed
+		// Just verify the handler doesn't crash and provides some info
+		const result = await handleYouTube("https://www.youtube.com/watch?v=kJQP7kiw5Fk", 30);
+		expect(result).not.toBeNull();
+
+		if (result?.method === "youtube") {
+			// Should still have basic metadata
+			expect(result.content).toContain("Video ID");
+		}
+	}, 30000);
+
+	it("returns appropriate response when yt-dlp is not available", async () => {
+		// We can't force yt-dlp to be unavailable in tests, but we can verify
+		// the return structure matches expectations for both cases
+		const result = await handleYouTube("https://www.youtube.com/watch?v=dQw4w9WgXcQ", 30);
+		expect(result).not.toBeNull();
+
+		// Should have one of these two methods
+		expect(["youtube", "youtube-no-ytdlp"]).toContain(result!.method);
+
+		// Both should have required fields
+		expect(result?.url).toBe("https://www.youtube.com/watch?v=dQw4w9WgXcQ");
+		expect(result?.finalUrl).toContain("youtube.com");
+		expect(result?.fetchedAt).toBeTruthy();
+		expect(typeof result?.truncated).toBe("boolean");
+		expect(Array.isArray(result?.notes)).toBe(true);
+	}, 30000);
+
+	it("normalizes video URLs to canonical format", async () => {
+		// Different input formats should normalize to same canonical URL
+		const result = await handleYouTube("https://youtu.be/dQw4w9WgXcQ", 30);
+		expect(result).not.toBeNull();
+		expect(result?.finalUrl).toBe("https://www.youtube.com/watch?v=dQw4w9WgXcQ");
+	}, 30000);
+
+	it("handles playlist URLs by extracting video ID", async () => {
+		const result = await handleYouTube(
+			"https://www.youtube.com/watch?v=dQw4w9WgXcQ&list=PLrAXtmErZgOeiKm4sgNOknGvNjby9efdf",
+			30,
+		);
+		expect(result).not.toBeNull();
+		expect(result?.content).toContain("dQw4w9WgXcQ");
+	}, 30000);
+
+	it("includes subtitle source information when available", async () => {
+		const result = await handleYouTube("https://www.youtube.com/watch?v=dQw4w9WgXcQ", 30);
+
+		if (result?.method === "youtube") {
+			// If transcript is present, should note the source
+			const hasManualNote = result.notes.includes("Using manual subtitles");
+			const hasAutoNote = result.notes.includes("Using auto-generated captions");
+			const hasNoSubsNote = result.notes.includes("No subtitles/captions available");
+
+			// Should have exactly one of these
+			const noteCount = [hasManualNote, hasAutoNote, hasNoSubsNote].filter(Boolean).length;
+			expect(noteCount).toBeGreaterThanOrEqual(1);
+		}
+	}, 30000);
+
+	it("formats duration in human readable format", async () => {
+		const result = await handleYouTube("https://www.youtube.com/watch?v=dQw4w9WgXcQ", 30);
+
+		if (result?.method === "youtube" && result.content.includes("Duration")) {
+			// Should have duration in M:SS or H:MM:SS format
+			expect(result.content).toMatch(/Duration.*\d+:\d{2}/);
+		}
+	}, 30000);
+
+	it("formats view count in readable format", async () => {
+		const result = await handleYouTube("https://www.youtube.com/watch?v=dQw4w9WgXcQ", 30);
+
+		if (result?.method === "youtube" && result.content.includes("Views")) {
+			// Should have views formatted (e.g., 1.5B, 100M, 10.5K)
+			expect(result.content).toMatch(/Views.*\d+(\.\d+)?[KM]?/);
+		}
+	}, 30000);
+
+	it("includes upload date when available", async () => {
+		const result = await handleYouTube("https://www.youtube.com/watch?v=dQw4w9WgXcQ", 30);
+
+		if (result?.method === "youtube" && result.content.includes("Uploaded")) {
+			// Should have date in YYYY-MM-DD format
+			expect(result.content).toMatch(/Uploaded.*\d{4}-\d{2}-\d{2}/);
+		}
+	}, 30000);
+
+	it("truncates long descriptions", async () => {
+		const result = await handleYouTube("https://www.youtube.com/watch?v=dQw4w9WgXcQ", 30);
+
+		if (result?.method === "youtube" && result.content.includes("Description")) {
+			// Description section should exist
+			expect(result.content).toContain("## Description");
+		}
+	}, 30000);
+
+	it("handles www prefix variations", async () => {
+		const withWww = await handleYouTube("https://www.youtube.com/watch?v=dQw4w9WgXcQ", 30);
+		const withoutWww = await handleYouTube("https://youtube.com/watch?v=dQw4w9WgXcQ", 30);
+
+		expect(withWww).not.toBeNull();
+		expect(withoutWww).not.toBeNull();
+		expect(withWww?.finalUrl).toBe(withoutWww?.finalUrl);
+	}, 30000);
+});