@oh-my-pi/pi-coding-agent 3.25.0 → 3.31.0

package/src/core/tools/web-scrapers/readthedocs.ts

@@ -0,0 +1,126 @@
+/**
+ * Read the Docs handler for web-fetch
+ */
+
+import { parse as parseHtml } from "node-html-parser";
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, htmlToBasicMarkdown, loadPage } from "./types";
+
+export const handleReadTheDocs: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	// Check if URL matches Read the Docs patterns
+	const urlObj = new URL(url);
+	const isReadTheDocs =
+		urlObj.hostname.endsWith(".readthedocs.io") ||
+		urlObj.hostname === "readthedocs.org" ||
+		urlObj.hostname === "www.readthedocs.org";
+
+	if (!isReadTheDocs) {
+		return null;
+	}
+
+	const notes: string[] = [];
+	const fetchedAt = new Date().toISOString();
+
+	// Fetch the page
+	const result = await loadPage(url, { timeout, signal });
+	if (!result.ok) {
+		return {
+			url,
+			finalUrl: result.finalUrl,
+			contentType: result.contentType,
+			method: "readthedocs",
+			content: `Failed to fetch Read the Docs page (status: ${result.status ?? "unknown"})`,
+			fetchedAt,
+			truncated: false,
+			notes,
+		};
+	}
+
+	// Parse HTML
+	const root = parseHtml(result.content);
+
+	// Extract main content from common Read the Docs selectors
+	let mainContent =
+		root.querySelector(".document") ||
+		root.querySelector('[role="main"]') ||
+		root.querySelector("main") ||
+		root.querySelector(".rst-content") ||
+		root.querySelector(".body");
+
+	if (!mainContent) {
+		// Fallback to body if no main content found
+		mainContent = root.querySelector("body");
+		notes.push("Using full body content (no main content div found)");
+	}
+
+	// Remove navigation, sidebar, footer elements
+	mainContent
+		?.querySelectorAll(
+			".headerlink, .viewcode-link, nav, .sidebar, footer, .related, .sphinxsidebar, .toctree-wrapper",
+		)
+		.forEach((el) => {
+			el.remove();
+		});
+
+	// Try to find Edit on GitHub/GitLab links for raw source
+	const editLinks = root.querySelectorAll('a[href*="github.com"], a[href*="gitlab.com"]');
+	let sourceUrl: string | null = null;
+
+	for (const link of editLinks) {
+		const href = link.getAttribute("href");
+		const text = link.textContent?.toLowerCase() || "";
+
+		if (href && (text.includes("edit") || text.includes("source"))) {
+			// Convert edit URL to raw URL
+			if (href.includes("github.com")) {
+				sourceUrl = href.replace("/blob/", "/raw/").replace("/edit/", "/raw/");
+			} else if (href.includes("gitlab.com")) {
+				sourceUrl = href.replace("/blob/", "/raw/").replace("/edit/", "/raw/");
+			}
+			break;
+		}
+	}
+
+	let content = "";
+
+	// Try to fetch raw source if available
+	if (sourceUrl) {
+		try {
+			const sourceResult = await loadPage(sourceUrl, { timeout: Math.min(timeout, 10), signal });
+			if (sourceResult.ok && sourceResult.content.length > 0 && sourceResult.content.length < 1_000_000) {
+				content = sourceResult.content;
+				notes.push(`Fetched raw source from ${sourceUrl}`);
+			}
+		} catch (_err) {
+			// Ignore errors, fall back to HTML
+		}
+	}
+
+	// If no raw source, convert HTML to markdown
+	if (!content && mainContent) {
+		const html = mainContent.innerHTML;
+		content = htmlToBasicMarkdown(html);
+	}
+
+	if (!content) {
+		content = "No content extracted from Read the Docs page";
+		notes.push("Failed to extract content");
+	}
+
+	const { content: finalContent, truncated } = finalizeOutput(content);
+
+	return {
+		url,
+		finalUrl: result.finalUrl,
+		contentType: sourceUrl ? "text/plain" : "text/html",
+		method: "readthedocs",
+		content: finalContent,
+		fetchedAt,
+		truncated,
+		notes,
+	};
+};

package/src/core/tools/web-scrapers/reddit.ts

@@ -0,0 +1,104 @@
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, loadPage } from "./types";
+
+interface RedditPost {
+	title: string;
+	selftext: string;
+	author: string;
+	score: number;
+	num_comments: number;
+	created_utc: number;
+	subreddit: string;
+	url: string;
+	is_self: boolean;
+}
+
+interface RedditComment {
+	body: string;
+	author: string;
+	score: number;
+	created_utc: number;
+	replies?: { data: { children: Array<{ data: RedditComment }> } };
+}
+
+/**
+ * Handle Reddit URLs via JSON API
+ */
+export const handleReddit: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		if (!parsed.hostname.includes("reddit.com")) return null;
+
+		const fetchedAt = new Date().toISOString();
+
+		// Append .json to get JSON response
+		let jsonUrl = `${url.replace(/\/$/, "")}.json`;
+		if (parsed.search) {
+			jsonUrl = `${url.replace(/\/$/, "").replace(parsed.search, "")}.json${parsed.search}`;
+		}
+
+		const result = await loadPage(jsonUrl, { timeout, signal });
+		if (!result.ok) return null;
+
+		const data = JSON.parse(result.content);
+		let md = "";
+
+		// Handle different Reddit URL types
+		if (Array.isArray(data) && data.length >= 1) {
+			// Post page (with comments)
+			const postData = data[0]?.data?.children?.[0]?.data as RedditPost | undefined;
+			if (postData) {
+				md = `# ${postData.title}\n\n`;
+				md += `**r/${postData.subreddit}** · u/${postData.author} · ${postData.score} points · ${postData.num_comments} comments\n`;
+				md += `*${new Date(postData.created_utc * 1000).toISOString().split("T")[0]}*\n\n`;
+
+				if (postData.is_self && postData.selftext) {
+					md += `---\n\n${postData.selftext}\n\n`;
+				} else if (!postData.is_self) {
+					md += `**Link:** ${postData.url}\n\n`;
+				}
+
+				// Add comments if available
+				if (data.length >= 2 && data[1]?.data?.children) {
+					md += `---\n\n## Top Comments\n\n`;
+					const comments = data[1].data.children.filter((c: { kind: string }) => c.kind === "t1").slice(0, 10);
+
+					for (const { data: comment } of comments as Array<{ data: RedditComment }>) {
+						md += `### u/${comment.author} · ${comment.score} points\n\n`;
+						md += `${comment.body}\n\n---\n\n`;
+					}
+				}
+			}
+		} else if (data?.data?.children) {
+			// Subreddit or listing page
+			const posts = data.data.children.slice(0, 20) as Array<{ data: RedditPost }>;
+			const subreddit = posts[0]?.data?.subreddit;
+
+			md = `# r/${subreddit || "Reddit"}\n\n`;
+			for (const { data: post } of posts) {
+				md += `- **${post.title}** (${post.score} pts, ${post.num_comments} comments)\n`;
+				md += `  by u/${post.author}\n\n`;
+			}
+		}
+
+		if (!md) return null;
+
+		const output = finalizeOutput(md);
+		return {
+			url,
+			finalUrl: url,
+			contentType: "text/markdown",
+			method: "reddit",
+			content: output.content,
+			fetchedAt,
+			truncated: output.truncated,
+			notes: ["Fetched via Reddit JSON API"],
+		};
+	} catch {}
+
+	return null;
+};

package/src/core/tools/web-scrapers/repology.ts

@@ -0,0 +1,262 @@
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, loadPage } from "./types";
+
+interface RepologyPackage {
+	repo: string;
+	subrepo?: string;
+	srcname?: string;
+	binname?: string;
+	visiblename?: string;
+	version: string;
+	origversion?: string;
+	status:
+		| "newest"
+		| "devel"
+		| "unique"
+		| "outdated"
+		| "legacy"
+		| "rolling"
+		| "noscheme"
+		| "incorrect"
+		| "untrusted"
+		| "ignored";
+	summary?: string;
+	categories?: string[];
+	licenses?: string[];
+	maintainers?: string[];
+}
+
+/**
+ * Get emoji indicator for version status
+ */
+function statusIndicator(status: string): string {
+	switch (status) {
+		case "newest":
+			return "\u2705"; // green check
+		case "devel":
+			return "\uD83D\uDEA7"; // construction
+		case "unique":
+			return "\uD83D\uDD35"; // blue circle
+		case "outdated":
+			return "\uD83D\uDD34"; // red circle
+		case "legacy":
+			return "\u26A0\uFE0F"; // warning
+		case "rolling":
+			return "\uD83D\uDD04"; // arrows
+		default:
+			return "\u2796"; // minus
+	}
+}
+
+/**
+ * Prettify repository name
+ */
+function prettifyRepo(repo: string): string {
+	const mapping: Record<string, string> = {
+		arch: "Arch Linux",
+		aur: "AUR",
+		debian_unstable: "Debian Unstable",
+		debian_stable: "Debian Stable",
+		ubuntu_24_04: "Ubuntu 24.04",
+		ubuntu_22_04: "Ubuntu 22.04",
+		fedora_rawhide: "Fedora Rawhide",
+		fedora_40: "Fedora 40",
+		gentoo: "Gentoo",
+		nix_unstable: "Nixpkgs Unstable",
+		nix_stable: "Nixpkgs Stable",
+		homebrew: "Homebrew",
+		macports: "MacPorts",
+		alpine_edge: "Alpine Edge",
+		freebsd: "FreeBSD",
+		openbsd: "OpenBSD",
+		void_x86_64: "Void Linux",
+		opensuse_tumbleweed: "openSUSE Tumbleweed",
+		msys2_mingw: "MSYS2",
+		chocolatey: "Chocolatey",
+		winget: "Winget",
+		scoop: "Scoop",
+		conda_main: "Conda",
+		pypi: "PyPI",
+		crates_io: "Crates.io",
+		npm: "npm",
+		rubygems: "RubyGems",
+		cpan: "CPAN",
+		hackage: "Hackage",
+	};
+
+	// Check exact match first
+	if (mapping[repo]) return mapping[repo];
+
+	// Check partial matches
+	for (const [key, value] of Object.entries(mapping)) {
+		if (repo.startsWith(key)) return value;
+	}
+
+	// Fallback: titlecase with underscores replaced
+	return repo
+		.split("_")
+		.map((w) => w.charAt(0).toUpperCase() + w.slice(1))
+		.join(" ");
+}
+
+/**
+ * Handle Repology URLs via API
+ */
+export const handleRepology: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		if (parsed.hostname !== "repology.org" && parsed.hostname !== "www.repology.org") return null;
+
+		// Extract package name from /project/{name}/versions or /project/{name}/information
+		const match = parsed.pathname.match(/^\/project\/([^/]+)/);
+		if (!match) return null;
+
+		const packageName = decodeURIComponent(match[1]);
+		const fetchedAt = new Date().toISOString();
+
+		// Fetch from Repology API
+		const apiUrl = `https://repology.org/api/v1/project/${encodeURIComponent(packageName)}`;
+		const result = await loadPage(apiUrl, {
+			timeout,
+			headers: { Accept: "application/json" },
+			signal,
+		});
+
+		if (!result.ok) return null;
+
+		let packages: RepologyPackage[];
+		try {
+			packages = JSON.parse(result.content);
+		} catch {
+			return null;
+		}
+
+		// Empty response means package not found
+		if (!Array.isArray(packages) || packages.length === 0) return null;
+
+		// Find newest version(s) and extract metadata
+		const newestVersions = new Set<string>();
+		let summary: string | undefined;
+		let licenses: string[] = [];
+		const categories = new Set<string>();
+
+		for (const pkg of packages) {
+			if (pkg.status === "newest" || pkg.status === "unique") {
+				newestVersions.add(pkg.version);
+			}
+			if (!summary && pkg.summary) summary = pkg.summary;
+			if (pkg.licenses?.length && !licenses.length) licenses = pkg.licenses;
+			if (pkg.categories) {
+				for (const cat of pkg.categories) categories.add(cat);
+			}
+		}
+
+		// If no newest found, find the highest version
+		if (newestVersions.size === 0) {
+			const versions = packages.map((p) => p.version);
+			if (versions.length > 0) newestVersions.add(versions[0]);
+		}
+
+		// Group packages by status for counting
+		const statusCounts: Record<string, number> = {};
+		for (const pkg of packages) {
+			statusCounts[pkg.status] = (statusCounts[pkg.status] || 0) + 1;
+		}
+
+		// Build markdown
+		let md = `# ${packageName}\n\n`;
+		if (summary) md += `${summary}\n\n`;
+
+		md += `**Newest Version:** ${Array.from(newestVersions).join(", ") || "unknown"}\n`;
+		md += `**Repositories:** ${packages.length}\n`;
+		if (licenses.length) md += `**License:** ${licenses.join(", ")}\n`;
+		if (categories.size) md += `**Categories:** ${Array.from(categories).join(", ")}\n`;
+		md += "\n";
+
+		// Status summary
+		md += "## Version Status Summary\n\n";
+		const statusOrder = [
+			"newest",
+			"unique",
+			"devel",
+			"rolling",
+			"outdated",
+			"legacy",
+			"noscheme",
+			"incorrect",
+			"untrusted",
+			"ignored",
+		];
+		for (const status of statusOrder) {
+			if (statusCounts[status]) {
+				md += `- ${statusIndicator(status)} **${status}**: ${statusCounts[status]} repos\n`;
+			}
+		}
+		md += "\n";
+
+		// Sort packages: newest first, then by repo name
+		const sortedPackages = [...packages].sort((a, b) => {
+			const statusPriority: Record<string, number> = {
+				newest: 0,
+				unique: 1,
+				devel: 2,
+				rolling: 3,
+				outdated: 4,
+				legacy: 5,
+				noscheme: 6,
+				incorrect: 7,
+				untrusted: 8,
+				ignored: 9,
+			};
+			const aPriority = statusPriority[a.status] ?? 10;
+			const bPriority = statusPriority[b.status] ?? 10;
+			if (aPriority !== bPriority) return aPriority - bPriority;
+			return a.repo.localeCompare(b.repo);
+		});
+
+		// Show top repositories (up to 15)
+		md += "## Package Versions by Repository\n\n";
+		md += "| Repository | Version | Status |\n";
+		md += "|------------|---------|--------|\n";
+
+		const shownRepos = new Set<string>();
+		let count = 0;
+		for (const pkg of sortedPackages) {
+			// Skip duplicate repos (some have multiple entries)
+			const repoKey = pkg.subrepo ? `${pkg.repo}/${pkg.subrepo}` : pkg.repo;
+			if (shownRepos.has(repoKey)) continue;
+			shownRepos.add(repoKey);
+
+			const repoName = prettifyRepo(pkg.repo);
+			const version = pkg.origversion || pkg.version;
+			md += `| ${repoName} | \`${version}\` | ${statusIndicator(pkg.status)} ${pkg.status} |\n`;
+
+			count++;
+			if (count >= 15) break;
+		}
+
+		if (packages.length > 15) {
+			md += `\n*...and ${packages.length - 15} more repositories*\n`;
+		}
+
+		md += `\n---\n\n[View on Repology](${url})\n`;
+
+		const output = finalizeOutput(md);
+		return {
+			url,
+			finalUrl: url,
+			contentType: "text/markdown",
+			method: "repology",
+			content: output.content,
+			fetchedAt,
+			truncated: output.truncated,
+			notes: ["Fetched via Repology API"],
+		};
+	} catch {}
+
+	return null;
+};

package/src/core/tools/web-scrapers/research.test.ts

@@ -0,0 +1,107 @@
+import { describe, expect, it } from "bun:test";
+import { handleBiorxiv } from "./biorxiv";
+import { handleOpenLibrary } from "./openlibrary";
+import { handleWikidata } from "./wikidata";
+
+const SKIP = !process.env.WEB_FETCH_INTEGRATION;
+
+describe.skipIf(SKIP)("handleWikidata", () => {
+	it("returns null for non-matching URLs", async () => {
+		const result = await handleWikidata("https://example.com", 20);
+		expect(result).toBeNull();
+	});
+
+	it("returns null for non-wikidata URLs", async () => {
+		const result = await handleWikidata("https://wikipedia.org/wiki/Apple_Inc", 20);
+		expect(result).toBeNull();
+	});
+
+	it("fetches Q312 - Apple Inc", async () => {
+		const result = await handleWikidata("https://www.wikidata.org/wiki/Q312", 20);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("wikidata");
+		expect(result?.content).toContain("Apple");
+		expect(result?.content).toContain("Q312");
+		expect(result?.contentType).toBe("text/markdown");
+		expect(result?.fetchedAt).toBeTruthy();
+		expect(result?.truncated).toBeDefined();
+	});
+
+	it("fetches Q5 - human (entity)", async () => {
+		const result = await handleWikidata("https://www.wikidata.org/entity/Q5", 20);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("wikidata");
+		expect(result?.content).toContain("human");
+		expect(result?.content).toContain("Q5");
+		expect(result?.contentType).toBe("text/markdown");
+		expect(result?.fetchedAt).toBeTruthy();
+		expect(result?.truncated).toBeDefined();
+	});
+});
+
+describe.skipIf(SKIP)("handleOpenLibrary", () => {
+	it("returns null for non-matching URLs", async () => {
+		const result = await handleOpenLibrary("https://example.com", 20);
+		expect(result).toBeNull();
+	});
+
+	it("returns null for non-openlibrary URLs", async () => {
+		const result = await handleOpenLibrary("https://amazon.com/books/123", 20);
+		expect(result).toBeNull();
+	});
+
+	it("fetches by ISBN - Fantastic Mr Fox", async () => {
+		const result = await handleOpenLibrary("https://openlibrary.org/isbn/9780140328721", 20);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("openlibrary");
+		expect(result?.content).toContain("Fantastic Mr");
+		expect(result?.content).toContain("Roald Dahl");
+		expect(result?.contentType).toBe("text/markdown");
+		expect(result?.fetchedAt).toBeTruthy();
+		expect(result?.truncated).toBeDefined();
+	});
+
+	it("fetches work OL45804W - The Lord of the Rings", async () => {
+		const result = await handleOpenLibrary("https://openlibrary.org/works/OL45804W", 20);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("openlibrary");
+		expect(result?.content).toContain("Lord of the Rings");
+		expect(result?.content).toContain("Tolkien");
+		expect(result?.contentType).toBe("text/markdown");
+		expect(result?.fetchedAt).toBeTruthy();
+		expect(result?.truncated).toBeDefined();
+	});
+});
+
+describe.skipIf(SKIP)("handleBiorxiv", () => {
+	it("returns null for non-matching URLs", async () => {
+		const result = await handleBiorxiv("https://example.com", 20);
+		expect(result).toBeNull();
+	});
+
+	it("returns null for non-biorxiv URLs", async () => {
+		const result = await handleBiorxiv("https://nature.com/articles/123", 20);
+		expect(result).toBeNull();
+	});
+
+	// Using the AlphaFold Protein Structure Database paper - highly cited and stable
+	it("fetches bioRxiv preprint - AlphaFold database", async () => {
+		const result = await handleBiorxiv("https://www.biorxiv.org/content/10.1101/2021.10.04.463034", 20);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("biorxiv");
+		expect(result?.content).toContain("AlphaFold");
+		expect(result?.content).toContain("Abstract");
+		expect(result?.contentType).toBe("text/markdown");
+		expect(result?.fetchedAt).toBeTruthy();
+		expect(result?.truncated).toBeDefined();
+	});
+
+	// Testing with version suffix handling
+	it("fetches bioRxiv preprint with version suffix", async () => {
+		const result = await handleBiorxiv("https://www.biorxiv.org/content/10.1101/2021.10.04.463034v1", 20);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("biorxiv");
+		expect(result?.content).toContain("AlphaFold");
+		expect(result?.contentType).toBe("text/markdown");
+	});
+});