@wahack/pi-coding-agent 15.11.0

package/src/web/scrapers/pypi.ts

@@ -0,0 +1,112 @@
+import { tryParseJson } from "@oh-my-pi/pi-utils";
+import { buildResult, formatNumber, loadPage, type RenderResult, type SpecialHandler } from "./types";
+
+/**
+ * Handle PyPI URLs via JSON API
+ */
+export const handlePyPI: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		if (parsed.hostname !== "pypi.org" && parsed.hostname !== "www.pypi.org") return null;
+
+		// Extract package name from /project/{package} or /project/{package}/{version}
+		const match = parsed.pathname.match(/^\/project\/([^/]+)/);
+		if (!match) return null;
+
+		const packageName = decodeURIComponent(match[1]);
+		const fetchedAt = new Date().toISOString();
+
+		// Fetch from PyPI JSON API
+		const apiUrl = `https://pypi.org/pypi/${packageName}/json`;
+		const downloadsUrl = `https://pypistats.org/api/packages/${packageName}/recent`;
+
+		// Fetch package info and download stats in parallel
+		const [result, downloadsResult] = await Promise.all([
+			loadPage(apiUrl, { timeout, signal }),
+			loadPage(downloadsUrl, { timeout: Math.min(timeout, 5), signal }),
+		]);
+
+		if (!result.ok) return null;
+
+		// Parse download stats
+		let weeklyDownloads: number | null = null;
+		if (downloadsResult.ok) {
+			const dlData = tryParseJson<{ data?: { last_week?: number } }>(downloadsResult.content);
+			if (dlData) weeklyDownloads = dlData.data?.last_week ?? null;
+		}
+
+		const pkg = tryParseJson<{
+			info: {
+				name: string;
+				version: string;
+				summary?: string;
+				description?: string;
+				author?: string;
+				author_email?: string;
+				license?: string;
+				home_page?: string;
+				project_urls?: Record<string, string>;
+				requires_python?: string;
+				keywords?: string;
+				classifiers?: string[];
+			};
+			urls?: Array<{ filename: string; size: number; upload_time: string }>;
+			releases?: Record<string, unknown>;
+			requires_dist?: string[];
+		}>(result.content);
+		if (!pkg) return null;
+
+		const info = pkg.info;
+		let md = `# ${info.name}\n\n`;
+		if (info.summary) md += `${info.summary}\n\n`;
+
+		md += `**Latest:** ${info.version}`;
+		if (info.license) md += ` · **License:** ${info.license}`;
+		md += "\n";
+
+		if (weeklyDownloads !== null) {
+			md += `**Weekly Downloads:** ${formatNumber(weeklyDownloads)}\n`;
+		}
+
+		md += "\n";
+
+		if (info.author) {
+			md += `**Author:** ${info.author}`;
+			if (info.author_email) md += ` <${info.author_email}>`;
+			md += "\n";
+		}
+
+		if (info.requires_python) md += `**Python:** ${info.requires_python}\n`;
+		if (info.home_page) md += `**Homepage:** ${info.home_page}\n`;
+
+		if (info.project_urls && Object.keys(info.project_urls).length > 0) {
+			md += "\n**Project URLs:**\n";
+			for (const [label, url] of Object.entries(info.project_urls)) {
+				md += `- ${label}: ${url}\n`;
+			}
+		}
+
+		if (info.keywords) md += `\n**Keywords:** ${info.keywords}\n`;
+
+		// Dependencies
+		if (pkg.requires_dist && pkg.requires_dist.length > 0) {
+			md += `\n## Dependencies\n\n`;
+			for (const dep of pkg.requires_dist) {
+				md += `- ${dep}\n`;
+			}
+		}
+
+		// README/Description
+		if (info.description) {
+			md += `\n---\n\n## Description\n\n${info.description}\n`;
+		}
+
+		return buildResult(md, { url, method: "pypi", fetchedAt, notes: ["Fetched via PyPI JSON API"] });
+	} catch {}
+
+	return null;
+};

package/src/web/scrapers/rawg.ts

@@ -0,0 +1,110 @@
+import { tryParseJson } from "@oh-my-pi/pi-utils";
+import { buildResult, htmlToBasicMarkdown, loadPage, type RenderResult, type SpecialHandler } from "./types";
+
+interface RawgPlatformEntry {
+	platform?: {
+		name?: string;
+	};
+}
+
+interface RawgGenreEntry {
+	name?: string;
+}
+
+interface RawgGameResponse {
+	name?: string;
+	released?: string;
+	rating?: number;
+	platforms?: RawgPlatformEntry[];
+	genres?: RawgGenreEntry[];
+	description?: string;
+	description_raw?: string;
+	detail?: string;
+	error?: string;
+}
+
+export const handleRawg: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		if (!isRawgHostname(parsed.hostname)) return null;
+
+		const slug = extractGameSlug(parsed.pathname);
+		if (!slug) return null;
+
+		const fetchedAt = new Date().toISOString();
+		const apiUrl = `https://api.rawg.io/api/games/${encodeURIComponent(slug)}`;
+		const result = await loadPage(apiUrl, { timeout, signal, headers: { Accept: "application/json" } });
+
+		if (!result.ok) return null;
+
+		const game = tryParseJson<RawgGameResponse>(result.content);
+		if (!game) return null;
+
+		if (requiresApiKey(game)) return null;
+
+		const title = game.name?.trim() || slug;
+		let md = `# ${title}\n\n`;
+
+		if (game.released) md += `**Released:** ${game.released}\n`;
+		if (typeof game.rating === "number" && !Number.isNaN(game.rating)) {
+			md += `**Rating:** ${game.rating.toFixed(2)} / 5\n`;
+		}
+
+		const platforms = collectNames(game.platforms?.map(entry => entry.platform?.name));
+		if (platforms.length) md += `**Platforms:** ${platforms.join(", ")}\n`;
+
+		const genres = collectNames(game.genres?.map(entry => entry.name));
+		if (genres.length) md += `**Genres:** ${genres.join(", ")}\n`;
+
+		md += `**RAWG:** https://rawg.io/games/${encodeURIComponent(slug)}\n`;
+		md += "\n";
+
+		const description = await extractDescription(game);
+		if (description) {
+			md += `## Description\n\n${description}\n`;
+		}
+
+		return buildResult(md, { url, method: "rawg", fetchedAt, notes: ["Fetched via RAWG API"] });
+	} catch {}
+
+	return null;
+};
+
+function isRawgHostname(hostname: string): boolean {
+	return hostname === "rawg.io" || hostname === "www.rawg.io";
+}
+
+function extractGameSlug(pathname: string): string | null {
+	const match = pathname.match(/^\/games\/([^/?#]+)/);
+	if (!match) return null;
+
+	const slug = decodeURIComponent(match[1]);
+	return slug ? slug.trim() : null;
+}
+
+function requiresApiKey(game: RawgGameResponse): boolean {
+	const detail = `${game.detail ?? ""} ${game.error ?? ""}`.toLowerCase();
+	return detail.includes("api key") || detail.includes("key is required") || detail.includes("apikey");
+}
+
+async function extractDescription(game: RawgGameResponse): Promise<string | null> {
+	if (game.description_raw) return game.description_raw.trim();
+	if (!game.description) return null;
+
+	const markdown = (await htmlToBasicMarkdown(game.description)).trim();
+	return markdown || null;
+}
+
+function collectNames(values?: Array<string | undefined>): string[] {
+	if (!values?.length) return [];
+	const names = new Set<string>();
+	for (const value of values) {
+		const trimmed = value?.trim();
+		if (trimmed) names.add(trimmed);
+	}
+	return Array.from(names);
+}

package/src/web/scrapers/readthedocs.ts

@@ -0,0 +1,120 @@
+/**
+ * Read the Docs handler for web-fetch
+ */
+import { buildResult, htmlToBasicMarkdown, loadPage, type RenderResult, type SpecialHandler } from "./types";
+
+export const handleReadTheDocs: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	// Check if URL matches Read the Docs patterns
+	const urlObj = new URL(url);
+	const isReadTheDocs =
+		urlObj.hostname.endsWith(".readthedocs.io") ||
+		urlObj.hostname === "readthedocs.org" ||
+		urlObj.hostname === "www.readthedocs.org";
+
+	if (!isReadTheDocs) {
+		return null;
+	}
+
+	const notes: string[] = [];
+	const fetchedAt = new Date().toISOString();
+
+	// Fetch the page
+	const result = await loadPage(url, { timeout, signal });
+	if (!result.ok) {
+		return {
+			url,
+			finalUrl: result.finalUrl,
+			contentType: result.contentType,
+			method: "readthedocs",
+			content: `Failed to fetch Read the Docs page (status: ${result.status ?? "unknown"})`,
+			fetchedAt,
+			truncated: false,
+			notes,
+		};
+	}
+
+	// Parse HTML
+	const { parseHTML } = await import("linkedom");
+	const root = parseHTML(result.content).document;
+
+	// Extract main content from common Read the Docs selectors
+	let mainContent =
+		root.querySelector(".document") ||
+		root.querySelector('[role="main"]') ||
+		root.querySelector("main") ||
+		root.querySelector(".rst-content") ||
+		root.querySelector(".body");
+
+	if (!mainContent) {
+		// Fallback to body if no main content found
+		mainContent = root.querySelector("body");
+		notes.push("Using full body content (no main content div found)");
+	}
+
+	// Remove navigation, sidebar, footer elements
+	mainContent
+		?.querySelectorAll(
+			".headerlink, .viewcode-link, nav, .sidebar, footer, .related, .sphinxsidebar, .toctree-wrapper",
+		)
+		.forEach((el: Element) => {
+			el.remove();
+		});
+
+	// Try to find Edit on GitHub/GitLab links for raw source
+	const editLinks = root.querySelectorAll('a[href*="github.com"], a[href*="gitlab.com"]');
+	let sourceUrl: string | null = null;
+
+	for (const link of editLinks) {
+		const href = link.getAttribute("href");
+		const text = link.textContent?.toLowerCase() || "";
+
+		if (href && (text.includes("edit") || text.includes("source"))) {
+			// Convert edit URL to raw URL
+			if (href.includes("github.com")) {
+				sourceUrl = href.replace("/blob/", "/raw/").replace("/edit/", "/raw/");
+			} else if (href.includes("gitlab.com")) {
+				sourceUrl = href.replace("/blob/", "/raw/").replace("/edit/", "/raw/");
+			}
+			break;
+		}
+	}
+
+	let content = "";
+
+	// Try to fetch raw source if available
+	if (sourceUrl) {
+		try {
+			const sourceResult = await loadPage(sourceUrl, { timeout: Math.min(timeout, 10), signal });
+			if (sourceResult.ok && sourceResult.content.length > 0 && sourceResult.content.length < 1_000_000) {
+				content = sourceResult.content;
+				notes.push(`Fetched raw source from ${sourceUrl}`);
+			}
+		} catch {
+			// Ignore errors, fall back to HTML
+		}
+	}
+
+	// If no raw source, convert HTML to markdown
+	if (!content && mainContent) {
+		const html = mainContent.innerHTML;
+		content = await htmlToBasicMarkdown(html);
+	}
+
+	if (!content) {
+		content = "No content extracted from Read the Docs page";
+		notes.push("Failed to extract content");
+	}
+
+	return buildResult(content, {
+		url,
+		finalUrl: result.finalUrl,
+		method: "readthedocs",
+		fetchedAt,
+		notes,
+		contentType: sourceUrl ? "text/plain" : "text/html",
+	});
+};

package/src/web/scrapers/reddit.ts

@@ -0,0 +1,95 @@
+import { tryParseJson } from "@oh-my-pi/pi-utils";
+import { buildResult, formatIsoDate, loadPage, type RenderResult, type SpecialHandler } from "./types";
+
+interface RedditPost {
+	title: string;
+	selftext: string;
+	author: string;
+	score: number;
+	num_comments: number;
+	created_utc: number;
+	subreddit: string;
+	url: string;
+	is_self: boolean;
+}
+
+interface RedditComment {
+	body: string;
+	author: string;
+	score: number;
+	created_utc: number;
+	replies?: { data: { children: Array<{ data: RedditComment }> } };
+}
+
+/**
+ * Handle Reddit URLs via JSON API
+ */
+export const handleReddit: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		if (!parsed.hostname.includes("reddit.com")) return null;
+
+		const fetchedAt = new Date().toISOString();
+
+		// Append .json to get JSON response
+		let jsonUrl = `${url.replace(/\/$/, "")}.json`;
+		if (parsed.search) {
+			jsonUrl = `${url.replace(/\/$/, "").replace(parsed.search, "")}.json${parsed.search}`;
+		}
+
+		const result = await loadPage(jsonUrl, { timeout, signal });
+		if (!result.ok) return null;
+
+		const data = tryParseJson<any>(result.content);
+		if (!data) return null;
+		let md = "";
+
+		// Handle different Reddit URL types
+		if (Array.isArray(data) && data.length >= 1) {
+			// Post page (with comments)
+			const postData = data[0]?.data?.children?.[0]?.data as RedditPost | undefined;
+			if (postData) {
+				md = `# ${postData.title}\n\n`;
+				md += `**r/${postData.subreddit}** · u/${postData.author} · ${postData.score} points · ${postData.num_comments} comments\n`;
+				md += `*${formatIsoDate(postData.created_utc * 1000)}*\n\n`;
+
+				if (postData.is_self && postData.selftext) {
+					md += `---\n\n${postData.selftext}\n\n`;
+				} else if (!postData.is_self) {
+					md += `**Link:** ${postData.url}\n\n`;
+				}
+
+				// Add comments if available
+				if (data.length >= 2 && data[1]?.data?.children) {
+					md += `---\n\n## Top Comments\n\n`;
+					const comments = data[1].data.children.filter((c: { kind: string }) => c.kind === "t1").slice(0, 10);
+
+					for (const { data: comment } of comments as Array<{ data: RedditComment }>) {
+						md += `### u/${comment.author} · ${comment.score} points\n\n`;
+						md += `${comment.body}\n\n---\n\n`;
+					}
+				}
+			}
+		} else if (data?.data?.children) {
+			// Subreddit or listing page
+			const posts = data.data.children.slice(0, 20) as Array<{ data: RedditPost }>;
+			const subreddit = posts[0]?.data?.subreddit;
+
+			md = `# r/${subreddit || "Reddit"}\n\n`;
+			for (const { data: post } of posts) {
+				md += `- **${post.title}** (${post.score} pts, ${post.num_comments} comments)\n`;
+				md += `  by u/${post.author}\n\n`;
+			}
+		}
+
+		if (!md) return null;
+
+		return buildResult(md, { url, method: "reddit", fetchedAt, notes: ["Fetched via Reddit JSON API"] });
+	} catch {}
+
+	return null;
+};

package/src/web/scrapers/repology.ts

@@ -0,0 +1,251 @@
+import { tryParseJson } from "@oh-my-pi/pi-utils";
+import { buildResult, loadPage, type RenderResult, type SpecialHandler } from "./types";
+
+interface RepologyPackage {
+	repo: string;
+	subrepo?: string;
+	srcname?: string;
+	binname?: string;
+	visiblename?: string;
+	version: string;
+	origversion?: string;
+	status:
+		| "newest"
+		| "devel"
+		| "unique"
+		| "outdated"
+		| "legacy"
+		| "rolling"
+		| "noscheme"
+		| "incorrect"
+		| "untrusted"
+		| "ignored";
+	summary?: string;
+	categories?: string[];
+	licenses?: string[];
+	maintainers?: string[];
+}
+
+/**
+ * Get emoji indicator for version status
+ */
+function statusIndicator(status: string): string {
+	switch (status) {
+		case "newest":
+			return "✅"; // green check
+		case "devel":
+			return "🚧"; // construction
+		case "unique":
+			return "🔵"; // blue circle
+		case "outdated":
+			return "🔴"; // red circle
+		case "legacy":
+			return "⚠\uFE0F"; // warning
+		case "rolling":
+			return "🔄"; // arrows
+		default:
+			return "➖"; // minus
+	}
+}
+
+/**
+ * Prettify repository name
+ */
+function prettifyRepo(repo: string): string {
+	const mapping: Record<string, string> = {
+		arch: "Arch Linux",
+		aur: "AUR",
+		debian_unstable: "Debian Unstable",
+		debian_stable: "Debian Stable",
+		ubuntu_24_04: "Ubuntu 24.04",
+		ubuntu_22_04: "Ubuntu 22.04",
+		fedora_rawhide: "Fedora Rawhide",
+		fedora_40: "Fedora 40",
+		gentoo: "Gentoo",
+		nix_unstable: "Nixpkgs Unstable",
+		nix_stable: "Nixpkgs Stable",
+		homebrew: "Homebrew",
+		macports: "MacPorts",
+		alpine_edge: "Alpine Edge",
+		freebsd: "FreeBSD",
+		openbsd: "OpenBSD",
+		void_x86_64: "Void Linux",
+		opensuse_tumbleweed: "openSUSE Tumbleweed",
+		msys2_mingw: "MSYS2",
+		chocolatey: "Chocolatey",
+		winget: "Winget",
+		scoop: "Scoop",
+		conda_main: "Conda",
+		pypi: "PyPI",
+		crates_io: "Crates.io",
+		npm: "npm",
+		rubygems: "RubyGems",
+		cpan: "CPAN",
+		hackage: "Hackage",
+	};
+
+	// Check exact match first
+	if (mapping[repo]) return mapping[repo];
+
+	// Check partial matches
+	for (const [key, value] of Object.entries(mapping)) {
+		if (repo.startsWith(key)) return value;
+	}
+
+	// Fallback: titlecase with underscores replaced
+	return repo
+		.split("_")
+		.map(w => w.charAt(0).toUpperCase() + w.slice(1))
+		.join(" ");
+}
+
+/**
+ * Handle Repology URLs via API
+ */
+export const handleRepology: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		if (parsed.hostname !== "repology.org" && parsed.hostname !== "www.repology.org") return null;
+
+		// Extract package name from /project/{name}/versions or /project/{name}/information
+		const match = parsed.pathname.match(/^\/project\/([^/]+)/);
+		if (!match) return null;
+
+		const packageName = decodeURIComponent(match[1]);
+		const fetchedAt = new Date().toISOString();
+
+		// Fetch from Repology API
+		const apiUrl = `https://repology.org/api/v1/project/${encodeURIComponent(packageName)}`;
+		const result = await loadPage(apiUrl, {
+			timeout,
+			headers: {
+				Accept: "application/json",
+				"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
+			},
+			signal,
+		});
+
+		if (!result.ok) return null;
+
+		const packages = tryParseJson<RepologyPackage[]>(result.content);
+		if (!packages) return null;
+
+		// Empty response means package not found
+		if (!Array.isArray(packages) || packages.length === 0) return null;
+
+		// Find newest version(s) and extract metadata
+		const newestVersions = new Set<string>();
+		let summary: string | undefined;
+		let licenses: string[] = [];
+		const categories = new Set<string>();
+
+		for (const pkg of packages) {
+			if (pkg.status === "newest" || pkg.status === "unique") {
+				newestVersions.add(pkg.version);
+			}
+			if (!summary && pkg.summary) summary = pkg.summary;
+			if (pkg.licenses?.length && !licenses.length) licenses = pkg.licenses;
+			if (pkg.categories) {
+				for (const cat of pkg.categories) categories.add(cat);
+			}
+		}
+
+		// If no newest found, find the highest version
+		if (newestVersions.size === 0) {
+			const versions = packages.map(p => p.version);
+			if (versions.length > 0) newestVersions.add(versions[0]);
+		}
+
+		// Group packages by status for counting
+		const statusCounts: Record<string, number> = {};
+		for (const pkg of packages) {
+			statusCounts[pkg.status] = (statusCounts[pkg.status] || 0) + 1;
+		}
+
+		// Build markdown
+		let md = `# ${packageName}\n\n`;
+		if (summary) md += `${summary}\n\n`;
+
+		md += `**Newest Version:** ${Array.from(newestVersions).join(", ") || "unknown"}\n`;
+		md += `**Repositories:** ${packages.length}\n`;
+		if (licenses.length) md += `**License:** ${licenses.join(", ")}\n`;
+		if (categories.size) md += `**Categories:** ${Array.from(categories).join(", ")}\n`;
+		md += "\n";
+
+		// Status summary
+		md += "## Version Status Summary\n\n";
+		const statusOrder = [
+			"newest",
+			"unique",
+			"devel",
+			"rolling",
+			"outdated",
+			"legacy",
+			"noscheme",
+			"incorrect",
+			"untrusted",
+			"ignored",
+		];
+		for (const status of statusOrder) {
+			if (statusCounts[status]) {
+				md += `- ${statusIndicator(status)} **${status}**: ${statusCounts[status]} repos\n`;
+			}
+		}
+		md += "\n";
+
+		// Sort packages: newest first, then by repo name
+		const sortedPackages = [...packages].sort((a, b) => {
+			const statusPriority: Record<string, number> = {
+				newest: 0,
+				unique: 1,
+				devel: 2,
+				rolling: 3,
+				outdated: 4,
+				legacy: 5,
+				noscheme: 6,
+				incorrect: 7,
+				untrusted: 8,
+				ignored: 9,
+			};
+			const aPriority = statusPriority[a.status] ?? 10;
+			const bPriority = statusPriority[b.status] ?? 10;
+			if (aPriority !== bPriority) return aPriority - bPriority;
+			return a.repo.localeCompare(b.repo);
+		});
+
+		// Show top repositories (up to 15)
+		md += "## Package Versions by Repository\n\n";
+		md += "| Repository | Version | Status |\n";
+		md += "|------------|---------|--------|\n";
+
+		const shownRepos = new Set<string>();
+		let count = 0;
+		for (const pkg of sortedPackages) {
+			// Skip duplicate repos (some have multiple entries)
+			const repoKey = pkg.subrepo ? `${pkg.repo}/${pkg.subrepo}` : pkg.repo;
+			if (shownRepos.has(repoKey)) continue;
+			shownRepos.add(repoKey);
+
+			const repoName = prettifyRepo(pkg.repo);
+			const version = pkg.origversion || pkg.version;
+			md += `| ${repoName} | \`${version}\` | ${statusIndicator(pkg.status)} ${pkg.status} |\n`;
+
+			count++;
+			if (count >= 15) break;
+		}
+
+		if (packages.length > 15) {
+			md += `\n*...and ${packages.length - 15} more repositories*\n`;
+		}
+
+		md += `\n---\n\n[View on Repology](${url})\n`;
+
+		return buildResult(md, { url, method: "repology", fetchedAt, notes: ["Fetched via Repology API"] });
+	} catch {}
+
+	return null;
+};