@oh-my-pi/pi-coding-agent 3.25.0 → 3.30.0

package/src/core/tools/web-fetch-handlers/repology.ts

@@ -0,0 +1,257 @@
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, loadPage } from "./types";
+
+interface RepologyPackage {
+	repo: string;
+	subrepo?: string;
+	srcname?: string;
+	binname?: string;
+	visiblename?: string;
+	version: string;
+	origversion?: string;
+	status:
+		| "newest"
+		| "devel"
+		| "unique"
+		| "outdated"
+		| "legacy"
+		| "rolling"
+		| "noscheme"
+		| "incorrect"
+		| "untrusted"
+		| "ignored";
+	summary?: string;
+	categories?: string[];
+	licenses?: string[];
+	maintainers?: string[];
+}
+
+/**
+ * Get emoji indicator for version status
+ */
+function statusIndicator(status: string): string {
+	switch (status) {
+		case "newest":
+			return "\u2705"; // green check
+		case "devel":
+			return "\uD83D\uDEA7"; // construction
+		case "unique":
+			return "\uD83D\uDD35"; // blue circle
+		case "outdated":
+			return "\uD83D\uDD34"; // red circle
+		case "legacy":
+			return "\u26A0\uFE0F"; // warning
+		case "rolling":
+			return "\uD83D\uDD04"; // arrows
+		default:
+			return "\u2796"; // minus
+	}
+}
+
+/**
+ * Prettify repository name
+ */
+function prettifyRepo(repo: string): string {
+	const mapping: Record<string, string> = {
+		arch: "Arch Linux",
+		aur: "AUR",
+		debian_unstable: "Debian Unstable",
+		debian_stable: "Debian Stable",
+		ubuntu_24_04: "Ubuntu 24.04",
+		ubuntu_22_04: "Ubuntu 22.04",
+		fedora_rawhide: "Fedora Rawhide",
+		fedora_40: "Fedora 40",
+		gentoo: "Gentoo",
+		nix_unstable: "Nixpkgs Unstable",
+		nix_stable: "Nixpkgs Stable",
+		homebrew: "Homebrew",
+		macports: "MacPorts",
+		alpine_edge: "Alpine Edge",
+		freebsd: "FreeBSD",
+		openbsd: "OpenBSD",
+		void_x86_64: "Void Linux",
+		opensuse_tumbleweed: "openSUSE Tumbleweed",
+		msys2_mingw: "MSYS2",
+		chocolatey: "Chocolatey",
+		winget: "Winget",
+		scoop: "Scoop",
+		conda_main: "Conda",
+		pypi: "PyPI",
+		crates_io: "Crates.io",
+		npm: "npm",
+		rubygems: "RubyGems",
+		cpan: "CPAN",
+		hackage: "Hackage",
+	};
+
+	// Check exact match first
+	if (mapping[repo]) return mapping[repo];
+
+	// Check partial matches
+	for (const [key, value] of Object.entries(mapping)) {
+		if (repo.startsWith(key)) return value;
+	}
+
+	// Fallback: titlecase with underscores replaced
+	return repo
+		.split("_")
+		.map((w) => w.charAt(0).toUpperCase() + w.slice(1))
+		.join(" ");
+}
+
+/**
+ * Handle Repology URLs via API
+ */
+export const handleRepology: SpecialHandler = async (url: string, timeout: number): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		if (parsed.hostname !== "repology.org" && parsed.hostname !== "www.repology.org") return null;
+
+		// Extract package name from /project/{name}/versions or /project/{name}/information
+		const match = parsed.pathname.match(/^\/project\/([^/]+)/);
+		if (!match) return null;
+
+		const packageName = decodeURIComponent(match[1]);
+		const fetchedAt = new Date().toISOString();
+
+		// Fetch from Repology API
+		const apiUrl = `https://repology.org/api/v1/project/${encodeURIComponent(packageName)}`;
+		const result = await loadPage(apiUrl, {
+			timeout,
+			headers: { Accept: "application/json" },
+		});
+
+		if (!result.ok) return null;
+
+		let packages: RepologyPackage[];
+		try {
+			packages = JSON.parse(result.content);
+		} catch {
+			return null;
+		}
+
+		// Empty response means package not found
+		if (!Array.isArray(packages) || packages.length === 0) return null;
+
+		// Find newest version(s) and extract metadata
+		const newestVersions = new Set<string>();
+		let summary: string | undefined;
+		let licenses: string[] = [];
+		const categories = new Set<string>();
+
+		for (const pkg of packages) {
+			if (pkg.status === "newest" || pkg.status === "unique") {
+				newestVersions.add(pkg.version);
+			}
+			if (!summary && pkg.summary) summary = pkg.summary;
+			if (pkg.licenses?.length && !licenses.length) licenses = pkg.licenses;
+			if (pkg.categories) {
+				for (const cat of pkg.categories) categories.add(cat);
+			}
+		}
+
+		// If no newest found, find the highest version
+		if (newestVersions.size === 0) {
+			const versions = packages.map((p) => p.version);
+			if (versions.length > 0) newestVersions.add(versions[0]);
+		}
+
+		// Group packages by status for counting
+		const statusCounts: Record<string, number> = {};
+		for (const pkg of packages) {
+			statusCounts[pkg.status] = (statusCounts[pkg.status] || 0) + 1;
+		}
+
+		// Build markdown
+		let md = `# ${packageName}\n\n`;
+		if (summary) md += `${summary}\n\n`;
+
+		md += `**Newest Version:** ${Array.from(newestVersions).join(", ") || "unknown"}\n`;
+		md += `**Repositories:** ${packages.length}\n`;
+		if (licenses.length) md += `**License:** ${licenses.join(", ")}\n`;
+		if (categories.size) md += `**Categories:** ${Array.from(categories).join(", ")}\n`;
+		md += "\n";
+
+		// Status summary
+		md += "## Version Status Summary\n\n";
+		const statusOrder = [
+			"newest",
+			"unique",
+			"devel",
+			"rolling",
+			"outdated",
+			"legacy",
+			"noscheme",
+			"incorrect",
+			"untrusted",
+			"ignored",
+		];
+		for (const status of statusOrder) {
+			if (statusCounts[status]) {
+				md += `- ${statusIndicator(status)} **${status}**: ${statusCounts[status]} repos\n`;
+			}
+		}
+		md += "\n";
+
+		// Sort packages: newest first, then by repo name
+		const sortedPackages = [...packages].sort((a, b) => {
+			const statusPriority: Record<string, number> = {
+				newest: 0,
+				unique: 1,
+				devel: 2,
+				rolling: 3,
+				outdated: 4,
+				legacy: 5,
+				noscheme: 6,
+				incorrect: 7,
+				untrusted: 8,
+				ignored: 9,
+			};
+			const aPriority = statusPriority[a.status] ?? 10;
+			const bPriority = statusPriority[b.status] ?? 10;
+			if (aPriority !== bPriority) return aPriority - bPriority;
+			return a.repo.localeCompare(b.repo);
+		});
+
+		// Show top repositories (up to 15)
+		md += "## Package Versions by Repository\n\n";
+		md += "| Repository | Version | Status |\n";
+		md += "|------------|---------|--------|\n";
+
+		const shownRepos = new Set<string>();
+		let count = 0;
+		for (const pkg of sortedPackages) {
+			// Skip duplicate repos (some have multiple entries)
+			const repoKey = pkg.subrepo ? `${pkg.repo}/${pkg.subrepo}` : pkg.repo;
+			if (shownRepos.has(repoKey)) continue;
+			shownRepos.add(repoKey);
+
+			const repoName = prettifyRepo(pkg.repo);
+			const version = pkg.origversion || pkg.version;
+			md += `| ${repoName} | \`${version}\` | ${statusIndicator(pkg.status)} ${pkg.status} |\n`;
+
+			count++;
+			if (count >= 15) break;
+		}
+
+		if (packages.length > 15) {
+			md += `\n*...and ${packages.length - 15} more repositories*\n`;
+		}
+
+		md += `\n---\n\n[View on Repology](${url})\n`;
+
+		const output = finalizeOutput(md);
+		return {
+			url,
+			finalUrl: url,
+			contentType: "text/markdown",
+			method: "repology",
+			content: output.content,
+			fetchedAt,
+			truncated: output.truncated,
+			notes: ["Fetched via Repology API"],
+		};
+	} catch {}
+
+	return null;
+};

package/src/core/tools/web-fetch-handlers/research.test.ts

@@ -0,0 +1,107 @@
+import { describe, expect, it } from "bun:test";
+import { handleBiorxiv } from "./biorxiv";
+import { handleOpenLibrary } from "./openlibrary";
+import { handleWikidata } from "./wikidata";
+
+const SKIP = !process.env.WEB_FETCH_INTEGRATION;
+
+describe.skipIf(SKIP)("handleWikidata", () => {
+	it("returns null for non-matching URLs", async () => {
+		const result = await handleWikidata("https://example.com", 20);
+		expect(result).toBeNull();
+	});
+
+	it("returns null for non-wikidata URLs", async () => {
+		const result = await handleWikidata("https://wikipedia.org/wiki/Apple_Inc", 20);
+		expect(result).toBeNull();
+	});
+
+	it("fetches Q312 - Apple Inc", async () => {
+		const result = await handleWikidata("https://www.wikidata.org/wiki/Q312", 20);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("wikidata");
+		expect(result?.content).toContain("Apple");
+		expect(result?.content).toContain("Q312");
+		expect(result?.contentType).toBe("text/markdown");
+		expect(result?.fetchedAt).toBeTruthy();
+		expect(result?.truncated).toBeDefined();
+	});
+
+	it("fetches Q5 - human (entity)", async () => {
+		const result = await handleWikidata("https://www.wikidata.org/entity/Q5", 20);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("wikidata");
+		expect(result?.content).toContain("human");
+		expect(result?.content).toContain("Q5");
+		expect(result?.contentType).toBe("text/markdown");
+		expect(result?.fetchedAt).toBeTruthy();
+		expect(result?.truncated).toBeDefined();
+	});
+});
+
+describe.skipIf(SKIP)("handleOpenLibrary", () => {
+	it("returns null for non-matching URLs", async () => {
+		const result = await handleOpenLibrary("https://example.com", 20);
+		expect(result).toBeNull();
+	});
+
+	it("returns null for non-openlibrary URLs", async () => {
+		const result = await handleOpenLibrary("https://amazon.com/books/123", 20);
+		expect(result).toBeNull();
+	});
+
+	it("fetches by ISBN - Fantastic Mr Fox", async () => {
+		const result = await handleOpenLibrary("https://openlibrary.org/isbn/9780140328721", 20);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("openlibrary");
+		expect(result?.content).toContain("Fantastic Mr");
+		expect(result?.content).toContain("Roald Dahl");
+		expect(result?.contentType).toBe("text/markdown");
+		expect(result?.fetchedAt).toBeTruthy();
+		expect(result?.truncated).toBeDefined();
+	});
+
+	it("fetches work OL45804W - The Lord of the Rings", async () => {
+		const result = await handleOpenLibrary("https://openlibrary.org/works/OL45804W", 20);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("openlibrary");
+		expect(result?.content).toContain("Lord of the Rings");
+		expect(result?.content).toContain("Tolkien");
+		expect(result?.contentType).toBe("text/markdown");
+		expect(result?.fetchedAt).toBeTruthy();
+		expect(result?.truncated).toBeDefined();
+	});
+});
+
+describe.skipIf(SKIP)("handleBiorxiv", () => {
+	it("returns null for non-matching URLs", async () => {
+		const result = await handleBiorxiv("https://example.com", 20);
+		expect(result).toBeNull();
+	});
+
+	it("returns null for non-biorxiv URLs", async () => {
+		const result = await handleBiorxiv("https://nature.com/articles/123", 20);
+		expect(result).toBeNull();
+	});
+
+	// Using the AlphaFold Protein Structure Database paper - highly cited and stable
+	it("fetches bioRxiv preprint - AlphaFold database", async () => {
+		const result = await handleBiorxiv("https://www.biorxiv.org/content/10.1101/2021.10.04.463034", 20);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("biorxiv");
+		expect(result?.content).toContain("AlphaFold");
+		expect(result?.content).toContain("Abstract");
+		expect(result?.contentType).toBe("text/markdown");
+		expect(result?.fetchedAt).toBeTruthy();
+		expect(result?.truncated).toBeDefined();
+	});
+
+	// Testing with version suffix handling
+	it("fetches bioRxiv preprint with version suffix", async () => {
+		const result = await handleBiorxiv("https://www.biorxiv.org/content/10.1101/2021.10.04.463034v1", 20);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("biorxiv");
+		expect(result?.content).toContain("AlphaFold");
+		expect(result?.contentType).toBe("text/markdown");
+	});
+});

package/src/core/tools/web-fetch-handlers/rfc.ts

@@ -0,0 +1,205 @@
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, loadPage } from "./types";
+
+interface RfcMetadata {
+	doc_id: string;
+	title: string;
+	authors?: Array<{ name: string; affiliation?: string }>;
+	pub_status?: string;
+	current_status?: string;
+	stream?: string;
+	area?: string;
+	wg_acronym?: string;
+	pub_date?: string;
+	page_count?: number;
+	abstract?: string;
+	keywords?: string[];
+	obsoletes?: string[];
+	obsoleted_by?: string[];
+	updates?: string[];
+	updated_by?: string[];
+	see_also?: string[];
+	errata_url?: string;
+}
+
+/**
+ * Extract RFC number from various URL patterns
+ */
+function extractRfcNumber(url: URL): string | null {
+	const { hostname, pathname } = url;
+
+	// https://www.rfc-editor.org/rfc/rfc{number}
+	// https://www.rfc-editor.org/rfc/rfc{number}.html
+	// https://www.rfc-editor.org/rfc/rfc{number}.txt
+	if (hostname === "www.rfc-editor.org" || hostname === "rfc-editor.org") {
+		const match = pathname.match(/\/rfc\/rfc(\d+)(?:\.(?:html|txt|pdf))?$/i);
+		if (match) return match[1];
+	}
+
+	// https://datatracker.ietf.org/doc/rfc{number}/
+	// https://datatracker.ietf.org/doc/html/rfc{number}
+	if (hostname === "datatracker.ietf.org") {
+		const match = pathname.match(/\/doc\/(?:html\/)?rfc(\d+)\/?$/i);
+		if (match) return match[1];
+	}
+
+	// https://tools.ietf.org/html/rfc{number}
+	if (hostname === "tools.ietf.org") {
+		const match = pathname.match(/\/html\/rfc(\d+)$/i);
+		if (match) return match[1];
+	}
+
+	return null;
+}
+
+/**
+ * Clean up RFC plain text - remove page headers/footers and extra formatting
+ */
+function cleanRfcText(text: string): string {
+	const lines = text.split("\n");
+	const cleaned: string[] = [];
+	let skipNext = 0;
+
+	for (let i = 0; i < lines.length; i++) {
+		const line = lines[i];
+
+		// Skip lines we've marked to skip (form feeds and surrounding blank lines)
+		if (skipNext > 0) {
+			skipNext--;
+			continue;
+		}
+
+		// Skip form feed characters and page headers (RFC NNNN ... Month Year pattern)
+		if (line.includes("\f")) {
+			// Skip the form feed line and typically 2-3 following header lines
+			skipNext = 3;
+			continue;
+		}
+
+		// Skip page footer lines (typically just a page number or "[Page N]")
+		if (/^\s*\[Page \d+\]\s*$/.test(line)) {
+			continue;
+		}
+
+		cleaned.push(line);
+	}
+
+	return cleaned.join("\n").replace(/\n{4,}/g, "\n\n\n");
+}
+
+/**
+ * Handle RFC Editor URLs - fetches IETF RFCs
+ */
+export const handleRfc: SpecialHandler = async (url: string, timeout: number): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		const rfcNumber = extractRfcNumber(parsed);
+
+		if (!rfcNumber) return null;
+
+		const fetchedAt = new Date().toISOString();
+		const notes: string[] = [];
+
+		// Fetch metadata JSON and plain text in parallel
+		const metadataUrl = `https://www.rfc-editor.org/rfc/rfc${rfcNumber}.json`;
+		const textUrl = `https://www.rfc-editor.org/rfc/rfc${rfcNumber}.txt`;
+
+		const [metaResult, textResult] = await Promise.all([
+			loadPage(metadataUrl, { timeout: Math.min(timeout, 10) }),
+			loadPage(textUrl, { timeout }),
+		]);
+
+		// We need at least the text content
+		if (!textResult.ok) return null;
+
+		let metadata: RfcMetadata | null = null;
+		if (metaResult.ok) {
+			try {
+				metadata = JSON.parse(metaResult.content);
+				notes.push("Metadata from RFC Editor JSON API");
+			} catch {
+				// JSON parse failed, continue without metadata
+			}
+		}
+
+		// Build markdown output
+		let md = "";
+
+		if (metadata) {
+			md += `# RFC ${rfcNumber}: ${metadata.title}\n\n`;
+
+			// Authors
+			if (metadata.authors?.length) {
+				const authorList = metadata.authors
+					.map((a) => (a.affiliation ? `${a.name} (${a.affiliation})` : a.name))
+					.join(", ");
+				md += `**Authors:** ${authorList}\n`;
+			}
+
+			// Publication info
+			if (metadata.pub_date) md += `**Published:** ${metadata.pub_date}\n`;
+			if (metadata.current_status) md += `**Status:** ${metadata.current_status}\n`;
+			if (metadata.stream) md += `**Stream:** ${metadata.stream}\n`;
+			if (metadata.area) md += `**Area:** ${metadata.area}\n`;
+			if (metadata.wg_acronym) md += `**Working Group:** ${metadata.wg_acronym}\n`;
+			if (metadata.page_count) md += `**Pages:** ${metadata.page_count}\n`;
+
+			// Related RFCs
+			if (metadata.obsoletes?.length) {
+				md += `**Obsoletes:** ${metadata.obsoletes.join(", ")}\n`;
+			}
+			if (metadata.obsoleted_by?.length) {
+				md += `**Obsoleted by:** ${metadata.obsoleted_by.join(", ")}\n`;
+			}
+			if (metadata.updates?.length) {
+				md += `**Updates:** ${metadata.updates.join(", ")}\n`;
+			}
+			if (metadata.updated_by?.length) {
+				md += `**Updated by:** ${metadata.updated_by.join(", ")}\n`;
+			}
+
+			// Keywords
+			if (metadata.keywords?.length) {
+				md += `**Keywords:** ${metadata.keywords.join(", ")}\n`;
+			}
+
+			// Errata
+			if (metadata.errata_url) {
+				md += `**Errata:** ${metadata.errata_url}\n`;
+			}
+
+			md += "\n";
+
+			// Abstract from metadata
+			if (metadata.abstract) {
+				md += `## Abstract\n\n${metadata.abstract}\n\n`;
+			}
+
+			md += "---\n\n";
+		} else {
+			// No metadata, use simple header
+			md += `# RFC ${rfcNumber}\n\n`;
+			notes.push("Metadata not available, showing plain text only");
+		}
+
+		// Add full text content
+		md += "## Full Text\n\n";
+		md += "```\n";
+		md += cleanRfcText(textResult.content);
+		md += "\n```\n";
+
+		const output = finalizeOutput(md);
+		return {
+			url,
+			finalUrl: `https://www.rfc-editor.org/rfc/rfc${rfcNumber}`,
+			contentType: "text/markdown",
+			method: "rfc",
+			content: output.content,
+			fetchedAt,
+			truncated: output.truncated,
+			notes: notes.length ? notes : ["Fetched from RFC Editor"],
+		};
+	} catch {}
+
+	return null;
+};

package/src/core/tools/web-fetch-handlers/rubygems.ts

@@ -0,0 +1,112 @@
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, formatCount, loadPage } from "./types";
+
+interface RubyGemsDependency {
+	name: string;
+	requirements: string;
+}
+
+interface RubyGemsResponse {
+	name: string;
+	version: string;
+	version_created_at?: string;
+	authors?: string;
+	info?: string;
+	licenses?: string[];
+	homepage_uri?: string;
+	source_code_uri?: string;
+	documentation_uri?: string;
+	project_uri?: string;
+	downloads: number;
+	version_downloads?: number;
+	gem_uri?: string;
+	dependencies?: {
+		development?: RubyGemsDependency[];
+		runtime?: RubyGemsDependency[];
+	};
+	metadata?: Record<string, string>;
+}
+
+/**
+ * Handle RubyGems URLs via API
+ */
+export const handleRubyGems: SpecialHandler = async (url: string, timeout: number): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		if (parsed.hostname !== "rubygems.org" && parsed.hostname !== "www.rubygems.org") return null;
+
+		// Extract gem name from /gems/{name}
+		const match = parsed.pathname.match(/^\/gems\/([^/]+)/);
+		if (!match) return null;
+
+		const gemName = decodeURIComponent(match[1]);
+		const fetchedAt = new Date().toISOString();
+
+		// Fetch from RubyGems API
+		const apiUrl = `https://rubygems.org/api/v1/gems/${encodeURIComponent(gemName)}.json`;
+		const result = await loadPage(apiUrl, {
+			timeout,
+			headers: { Accept: "application/json" },
+		});
+
+		if (!result.ok) return null;
+
+		let gem: RubyGemsResponse;
+		try {
+			gem = JSON.parse(result.content);
+		} catch {
+			return null;
+		}
+
+		let md = `# ${gem.name}\n\n`;
+		if (gem.info) md += `${gem.info}\n\n`;
+
+		// Version and license
+		md += `**Version:** ${gem.version}`;
+		if (gem.licenses?.length) md += ` · **License:** ${gem.licenses.join(", ")}`;
+		md += "\n";
+
+		// Downloads
+		md += `**Total Downloads:** ${formatCount(gem.downloads)}`;
+		if (gem.version_downloads) md += ` · **Version Downloads:** ${formatCount(gem.version_downloads)}`;
+		md += "\n\n";
+
+		// Links
+		if (gem.homepage_uri) md += `**Homepage:** ${gem.homepage_uri}\n`;
+		if (gem.source_code_uri) md += `**Source Code:** ${gem.source_code_uri}\n`;
+		if (gem.documentation_uri) md += `**Documentation:** ${gem.documentation_uri}\n`;
+		if (gem.authors) md += `**Authors:** ${gem.authors}\n`;
+
+		// Runtime dependencies
+		const runtimeDeps = gem.dependencies?.runtime;
+		if (runtimeDeps && runtimeDeps.length > 0) {
+			md += `\n## Runtime Dependencies\n\n`;
+			for (const dep of runtimeDeps) {
+				md += `- ${dep.name} ${dep.requirements}\n`;
+			}
+		}
+
+		// Development dependencies
+		const devDeps = gem.dependencies?.development;
+		if (devDeps && devDeps.length > 0) {
+			md += `\n## Development Dependencies\n\n`;
+			for (const dep of devDeps) {
+				md += `- ${dep.name} ${dep.requirements}\n`;
+			}
+		}
+
+		const output = finalizeOutput(md);
+		return {
+			url,
+			finalUrl: url,
+			contentType: "text/markdown",
+			method: "rubygems",
+			content: output.content,
+			fetchedAt,
+			truncated: output.truncated,
+			notes: ["Fetched via RubyGems API"],
+		};
+	} catch {}
+
+	return null;
+};