@oh-my-pi/pi-coding-agent 3.25.0 → 3.31.0

package/src/core/tools/web-scrapers/academic.test.ts

@@ -0,0 +1,239 @@
+import { describe, expect, it } from "bun:test";
+import { handleArxiv } from "./arxiv";
+import { handleIacr } from "./iacr";
+import { handlePubMed } from "./pubmed";
+import { handleSemanticScholar } from "./semantic-scholar";
+
+const SKIP = !process.env.WEB_FETCH_INTEGRATION;
+
+describe.skipIf(SKIP)("handleSemanticScholar", () => {
+	it("returns null for non-S2 URLs", async () => {
+		const result = await handleSemanticScholar("https://example.com", 10);
+		expect(result).toBeNull();
+	});
+
+	it("fetches a known paper", async () => {
+		// "Attention Is All You Need" paper
+		const result = await handleSemanticScholar(
+			"https://www.semanticscholar.org/paper/Attention-is-All-you-Need-Vaswani-Shazeer/204e3073870fae3d05bcbc2f6a8e263d9b72e776",
+			20,
+		);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("semantic-scholar");
+		// API may be rate-limited or fail, verify handler structure
+		if (
+			!result?.content.includes("Too Many Requests") &&
+			!result?.content.includes("Failed to fetch") &&
+			!result?.content.includes("Failed to parse")
+		) {
+			expect(result?.content).toContain("Attention");
+			expect(result?.contentType).toBe("text/markdown");
+		}
+		expect(result?.truncated).toBe(false);
+	});
+
+	it("handles invalid paper ID format", async () => {
+		const result = await handleSemanticScholar("https://www.semanticscholar.org/paper/invalid", 20);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("semantic-scholar");
+		expect(result?.content).toContain("Failed to extract paper ID");
+		expect(result?.notes).toContain("Invalid URL format");
+	});
+
+	it("extracts paper ID from various URL formats", async () => {
+		const paperId = "204e3073870fae3d05bcbc2f6a8e263d9b72e776";
+		const urls = [
+			`https://www.semanticscholar.org/paper/Attention-is-All-you-Need-Vaswani-Shazeer/${paperId}`,
+			`https://www.semanticscholar.org/paper/${paperId}`,
+		];
+
+		for (const url of urls) {
+			const result = await handleSemanticScholar(url, 20);
+			expect(result).not.toBeNull();
+			expect(result?.method).toBe("semantic-scholar");
+			// API may be rate-limited or fail
+			if (
+				!result?.content.includes("Too Many Requests") &&
+				!result?.content.includes("Failed to fetch") &&
+				!result?.content.includes("Failed to parse")
+			) {
+				expect(result?.content).toContain("Attention");
+			}
+		}
+	});
+
+	it("includes metadata in formatted output", async () => {
+		const result = await handleSemanticScholar(
+			"https://www.semanticscholar.org/paper/Attention-is-All-you-Need-Vaswani-Shazeer/204e3073870fae3d05bcbc2f6a8e263d9b72e776",
+			20,
+		);
+		expect(result).not.toBeNull();
+		// Only check metadata if API call succeeded (not rate-limited)
+		if (!result?.content.includes("Too Many Requests") && !result?.content.includes("Failed to fetch")) {
+			expect(result?.content).toMatch(/Year:/);
+			expect(result?.content).toMatch(/Citations:/);
+			expect(result?.content).toMatch(/Authors:/);
+			expect(result?.content).toContain("Vaswani");
+		}
+	});
+});
+
+describe.skipIf(SKIP)("handlePubMed", () => {
+	it("returns null for non-PubMed URLs", async () => {
+		const result = await handlePubMed("https://example.com", 10);
+		expect(result).toBeNull();
+	});
+
+	it("fetches a known article from pubmed.ncbi.nlm.nih.gov", async () => {
+		// PMID 33782455 - COVID-19 vaccine paper
+		const result = await handlePubMed("https://pubmed.ncbi.nlm.nih.gov/33782455/", 20);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("pubmed");
+		expect(result?.contentType).toBe("text/markdown");
+		expect(result?.truncated).toBe(false);
+	});
+
+	it("fetches from ncbi.nlm.nih.gov/pubmed format", async () => {
+		const result = await handlePubMed("https://ncbi.nlm.nih.gov/pubmed/33782455", 20);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("pubmed");
+	});
+
+	it("includes PMID in output", async () => {
+		const result = await handlePubMed("https://pubmed.ncbi.nlm.nih.gov/33782455/", 20);
+		expect(result).not.toBeNull();
+		expect(result?.content).toContain("PMID:");
+		expect(result?.content).toContain("33782455");
+	});
+
+	it("includes abstract section", async () => {
+		const result = await handlePubMed("https://pubmed.ncbi.nlm.nih.gov/33782455/", 20);
+		expect(result).not.toBeNull();
+		expect(result?.content).toContain("## Abstract");
+	});
+
+	it("includes metadata fields", async () => {
+		const result = await handlePubMed("https://pubmed.ncbi.nlm.nih.gov/33782455/", 20);
+		expect(result).not.toBeNull();
+		expect(result?.content).toMatch(/Authors:/);
+		expect(result?.content).toMatch(/Journal:/);
+	});
+
+	it("returns null for invalid PMID format", async () => {
+		const result = await handlePubMed("https://pubmed.ncbi.nlm.nih.gov/invalid/", 20);
+		expect(result).toBeNull();
+	});
+
+	it("handles non-existent PMID gracefully", async () => {
+		const result = await handlePubMed("https://pubmed.ncbi.nlm.nih.gov/99999999999/", 20);
+		// NCBI API returns a response even for non-existent PMIDs with minimal data
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("pubmed");
+	});
+});
+
+describe.skipIf(SKIP)("handleArxiv", () => {
+	it("returns null for non-arXiv URLs", async () => {
+		const result = await handleArxiv("https://example.com", 10000);
+		expect(result).toBeNull();
+	});
+
+	it("fetches a known paper", async () => {
+		// "Attention Is All You Need" paper
+		const result = await handleArxiv("https://arxiv.org/abs/1706.03762", 30000);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("arxiv");
+		expect(result?.contentType).toBe("text/markdown");
+		// API may be rate-limited or fail
+		if (!result?.content.includes("Too Many Requests") && !result?.content.includes("Failed to fetch")) {
+			expect(result?.content).toContain("Attention");
+			expect(result?.content).toContain("arXiv:");
+			expect(result?.content).toContain("1706.03762");
+		}
+		expect(result?.truncated).toBe(false);
+	});
+
+	it("handles /pdf/ URLs", async () => {
+		const result = await handleArxiv("https://arxiv.org/pdf/1706.03762", 30000);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("arxiv");
+		if (!result?.content.includes("Too Many Requests") && !result?.content.includes("Failed to fetch")) {
+			expect(result?.content).toContain("Attention");
+			expect(result?.notes?.some((n) => n.includes("PDF"))).toBe(true);
+		}
+	});
+
+	it("handles arxiv.org/abs/ format", async () => {
+		const result = await handleArxiv("https://arxiv.org/abs/1706.03762", 30000);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("arxiv");
+		if (!result?.content.includes("Too Many Requests") && !result?.content.includes("Failed to fetch")) {
+			expect(result?.content).toContain("1706.03762");
+		}
+	});
+
+	it("includes paper metadata", async () => {
+		const result = await handleArxiv("https://arxiv.org/abs/1706.03762", 30000);
+		expect(result).not.toBeNull();
+		if (!result?.content.includes("Too Many Requests") && !result?.content.includes("Failed to fetch")) {
+			expect(result?.content).toMatch(/Authors:/);
+			expect(result?.content).toContain("Vaswani");
+			expect(result?.content).toMatch(/Abstract/);
+			expect(result?.content).toMatch(/Published:/);
+		}
+	});
+
+	it("handles rate limiting gracefully", async () => {
+		const result = await handleArxiv("https://arxiv.org/abs/1706.03762", 5000);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("arxiv");
+		// Should return something, even if rate limited
+		expect(result?.content).toBeTruthy();
+	});
+});
+
+describe.skipIf(SKIP)("handleIacr", () => {
+	it("returns null for non-IACR URLs", async () => {
+		const result = await handleIacr("https://example.com", 10000);
+		expect(result).toBeNull();
+	});
+
+	it("fetches a known ePrint", async () => {
+		// Using a well-known paper
+		const result = await handleIacr("https://eprint.iacr.org/2023/123", 30000);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("iacr");
+		expect(result?.contentType).toBe("text/markdown");
+		if (!result?.content.includes("Too Many Requests") && !result?.content.includes("Failed to fetch")) {
+			expect(result?.content).toContain("ePrint:");
+			expect(result?.content).toContain("2023/123");
+		}
+		expect(result?.truncated).toBe(false);
+	});
+
+	it("includes paper metadata", async () => {
+		const result = await handleIacr("https://eprint.iacr.org/2023/123", 30000);
+		expect(result).not.toBeNull();
+		if (!result?.content.includes("Too Many Requests") && !result?.content.includes("Failed to fetch")) {
+			expect(result?.content).toMatch(/Authors:/);
+			expect(result?.content).toMatch(/Abstract/);
+		}
+	});
+
+	it("handles rate limiting gracefully", async () => {
+		const result = await handleIacr("https://eprint.iacr.org/2023/123", 5000);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("iacr");
+		// Should return something, even if rate limited
+		expect(result?.content).toBeTruthy();
+	});
+
+	it("handles PDF URLs", async () => {
+		const result = await handleIacr("https://eprint.iacr.org/2023/123.pdf", 30000);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("iacr");
+		if (!result?.content.includes("Too Many Requests") && !result?.content.includes("Failed to fetch")) {
+			expect(result?.notes?.some((n) => n.includes("PDF"))).toBe(true);
+		}
+	});
+});

package/src/core/tools/web-scrapers/artifacthub.ts

@@ -0,0 +1,215 @@
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, formatCount, loadPage } from "./types";
+
+interface ArtifactHubMaintainer {
+	name: string;
+	email?: string;
+}
+
+interface ArtifactHubLink {
+	name: string;
+	url: string;
+}
+
+interface ArtifactHubRepository {
+	name: string;
+	display_name?: string;
+	url: string;
+	organization_name?: string;
+	organization_display_name?: string;
+}
+
+interface ArtifactHubPackage {
+	package_id: string;
+	name: string;
+	normalized_name: string;
+	display_name?: string;
+	description?: string;
+	version: string;
+	app_version?: string;
+	license?: string;
+	home_url?: string;
+	readme?: string;
+	install?: string;
+	keywords?: string[];
+	maintainers?: ArtifactHubMaintainer[];
+	links?: ArtifactHubLink[];
+	repository: ArtifactHubRepository;
+	ts: number;
+	created_at: number;
+	stars?: number;
+	official?: boolean;
+	signed?: boolean;
+	security_report_summary?: {
+		low?: number;
+		medium?: number;
+		high?: number;
+		critical?: number;
+	};
+	available_versions?: Array<{ version: string; ts: number }>;
+}
+
+/**
+ * Handle Artifact Hub URLs via API
+ * Supports Helm charts, OLM operators, Falco rules, OPA policies, etc.
+ */
+export const handleArtifactHub: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		if (parsed.hostname !== "artifacthub.io" && parsed.hostname !== "www.artifacthub.io") return null;
+
+		// Extract kind, repo, and package name from /packages/{kind}/{repo}/{name}
+		const match = parsed.pathname.match(/^\/packages\/([^/]+)\/([^/]+)\/([^/]+)/);
+		if (!match) return null;
+
+		const [, kind, repo, name] = match;
+		const fetchedAt = new Date().toISOString();
+
+		// Fetch from Artifact Hub API
+		const apiUrl = `https://artifacthub.io/api/v1/packages/${kind}/${repo}/${name}`;
+		const result = await loadPage(apiUrl, {
+			timeout,
+			headers: { Accept: "application/json" },
+			signal,
+		});
+
+		if (!result.ok) return null;
+
+		let pkg: ArtifactHubPackage;
+		try {
+			pkg = JSON.parse(result.content);
+		} catch {
+			return null;
+		}
+
+		const displayName = pkg.display_name || pkg.name;
+		const kindLabel = formatKindLabel(kind);
+
+		let md = `# ${displayName}\n\n`;
+		if (pkg.description) md += `${pkg.description}\n\n`;
+
+		// Basic info line
+		md += `**Type:** ${kindLabel}`;
+		md += ` · **Version:** ${pkg.version}`;
+		if (pkg.app_version) md += ` · **App Version:** ${pkg.app_version}`;
+		if (pkg.license) md += ` · **License:** ${pkg.license}`;
+		md += "\n";
+
+		// Stats and badges
+		const badges: string[] = [];
+		if (pkg.official) badges.push("Official");
+		if (pkg.signed) badges.push("Signed");
+		if (pkg.stars) badges.push(`${formatCount(pkg.stars)} stars`);
+		if (badges.length > 0) md += `**${badges.join(" · ")}**\n`;
+		md += "\n";
+
+		// Repository info
+		const repoDisplay =
+			pkg.repository.organization_display_name || pkg.repository.display_name || pkg.repository.name;
+		md += `**Repository:** ${repoDisplay}`;
+		if (pkg.repository.url) md += ` ([${pkg.repository.url}](${pkg.repository.url}))`;
+		md += "\n";
+
+		if (pkg.home_url) md += `**Homepage:** ${pkg.home_url}\n`;
+		if (pkg.keywords?.length) md += `**Keywords:** ${pkg.keywords.join(", ")}\n`;
+
+		// Maintainers
+		if (pkg.maintainers?.length) {
+			const maintainerNames = pkg.maintainers.map((m) => m.name).join(", ");
+			md += `**Maintainers:** ${maintainerNames}\n`;
+		}
+
+		// Security report summary
+		if (pkg.security_report_summary) {
+			const sec = pkg.security_report_summary;
+			const parts: string[] = [];
+			if (sec.critical) parts.push(`${sec.critical} critical`);
+			if (sec.high) parts.push(`${sec.high} high`);
+			if (sec.medium) parts.push(`${sec.medium} medium`);
+			if (sec.low) parts.push(`${sec.low} low`);
+			if (parts.length > 0) {
+				md += `**Security:** ${parts.join(", ")}\n`;
+			}
+		}
+
+		// Links
+		if (pkg.links?.length) {
+			md += `\n## Links\n\n`;
+			for (const link of pkg.links) {
+				md += `- [${link.name}](${link.url})\n`;
+			}
+		}
+
+		// Install instructions
+		if (pkg.install) {
+			md += `\n## Installation\n\n\`\`\`bash\n${pkg.install.trim()}\n\`\`\`\n`;
+		}
+
+		// Recent versions
+		if (pkg.available_versions?.length) {
+			md += `\n## Recent Versions\n\n`;
+			for (const ver of pkg.available_versions.slice(0, 5)) {
+				const date = new Date(ver.ts * 1000).toISOString().split("T")[0];
+				md += `- **${ver.version}** (${date})\n`;
+			}
+		}
+
+		// README
+		if (pkg.readme) {
+			md += `\n---\n\n## README\n\n${pkg.readme}\n`;
+		}
+
+		const output = finalizeOutput(md);
+		return {
+			url,
+			finalUrl: url,
+			contentType: "text/markdown",
+			method: "artifacthub",
+			content: output.content,
+			fetchedAt,
+			truncated: output.truncated,
+			notes: [`Fetched via Artifact Hub API (${kindLabel})`],
+		};
+	} catch {}
+
+	return null;
+};
+
+/**
+ * Convert kind slug to display label
+ */
+function formatKindLabel(kind: string): string {
+	const labels: Record<string, string> = {
+		helm: "Helm Chart",
+		"helm-plugin": "Helm Plugin",
+		falco: "Falco Rules",
+		opa: "OPA Policy",
+		olm: "OLM Operator",
+		tbaction: "Tinkerbell Action",
+		krew: "Krew Plugin",
+		tekton: "Tekton Task",
+		"tekton-pipeline": "Tekton Pipeline",
+		keda: "KEDA Scaler",
+		coredns: "CoreDNS Plugin",
+		keptn: "Keptn Integration",
+		container: "Container Image",
+		kubewarden: "Kubewarden Policy",
+		gatekeeper: "Gatekeeper Policy",
+		kyverno: "Kyverno Policy",
+		"knative-client": "Knative Client Plugin",
+		backstage: "Backstage Plugin",
+		argo: "Argo Template",
+		kubearmor: "KubeArmor Policy",
+		kcl: "KCL Module",
+		headlamp: "Headlamp Plugin",
+		inspektor: "Inspektor Gadget",
+		"meshery-design": "Meshery Design",
+		"opencost-plugin": "OpenCost Plugin",
+		radius: "Radius Recipe",
+	};
+	return labels[kind] || kind.charAt(0).toUpperCase() + kind.slice(1);
+}

package/src/core/tools/web-scrapers/arxiv.ts

@@ -0,0 +1,88 @@
+import { parse as parseHtml } from "node-html-parser";
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, loadPage } from "./types";
+import { convertWithMarkitdown, fetchBinary } from "./utils";
+
+/**
+ * Handle arXiv URLs via arXiv API
+ */
+export const handleArxiv: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		if (parsed.hostname !== "arxiv.org") return null;
+
+		// Extract paper ID from various URL formats
+		// /abs/1234.56789, /pdf/1234.56789, /abs/cs/0123456
+		const match = parsed.pathname.match(/\/(abs|pdf)\/(.+?)(?:\.pdf)?$/);
+		if (!match) return null;
+
+		const paperId = match[2];
+		const fetchedAt = new Date().toISOString();
+		const notes: string[] = [];
+
+		// Fetch metadata via arXiv API
+		const apiUrl = `https://export.arxiv.org/api/query?id_list=${paperId}`;
+		const result = await loadPage(apiUrl, { timeout, signal });
+
+		if (!result.ok) return null;
+
+		// Parse the Atom feed response
+		const doc = parseHtml(result.content, { parseNoneClosedTags: true });
+		const entry = doc.querySelector("entry");
+
+		if (!entry) return null;
+
+		const title = entry.querySelector("title")?.text?.trim()?.replace(/\s+/g, " ");
+		const summary = entry.querySelector("summary")?.text?.trim();
+		const authors = entry
+			.querySelectorAll("author name")
+			.map((n) => n.text?.trim())
+			.filter(Boolean);
+		const published = entry.querySelector("published")?.text?.trim()?.split("T")[0];
+		const categories = entry
+			.querySelectorAll("category")
+			.map((c) => c.getAttribute("term"))
+			.filter(Boolean);
+		const pdfLink = entry.querySelector('link[title="pdf"]')?.getAttribute("href");
+
+		let md = `# ${title || "arXiv Paper"}\n\n`;
+		if (authors.length) md += `**Authors:** ${authors.join(", ")}\n`;
+		if (published) md += `**Published:** ${published}\n`;
+		if (categories.length) md += `**Categories:** ${categories.join(", ")}\n`;
+		md += `**arXiv:** ${paperId}\n\n`;
+		md += `---\n\n## Abstract\n\n${summary || "No abstract available."}\n\n`;
+
+		// If it was a PDF link or we want full content, try to fetch and convert PDF
+		if (match[1] === "pdf" || parsed.pathname.includes(".pdf")) {
+			if (pdfLink) {
+				notes.push("Fetching PDF for full content...");
+				const pdfResult = await fetchBinary(pdfLink, timeout, signal);
+				if (pdfResult.ok) {
+					const converted = await convertWithMarkitdown(pdfResult.buffer, ".pdf", timeout, signal);
+					if (converted.ok && converted.content.length > 500) {
+						md += `---\n\n## Full Paper\n\n${converted.content}\n`;
+						notes.push("PDF converted via markitdown");
+					}
+				}
+			}
+		}
+
+		const output = finalizeOutput(md);
+		return {
+			url,
+			finalUrl: url,
+			contentType: "text/markdown",
+			method: "arxiv",
+			content: output.content,
+			fetchedAt,
+			truncated: output.truncated,
+			notes: notes.length ? notes : ["Fetched via arXiv API"],
+		};
+	} catch {}
+
+	return null;
+};