@oh-my-pi/pi-coding-agent 3.24.0 → 3.30.0

package/src/core/tools/web-fetch-handlers/pub-dev.ts

@@ -0,0 +1,185 @@
+import { finalizeOutput, formatCount, loadPage, type SpecialHandler } from "./types";
+
+/**
+ * Handle pub.dev URLs via API
+ */
+export const handlePubDev: SpecialHandler = async (url: string, timeout: number) => {
+	try {
+		const parsed = new URL(url);
+		if (parsed.hostname !== "pub.dev" && parsed.hostname !== "www.pub.dev") return null;
+
+		// Extract package name from /packages/{package}
+		const match = parsed.pathname.match(/^\/packages\/([^/]+)/);
+		if (!match) return null;
+
+		const packageName = decodeURIComponent(match[1]);
+		const fetchedAt = new Date().toISOString();
+
+		// Fetch from pub.dev API
+		const apiUrl = `https://pub.dev/api/packages/${encodeURIComponent(packageName)}`;
+		const result = await loadPage(apiUrl, { timeout });
+
+		if (!result.ok) return null;
+
+		let data: {
+			name: string;
+			latest: {
+				version: string;
+				pubspec: {
+					description?: string;
+					homepage?: string;
+					repository?: string;
+					documentation?: string;
+					environment?: Record<string, string>;
+					dependencies?: Record<string, unknown>;
+					dev_dependencies?: Record<string, unknown>;
+				};
+			};
+			publisherId?: string;
+			metrics?: {
+				score?: {
+					likeCount?: number;
+					grantedPoints?: number;
+					maxPoints?: number;
+					popularityScore?: number;
+				};
+			};
+		};
+
+		try {
+			data = JSON.parse(result.content);
+		} catch {
+			return null;
+		}
+
+		const { name, latest, publisherId, metrics } = data;
+		const pubspec = latest.pubspec;
+
+		let md = `# ${name}\n\n`;
+		if (pubspec.description) md += `${pubspec.description}\n\n`;
+
+		md += `**Latest:** ${latest.version}`;
+		if (publisherId) md += ` · **Publisher:** ${publisherId}`;
+		md += "\n";
+
+		// Add metrics if available
+		const score = metrics?.score;
+		if (score) {
+			const likes = score.likeCount;
+			const points = score.grantedPoints;
+			const maxPoints = score.maxPoints;
+			const popularity = score.popularityScore;
+
+			if (likes !== undefined) md += `**Likes:** ${formatCount(likes)}`;
+			if (points !== undefined && maxPoints !== undefined) {
+				md += ` · **Pub Points:** ${points}/${maxPoints}`;
+			}
+			if (popularity !== undefined) {
+				md += ` · **Popularity:** ${Math.round(popularity * 100)}%`;
+			}
+			md += "\n";
+		}
+
+		md += "\n";
+
+		if (pubspec.homepage) md += `**Homepage:** ${pubspec.homepage}\n`;
+		if (pubspec.repository) md += `**Repository:** ${pubspec.repository}\n`;
+		if (pubspec.documentation) md += `**Documentation:** ${pubspec.documentation}\n`;
+
+		// SDK constraints
+		if (pubspec.environment) {
+			const constraints: string[] = [];
+			for (const [key, value] of Object.entries(pubspec.environment)) {
+				constraints.push(`${key}: ${value}`);
+			}
+			if (constraints.length > 0) {
+				md += `**SDK:** ${constraints.join(", ")}\n`;
+			}
+		}
+
+		md += "\n";
+
+		// Dependencies
+		if (pubspec.dependencies) {
+			const deps = Object.keys(pubspec.dependencies);
+			if (deps.length > 0) {
+				md += `## Dependencies (${deps.length})\n\n`;
+				for (const dep of deps.slice(0, 20)) {
+					const constraint = pubspec.dependencies[dep];
+					const constraintStr =
+						typeof constraint === "string" ? constraint : typeof constraint === "object" ? "complex" : "";
+					md += `- ${dep}`;
+					if (constraintStr) md += `: ${constraintStr}`;
+					md += "\n";
+				}
+				if (deps.length > 20) {
+					md += `\n*...and ${deps.length - 20} more*\n`;
+				}
+				md += "\n";
+			}
+		}
+
+		// Try to fetch README from pub.dev
+		const readmeUrl = `https://pub.dev/packages/${encodeURIComponent(packageName)}/versions/${encodeURIComponent(latest.version)}/readme`;
+		try {
+			const readmeResult = await loadPage(readmeUrl, { timeout: Math.min(timeout, 10) });
+			if (readmeResult.ok) {
+				// Extract README content from HTML
+				const readmeMatch = readmeResult.content.match(
+					/<div[^>]*class="[^"]*markdown-body[^"]*"[^>]*>([\s\S]*?)<\/div>/i,
+				);
+				if (readmeMatch) {
+					// Basic HTML to markdown conversion for README
+					const readme = readmeMatch[1]
+						.replace(/<h(\d)[^>]*>(.*?)<\/h\d>/gi, (_, level, text) => {
+							const stripped = text.replace(/<[^>]+>/g, "");
+							return `${"#".repeat(parseInt(level, 10))} ${stripped}\n\n`;
+						})
+						.replace(/<pre><code[^>]*>([\s\S]*?)<\/code><\/pre>/gi, (_, code) => {
+							const decoded = code
+								.replace(/&lt;/g, "<")
+								.replace(/&gt;/g, ">")
+								.replace(/&amp;/g, "&")
+								.replace(/&quot;/g, '"');
+							return `\n\`\`\`\n${decoded}\n\`\`\`\n\n`;
+						})
+						.replace(/<code[^>]*>(.*?)<\/code>/gi, "`$1`")
+						.replace(/<a[^>]*href="([^"]+)"[^>]*>(.*?)<\/a>/gi, "[$2]($1)")
+						.replace(/<strong[^>]*>(.*?)<\/strong>/gi, "**$1**")
+						.replace(/<em[^>]*>(.*?)<\/em>/gi, "*$1*")
+						.replace(/<li[^>]*>(.*?)<\/li>/gi, "- $1\n")
+						.replace(/<\/?(ul|ol|p|br)[^>]*>/gi, "\n")
+						.replace(/<[^>]+>/g, "")
+						.replace(/&lt;/g, "<")
+						.replace(/&gt;/g, ">")
+						.replace(/&amp;/g, "&")
+						.replace(/&quot;/g, '"')
+						.replace(/&#39;/g, "'")
+						.replace(/&nbsp;/g, " ")
+						.replace(/\n{3,}/g, "\n\n")
+						.trim();
+
+					if (readme.length > 100) {
+						md += `## README\n\n${readme}\n`;
+					}
+				}
+			}
+		} catch {
+			// README fetch failed, continue without it
+		}
+
+		const output = finalizeOutput(md);
+		return {
+			url,
+			finalUrl: url,
+			contentType: "text/markdown",
+			method: "pub.dev",
+			content: output.content,
+			fetchedAt,
+			truncated: output.truncated,
+			notes: ["Fetched via pub.dev API"],
+		};
+	} catch {}
+
+	return null;
+};

package/src/core/tools/web-fetch-handlers/pubmed.ts

@@ -0,0 +1,174 @@
+/**
+ * PubMed handler for web-fetch
+ */
+
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, loadPage } from "./types";
+
+/**
+ * Handle PubMed URLs - fetch article metadata, abstract, MeSH terms
+ */
+export const handlePubMed: SpecialHandler = async (url: string, timeout: number): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+
+		// Match pubmed.ncbi.nlm.nih.gov/{pmid} or ncbi.nlm.nih.gov/pubmed/{pmid}
+		if (
+			parsed.hostname !== "pubmed.ncbi.nlm.nih.gov" &&
+			!(parsed.hostname === "ncbi.nlm.nih.gov" && parsed.pathname.startsWith("/pubmed"))
+		) {
+			return null;
+		}
+
+		// Extract PMID from URL
+		let pmid: string | null = null;
+		if (parsed.hostname === "pubmed.ncbi.nlm.nih.gov") {
+			// Format: pubmed.ncbi.nlm.nih.gov/12345678/
+			const match = parsed.pathname.match(/\/(\d+)/);
+			if (match) pmid = match[1];
+		} else {
+			// Format: ncbi.nlm.nih.gov/pubmed/12345678
+			const match = parsed.pathname.match(/\/pubmed\/(\d+)/);
+			if (match) pmid = match[1];
+		}
+
+		if (!pmid) return null;
+
+		const fetchedAt = new Date().toISOString();
+		const notes: string[] = [];
+
+		// Fetch summary metadata
+		const summaryUrl = `https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esummary.fcgi?db=pubmed&id=${pmid}&retmode=json`;
+		const summaryResult = await loadPage(summaryUrl, { timeout });
+
+		if (!summaryResult.ok) return null;
+
+		let summaryData: {
+			result?: {
+				[pmid: string]: {
+					title?: string;
+					authors?: Array<{ name: string }>;
+					fulljournalname?: string;
+					pubdate?: string;
+					volume?: string;
+					issue?: string;
+					pages?: string;
+					elocationid?: string; // DOI
+					articleids?: Array<{ idtype: string; value: string }>;
+				};
+			};
+		};
+
+		try {
+			summaryData = JSON.parse(summaryResult.content);
+		} catch {
+			return null;
+		}
+
+		const article = summaryData.result?.[pmid];
+		if (!article) return null;
+
+		// Fetch abstract
+		const abstractUrl = `https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?db=pubmed&id=${pmid}&rettype=abstract&retmode=text`;
+		const abstractResult = await loadPage(abstractUrl, { timeout });
+
+		let abstractText = "";
+		if (abstractResult.ok) {
+			abstractText = abstractResult.content.trim();
+			notes.push("Fetched abstract via NCBI E-utilities");
+		}
+
+		// Extract DOI and PMCID
+		let doi = "";
+		let pmcid = "";
+		if (article.articleids) {
+			for (const id of article.articleids) {
+				if (id.idtype === "doi") doi = id.value;
+				if (id.idtype === "pmc") pmcid = id.value;
+			}
+		}
+		if (!doi && article.elocationid) {
+			doi = article.elocationid;
+		}
+
+		// Build markdown output
+		let md = `# ${article.title || "PubMed Article"}\n\n`;
+
+		// Authors
+		if (article.authors && article.authors.length > 0) {
+			const authorNames = article.authors.map((a) => a.name).join(", ");
+			md += `**Authors:** ${authorNames}\n`;
+		}
+
+		// Journal info
+		if (article.fulljournalname) {
+			md += `**Journal:** ${article.fulljournalname}`;
+			if (article.pubdate) md += ` (${article.pubdate})`;
+			md += "\n";
+		}
+
+		// Volume/Issue/Pages
+		const citation: string[] = [];
+		if (article.volume) citation.push(`Vol ${article.volume}`);
+		if (article.issue) citation.push(`Issue ${article.issue}`);
+		if (article.pages) citation.push(`pp ${article.pages}`);
+		if (citation.length > 0) {
+			md += `**Citation:** ${citation.join(", ")}\n`;
+		}
+
+		// IDs
+		md += `**PMID:** ${pmid}\n`;
+		if (doi) md += `**DOI:** ${doi}\n`;
+		if (pmcid) md += `**PMCID:** ${pmcid}\n`;
+
+		md += "\n---\n\n";
+
+		// Abstract section
+		if (abstractText) {
+			md += `## Abstract\n\n${abstractText}\n`;
+		} else {
+			md += `## Abstract\n\nNo abstract available.\n`;
+		}
+
+		// Try to fetch MeSH terms
+		try {
+			const meshUrl = `https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?db=pubmed&id=${pmid}&rettype=medline&retmode=text`;
+			const meshResult = await loadPage(meshUrl, { timeout: Math.min(timeout, 5) });
+
+			if (meshResult.ok) {
+				const meshTerms: string[] = [];
+				const lines = meshResult.content.split("\n");
+				for (const line of lines) {
+					if (line.startsWith("MH  - ")) {
+						const term = line.slice(6).trim();
+						meshTerms.push(term);
+					}
+				}
+
+				if (meshTerms.length > 0) {
+					md += `\n## MeSH Terms\n\n`;
+					for (const term of meshTerms) {
+						md += `- ${term}\n`;
+					}
+					notes.push("Fetched MeSH terms via NCBI E-utilities");
+				}
+			}
+		} catch {
+			// MeSH terms are optional
+		}
+
+		const output = finalizeOutput(md);
+		return {
+			url,
+			finalUrl: url,
+			contentType: "text/markdown",
+			method: "pubmed",
+			content: output.content,
+			fetchedAt,
+			truncated: output.truncated,
+			notes: notes.length > 0 ? notes : ["Fetched via NCBI E-utilities"],
+		};
+	} catch {
+		return null;
+	}
+};

package/src/core/tools/web-fetch-handlers/pypi.ts

@@ -0,0 +1,125 @@
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, formatCount, loadPage } from "./types";
+
+/**
+ * Handle PyPI URLs via JSON API
+ */
+export const handlePyPI: SpecialHandler = async (url: string, timeout: number): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		if (parsed.hostname !== "pypi.org" && parsed.hostname !== "www.pypi.org") return null;
+
+		// Extract package name from /project/{package} or /project/{package}/{version}
+		const match = parsed.pathname.match(/^\/project\/([^/]+)/);
+		if (!match) return null;
+
+		const packageName = decodeURIComponent(match[1]);
+		const fetchedAt = new Date().toISOString();
+
+		// Fetch from PyPI JSON API
+		const apiUrl = `https://pypi.org/pypi/${packageName}/json`;
+		const downloadsUrl = `https://pypistats.org/api/packages/${packageName}/recent`;
+
+		// Fetch package info and download stats in parallel
+		const [result, downloadsResult] = await Promise.all([
+			loadPage(apiUrl, { timeout }),
+			loadPage(downloadsUrl, { timeout: Math.min(timeout, 5) }),
+		]);
+
+		if (!result.ok) return null;
+
+		// Parse download stats
+		let weeklyDownloads: number | null = null;
+		if (downloadsResult.ok) {
+			try {
+				const dlData = JSON.parse(downloadsResult.content) as { data?: { last_week?: number } };
+				weeklyDownloads = dlData.data?.last_week ?? null;
+			} catch {}
+		}
+
+		let pkg: {
+			info: {
+				name: string;
+				version: string;
+				summary?: string;
+				description?: string;
+				author?: string;
+				author_email?: string;
+				license?: string;
+				home_page?: string;
+				project_urls?: Record<string, string>;
+				requires_python?: string;
+				keywords?: string;
+				classifiers?: string[];
+			};
+			urls?: Array<{ filename: string; size: number; upload_time: string }>;
+			releases?: Record<string, unknown>;
+			requires_dist?: string[];
+		};
+
+		try {
+			pkg = JSON.parse(result.content);
+		} catch {
+			return null; // JSON parse failed
+		}
+
+		const info = pkg.info;
+		let md = `# ${info.name}\n\n`;
+		if (info.summary) md += `${info.summary}\n\n`;
+
+		md += `**Latest:** ${info.version}`;
+		if (info.license) md += ` · **License:** ${info.license}`;
+		md += "\n";
+
+		if (weeklyDownloads !== null) {
+			md += `**Weekly Downloads:** ${formatCount(weeklyDownloads)}\n`;
+		}
+
+		md += "\n";
+
+		if (info.author) {
+			md += `**Author:** ${info.author}`;
+			if (info.author_email) md += ` <${info.author_email}>`;
+			md += "\n";
+		}
+
+		if (info.requires_python) md += `**Python:** ${info.requires_python}\n`;
+		if (info.home_page) md += `**Homepage:** ${info.home_page}\n`;
+
+		if (info.project_urls && Object.keys(info.project_urls).length > 0) {
+			md += "\n**Project URLs:**\n";
+			for (const [label, url] of Object.entries(info.project_urls)) {
+				md += `- ${label}: ${url}\n`;
+			}
+		}
+
+		if (info.keywords) md += `\n**Keywords:** ${info.keywords}\n`;
+
+		// Dependencies
+		if (pkg.requires_dist && pkg.requires_dist.length > 0) {
+			md += `\n## Dependencies\n\n`;
+			for (const dep of pkg.requires_dist) {
+				md += `- ${dep}\n`;
+			}
+		}
+
+		// README/Description
+		if (info.description) {
+			md += `\n---\n\n## Description\n\n${info.description}\n`;
+		}
+
+		const output = finalizeOutput(md);
+		return {
+			url,
+			finalUrl: url,
+			contentType: "text/markdown",
+			method: "pypi",
+			content: output.content,
+			fetchedAt,
+			truncated: output.truncated,
+			notes: ["Fetched via PyPI JSON API"],
+		};
+	} catch {}
+
+	return null;
+};

package/src/core/tools/web-fetch-handlers/readthedocs.ts

@@ -0,0 +1,122 @@
+/**
+ * Read the Docs handler for web-fetch
+ */
+
+import { parse as parseHtml } from "node-html-parser";
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, htmlToBasicMarkdown, loadPage } from "./types";
+
+export const handleReadTheDocs: SpecialHandler = async (url: string, timeout: number): Promise<RenderResult | null> => {
+	// Check if URL matches Read the Docs patterns
+	const urlObj = new URL(url);
+	const isReadTheDocs =
+		urlObj.hostname.endsWith(".readthedocs.io") ||
+		urlObj.hostname === "readthedocs.org" ||
+		urlObj.hostname === "www.readthedocs.org";
+
+	if (!isReadTheDocs) {
+		return null;
+	}
+
+	const notes: string[] = [];
+	const fetchedAt = new Date().toISOString();
+
+	// Fetch the page
+	const result = await loadPage(url, { timeout });
+	if (!result.ok) {
+		return {
+			url,
+			finalUrl: result.finalUrl,
+			contentType: result.contentType,
+			method: "readthedocs",
+			content: `Failed to fetch Read the Docs page (status: ${result.status ?? "unknown"})`,
+			fetchedAt,
+			truncated: false,
+			notes,
+		};
+	}
+
+	// Parse HTML
+	const root = parseHtml(result.content);
+
+	// Extract main content from common Read the Docs selectors
+	let mainContent =
+		root.querySelector(".document") ||
+		root.querySelector('[role="main"]') ||
+		root.querySelector("main") ||
+		root.querySelector(".rst-content") ||
+		root.querySelector(".body");
+
+	if (!mainContent) {
+		// Fallback to body if no main content found
+		mainContent = root.querySelector("body");
+		notes.push("Using full body content (no main content div found)");
+	}
+
+	// Remove navigation, sidebar, footer elements
+	mainContent
+		?.querySelectorAll(
+			".headerlink, .viewcode-link, nav, .sidebar, footer, .related, .sphinxsidebar, .toctree-wrapper",
+		)
+		.forEach((el) => {
+			el.remove();
+		});
+
+	// Try to find Edit on GitHub/GitLab links for raw source
+	const editLinks = root.querySelectorAll('a[href*="github.com"], a[href*="gitlab.com"]');
+	let sourceUrl: string | null = null;
+
+	for (const link of editLinks) {
+		const href = link.getAttribute("href");
+		const text = link.textContent?.toLowerCase() || "";
+
+		if (href && (text.includes("edit") || text.includes("source"))) {
+			// Convert edit URL to raw URL
+			if (href.includes("github.com")) {
+				sourceUrl = href.replace("/blob/", "/raw/").replace("/edit/", "/raw/");
+			} else if (href.includes("gitlab.com")) {
+				sourceUrl = href.replace("/blob/", "/raw/").replace("/edit/", "/raw/");
+			}
+			break;
+		}
+	}
+
+	let content = "";
+
+	// Try to fetch raw source if available
+	if (sourceUrl) {
+		try {
+			const sourceResult = await loadPage(sourceUrl, { timeout: Math.min(timeout, 10) });
+			if (sourceResult.ok && sourceResult.content.length > 0 && sourceResult.content.length < 1_000_000) {
+				content = sourceResult.content;
+				notes.push(`Fetched raw source from ${sourceUrl}`);
+			}
+		} catch (_err) {
+			// Ignore errors, fall back to HTML
+		}
+	}
+
+	// If no raw source, convert HTML to markdown
+	if (!content && mainContent) {
+		const html = mainContent.innerHTML;
+		content = htmlToBasicMarkdown(html);
+	}
+
+	if (!content) {
+		content = "No content extracted from Read the Docs page";
+		notes.push("Failed to extract content");
+	}
+
+	const { content: finalContent, truncated } = finalizeOutput(content);
+
+	return {
+		url,
+		finalUrl: result.finalUrl,
+		contentType: sourceUrl ? "text/plain" : "text/html",
+		method: "readthedocs",
+		content: finalContent,
+		fetchedAt,
+		truncated,
+		notes,
+	};
+};

package/src/core/tools/web-fetch-handlers/reddit.ts

@@ -0,0 +1,100 @@
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, loadPage } from "./types";
+
+interface RedditPost {
+	title: string;
+	selftext: string;
+	author: string;
+	score: number;
+	num_comments: number;
+	created_utc: number;
+	subreddit: string;
+	url: string;
+	is_self: boolean;
+}
+
+interface RedditComment {
+	body: string;
+	author: string;
+	score: number;
+	created_utc: number;
+	replies?: { data: { children: Array<{ data: RedditComment }> } };
+}
+
+/**
+ * Handle Reddit URLs via JSON API
+ */
+export const handleReddit: SpecialHandler = async (url: string, timeout: number): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		if (!parsed.hostname.includes("reddit.com")) return null;
+
+		const fetchedAt = new Date().toISOString();
+
+		// Append .json to get JSON response
+		let jsonUrl = `${url.replace(/\/$/, "")}.json`;
+		if (parsed.search) {
+			jsonUrl = `${url.replace(/\/$/, "").replace(parsed.search, "")}.json${parsed.search}`;
+		}
+
+		const result = await loadPage(jsonUrl, { timeout });
+		if (!result.ok) return null;
+
+		const data = JSON.parse(result.content);
+		let md = "";
+
+		// Handle different Reddit URL types
+		if (Array.isArray(data) && data.length >= 1) {
+			// Post page (with comments)
+			const postData = data[0]?.data?.children?.[0]?.data as RedditPost | undefined;
+			if (postData) {
+				md = `# ${postData.title}\n\n`;
+				md += `**r/${postData.subreddit}** · u/${postData.author} · ${postData.score} points · ${postData.num_comments} comments\n`;
+				md += `*${new Date(postData.created_utc * 1000).toISOString().split("T")[0]}*\n\n`;
+
+				if (postData.is_self && postData.selftext) {
+					md += `---\n\n${postData.selftext}\n\n`;
+				} else if (!postData.is_self) {
+					md += `**Link:** ${postData.url}\n\n`;
+				}
+
+				// Add comments if available
+				if (data.length >= 2 && data[1]?.data?.children) {
+					md += `---\n\n## Top Comments\n\n`;
+					const comments = data[1].data.children.filter((c: { kind: string }) => c.kind === "t1").slice(0, 10);
+
+					for (const { data: comment } of comments as Array<{ data: RedditComment }>) {
+						md += `### u/${comment.author} · ${comment.score} points\n\n`;
+						md += `${comment.body}\n\n---\n\n`;
+					}
+				}
+			}
+		} else if (data?.data?.children) {
+			// Subreddit or listing page
+			const posts = data.data.children.slice(0, 20) as Array<{ data: RedditPost }>;
+			const subreddit = posts[0]?.data?.subreddit;
+
+			md = `# r/${subreddit || "Reddit"}\n\n`;
+			for (const { data: post } of posts) {
+				md += `- **${post.title}** (${post.score} pts, ${post.num_comments} comments)\n`;
+				md += `  by u/${post.author}\n\n`;
+			}
+		}
+
+		if (!md) return null;
+
+		const output = finalizeOutput(md);
+		return {
+			url,
+			finalUrl: url,
+			contentType: "text/markdown",
+			method: "reddit",
+			content: output.content,
+			fetchedAt,
+			truncated: output.truncated,
+			notes: ["Fetched via Reddit JSON API"],
+		};
+	} catch {}
+
+	return null;
+};