@oh-my-pi/pi-coding-agent 3.25.0 → 3.31.0

package/src/core/tools/web-scrapers/clojars.ts

@@ -0,0 +1,180 @@
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, formatCount, loadPage } from "./types";
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+	return typeof value === "object" && value !== null;
+}
+
+function asString(value: unknown): string | null {
+	if (typeof value !== "string") return null;
+	const trimmed = value.trim();
+	return trimmed.length > 0 ? trimmed : null;
+}
+
+function asNumber(value: unknown): number | null {
+	return typeof value === "number" && Number.isFinite(value) ? value : null;
+}
+
+function formatLicenses(licenses: unknown): string[] {
+	if (!Array.isArray(licenses)) return [];
+	const output: string[] = [];
+	for (const license of licenses) {
+		if (typeof license === "string") {
+			const trimmed = license.trim();
+			if (trimmed) output.push(trimmed);
+			continue;
+		}
+		if (isRecord(license)) {
+			const name = asString(license.name);
+			const url = asString(license.url);
+			if (name && url) {
+				output.push(`${name} (${url})`);
+			} else if (name) {
+				output.push(name);
+			} else if (url) {
+				output.push(url);
+			}
+		}
+	}
+	return output;
+}
+
+function formatDependencies(deps: unknown): string[] {
+	const output: string[] = [];
+	if (Array.isArray(deps)) {
+		for (const dep of deps) {
+			if (typeof dep === "string") {
+				const trimmed = dep.trim();
+				if (trimmed) output.push(trimmed);
+				continue;
+			}
+			if (Array.isArray(dep)) {
+				const name = asString(dep[0]);
+				const version = asString(dep[1]);
+				if (name && version) {
+					output.push(`${name}: ${version}`);
+				} else if (name) {
+					output.push(name);
+				}
+				continue;
+			}
+			if (isRecord(dep)) {
+				const name = asString(dep.name) ?? asString(dep.artifact) ?? asString(dep.jar_name);
+				const version = asString(dep.version);
+				if (name && version) {
+					output.push(`${name}: ${version}`);
+				} else if (name) {
+					output.push(name);
+				}
+			}
+		}
+		return output;
+	}
+
+	if (isRecord(deps)) {
+		for (const [name, version] of Object.entries(deps)) {
+			const versionText = asString(version);
+			if (versionText) {
+				output.push(`${name}: ${versionText}`);
+			} else if (name.trim()) {
+				output.push(name);
+			}
+		}
+	}
+
+	return output;
+}
+
+/**
+ * Handle Clojars URLs via API
+ */
+export const handleClojars: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		if (parsed.hostname !== "clojars.org" && parsed.hostname !== "www.clojars.org") return null;
+
+		const path = parsed.pathname.replace(/^\/+|\/+$/g, "");
+		if (!path) return null;
+
+		const segments = path.split("/").filter(Boolean);
+		if (segments.length < 1 || segments.length > 2) return null;
+
+		const groupFromUrl = segments.length === 2 ? decodeURIComponent(segments[0]) : null;
+		const artifactFromUrl = decodeURIComponent(segments[segments.length - 1]);
+
+		const apiUrl =
+			segments.length === 2
+				? `https://clojars.org/api/artifacts/${encodeURIComponent(groupFromUrl ?? "")}/${encodeURIComponent(artifactFromUrl)}`
+				: `https://clojars.org/api/artifacts/${encodeURIComponent(artifactFromUrl)}`;
+
+		const fetchedAt = new Date().toISOString();
+
+		const result = await loadPage(apiUrl, {
+			timeout,
+			headers: { Accept: "application/json" },
+			signal,
+		});
+
+		if (!result.ok) return null;
+
+		let payload: unknown;
+		try {
+			payload = JSON.parse(result.content);
+		} catch {
+			return null;
+		}
+
+		const data = Array.isArray(payload) ? payload[0] : payload;
+		if (!isRecord(data)) return null;
+
+		const groupName = asString(data.group_name) ?? asString(data.group) ?? groupFromUrl;
+		const artifactName = asString(data.jar_name) ?? asString(data.artifact) ?? asString(data.name) ?? artifactFromUrl;
+		const version = asString(data.latest_version) ?? asString(data.version);
+		const description = asString(data.description) ?? asString(data.summary);
+		const downloads =
+			asNumber(data.downloads) ?? asNumber(data.downloads_total) ?? asNumber(data.total_downloads) ?? null;
+		const homepage = asString(data.homepage) ?? asString(data.url);
+		const licenses = formatLicenses(data.licenses);
+		const dependencies = formatDependencies(data.dependencies ?? data.deps);
+
+		const displayName =
+			groupName && artifactName && groupName !== artifactName
+				? `${groupName}/${artifactName}`
+				: (artifactName ?? groupName ?? "Clojars artifact");
+
+		let md = `# ${displayName}\n\n`;
+		if (description) md += `${description}\n\n`;
+
+		if (groupName) md += `**Group:** ${groupName}\n`;
+		if (artifactName) md += `**Artifact:** ${artifactName}\n`;
+		if (version) md += `**Latest:** ${version}\n`;
+		if (downloads !== null) md += `**Downloads:** ${formatCount(downloads)}\n`;
+		if (homepage) md += `**Homepage:** ${homepage}\n`;
+		if (licenses.length > 0) md += `**Licenses:** ${licenses.join(", ")}\n`;
+
+		if (dependencies.length > 0) {
+			md += "\n## Dependencies\n\n";
+			for (const dep of dependencies) {
+				md += `- ${dep}\n`;
+			}
+		}
+
+		const output = finalizeOutput(md);
+		return {
+			url,
+			finalUrl: url,
+			contentType: "text/markdown",
+			method: "clojars",
+			content: output.content,
+			fetchedAt,
+			truncated: output.truncated,
+			notes: ["Fetched via Clojars API"],
+		};
+	} catch {}
+
+	return null;
+};

package/src/core/tools/web-scrapers/coingecko.ts

@@ -0,0 +1,184 @@
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, formatCount, loadPage } from "./types";
+
+interface CoinGeckoResponse {
+	id: string;
+	symbol: string;
+	name: string;
+	description?: { en?: string };
+	links?: {
+		homepage?: string[];
+		blockchain_site?: string[];
+		repos_url?: { github?: string[] };
+	};
+	market_data?: {
+		current_price?: { usd?: number };
+		market_cap?: { usd?: number };
+		total_volume?: { usd?: number };
+		price_change_percentage_24h?: number;
+		ath?: { usd?: number };
+		ath_date?: { usd?: string };
+		circulating_supply?: number;
+		total_supply?: number;
+		max_supply?: number;
+	};
+	categories?: string[];
+	genesis_date?: string;
+}
+
+/**
+ * Handle CoinGecko cryptocurrency URLs via API
+ */
+export const handleCoinGecko: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		if (!parsed.hostname.includes("coingecko.com")) return null;
+
+		// Extract coin ID from /coins/{id} or /en/coins/{id}
+		const match = parsed.pathname.match(/^(?:\/[a-z]{2})?\/coins\/([^/?#]+)/);
+		if (!match) return null;
+
+		const coinId = decodeURIComponent(match[1]);
+		const fetchedAt = new Date().toISOString();
+
+		// Fetch from CoinGecko API
+		const apiUrl = `https://api.coingecko.com/api/v3/coins/${coinId}?localization=false&tickers=false&community_data=false&developer_data=false`;
+		const result = await loadPage(apiUrl, {
+			timeout,
+			headers: { Accept: "application/json" },
+			signal,
+		});
+
+		if (!result.ok) return null;
+
+		let coin: CoinGeckoResponse;
+		try {
+			coin = JSON.parse(result.content);
+		} catch {
+			return null;
+		}
+
+		const market = coin.market_data;
+
+		let md = `# ${coin.name} (${coin.symbol.toUpperCase()})\n\n`;
+
+		// Price and market data
+		if (market?.current_price?.usd !== undefined) {
+			md += `**Price:** $${formatPrice(market.current_price.usd)}`;
+			if (market.price_change_percentage_24h !== undefined) {
+				const change = market.price_change_percentage_24h;
+				const sign = change >= 0 ? "+" : "";
+				md += ` (${sign}${change.toFixed(2)}% 24h)`;
+			}
+			md += "\n";
+		}
+
+		if (market?.market_cap?.usd) {
+			md += `**Market Cap:** $${formatLargeNumber(market.market_cap.usd)}\n`;
+		}
+
+		if (market?.total_volume?.usd) {
+			md += `**24h Volume:** $${formatLargeNumber(market.total_volume.usd)}\n`;
+		}
+
+		if (market?.ath?.usd !== undefined) {
+			md += `**All-Time High:** $${formatPrice(market.ath.usd)}`;
+			if (market.ath_date?.usd) {
+				const athDate = new Date(market.ath_date.usd).toLocaleDateString("en-US", {
+					year: "numeric",
+					month: "short",
+					day: "numeric",
+				});
+				md += ` (${athDate})`;
+			}
+			md += "\n";
+		}
+
+		md += "\n";
+
+		// Supply info
+		if (market?.circulating_supply) {
+			md += `**Circulating Supply:** ${formatCount(Math.round(market.circulating_supply))}`;
+			if (market.max_supply) {
+				const percent = ((market.circulating_supply / market.max_supply) * 100).toFixed(1);
+				md += ` / ${formatCount(Math.round(market.max_supply))} (${percent}%)`;
+			} else if (market.total_supply) {
+				md += ` / ${formatCount(Math.round(market.total_supply))} total`;
+			}
+			md += "\n";
+		}
+
+		if (coin.genesis_date) {
+			md += `**Launch Date:** ${coin.genesis_date}\n`;
+		}
+
+		if (coin.categories?.length) {
+			md += `**Categories:** ${coin.categories.join(", ")}\n`;
+		}
+
+		// Links
+		const links: string[] = [];
+		if (coin.links?.homepage?.[0]) {
+			links.push(`[Website](${coin.links.homepage[0]})`);
+		}
+		if (coin.links?.blockchain_site?.[0]) {
+			links.push(`[Explorer](${coin.links.blockchain_site[0]})`);
+		}
+		if (coin.links?.repos_url?.github?.[0]) {
+			links.push(`[GitHub](${coin.links.repos_url.github[0]})`);
+		}
+		if (links.length) {
+			md += `**Links:** ${links.join(" · ")}\n`;
+		}
+
+		// Description
+		if (coin.description?.en) {
+			const desc = coin.description.en
+				.replace(/<[^>]+>/g, "") // Strip HTML
+				.replace(/\r\n/g, "\n")
+				.trim();
+			if (desc) {
+				md += `\n## About\n\n${desc}\n`;
+			}
+		}
+
+		const output = finalizeOutput(md);
+		return {
+			url,
+			finalUrl: url,
+			contentType: "text/markdown",
+			method: "coingecko",
+			content: output.content,
+			fetchedAt,
+			truncated: output.truncated,
+			notes: ["Fetched via CoinGecko API"],
+		};
+	} catch {}
+
+	return null;
+};
+
+/**
+ * Format price with appropriate decimal places
+ */
+function formatPrice(price: number): string {
+	if (price >= 1000) return price.toLocaleString("en-US", { maximumFractionDigits: 2 });
+	if (price >= 1) return price.toFixed(2);
+	if (price >= 0.01) return price.toFixed(4);
+	if (price >= 0.0001) return price.toFixed(6);
+	return price.toFixed(8);
+}
+
+/**
+ * Format large numbers with B/M/K suffixes
+ */
+function formatLargeNumber(n: number): string {
+	if (n >= 1_000_000_000) return `${(n / 1_000_000_000).toFixed(2)}B`;
+	if (n >= 1_000_000) return `${(n / 1_000_000).toFixed(2)}M`;
+	if (n >= 1_000) return `${(n / 1_000).toFixed(2)}K`;
+	return n.toFixed(2);
+}

package/src/core/tools/web-scrapers/crates-io.ts

@@ -0,0 +1,128 @@
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, loadPage } from "./types";
+
+/**
+ * Check if content looks like HTML
+ */
+function looksLikeHtml(content: string): boolean {
+	const trimmed = content.trim().toLowerCase();
+	return (
+		trimmed.startsWith("<!doctype") ||
+		trimmed.startsWith("<html") ||
+		trimmed.startsWith("<head") ||
+		trimmed.startsWith("<body")
+	);
+}
+
+/**
+ * Handle crates.io URLs via API
+ */
+export const handleCratesIo: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		if (parsed.hostname !== "crates.io" && parsed.hostname !== "www.crates.io") return null;
+
+		// Extract crate name from /crates/name or /crates/name/version
+		const match = parsed.pathname.match(/^\/crates\/([^/]+)/);
+		if (!match) return null;
+
+		const crateName = decodeURIComponent(match[1]);
+		const fetchedAt = new Date().toISOString();
+
+		// Fetch from crates.io API
+		const apiUrl = `https://crates.io/api/v1/crates/${crateName}`;
+		const result = await loadPage(apiUrl, {
+			timeout,
+			signal,
+			headers: { "User-Agent": "omp-web-fetch/1.0 (https://github.com/anthropics)" },
+		});
+
+		if (!result.ok) return null;
+
+		let data: {
+			crate: {
+				name: string;
+				description: string | null;
+				downloads: number;
+				recent_downloads: number;
+				max_version: string;
+				repository: string | null;
+				homepage: string | null;
+				documentation: string | null;
+				categories: string[];
+				keywords: string[];
+				created_at: string;
+				updated_at: string;
+			};
+			versions: Array<{
+				num: string;
+				downloads: number;
+				created_at: string;
+				license: string | null;
+				rust_version: string | null;
+			}>;
+		};
+
+		try {
+			data = JSON.parse(result.content);
+		} catch {
+			return null;
+		}
+
+		const crate = data.crate;
+		const latestVersion = data.versions?.[0];
+
+		// Format download counts
+		const formatDownloads = (n: number): string =>
+			n >= 1_000_000 ? `${(n / 1_000_000).toFixed(1)}M` : n >= 1_000 ? `${(n / 1_000).toFixed(1)}K` : String(n);
+
+		let md = `# ${crate.name}\n\n`;
+		if (crate.description) md += `${crate.description}\n\n`;
+
+		md += `**Latest:** ${crate.max_version}`;
+		if (latestVersion?.license) md += ` · **License:** ${latestVersion.license}`;
+		if (latestVersion?.rust_version) md += ` · **MSRV:** ${latestVersion.rust_version}`;
+		md += "\n";
+		md += `**Downloads:** ${formatDownloads(crate.downloads)} total · ${formatDownloads(crate.recent_downloads)} recent\n\n`;
+
+		if (crate.repository) md += `**Repository:** ${crate.repository}\n`;
+		if (crate.homepage && crate.homepage !== crate.repository) md += `**Homepage:** ${crate.homepage}\n`;
+		if (crate.documentation) md += `**Docs:** ${crate.documentation}\n`;
+		if (crate.keywords?.length) md += `**Keywords:** ${crate.keywords.join(", ")}\n`;
+		if (crate.categories?.length) md += `**Categories:** ${crate.categories.join(", ")}\n`;
+
+		// Show recent versions
+		if (data.versions?.length > 0) {
+			md += `\n## Recent Versions\n\n`;
+			for (const ver of data.versions.slice(0, 5)) {
+				const date = ver.created_at.split("T")[0];
+				md += `- **${ver.num}** (${date}) - ${formatDownloads(ver.downloads)} downloads\n`;
+			}
+		}
+
+		// Try to fetch README from docs.rs or repository
+		const docsRsUrl = `https://docs.rs/crate/${crateName}/${crate.max_version}/source/README.md`;
+		const readmeResult = await loadPage(docsRsUrl, { timeout: Math.min(timeout, 5), signal });
+		if (readmeResult.ok && readmeResult.content.length > 100 && !looksLikeHtml(readmeResult.content)) {
+			md += `\n---\n\n## README\n\n${readmeResult.content}\n`;
+		}
+
+		const output = finalizeOutput(md);
+		return {
+			url,
+			finalUrl: url,
+			contentType: "text/markdown",
+			method: "crates.io",
+			content: output.content,
+			fetchedAt,
+			truncated: output.truncated,
+			notes: ["Fetched via crates.io API"],
+		};
+	} catch {}
+
+	return null;
+};

package/src/core/tools/web-scrapers/crossref.ts

@@ -0,0 +1,149 @@
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, htmlToBasicMarkdown, loadPage } from "./types";
+
+interface CrossrefAuthor {
+	given?: string;
+	family?: string;
+	name?: string;
+}
+
+interface CrossrefDate {
+	"date-parts"?: number[][];
+}
+
+interface CrossrefMessage {
+	title?: string[];
+	author?: CrossrefAuthor[];
+	"container-title"?: string[];
+	"short-container-title"?: string[];
+	publisher?: string;
+	published?: CrossrefDate;
+	"published-print"?: CrossrefDate;
+	"published-online"?: CrossrefDate;
+	issued?: CrossrefDate;
+	created?: CrossrefDate;
+	DOI?: string;
+	abstract?: string;
+	type?: string;
+}
+
+interface CrossrefResponse {
+	message?: CrossrefMessage;
+}
+
+const DOI_HOSTS = new Set(["doi.org", "dx.doi.org", "www.doi.org"]);
+
+function extractDoi(pathname: string): string | null {
+	const raw = pathname.replace(/^\/+/, "");
+	if (!raw) return null;
+	return decodeURIComponent(raw);
+}
+
+function formatAuthors(authors?: CrossrefAuthor[]): string | null {
+	if (!authors || authors.length === 0) return null;
+	const names = authors
+		.map((author) => {
+			if (author.name) return author.name;
+			const parts = [author.given, author.family].filter(Boolean);
+			return parts.length > 0 ? parts.join(" ") : null;
+		})
+		.filter((name): name is string => Boolean(name));
+	if (names.length === 0) return null;
+	return names.join(", ");
+}
+
+function formatDate(date?: CrossrefDate): string | null {
+	const parts = date?.["date-parts"]?.[0];
+	if (!parts || parts.length === 0) return null;
+	const [year, month, day] = parts;
+	if (!year) return null;
+	const formatted = [
+		String(year),
+		month ? String(month).padStart(2, "0") : "",
+		day ? String(day).padStart(2, "0") : "",
+	].filter(Boolean);
+	return formatted.join("-");
+}
+
+function formatAbstract(abstract?: string): string | null {
+	if (!abstract) return null;
+	const normalized = abstract.replace(/<\/?jats:p[^>]*>/g, (match) => (match.startsWith("</") ? "</p>" : "<p>"));
+	const markdown = htmlToBasicMarkdown(normalized);
+	return markdown.trim().length > 0 ? markdown : null;
+}
+
+export const handleCrossref: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		if (!DOI_HOSTS.has(parsed.hostname.toLowerCase())) return null;
+
+		const doi = extractDoi(parsed.pathname);
+		if (!doi) return null;
+
+		const fetchedAt = new Date().toISOString();
+		const apiUrl = `https://api.crossref.org/works/${encodeURIComponent(doi)}`;
+		const result = await loadPage(apiUrl, {
+			timeout,
+			signal,
+			headers: {
+				Accept: "application/json",
+			},
+		});
+
+		if (!result.ok) return null;
+
+		let data: CrossrefResponse;
+		try {
+			data = JSON.parse(result.content);
+		} catch {
+			return null;
+		}
+
+		const message = data.message;
+		if (!message) return null;
+
+		const title = message.title?.[0]?.trim() || "CrossRef Record";
+		const authors = formatAuthors(message.author);
+		const journal = message["container-title"]?.[0] || message["short-container-title"]?.[0];
+		const publisher = message.publisher;
+		const published =
+			formatDate(message.published) ||
+			formatDate(message["published-print"]) ||
+			formatDate(message["published-online"]) ||
+			formatDate(message.issued) ||
+			formatDate(message.created);
+		const doiValue = message.DOI || doi;
+		const abstract = formatAbstract(message.abstract);
+		const type = message.type?.replace(/-/g, " ");
+
+		let md = `# ${title}\n\n`;
+		if (authors) md += `**Authors:** ${authors}\n`;
+		if (journal) md += `**Journal:** ${journal}\n`;
+		if (publisher) md += `**Publisher:** ${publisher}\n`;
+		if (published) md += `**Published:** ${published}\n`;
+		md += `**DOI:** ${doiValue}\n`;
+		if (type) md += `**Type:** ${type}\n`;
+		md += "\n---\n\n";
+		md += "## Abstract\n\n";
+		md += abstract || "No abstract available.";
+		md += "\n";
+
+		const output = finalizeOutput(md);
+		return {
+			url,
+			finalUrl: url,
+			contentType: "text/markdown",
+			method: "crossref",
+			content: output.content,
+			fetchedAt,
+			truncated: output.truncated,
+			notes: ["Fetched via CrossRef API"],
+		};
+	} catch {}
+
+	return null;
+};