@oh-my-pi/pi-coding-agent 3.25.0 → 3.31.0

package/src/core/tools/web-scrapers/aur.ts

@@ -0,0 +1,175 @@
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, formatCount, loadPage } from "./types";
+
+interface AurPackage {
+	Name: string;
+	Version: string;
+	Description?: string;
+	Maintainer?: string;
+	NumVotes: number;
+	Popularity: number;
+	Depends?: string[];
+	MakeDepends?: string[];
+	OptDepends?: string[];
+	CheckDepends?: string[];
+	LastModified: number;
+	FirstSubmitted: number;
+	URL?: string;
+	URLPath?: string;
+	PackageBase: string;
+	OutOfDate?: number | null;
+	License?: string[];
+	Keywords?: string[];
+	Conflicts?: string[];
+	Provides?: string[];
+	Replaces?: string[];
+}
+
+interface AurResponse {
+	version: number;
+	type: string;
+	resultcount: number;
+	results: AurPackage[];
+}
+
+/**
+ * Handle AUR (Arch User Repository) URLs via RPC API
+ */
+export const handleAur: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		if (parsed.hostname !== "aur.archlinux.org") return null;
+
+		// Extract package name from /packages/{name}
+		const match = parsed.pathname.match(/^\/packages\/([^/?#]+)/);
+		if (!match) return null;
+
+		const packageName = decodeURIComponent(match[1]);
+		const fetchedAt = new Date().toISOString();
+
+		// Fetch from AUR RPC API
+		const apiUrl = `https://aur.archlinux.org/rpc/?v=5&type=info&arg=${encodeURIComponent(packageName)}`;
+		const result = await loadPage(apiUrl, { timeout, signal });
+
+		if (!result.ok) return null;
+
+		let data: AurResponse;
+		try {
+			data = JSON.parse(result.content);
+		} catch {
+			return null;
+		}
+
+		if (data.resultcount === 0 || !data.results[0]) return null;
+
+		const pkg = data.results[0];
+
+		let md = `# ${pkg.Name}\n\n`;
+		if (pkg.Description) md += `${pkg.Description}\n\n`;
+
+		// Package info
+		md += `**Version:** ${pkg.Version}`;
+		if (pkg.OutOfDate) {
+			const outOfDateDate = new Date(pkg.OutOfDate * 1000).toISOString().split("T")[0];
+			md += ` (flagged out-of-date: ${outOfDateDate})`;
+		}
+		md += "\n";
+
+		if (pkg.Maintainer) {
+			md += `**Maintainer:** [${pkg.Maintainer}](https://aur.archlinux.org/account/${pkg.Maintainer})\n`;
+		} else {
+			md += "**Maintainer:** Orphaned\n";
+		}
+
+		md += `**Votes:** ${formatCount(pkg.NumVotes)} · **Popularity:** ${pkg.Popularity.toFixed(2)}\n`;
+
+		// Timestamps
+		const lastModified = new Date(pkg.LastModified * 1000).toISOString().split("T")[0];
+		const firstSubmitted = new Date(pkg.FirstSubmitted * 1000).toISOString().split("T")[0];
+		md += `**Last Updated:** ${lastModified} · **First Submitted:** ${firstSubmitted}\n`;
+
+		if (pkg.License?.length) md += `**License:** ${pkg.License.join(", ")}\n`;
+		if (pkg.URL) md += `**Upstream:** ${pkg.URL}\n`;
+		if (pkg.Keywords?.length) md += `**Keywords:** ${pkg.Keywords.join(", ")}\n`;
+
+		// Dependencies
+		if (pkg.Depends?.length) {
+			md += `\n## Dependencies (${pkg.Depends.length})\n\n`;
+			for (const dep of pkg.Depends) {
+				md += `- ${dep}\n`;
+			}
+		}
+
+		if (pkg.MakeDepends?.length) {
+			md += `\n## Make Dependencies (${pkg.MakeDepends.length})\n\n`;
+			for (const dep of pkg.MakeDepends) {
+				md += `- ${dep}\n`;
+			}
+		}
+
+		if (pkg.OptDepends?.length) {
+			md += `\n## Optional Dependencies\n\n`;
+			for (const dep of pkg.OptDepends) {
+				md += `- ${dep}\n`;
+			}
+		}
+
+		if (pkg.CheckDepends?.length) {
+			md += `\n## Check Dependencies\n\n`;
+			for (const dep of pkg.CheckDepends) {
+				md += `- ${dep}\n`;
+			}
+		}
+
+		// Package relationships
+		if (pkg.Provides?.length) {
+			md += `\n## Provides\n\n`;
+			for (const p of pkg.Provides) {
+				md += `- ${p}\n`;
+			}
+		}
+
+		if (pkg.Conflicts?.length) {
+			md += `\n## Conflicts\n\n`;
+			for (const c of pkg.Conflicts) {
+				md += `- ${c}\n`;
+			}
+		}
+
+		if (pkg.Replaces?.length) {
+			md += `\n## Replaces\n\n`;
+			for (const r of pkg.Replaces) {
+				md += `- ${r}\n`;
+			}
+		}
+
+		// Installation instructions
+		md += `\n---\n\n## Installation\n\n`;
+		md += "```bash\n";
+		md += `# Using an AUR helper (e.g., yay, paru)\n`;
+		md += `yay -S ${pkg.Name}\n\n`;
+		md += `# Manual installation\n`;
+		md += `git clone https://aur.archlinux.org/${pkg.PackageBase}.git\n`;
+		md += `cd ${pkg.PackageBase}\n`;
+		md += `makepkg -si\n`;
+		md += "```\n";
+
+		const output = finalizeOutput(md);
+		return {
+			url,
+			finalUrl: url,
+			contentType: "text/markdown",
+			method: "aur",
+			content: output.content,
+			fetchedAt,
+			truncated: output.truncated,
+			notes: ["Fetched via AUR RPC API"],
+		};
+	} catch {}
+
+	return null;
+};

package/src/core/tools/web-scrapers/biorxiv.ts

@@ -0,0 +1,141 @@
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, loadPage } from "./types";
+
+interface BiorxivPaper {
+	biorxiv_doi?: string;
+	medrxiv_doi?: string;
+	title?: string;
+	authors?: string;
+	author_corresponding?: string;
+	author_corresponding_institution?: string;
+	abstract?: string;
+	date?: string;
+	category?: string;
+	version?: string;
+	type?: string;
+	license?: string;
+	jatsxml?: string;
+	published?: string; // Journal DOI if published
+	server?: string;
+}
+
+interface BiorxivResponse {
+	collection?: BiorxivPaper[];
+	messages?: { status: string; count: number }[];
+}
+
+/**
+ * Handle bioRxiv and medRxiv preprint URLs via their API
+ */
+export const handleBiorxiv: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		const hostname = parsed.hostname.toLowerCase();
+
+		// Check if it's bioRxiv or medRxiv
+		const isBiorxiv = hostname === "www.biorxiv.org" || hostname === "biorxiv.org";
+		const isMedrxiv = hostname === "www.medrxiv.org" || hostname === "medrxiv.org";
+
+		if (!isBiorxiv && !isMedrxiv) return null;
+
+		// Extract DOI from URL path: /content/10.1101/2024.01.01.123456
+		const match = parsed.pathname.match(/\/content\/(10\.\d{4,}\/[^\s?#]+)/);
+		if (!match) return null;
+
+		let doi = match[1];
+		// Remove version suffix if present (e.g., v1, v2)
+		doi = doi.replace(/v\d+$/, "");
+		// Remove trailing .full or .full.pdf
+		doi = doi.replace(/\.full(\.pdf)?$/, "");
+
+		const server = isBiorxiv ? "biorxiv" : "medrxiv";
+		const apiUrl = `https://api.${server}.org/details/${server}/${doi}/na/json`;
+
+		const result = await loadPage(apiUrl, {
+			timeout,
+			headers: { Accept: "application/json" },
+			signal,
+		});
+
+		if (!result.ok) return null;
+
+		let data: BiorxivResponse;
+		try {
+			data = JSON.parse(result.content);
+		} catch {
+			return null;
+		}
+
+		if (!data.collection || data.collection.length === 0) return null;
+
+		// Get the latest version (last in array)
+		const paper = data.collection[data.collection.length - 1];
+		if (!paper) return null;
+
+		const serverName = isBiorxiv ? "bioRxiv" : "medRxiv";
+		const paperDoi = paper.biorxiv_doi || paper.medrxiv_doi || doi;
+
+		// Build markdown output
+		let md = `# ${paper.title || "Untitled Preprint"}\n\n`;
+
+		// Metadata section
+		if (paper.authors) {
+			md += `**Authors:** ${paper.authors}\n`;
+		}
+		if (paper.author_corresponding) {
+			let correspondingLine = `**Corresponding Author:** ${paper.author_corresponding}`;
+			if (paper.author_corresponding_institution) {
+				correspondingLine += ` (${paper.author_corresponding_institution})`;
+			}
+			md += `${correspondingLine}\n`;
+		}
+		if (paper.date) {
+			md += `**Posted:** ${paper.date}\n`;
+		}
+		if (paper.category) {
+			md += `**Category:** ${paper.category}\n`;
+		}
+		if (paper.version) {
+			md += `**Version:** ${paper.version}\n`;
+		}
+		if (paper.license) {
+			md += `**License:** ${paper.license}\n`;
+		}
+		md += `**DOI:** [${paperDoi}](https://doi.org/${paperDoi})\n`;
+		md += `**Server:** ${serverName}\n`;
+
+		// Published status
+		if (paper.published) {
+			md += `\n> **Published in journal:** [${paper.published}](https://doi.org/${paper.published})\n`;
+		}
+
+		// Abstract
+		md += `\n---\n\n## Abstract\n\n${paper.abstract || "No abstract available."}\n`;
+
+		// Links section
+		md += `\n---\n\n## Links\n\n`;
+		md += `- [View on ${serverName}](https://www.${server}.org/content/${paperDoi})\n`;
+		md += `- [PDF](https://www.${server}.org/content/${paperDoi}.full.pdf)\n`;
+		if (paper.jatsxml) {
+			md += `- [JATS XML](${paper.jatsxml})\n`;
+		}
+
+		const output = finalizeOutput(md);
+		return {
+			url,
+			finalUrl: url,
+			contentType: "text/markdown",
+			method: server,
+			content: output.content,
+			fetchedAt: new Date().toISOString(),
+			truncated: output.truncated,
+			notes: [`Fetched via ${serverName} API`],
+		};
+	} catch {}
+
+	return null;
+};

package/src/core/tools/web-scrapers/bluesky.ts

@@ -0,0 +1,284 @@
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, formatCount, loadPage } from "./types";
+
+const API_BASE = "https://public.api.bsky.app/xrpc";
+
+interface BlueskyProfile {
+	did: string;
+	handle: string;
+	displayName?: string;
+	description?: string;
+	avatar?: string;
+	followersCount?: number;
+	followsCount?: number;
+	postsCount?: number;
+	createdAt?: string;
+}
+
+interface BlueskyPost {
+	uri: string;
+	cid: string;
+	author: BlueskyProfile;
+	record: {
+		text: string;
+		createdAt: string;
+		embed?: {
+			$type: string;
+			external?: { uri: string; title?: string; description?: string };
+			images?: Array<{ alt?: string; image: unknown }>;
+			record?: { uri: string };
+		};
+		facets?: Array<{
+			features: Array<{ $type: string; uri?: string; tag?: string; did?: string }>;
+			index: { byteStart: number; byteEnd: number };
+		}>;
+	};
+	likeCount?: number;
+	repostCount?: number;
+	replyCount?: number;
+	quoteCount?: number;
+	embed?: {
+		$type: string;
+		external?: { uri: string; title?: string; description?: string };
+		images?: Array<{ alt?: string; fullsize?: string; thumb?: string }>;
+		record?: { uri: string; value?: { text?: string }; author?: BlueskyProfile };
+	};
+}
+
+interface ThreadViewPost {
+	post: BlueskyPost;
+	parent?: ThreadViewPost | { $type: string };
+	replies?: Array<ThreadViewPost | { $type: string }>;
+}
+
+/**
+ * Resolve a handle to DID using the profile API
+ */
+async function resolveHandle(handle: string, timeout: number, signal?: AbortSignal): Promise<string | null> {
+	const url = `${API_BASE}/app.bsky.actor.getProfile?actor=${encodeURIComponent(handle)}`;
+	const result = await loadPage(url, {
+		timeout,
+		headers: { Accept: "application/json" },
+		signal,
+	});
+
+	if (!result.ok) return null;
+
+	try {
+		const data = JSON.parse(result.content) as BlueskyProfile;
+		return data.did;
+	} catch {
+		return null;
+	}
+}
+
+/**
+ * Format a post as markdown
+ */
+function formatPost(post: BlueskyPost, isQuote = false): string {
+	const author = post.author;
+	const name = author.displayName || author.handle;
+	const handle = `@${author.handle}`;
+	const date = new Date(post.record.createdAt).toLocaleString("en-US", {
+		year: "numeric",
+		month: "short",
+		day: "numeric",
+		hour: "2-digit",
+		minute: "2-digit",
+	});
+
+	let md = "";
+
+	if (isQuote) {
+		md += `> **${name}** (${handle}) - ${date}\n>\n`;
+		md += post.record.text
+			.split("\n")
+			.map((line) => `> ${line}`)
+			.join("\n");
+		md += "\n";
+	} else {
+		md += `**${name}** (${handle})\n`;
+		md += `*${date}*\n\n`;
+		md += `${post.record.text}\n`;
+	}
+
+	// Handle embeds
+	const embed = post.embed;
+	if (embed) {
+		if (embed.$type === "app.bsky.embed.external#view" && embed.external) {
+			const ext = embed.external;
+			md += `\n📎 [${ext.title || ext.uri}](${ext.uri})`;
+			if (ext.description) md += `\n*${ext.description}*`;
+			md += "\n";
+		} else if (embed.$type === "app.bsky.embed.images#view" && embed.images) {
+			md += `\n🖼️ ${embed.images.length} image(s)`;
+			for (const img of embed.images) {
+				if (img.alt) md += `\n- Alt: "${img.alt}"`;
+			}
+			md += "\n";
+		} else if (
+			(embed.$type === "app.bsky.embed.record#view" || embed.$type === "app.bsky.embed.recordWithMedia#view") &&
+			embed.record
+		) {
+			const rec = embed.record;
+			if (rec.value?.text && rec.author) {
+				md += "\n**Quoted post:**\n";
+				md += `> **${rec.author.displayName || rec.author.handle}** (@${rec.author.handle})\n`;
+				md += rec.value.text
+					.split("\n")
+					.map((line) => `> ${line}`)
+					.join("\n");
+				md += "\n";
+			}
+		}
+	}
+
+	// Stats
+	if (!isQuote) {
+		const stats: string[] = [];
+		if (post.likeCount) stats.push(`❤️ ${formatCount(post.likeCount)}`);
+		if (post.repostCount) stats.push(`🔁 ${formatCount(post.repostCount)}`);
+		if (post.replyCount) stats.push(`💬 ${formatCount(post.replyCount)}`);
+		if (post.quoteCount) stats.push(`📝 ${formatCount(post.quoteCount)}`);
+		if (stats.length) md += `\n${stats.join(" • ")}\n`;
+	}
+
+	return md;
+}
+
+/**
+ * Handle Bluesky post URLs
+ */
+export const handleBluesky: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		if (!["bsky.app", "www.bsky.app"].includes(parsed.hostname)) {
+			return null;
+		}
+
+		const fetchedAt = new Date().toISOString();
+		const pathParts = parsed.pathname.split("/").filter(Boolean);
+
+		// /profile/{handle}
+		if (pathParts[0] === "profile" && pathParts[1]) {
+			const handle = pathParts[1];
+
+			// /profile/{handle}/post/{rkey}
+			if (pathParts[2] === "post" && pathParts[3]) {
+				const rkey = pathParts[3];
+
+				// First resolve handle to DID
+				const did = await resolveHandle(handle, timeout, signal);
+				if (!did) return null;
+
+				// Construct AT URI and fetch thread
+				const atUri = `at://${did}/app.bsky.feed.post/${rkey}`;
+				const threadUrl = `${API_BASE}/app.bsky.feed.getPostThread?uri=${encodeURIComponent(atUri)}&depth=6&parentHeight=3`;
+
+				const result = await loadPage(threadUrl, {
+					timeout,
+					headers: { Accept: "application/json" },
+					signal,
+				});
+
+				if (!result.ok) return null;
+
+				const data = JSON.parse(result.content) as { thread: ThreadViewPost };
+				const thread = data.thread;
+
+				if (!thread.post) return null;
+
+				let md = `# Bluesky Post\n\n`;
+
+				// Show parent context if exists
+				if (thread.parent && "post" in thread.parent) {
+					md += "**Replying to:**\n";
+					md += formatPost(thread.parent.post, true);
+					md += "\n---\n\n";
+				}
+
+				// Main post
+				md += formatPost(thread.post);
+
+				// Show replies
+				if (thread.replies?.length) {
+					md += "\n---\n\n## Replies\n\n";
+					let replyCount = 0;
+					for (const reply of thread.replies) {
+						if (replyCount >= 10) break;
+						if ("post" in reply) {
+							md += formatPost(reply.post);
+							md += "\n---\n\n";
+							replyCount++;
+						}
+					}
+				}
+
+				const output = finalizeOutput(md);
+				return {
+					url,
+					finalUrl: url,
+					contentType: "text/markdown",
+					method: "bluesky-api",
+					content: output.content,
+					fetchedAt,
+					truncated: output.truncated,
+					notes: [`AT URI: ${atUri}`],
+				};
+			}
+
+			// Profile only
+			const profileUrl = `${API_BASE}/app.bsky.actor.getProfile?actor=${encodeURIComponent(handle)}`;
+			const result = await loadPage(profileUrl, {
+				timeout,
+				headers: { Accept: "application/json" },
+				signal,
+			});
+
+			if (!result.ok) return null;
+
+			const profile = JSON.parse(result.content) as BlueskyProfile;
+
+			let md = `# ${profile.displayName || profile.handle}\n\n`;
+			md += `**@${profile.handle}**\n\n`;
+
+			if (profile.description) {
+				md += `${profile.description}\n\n`;
+			}
+
+			md += "---\n\n";
+			md += `- **Followers:** ${formatCount(profile.followersCount || 0)}\n`;
+			md += `- **Following:** ${formatCount(profile.followsCount || 0)}\n`;
+			md += `- **Posts:** ${formatCount(profile.postsCount || 0)}\n`;
+
+			if (profile.createdAt) {
+				const joined = new Date(profile.createdAt).toLocaleDateString("en-US", {
+					year: "numeric",
+					month: "long",
+					day: "numeric",
+				});
+				md += `- **Joined:** ${joined}\n`;
+			}
+
+			md += `\n**DID:** \`${profile.did}\`\n`;
+
+			const output = finalizeOutput(md);
+			return {
+				url,
+				finalUrl: url,
+				contentType: "text/markdown",
+				method: "bluesky-api",
+				content: output.content,
+				fetchedAt,
+				truncated: output.truncated,
+				notes: ["Fetched via AT Protocol API"],
+			};
+		}
+	} catch {}
+
+	return null;
+};