@wahack/pi-coding-agent 15.11.0

package/src/web/scrapers/hackage.ts

@@ -0,0 +1,140 @@
+import { tryParseJson } from "@oh-my-pi/pi-utils";
+import type { RenderResult, SpecialHandler } from "./types";
+import { buildResult, loadPage } from "./types";
+
+interface HackageVersionMap {
+	[version: string]: string;
+}
+
+interface ParsedCabal {
+	name?: string;
+	version?: string;
+	synopsis?: string;
+	description?: string;
+	license?: string;
+	author?: string;
+	maintainer?: string;
+	homepage?: string;
+	bugReports?: string;
+	category?: string;
+	stability?: string;
+}
+
+function compareVersions(a: string, b: string): number {
+	const aParts = a.split(".").map(part => Number.parseInt(part, 10) || 0);
+	const bParts = b.split(".").map(part => Number.parseInt(part, 10) || 0);
+	const max = Math.max(aParts.length, bParts.length);
+	for (let i = 0; i < max; i++) {
+		const delta = (aParts[i] || 0) - (bParts[i] || 0);
+		if (delta !== 0) return delta;
+	}
+	return 0;
+}
+
+function extractCabalField(content: string, fieldName: string): string | undefined {
+	const pattern = new RegExp(`^${fieldName}:\\s*(.*)$`, "im");
+	const match = content.match(pattern);
+	if (!match) return undefined;
+	return match[1].trim();
+}
+
+function extractCabalDescription(content: string): string | undefined {
+	const lines = content.split("\n");
+	const start = lines.findIndex(line => line.toLowerCase().startsWith("description:"));
+	if (start < 0) return undefined;
+	const value = lines[start].replace(/^description:\s*/i, "").trim();
+	const chunks: string[] = [value];
+	for (let i = start + 1; i < lines.length; i++) {
+		const line = lines[i];
+		if (!line.startsWith("  ")) break;
+		chunks.push(line.trim());
+	}
+	const description = chunks.join("\n").trim();
+	return description || undefined;
+}
+
+function parseCabal(content: string): ParsedCabal {
+	return {
+		name: extractCabalField(content, "name"),
+		version: extractCabalField(content, "version"),
+		synopsis: extractCabalField(content, "synopsis"),
+		description: extractCabalDescription(content),
+		license: extractCabalField(content, "license"),
+		author: extractCabalField(content, "author"),
+		maintainer: extractCabalField(content, "maintainer"),
+		homepage: extractCabalField(content, "homepage"),
+		bugReports: extractCabalField(content, "bug-reports"),
+		category: extractCabalField(content, "category"),
+		stability: extractCabalField(content, "stability"),
+	};
+}
+
+/**
+ * Handle Hackage (Haskell package registry) URLs via JSON API
+ */
+export const handleHackage: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		if (parsed.hostname !== "hackage.haskell.org") return null;
+
+		// Match /package/{name} or /package/{name}-{version}
+		const match = parsed.pathname.match(/^\/package\/([^/]+)(?:\/|$)/);
+		if (!match) return null;
+
+		const packageId = decodeURIComponent(match[1]);
+		const fetchedAt = new Date().toISOString();
+
+		// Version endpoint returns a map of version -> status.
+		const versionUrl = `https://hackage.haskell.org/package/${encodeURIComponent(packageId)}.json`;
+		const versionResult = await loadPage(versionUrl, {
+			timeout,
+			headers: { Accept: "application/json" },
+			signal,
+		});
+
+		if (!versionResult.ok) return null;
+
+		const versionMap = tryParseJson<HackageVersionMap>(versionResult.content);
+		if (!versionMap) return null;
+		const latestVersion = Object.keys(versionMap).sort(compareVersions).at(-1);
+		if (!latestVersion) return null;
+
+		// Fetch the latest cabal file for package metadata.
+		const cabalUrl = `https://hackage.haskell.org/package/${encodeURIComponent(packageId)}-${latestVersion}/${encodeURIComponent(packageId)}.cabal`;
+		const cabalResult = await loadPage(cabalUrl, {
+			timeout,
+			headers: { Accept: "text/plain" },
+			signal,
+		});
+
+		if (!cabalResult.ok) return null;
+
+		const pkg = parseCabal(cabalResult.content);
+
+		let md = `# ${pkg.name || packageId}\n\n`;
+		if (pkg.synopsis) md += `${pkg.synopsis}\n\n`;
+
+		md += `**Version:** ${pkg.version || latestVersion}`;
+		if (pkg.license) md += ` · **License:** ${pkg.license}`;
+		md += "\n";
+
+		if (pkg.author) md += `**Author:** ${pkg.author}\n`;
+		if (pkg.maintainer) md += `**Maintainer:** ${pkg.maintainer}\n`;
+		if (pkg.category) md += `**Category:** ${pkg.category}\n`;
+		if (pkg.stability) md += `**Stability:** ${pkg.stability}\n`;
+		if (pkg.homepage) md += `**Homepage:** ${pkg.homepage}\n`;
+		if (pkg.bugReports) md += `**Bug Reports:** ${pkg.bugReports}\n`;
+
+		if (pkg.description) {
+			md += `\n## Description\n\n${pkg.description}\n`;
+		}
+
+		return buildResult(md, { url, method: "hackage", fetchedAt, notes: ["Fetched via Hackage API"] });
+	} catch {}
+
+	return null;
+};

package/src/web/scrapers/hackernews.ts

@@ -0,0 +1,189 @@
+import { tryParseJson } from "@oh-my-pi/pi-utils";
+import type { SpecialHandler } from "./types";
+import { buildResult, decodeHtmlEntities, formatIsoDate, loadPage } from "./types";
+
+interface HNItem {
+	id: number;
+	deleted?: boolean;
+	type?: "job" | "story" | "comment" | "poll" | "pollopt";
+	by?: string;
+	time?: number;
+	text?: string;
+	dead?: boolean;
+	parent?: number;
+	poll?: number;
+	kids?: number[];
+	url?: string;
+	score?: number;
+	title?: string;
+	parts?: number[];
+	descendants?: number;
+}
+
+const API_BASE = "https://hacker-news.firebaseio.com/v0";
+
+async function fetchItem(id: number, timeout: number, signal?: AbortSignal): Promise<HNItem | null> {
+	const url = `${API_BASE}/item/${id}.json`;
+	const { content, ok } = await loadPage(url, { timeout, signal });
+	if (!ok) return null;
+	return tryParseJson<HNItem>(content);
+}
+
+async function fetchItems(ids: number[], timeout: number, limit = 20, signal?: AbortSignal): Promise<HNItem[]> {
+	const promises = ids.slice(0, limit).map(id => fetchItem(id, timeout, signal));
+	const results = await Promise.all(promises);
+	return results.filter((item): item is HNItem => item !== null && !item.deleted && !item.dead);
+}
+
+function decodeHNText(html: string): string {
+	return decodeHtmlEntities(
+		html
+			.replace(/<p>/g, "\n\n")
+			.replace(/<\/p>/g, "")
+			.replace(/<pre><code>/g, "\n```\n")
+			.replace(/<\/code><\/pre>/g, "\n```\n")
+			.replace(/<code>/g, "`")
+			.replace(/<\/code>/g, "`")
+			.replace(/<i>/g, "*")
+			.replace(/<\/i>/g, "*")
+			.replace(/<a href="([^"]+)"[^>]*>([^<]*)<\/a>/g, "[$2]($1)")
+			.replace(/<[^>]+>/g, ""),
+	).trim();
+}
+
+function formatTimestamp(unixTime: number): string {
+	const date = new Date(unixTime * 1000);
+	const now = Date.now();
+	const diff = now - date.getTime();
+	const hours = Math.floor(diff / (1000 * 60 * 60));
+	const days = Math.floor(hours / 24);
+
+	if (days > 7) return formatIsoDate(unixTime * 1000);
+	if (days > 0) return `${days}d ago`;
+	if (hours > 0) return `${hours}h ago`;
+	const minutes = Math.floor(diff / (1000 * 60));
+	return `${minutes}m ago`;
+}
+
+async function renderStory(item: HNItem, timeout: number, depth = 0, signal?: AbortSignal): Promise<string> {
+	let output = "";
+
+	if (depth === 0) {
+		output += `# ${item.title}\n\n`;
+		if (item.url) {
+			output += `**URL:** ${item.url}\n\n`;
+		}
+		output += `**Posted by:** ${item.by} | **Score:** ${item.score ?? 0} | **Time:** ${formatTimestamp(item.time ?? 0)}`;
+		if (item.descendants) {
+			output += ` | **Comments:** ${item.descendants}`;
+		}
+		output += "\n\n";
+	}
+
+	if (item.text) {
+		output += `${decodeHNText(item.text)}\n\n`;
+	}
+
+	if (item.kids && item.kids.length > 0 && depth < 2) {
+		const topComments = item.kids.slice(0, depth === 0 ? 20 : 10);
+		const comments = await fetchItems(topComments, timeout, topComments.length, signal);
+
+		if (comments.length > 0) {
+			if (depth === 0) output += "---\n\n## Comments\n\n";
+
+			for (const comment of comments) {
+				const indent = "  ".repeat(depth);
+				output += `${indent}**${comment.by}** (${formatTimestamp(comment.time ?? 0)})`;
+				if (comment.score !== undefined) output += ` [${comment.score}]`;
+				output += "\n";
+				if (comment.text) {
+					const text = decodeHNText(comment.text);
+					const lines = text.split("\n");
+					output += `${lines.map(line => `${indent}${line}`).join("\n")}\n\n`;
+				}
+
+				if (comment.kids && comment.kids.length > 0 && depth < 1) {
+					const childOutput = await renderStory(comment, timeout, depth + 1, signal);
+					output += childOutput;
+				}
+			}
+		}
+	}
+
+	return output;
+}
+
+async function renderListing(ids: number[], timeout: number, title: string, signal?: AbortSignal): Promise<string> {
+	let output = `# ${title}\n\n`;
+	const stories = await fetchItems(ids, timeout, 20, signal);
+
+	for (let i = 0; i < stories.length; i++) {
+		const story = stories[i];
+		output += `${i + 1}. **${story.title}**\n`;
+		if (story.url) {
+			output += `   ${story.url}\n`;
+		}
+		output += `   ${story.score ?? 0} points by ${story.by} | ${formatTimestamp(story.time ?? 0)}`;
+		if (story.descendants) {
+			output += ` | ${story.descendants} comments`;
+		}
+		output += `\n   https://news.ycombinator.com/item?id=${story.id}\n\n`;
+	}
+
+	return output;
+}
+
+export const handleHackerNews: SpecialHandler = async (url, timeout, signal) => {
+	const parsed = new URL(url);
+	if (!parsed.hostname.includes("news.ycombinator.com")) return null;
+
+	const notes: string[] = [];
+	let content = "";
+	const fetchedAt = new Date().toISOString();
+
+	try {
+		const itemId = parsed.searchParams.get("id");
+
+		if (itemId) {
+			const item = await fetchItem(parseInt(itemId, 10), timeout, signal);
+			if (!item) throw new Error(`Failed to fetch item ${itemId}`);
+
+			content = await renderStory(item, timeout, 0, signal);
+			notes.push(`Fetched HN item ${itemId} with top-level comments (depth 2)`);
+		} else if (parsed.pathname === "/" || parsed.pathname === "/news") {
+			const { content: raw, ok } = await loadPage(`${API_BASE}/topstories.json`, { timeout, signal });
+			if (!ok) throw new Error("Failed to fetch top stories");
+			const ids = tryParseJson<number[]>(raw);
+			if (!ids) throw new Error("Failed to parse top stories");
+			content = await renderListing(ids, timeout, "Hacker News - Top Stories", signal);
+			notes.push("Fetched top 20 stories from HN front page");
+		} else if (parsed.pathname === "/newest") {
+			const { content: raw, ok } = await loadPage(`${API_BASE}/newstories.json`, { timeout, signal });
+			if (!ok) throw new Error("Failed to fetch new stories");
+			const ids = tryParseJson<number[]>(raw);
+			if (!ids) throw new Error("Failed to parse new stories");
+			content = await renderListing(ids, timeout, "Hacker News - New Stories", signal);
+			notes.push("Fetched top 20 new stories");
+		} else if (parsed.pathname === "/best") {
+			const { content: raw, ok } = await loadPage(`${API_BASE}/beststories.json`, { timeout, signal });
+			if (!ok) throw new Error("Failed to fetch best stories");
+			const ids = tryParseJson<number[]>(raw);
+			if (!ids) throw new Error("Failed to parse best stories");
+			content = await renderListing(ids, timeout, "Hacker News - Best Stories", signal);
+			notes.push("Fetched top 20 best stories");
+		} else {
+			return null;
+		}
+
+		return buildResult(content, { url, method: "hackernews", fetchedAt, notes });
+	} catch (err) {
+		const errorMsg = err instanceof Error ? err.message : String(err);
+		notes.push(`Error: ${errorMsg}`);
+		return buildResult(`# Error fetching Hacker News content\n\n${errorMsg}`, {
+			url,
+			method: "hackernews",
+			fetchedAt,
+			notes,
+		});
+	}
+};

package/src/web/scrapers/hex.ts

@@ -0,0 +1,105 @@
+import { tryParseJson } from "@oh-my-pi/pi-utils";
+import type { SpecialHandler } from "./types";
+import { buildResult, formatIsoDate, formatNumber, loadPage } from "./types";
+
+/**
+ * Handle Hex.pm (Elixir package registry) URLs via API
+ */
+export const handleHex: SpecialHandler = async (url, timeout, signal) => {
+	try {
+		const parsed = new URL(url);
+		if (parsed.hostname !== "hex.pm" && parsed.hostname !== "www.hex.pm") return null;
+
+		// Extract package name from /packages/name or /packages/name/version
+		const match = parsed.pathname.match(/^\/packages\/([^/]+)/);
+		if (!match) return null;
+
+		const packageName = decodeURIComponent(match[1]);
+		const fetchedAt = new Date().toISOString();
+
+		// Fetch from Hex.pm API
+		const apiUrl = `https://hex.pm/api/packages/${packageName}`;
+		const result = await loadPage(apiUrl, { timeout, signal });
+
+		if (!result.ok) return null;
+
+		const data = tryParseJson<{
+			name: string;
+			meta?: {
+				description?: string;
+				links?: Record<string, string>;
+				licenses?: string[];
+			};
+			releases?: Array<{
+				version: string;
+				inserted_at: string;
+			}>;
+			downloads?: {
+				all?: number;
+				week?: number;
+				day?: number;
+			};
+			latest_version?: string;
+			latest_stable_version?: string;
+		}>(result.content);
+		if (!data) return null;
+
+		let md = `# ${data.name}\n\n`;
+		if (data.meta?.description) md += `${data.meta.description}\n\n`;
+
+		const version = data.latest_stable_version || data.latest_version || "unknown";
+		md += `**Latest:** ${version}`;
+		if (data.meta?.licenses?.length) md += ` · **License:** ${data.meta.licenses.join(", ")}`;
+		md += "\n";
+
+		if (data.downloads?.all) {
+			md += `**Total Downloads:** ${formatNumber(data.downloads.all)}`;
+			if (data.downloads.week) md += ` · **This Week:** ${formatNumber(data.downloads.week)}`;
+			md += "\n";
+		}
+		md += "\n";
+
+		if (data.meta?.links && Object.keys(data.meta.links).length > 0) {
+			md += `## Links\n\n`;
+			for (const [key, value] of Object.entries(data.meta.links)) {
+				md += `- **${key}:** ${value}\n`;
+			}
+			md += "\n";
+		}
+
+		// Fetch releases if available
+		if (data.releases?.length) {
+			const releasesUrl = `https://hex.pm/api/packages/${packageName}/releases/${version}`;
+			const releaseResult = await loadPage(releasesUrl, { timeout: Math.min(timeout, 5), signal });
+
+			if (releaseResult.ok) {
+				const releaseData = tryParseJson<{
+					requirements?: Record<string, { app?: string; optional: boolean; requirement: string }>;
+				}>(releaseResult.content);
+
+				if (releaseData?.requirements && Object.keys(releaseData.requirements).length > 0) {
+					md += `## Dependencies (${version})\n\n`;
+					for (const [dep, info] of Object.entries(releaseData.requirements)) {
+						const optional = info.optional ? " (optional)" : "";
+						md += `- ${dep}: ${info.requirement}${optional}\n`;
+					}
+					md += "\n";
+				}
+			}
+
+			// Show recent releases
+			const recentReleases = data.releases.slice(0, 10);
+			if (recentReleases.length > 0) {
+				md += `## Recent Releases\n\n`;
+				for (const release of recentReleases) {
+					const date = formatIsoDate(release.inserted_at);
+					md += `- **${release.version}** (${date})\n`;
+				}
+			}
+		}
+
+		return buildResult(md, { url, method: "hex", fetchedAt, notes: ["Fetched via Hex.pm API"] });
+	} catch {}
+
+	return null;
+};