@nghyane/arcane 0.1.0

package/src/web/scrapers/go-pkg.ts

@@ -0,0 +1,275 @@
+import { parse as parseHtml } from "node-html-parser";
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, htmlToBasicMarkdown, loadPage } from "./types";
+
+interface GoModuleInfo {
+	Version: string;
+	Time: string;
+}
+
+/**
+ * Handle pkg.go.dev URLs via proxy API and page parsing
+ */
+export const handleGoPkg: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		if (parsed.hostname !== "pkg.go.dev") return null;
+
+		// Extract module path and version from URL
+		// Patterns: /module, /module@version, /module/subpackage
+		const pathname = parsed.pathname.slice(1); // remove leading /
+		if (!pathname) return null;
+
+		let modulePath: string;
+		let version = "latest";
+		let _subpackage = "";
+
+		// Parse @version if present
+		const atIndex = pathname.indexOf("@");
+		if (atIndex !== -1) {
+			const beforeAt = pathname.slice(0, atIndex);
+			const afterAt = pathname.slice(atIndex + 1);
+
+			// Check if there's a subpackage after version
+			const slashIndex = afterAt.indexOf("/");
+			if (slashIndex !== -1) {
+				version = afterAt.slice(0, slashIndex);
+				const remainder = afterAt.slice(slashIndex + 1);
+				modulePath = beforeAt;
+				_subpackage = remainder;
+			} else {
+				version = afterAt;
+				modulePath = beforeAt;
+			}
+		} else {
+			// No version specified, check for subpackage
+			// Need to determine where module ends and subpackage begins
+			// For now, treat the whole path as module path (we'll refine from proxy response)
+			modulePath = pathname;
+		}
+
+		const notes: string[] = [];
+		const sections: string[] = [];
+
+		// Fetch module info from proxy
+		let moduleInfo: GoModuleInfo | null = null;
+		let actualModulePath = modulePath;
+
+		if (version === "latest") {
+			try {
+				const proxyUrl = `https://proxy.golang.org/${encodeURIComponent(modulePath)}/@latest`;
+				const proxyResult = await loadPage(proxyUrl, { timeout, signal });
+
+				if (proxyResult.ok) {
+					moduleInfo = JSON.parse(proxyResult.content) as GoModuleInfo;
+					version = moduleInfo.Version;
+				}
+			} catch {
+				// If @latest fails, might be a subpackage - will extract from page
+			}
+		} else {
+			try {
+				const proxyUrl = `https://proxy.golang.org/${encodeURIComponent(modulePath)}/@v/${encodeURIComponent(version)}.info`;
+				const proxyResult = await loadPage(proxyUrl, { timeout, signal });
+
+				if (proxyResult.ok) {
+					moduleInfo = JSON.parse(proxyResult.content) as GoModuleInfo;
+				}
+			} catch {
+				// Proxy lookup failed, will rely on page data
+			}
+		}
+
+		// Fetch the pkg.go.dev page
+		const pageResult = await loadPage(url, { timeout, signal });
+		if (!pageResult.ok) {
+			return {
+				url,
+				finalUrl: pageResult.finalUrl,
+				contentType: "text/plain",
+				method: "go-pkg",
+				content: `Failed to fetch pkg.go.dev page (status: ${pageResult.status ?? "unknown"})`,
+				fetchedAt: new Date().toISOString(),
+				truncated: false,
+				notes: ["error"],
+			};
+		}
+
+		const doc = parseHtml(pageResult.content);
+
+		// Extract module/package information
+		const breadcrumb = doc.querySelector(".go-Breadcrumb");
+		const _headerDiv = doc.querySelector(".go-Main-header");
+
+		// Extract actual module path from breadcrumb or header
+		if (breadcrumb) {
+			const moduleLink = breadcrumb.querySelector("a[href^='/']");
+			if (moduleLink) {
+				const href = moduleLink.getAttribute("href");
+				if (href) {
+					actualModulePath = href.slice(1).split("@")[0];
+				}
+			}
+		}
+
+		// Extract version if not from proxy
+		if (!moduleInfo) {
+			const versionBadge = doc.querySelector(".go-Chip");
+			if (versionBadge) {
+				const versionText = versionBadge.textContent?.trim();
+				if (versionText?.startsWith("v")) {
+					version = versionText;
+				}
+			}
+		}
+
+		// Extract license
+		const licenseLink = doc.querySelector("a[data-test-id='UnitHeader-license']");
+		const license = licenseLink?.textContent?.trim() || "Unknown";
+
+		// Extract import path
+		const importPathInput = doc.querySelector("input[data-test-id='UnitHeader-importPath']");
+		const importPath = importPathInput?.getAttribute("value") || actualModulePath;
+
+		// Build header
+		sections.push(`# ${importPath}`);
+		sections.push("");
+		sections.push(`**Module:** ${actualModulePath}`);
+		sections.push(`**Version:** ${version}`);
+		sections.push(`**License:** ${license}`);
+		sections.push("");
+
+		// Extract package synopsis
+		const synopsis = doc.querySelector(".go-Main-headerContent p");
+		if (synopsis) {
+			const synopsisText = synopsis.textContent?.trim();
+			if (synopsisText) {
+				sections.push(`## Synopsis`);
+				sections.push("");
+				sections.push(synopsisText);
+				sections.push("");
+			}
+		}
+
+		// Extract documentation overview
+		const docSection = doc.querySelector("#section-documentation");
+		if (docSection) {
+			sections.push("## Documentation");
+			sections.push("");
+
+			// Get overview paragraph
+			const overview = docSection.querySelector(".go-Message");
+			if (overview) {
+				const overviewMd = htmlToBasicMarkdown(overview.innerHTML);
+				sections.push(overviewMd);
+				sections.push("");
+			}
+
+			// Get package-level documentation
+			const docContent = docSection.querySelector(".Documentation-content");
+			if (docContent) {
+				// Extract first few paragraphs
+				const paragraphs = docContent.querySelectorAll("p");
+				const docParts: string[] = [];
+				for (let i = 0; i < Math.min(3, paragraphs.length); i++) {
+					const p = paragraphs[i];
+					const text = htmlToBasicMarkdown(p.innerHTML).trim();
+					if (text) {
+						docParts.push(text);
+					}
+				}
+
+				if (docParts.length > 0) {
+					sections.push(docParts.join("\n\n"));
+					sections.push("");
+				}
+			}
+		}
+
+		// Extract index of exported identifiers
+		const indexSection = doc.querySelector("#section-index");
+		if (indexSection) {
+			const indexList = indexSection.querySelector(".Documentation-indexList");
+			if (indexList) {
+				sections.push("## Index");
+				sections.push("");
+
+				const items = indexList.querySelectorAll("li");
+				const exported: string[] = [];
+
+				for (const item of items) {
+					const link = item.querySelector("a");
+					if (link) {
+						const name = link.textContent?.trim();
+						if (name) {
+							exported.push(`- ${name}`);
+						}
+					}
+				}
+
+				if (exported.length > 0) {
+					// Limit to first 50 exports
+					sections.push(exported.slice(0, 50).join("\n"));
+					if (exported.length > 50) {
+						notes.push(`showing 50 of ${exported.length} exports`);
+						sections.push(`\n... and ${exported.length - 50} more`);
+					}
+					sections.push("");
+				}
+			}
+		}
+
+		// Extract dependencies/imports
+		const importsSection = doc.querySelector("#section-imports");
+		if (importsSection) {
+			const importsList = importsSection.querySelector(".go-Message");
+			if (importsList) {
+				sections.push("## Imports");
+				sections.push("");
+
+				const links = importsList.querySelectorAll("a");
+				const imports: string[] = [];
+
+				for (const link of links) {
+					const imp = link.textContent?.trim();
+					if (imp) {
+						imports.push(`- ${imp}`);
+					}
+				}
+
+				if (imports.length > 0) {
+					sections.push(imports.slice(0, 20).join("\n"));
+					if (imports.length > 20) {
+						notes.push(`showing 20 of ${imports.length} imports`);
+						sections.push(`\n... and ${imports.length - 20} more`);
+					}
+					sections.push("");
+				}
+			}
+		}
+
+		if (moduleInfo) {
+			notes.push(`published ${moduleInfo.Time}`);
+		}
+
+		const content = sections.join("\n");
+		const { content: finalContent, truncated } = finalizeOutput(content);
+
+		return {
+			url,
+			finalUrl: pageResult.finalUrl,
+			contentType: "text/markdown",
+			method: "go-pkg",
+			content: finalContent,
+			fetchedAt: new Date().toISOString(),
+			truncated,
+			notes,
+		};
+	} catch {
+		return null;
+	}
+};

package/src/web/scrapers/hackage.ts

@@ -0,0 +1,94 @@
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, loadPage } from "./types";
+
+interface HackagePackage {
+	name: string;
+	synopsis: string;
+	description: string;
+	license: string;
+	author: string;
+	maintainer: string;
+	version: string;
+	homepage?: string;
+	"bug-reports"?: string;
+	category?: string;
+	stability?: string;
+	dependencies?: Record<string, string>;
+}
+
+/**
+ * Handle Hackage (Haskell package registry) URLs via JSON API
+ */
+export const handleHackage: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		if (parsed.hostname !== "hackage.haskell.org") return null;
+
+		// Match /package/{name} or /package/{name}-{version}
+		const match = parsed.pathname.match(/^\/package\/([^/]+)(?:\/|$)/);
+		if (!match) return null;
+
+		const packageId = decodeURIComponent(match[1]);
+		const fetchedAt = new Date().toISOString();
+
+		// Fetch package info with JSON accept header
+		const apiUrl = `https://hackage.haskell.org/package/${encodeURIComponent(packageId)}`;
+		const result = await loadPage(apiUrl, {
+			timeout,
+			headers: { Accept: "application/json" },
+			signal,
+		});
+
+		if (!result.ok) return null;
+
+		let pkg: HackagePackage;
+		try {
+			pkg = JSON.parse(result.content);
+		} catch {
+			return null;
+		}
+
+		let md = `# ${pkg.name}\n\n`;
+		if (pkg.synopsis) md += `${pkg.synopsis}\n\n`;
+
+		md += `**Version:** ${pkg.version}`;
+		if (pkg.license) md += ` · **License:** ${pkg.license}`;
+		md += "\n";
+
+		if (pkg.author) md += `**Author:** ${pkg.author}\n`;
+		if (pkg.maintainer) md += `**Maintainer:** ${pkg.maintainer}\n`;
+		if (pkg.category) md += `**Category:** ${pkg.category}\n`;
+		if (pkg.stability) md += `**Stability:** ${pkg.stability}\n`;
+		if (pkg.homepage) md += `**Homepage:** ${pkg.homepage}\n`;
+		if (pkg["bug-reports"]) md += `**Bug Reports:** ${pkg["bug-reports"]}\n`;
+
+		if (pkg.description) {
+			md += `\n## Description\n\n${pkg.description}\n`;
+		}
+
+		if (pkg.dependencies && Object.keys(pkg.dependencies).length > 0) {
+			md += `\n## Dependencies\n\n`;
+			for (const [dep, version] of Object.entries(pkg.dependencies)) {
+				md += `- ${dep}: ${version}\n`;
+			}
+		}
+
+		const output = finalizeOutput(md);
+		return {
+			url,
+			finalUrl: url,
+			contentType: "text/markdown",
+			method: "hackage",
+			content: output.content,
+			fetchedAt,
+			truncated: output.truncated,
+			notes: ["Fetched via Hackage API"],
+		};
+	} catch {}
+
+	return null;
+};

package/src/web/scrapers/hackernews.ts

@@ -0,0 +1,208 @@
+import type { SpecialHandler } from "./types";
+import { finalizeOutput, loadPage } from "./types";
+
+interface HNItem {
+	id: number;
+	deleted?: boolean;
+	type?: "job" | "story" | "comment" | "poll" | "pollopt";
+	by?: string;
+	time?: number;
+	text?: string;
+	dead?: boolean;
+	parent?: number;
+	poll?: number;
+	kids?: number[];
+	url?: string;
+	score?: number;
+	title?: string;
+	parts?: number[];
+	descendants?: number;
+}
+
+const API_BASE = "https://hacker-news.firebaseio.com/v0";
+
+async function fetchItem(id: number, timeout: number, signal?: AbortSignal): Promise<HNItem | null> {
+	const url = `${API_BASE}/item/${id}.json`;
+	const { content, ok } = await loadPage(url, { timeout, signal });
+	if (!ok) return null;
+	return JSON.parse(content) as HNItem;
+}
+
+async function fetchItems(ids: number[], timeout: number, limit = 20, signal?: AbortSignal): Promise<HNItem[]> {
+	const promises = ids.slice(0, limit).map(id => fetchItem(id, timeout, signal));
+	const results = await Promise.all(promises);
+	return results.filter((item): item is HNItem => item !== null && !item.deleted && !item.dead);
+}
+
+function decodeHNText(html: string): string {
+	return html
+		.replace(/<p>/g, "\n\n")
+		.replace(/<\/p>/g, "")
+		.replace(/<pre><code>/g, "\n```\n")
+		.replace(/<\/code><\/pre>/g, "\n```\n")
+		.replace(/<code>/g, "`")
+		.replace(/<\/code>/g, "`")
+		.replace(/<i>/g, "*")
+		.replace(/<\/i>/g, "*")
+		.replace(/<a href="([^"]+)"[^>]*>([^<]*)<\/a>/g, "[$2]($1)")
+		.replace(/<[^>]+>/g, "")
+		.replace(/&quot;/g, '"')
+		.replace(/&#x27;/g, "'")
+		.replace(/&#x2F;/g, "/")
+		.replace(/&lt;/g, "<")
+		.replace(/&gt;/g, ">")
+		.replace(/&amp;/g, "&")
+		.trim();
+}
+
+function formatTimestamp(unixTime: number): string {
+	const date = new Date(unixTime * 1000);
+	const now = Date.now();
+	const diff = now - date.getTime();
+	const hours = Math.floor(diff / (1000 * 60 * 60));
+	const days = Math.floor(hours / 24);
+
+	if (days > 7) return date.toISOString().split("T")[0];
+	if (days > 0) return `${days}d ago`;
+	if (hours > 0) return `${hours}h ago`;
+	const minutes = Math.floor(diff / (1000 * 60));
+	return `${minutes}m ago`;
+}
+
+async function renderStory(item: HNItem, timeout: number, depth = 0, signal?: AbortSignal): Promise<string> {
+	let output = "";
+
+	if (depth === 0) {
+		output += `# ${item.title}\n\n`;
+		if (item.url) {
+			output += `**URL:** ${item.url}\n\n`;
+		}
+		output += `**Posted by:** ${item.by} | **Score:** ${item.score ?? 0} | **Time:** ${formatTimestamp(item.time ?? 0)}`;
+		if (item.descendants) {
+			output += ` | **Comments:** ${item.descendants}`;
+		}
+		output += "\n\n";
+	}
+
+	if (item.text) {
+		output += `${decodeHNText(item.text)}\n\n`;
+	}
+
+	if (item.kids && item.kids.length > 0 && depth < 2) {
+		const topComments = item.kids.slice(0, depth === 0 ? 20 : 10);
+		const comments = await fetchItems(topComments, timeout, topComments.length, signal);
+
+		if (comments.length > 0) {
+			if (depth === 0) output += "---\n\n## Comments\n\n";
+
+			for (const comment of comments) {
+				const indent = "  ".repeat(depth);
+				output += `${indent}**${comment.by}** (${formatTimestamp(comment.time ?? 0)})`;
+				if (comment.score !== undefined) output += ` [${comment.score}]`;
+				output += "\n";
+				if (comment.text) {
+					const text = decodeHNText(comment.text);
+					const lines = text.split("\n");
+					output += `${lines.map(line => `${indent}${line}`).join("\n")}\n\n`;
+				}
+
+				if (comment.kids && comment.kids.length > 0 && depth < 1) {
+					const childOutput = await renderStory(comment, timeout, depth + 1, signal);
+					output += childOutput;
+				}
+			}
+		}
+	}
+
+	return output;
+}
+
+async function renderListing(ids: number[], timeout: number, title: string, signal?: AbortSignal): Promise<string> {
+	let output = `# ${title}\n\n`;
+	const stories = await fetchItems(ids, timeout, 20, signal);
+
+	for (let i = 0; i < stories.length; i++) {
+		const story = stories[i];
+		output += `${i + 1}. **${story.title}**\n`;
+		if (story.url) {
+			output += `   ${story.url}\n`;
+		}
+		output += `   ${story.score ?? 0} points by ${story.by} | ${formatTimestamp(story.time ?? 0)}`;
+		if (story.descendants) {
+			output += ` | ${story.descendants} comments`;
+		}
+		output += `\n   https://news.ycombinator.com/item?id=${story.id}\n\n`;
+	}
+
+	return output;
+}
+
+export const handleHackerNews: SpecialHandler = async (url, timeout, signal) => {
+	const parsed = new URL(url);
+	if (!parsed.hostname.includes("news.ycombinator.com")) return null;
+
+	const notes: string[] = [];
+	let content = "";
+	const fetchedAt = new Date().toISOString();
+
+	try {
+		const itemId = parsed.searchParams.get("id");
+
+		if (itemId) {
+			const item = await fetchItem(parseInt(itemId, 10), timeout, signal);
+			if (!item) throw new Error(`Failed to fetch item ${itemId}`);
+
+			content = await renderStory(item, timeout, 0, signal);
+			notes.push(`Fetched HN item ${itemId} with top-level comments (depth 2)`);
+		} else if (parsed.pathname === "/" || parsed.pathname === "/news") {
+			const { content: raw, ok } = await loadPage(`${API_BASE}/topstories.json`, { timeout, signal });
+			if (!ok) throw new Error("Failed to fetch top stories");
+			const ids = JSON.parse(raw) as number[];
+			content = await renderListing(ids, timeout, "Hacker News - Top Stories", signal);
+			notes.push("Fetched top 20 stories from HN front page");
+		} else if (parsed.pathname === "/newest") {
+			const { content: raw, ok } = await loadPage(`${API_BASE}/newstories.json`, { timeout, signal });
+			if (!ok) throw new Error("Failed to fetch new stories");
+			const ids = JSON.parse(raw) as number[];
+			content = await renderListing(ids, timeout, "Hacker News - New Stories", signal);
+			notes.push("Fetched top 20 new stories");
+		} else if (parsed.pathname === "/best") {
+			const { content: raw, ok } = await loadPage(`${API_BASE}/beststories.json`, { timeout, signal });
+			if (!ok) throw new Error("Failed to fetch best stories");
+			const ids = JSON.parse(raw) as number[];
+			content = await renderListing(ids, timeout, "Hacker News - Best Stories", signal);
+			notes.push("Fetched top 20 best stories");
+		} else {
+			return null;
+		}
+
+		const { content: finalContent, truncated } = finalizeOutput(content);
+
+		return {
+			url,
+			finalUrl: url,
+			contentType: "text/markdown",
+			method: "hackernews",
+			content: finalContent,
+			fetchedAt,
+			truncated,
+			notes,
+		};
+	} catch (err) {
+		const errorMsg = err instanceof Error ? err.message : String(err);
+		notes.push(`Error: ${errorMsg}`);
+		const { content: finalContent, truncated } = finalizeOutput(
+			`# Error fetching Hacker News content\n\n${errorMsg}`,
+		);
+		return {
+			url,
+			finalUrl: url,
+			contentType: "text/markdown",
+			method: "hackernews",
+			content: finalContent,
+			fetchedAt,
+			truncated,
+			notes,
+		};
+	}
+};

package/src/web/scrapers/hex.ts

@@ -0,0 +1,121 @@
+import type { SpecialHandler } from "./types";
+import { finalizeOutput, formatCount, loadPage } from "./types";
+
+/**
+ * Handle Hex.pm (Elixir package registry) URLs via API
+ */
+export const handleHex: SpecialHandler = async (url, timeout, signal) => {
+	try {
+		const parsed = new URL(url);
+		if (parsed.hostname !== "hex.pm" && parsed.hostname !== "www.hex.pm") return null;
+
+		// Extract package name from /packages/name or /packages/name/version
+		const match = parsed.pathname.match(/^\/packages\/([^/]+)/);
+		if (!match) return null;
+
+		const packageName = decodeURIComponent(match[1]);
+		const fetchedAt = new Date().toISOString();
+
+		// Fetch from Hex.pm API
+		const apiUrl = `https://hex.pm/api/packages/${packageName}`;
+		const result = await loadPage(apiUrl, { timeout, signal });
+
+		if (!result.ok) return null;
+
+		let data: {
+			name: string;
+			meta?: {
+				description?: string;
+				links?: Record<string, string>;
+				licenses?: string[];
+			};
+			releases?: Array<{
+				version: string;
+				inserted_at: string;
+			}>;
+			downloads?: {
+				all?: number;
+				week?: number;
+				day?: number;
+			};
+			latest_version?: string;
+			latest_stable_version?: string;
+		};
+
+		try {
+			data = JSON.parse(result.content);
+		} catch {
+			return null;
+		}
+
+		let md = `# ${data.name}\n\n`;
+		if (data.meta?.description) md += `${data.meta.description}\n\n`;
+
+		const version = data.latest_stable_version || data.latest_version || "unknown";
+		md += `**Latest:** ${version}`;
+		if (data.meta?.licenses?.length) md += ` · **License:** ${data.meta.licenses.join(", ")}`;
+		md += "\n";
+
+		if (data.downloads?.all) {
+			md += `**Total Downloads:** ${formatCount(data.downloads.all)}`;
+			if (data.downloads.week) md += ` · **This Week:** ${formatCount(data.downloads.week)}`;
+			md += "\n";
+		}
+		md += "\n";
+
+		if (data.meta?.links && Object.keys(data.meta.links).length > 0) {
+			md += `## Links\n\n`;
+			for (const [key, value] of Object.entries(data.meta.links)) {
+				md += `- **${key}:** ${value}\n`;
+			}
+			md += "\n";
+		}
+
+		// Fetch releases if available
+		if (data.releases?.length) {
+			const releasesUrl = `https://hex.pm/api/packages/${packageName}/releases/${version}`;
+			const releaseResult = await loadPage(releasesUrl, { timeout: Math.min(timeout, 5), signal });
+
+			if (releaseResult.ok) {
+				try {
+					const releaseData = JSON.parse(releaseResult.content) as {
+						requirements?: Record<string, { app?: string; optional: boolean; requirement: string }>;
+					};
+
+					if (releaseData.requirements && Object.keys(releaseData.requirements).length > 0) {
+						md += `## Dependencies (${version})\n\n`;
+						for (const [dep, info] of Object.entries(releaseData.requirements)) {
+							const optional = info.optional ? " (optional)" : "";
+							md += `- ${dep}: ${info.requirement}${optional}\n`;
+						}
+						md += "\n";
+					}
+				} catch {}
+			}
+
+			// Show recent releases
+			const recentReleases = data.releases.slice(0, 10);
+			if (recentReleases.length > 0) {
+				md += `## Recent Releases\n\n`;
+				for (const release of recentReleases) {
+					const date = new Date(release.inserted_at).toISOString().split("T")[0];
+					md += `- **${release.version}** (${date})\n`;
+				}
+			}
+		}
+
+		const output = finalizeOutput(md);
+		return {
+			url,
+			finalUrl: url,
+			contentType: "text/markdown",
+			method: "hex",
+			content: output.content,
+			fetchedAt,
+			truncated: output.truncated,
+			notes: ["Fetched via Hex.pm API"],
+		};
+	} catch {}
+
+	return null;
+};