@nghyane/arcane 0.1.0

package/src/web/scrapers/readthedocs.ts

@@ -0,0 +1,125 @@
+/**
+ * Read the Docs handler for web-fetch
+ */
+import { parse as parseHtml } from "node-html-parser";
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, htmlToBasicMarkdown, loadPage } from "./types";
+
+export const handleReadTheDocs: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	// Check if URL matches Read the Docs patterns
+	const urlObj = new URL(url);
+	const isReadTheDocs =
+		urlObj.hostname.endsWith(".readthedocs.io") ||
+		urlObj.hostname === "readthedocs.org" ||
+		urlObj.hostname === "www.readthedocs.org";
+
+	if (!isReadTheDocs) {
+		return null;
+	}
+
+	const notes: string[] = [];
+	const fetchedAt = new Date().toISOString();
+
+	// Fetch the page
+	const result = await loadPage(url, { timeout, signal });
+	if (!result.ok) {
+		return {
+			url,
+			finalUrl: result.finalUrl,
+			contentType: result.contentType,
+			method: "readthedocs",
+			content: `Failed to fetch Read the Docs page (status: ${result.status ?? "unknown"})`,
+			fetchedAt,
+			truncated: false,
+			notes,
+		};
+	}
+
+	// Parse HTML
+	const root = parseHtml(result.content);
+
+	// Extract main content from common Read the Docs selectors
+	let mainContent =
+		root.querySelector(".document") ||
+		root.querySelector('[role="main"]') ||
+		root.querySelector("main") ||
+		root.querySelector(".rst-content") ||
+		root.querySelector(".body");
+
+	if (!mainContent) {
+		// Fallback to body if no main content found
+		mainContent = root.querySelector("body");
+		notes.push("Using full body content (no main content div found)");
+	}
+
+	// Remove navigation, sidebar, footer elements
+	mainContent
+		?.querySelectorAll(
+			".headerlink, .viewcode-link, nav, .sidebar, footer, .related, .sphinxsidebar, .toctree-wrapper",
+		)
+		.forEach(el => {
+			el.remove();
+		});
+
+	// Try to find Edit on GitHub/GitLab links for raw source
+	const editLinks = root.querySelectorAll('a[href*="github.com"], a[href*="gitlab.com"]');
+	let sourceUrl: string | null = null;
+
+	for (const link of editLinks) {
+		const href = link.getAttribute("href");
+		const text = link.textContent?.toLowerCase() || "";
+
+		if (href && (text.includes("edit") || text.includes("source"))) {
+			// Convert edit URL to raw URL
+			if (href.includes("github.com")) {
+				sourceUrl = href.replace("/blob/", "/raw/").replace("/edit/", "/raw/");
+			} else if (href.includes("gitlab.com")) {
+				sourceUrl = href.replace("/blob/", "/raw/").replace("/edit/", "/raw/");
+			}
+			break;
+		}
+	}
+
+	let content = "";
+
+	// Try to fetch raw source if available
+	if (sourceUrl) {
+		try {
+			const sourceResult = await loadPage(sourceUrl, { timeout: Math.min(timeout, 10), signal });
+			if (sourceResult.ok && sourceResult.content.length > 0 && sourceResult.content.length < 1_000_000) {
+				content = sourceResult.content;
+				notes.push(`Fetched raw source from ${sourceUrl}`);
+			}
+		} catch {
+			// Ignore errors, fall back to HTML
+		}
+	}
+
+	// If no raw source, convert HTML to markdown
+	if (!content && mainContent) {
+		const html = mainContent.innerHTML;
+		content = htmlToBasicMarkdown(html);
+	}
+
+	if (!content) {
+		content = "No content extracted from Read the Docs page";
+		notes.push("Failed to extract content");
+	}
+
+	const { content: finalContent, truncated } = finalizeOutput(content);
+
+	return {
+		url,
+		finalUrl: result.finalUrl,
+		contentType: sourceUrl ? "text/plain" : "text/html",
+		method: "readthedocs",
+		content: finalContent,
+		fetchedAt,
+		truncated,
+		notes,
+	};
+};

package/src/web/scrapers/reddit.ts

@@ -0,0 +1,104 @@
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, loadPage } from "./types";
+
+interface RedditPost {
+	title: string;
+	selftext: string;
+	author: string;
+	score: number;
+	num_comments: number;
+	created_utc: number;
+	subreddit: string;
+	url: string;
+	is_self: boolean;
+}
+
+interface RedditComment {
+	body: string;
+	author: string;
+	score: number;
+	created_utc: number;
+	replies?: { data: { children: Array<{ data: RedditComment }> } };
+}
+
+/**
+ * Handle Reddit URLs via JSON API
+ */
+export const handleReddit: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		if (!parsed.hostname.includes("reddit.com")) return null;
+
+		const fetchedAt = new Date().toISOString();
+
+		// Append .json to get JSON response
+		let jsonUrl = `${url.replace(/\/$/, "")}.json`;
+		if (parsed.search) {
+			jsonUrl = `${url.replace(/\/$/, "").replace(parsed.search, "")}.json${parsed.search}`;
+		}
+
+		const result = await loadPage(jsonUrl, { timeout, signal });
+		if (!result.ok) return null;
+
+		const data = JSON.parse(result.content);
+		let md = "";
+
+		// Handle different Reddit URL types
+		if (Array.isArray(data) && data.length >= 1) {
+			// Post page (with comments)
+			const postData = data[0]?.data?.children?.[0]?.data as RedditPost | undefined;
+			if (postData) {
+				md = `# ${postData.title}\n\n`;
+				md += `**r/${postData.subreddit}** · u/${postData.author} · ${postData.score} points · ${postData.num_comments} comments\n`;
+				md += `*${new Date(postData.created_utc * 1000).toISOString().split("T")[0]}*\n\n`;
+
+				if (postData.is_self && postData.selftext) {
+					md += `---\n\n${postData.selftext}\n\n`;
+				} else if (!postData.is_self) {
+					md += `**Link:** ${postData.url}\n\n`;
+				}
+
+				// Add comments if available
+				if (data.length >= 2 && data[1]?.data?.children) {
+					md += `---\n\n## Top Comments\n\n`;
+					const comments = data[1].data.children.filter((c: { kind: string }) => c.kind === "t1").slice(0, 10);
+
+					for (const { data: comment } of comments as Array<{ data: RedditComment }>) {
+						md += `### u/${comment.author} · ${comment.score} points\n\n`;
+						md += `${comment.body}\n\n---\n\n`;
+					}
+				}
+			}
+		} else if (data?.data?.children) {
+			// Subreddit or listing page
+			const posts = data.data.children.slice(0, 20) as Array<{ data: RedditPost }>;
+			const subreddit = posts[0]?.data?.subreddit;
+
+			md = `# r/${subreddit || "Reddit"}\n\n`;
+			for (const { data: post } of posts) {
+				md += `- **${post.title}** (${post.score} pts, ${post.num_comments} comments)\n`;
+				md += `  by u/${post.author}\n\n`;
+			}
+		}
+
+		if (!md) return null;
+
+		const output = finalizeOutput(md);
+		return {
+			url,
+			finalUrl: url,
+			contentType: "text/markdown",
+			method: "reddit",
+			content: output.content,
+			fetchedAt,
+			truncated: output.truncated,
+			notes: ["Fetched via Reddit JSON API"],
+		};
+	} catch {}
+
+	return null;
+};

package/src/web/scrapers/repology.ts

@@ -0,0 +1,262 @@
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, loadPage } from "./types";
+
+interface RepologyPackage {
+	repo: string;
+	subrepo?: string;
+	srcname?: string;
+	binname?: string;
+	visiblename?: string;
+	version: string;
+	origversion?: string;
+	status:
+		| "newest"
+		| "devel"
+		| "unique"
+		| "outdated"
+		| "legacy"
+		| "rolling"
+		| "noscheme"
+		| "incorrect"
+		| "untrusted"
+		| "ignored";
+	summary?: string;
+	categories?: string[];
+	licenses?: string[];
+	maintainers?: string[];
+}
+
+/**
+ * Get emoji indicator for version status
+ */
+function statusIndicator(status: string): string {
+	switch (status) {
+		case "newest":
+			return "\u2705"; // green check
+		case "devel":
+			return "\uD83D\uDEA7"; // construction
+		case "unique":
+			return "\uD83D\uDD35"; // blue circle
+		case "outdated":
+			return "\uD83D\uDD34"; // red circle
+		case "legacy":
+			return "\u26A0\uFE0F"; // warning
+		case "rolling":
+			return "\uD83D\uDD04"; // arrows
+		default:
+			return "\u2796"; // minus
+	}
+}
+
+/**
+ * Prettify repository name
+ */
+function prettifyRepo(repo: string): string {
+	const mapping: Record<string, string> = {
+		arch: "Arch Linux",
+		aur: "AUR",
+		debian_unstable: "Debian Unstable",
+		debian_stable: "Debian Stable",
+		ubuntu_24_04: "Ubuntu 24.04",
+		ubuntu_22_04: "Ubuntu 22.04",
+		fedora_rawhide: "Fedora Rawhide",
+		fedora_40: "Fedora 40",
+		gentoo: "Gentoo",
+		nix_unstable: "Nixpkgs Unstable",
+		nix_stable: "Nixpkgs Stable",
+		homebrew: "Homebrew",
+		macports: "MacPorts",
+		alpine_edge: "Alpine Edge",
+		freebsd: "FreeBSD",
+		openbsd: "OpenBSD",
+		void_x86_64: "Void Linux",
+		opensuse_tumbleweed: "openSUSE Tumbleweed",
+		msys2_mingw: "MSYS2",
+		chocolatey: "Chocolatey",
+		winget: "Winget",
+		scoop: "Scoop",
+		conda_main: "Conda",
+		pypi: "PyPI",
+		crates_io: "Crates.io",
+		npm: "npm",
+		rubygems: "RubyGems",
+		cpan: "CPAN",
+		hackage: "Hackage",
+	};
+
+	// Check exact match first
+	if (mapping[repo]) return mapping[repo];
+
+	// Check partial matches
+	for (const [key, value] of Object.entries(mapping)) {
+		if (repo.startsWith(key)) return value;
+	}
+
+	// Fallback: titlecase with underscores replaced
+	return repo
+		.split("_")
+		.map(w => w.charAt(0).toUpperCase() + w.slice(1))
+		.join(" ");
+}
+
+/**
+ * Handle Repology URLs via API
+ */
+export const handleRepology: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		if (parsed.hostname !== "repology.org" && parsed.hostname !== "www.repology.org") return null;
+
+		// Extract package name from /project/{name}/versions or /project/{name}/information
+		const match = parsed.pathname.match(/^\/project\/([^/]+)/);
+		if (!match) return null;
+
+		const packageName = decodeURIComponent(match[1]);
+		const fetchedAt = new Date().toISOString();
+
+		// Fetch from Repology API
+		const apiUrl = `https://repology.org/api/v1/project/${encodeURIComponent(packageName)}`;
+		const result = await loadPage(apiUrl, {
+			timeout,
+			headers: { Accept: "application/json" },
+			signal,
+		});
+
+		if (!result.ok) return null;
+
+		let packages: RepologyPackage[];
+		try {
+			packages = JSON.parse(result.content);
+		} catch {
+			return null;
+		}
+
+		// Empty response means package not found
+		if (!Array.isArray(packages) || packages.length === 0) return null;
+
+		// Find newest version(s) and extract metadata
+		const newestVersions = new Set<string>();
+		let summary: string | undefined;
+		let licenses: string[] = [];
+		const categories = new Set<string>();
+
+		for (const pkg of packages) {
+			if (pkg.status === "newest" || pkg.status === "unique") {
+				newestVersions.add(pkg.version);
+			}
+			if (!summary && pkg.summary) summary = pkg.summary;
+			if (pkg.licenses?.length && !licenses.length) licenses = pkg.licenses;
+			if (pkg.categories) {
+				for (const cat of pkg.categories) categories.add(cat);
+			}
+		}
+
+		// If no newest found, find the highest version
+		if (newestVersions.size === 0) {
+			const versions = packages.map(p => p.version);
+			if (versions.length > 0) newestVersions.add(versions[0]);
+		}
+
+		// Group packages by status for counting
+		const statusCounts: Record<string, number> = {};
+		for (const pkg of packages) {
+			statusCounts[pkg.status] = (statusCounts[pkg.status] || 0) + 1;
+		}
+
+		// Build markdown
+		let md = `# ${packageName}\n\n`;
+		if (summary) md += `${summary}\n\n`;
+
+		md += `**Newest Version:** ${Array.from(newestVersions).join(", ") || "unknown"}\n`;
+		md += `**Repositories:** ${packages.length}\n`;
+		if (licenses.length) md += `**License:** ${licenses.join(", ")}\n`;
+		if (categories.size) md += `**Categories:** ${Array.from(categories).join(", ")}\n`;
+		md += "\n";
+
+		// Status summary
+		md += "## Version Status Summary\n\n";
+		const statusOrder = [
+			"newest",
+			"unique",
+			"devel",
+			"rolling",
+			"outdated",
+			"legacy",
+			"noscheme",
+			"incorrect",
+			"untrusted",
+			"ignored",
+		];
+		for (const status of statusOrder) {
+			if (statusCounts[status]) {
+				md += `- ${statusIndicator(status)} **${status}**: ${statusCounts[status]} repos\n`;
+			}
+		}
+		md += "\n";
+
+		// Sort packages: newest first, then by repo name
+		const sortedPackages = [...packages].sort((a, b) => {
+			const statusPriority: Record<string, number> = {
+				newest: 0,
+				unique: 1,
+				devel: 2,
+				rolling: 3,
+				outdated: 4,
+				legacy: 5,
+				noscheme: 6,
+				incorrect: 7,
+				untrusted: 8,
+				ignored: 9,
+			};
+			const aPriority = statusPriority[a.status] ?? 10;
+			const bPriority = statusPriority[b.status] ?? 10;
+			if (aPriority !== bPriority) return aPriority - bPriority;
+			return a.repo.localeCompare(b.repo);
+		});
+
+		// Show top repositories (up to 15)
+		md += "## Package Versions by Repository\n\n";
+		md += "| Repository | Version | Status |\n";
+		md += "|------------|---------|--------|\n";
+
+		const shownRepos = new Set<string>();
+		let count = 0;
+		for (const pkg of sortedPackages) {
+			// Skip duplicate repos (some have multiple entries)
+			const repoKey = pkg.subrepo ? `${pkg.repo}/${pkg.subrepo}` : pkg.repo;
+			if (shownRepos.has(repoKey)) continue;
+			shownRepos.add(repoKey);
+
+			const repoName = prettifyRepo(pkg.repo);
+			const version = pkg.origversion || pkg.version;
+			md += `| ${repoName} | \`${version}\` | ${statusIndicator(pkg.status)} ${pkg.status} |\n`;
+
+			count++;
+			if (count >= 15) break;
+		}
+
+		if (packages.length > 15) {
+			md += `\n*...and ${packages.length - 15} more repositories*\n`;
+		}
+
+		md += `\n---\n\n[View on Repology](${url})\n`;
+
+		const output = finalizeOutput(md);
+		return {
+			url,
+			finalUrl: url,
+			contentType: "text/markdown",
+			method: "repology",
+			content: output.content,
+			fetchedAt,
+			truncated: output.truncated,
+			notes: ["Fetched via Repology API"],
+		};
+	} catch {}
+
+	return null;
+};

package/src/web/scrapers/rfc.ts

@@ -0,0 +1,209 @@
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, loadPage } from "./types";
+
+interface RfcMetadata {
+	doc_id: string;
+	title: string;
+	authors?: Array<{ name: string; affiliation?: string }>;
+	pub_status?: string;
+	current_status?: string;
+	stream?: string;
+	area?: string;
+	wg_acronym?: string;
+	pub_date?: string;
+	page_count?: number;
+	abstract?: string;
+	keywords?: string[];
+	obsoletes?: string[];
+	obsoleted_by?: string[];
+	updates?: string[];
+	updated_by?: string[];
+	see_also?: string[];
+	errata_url?: string;
+}
+
+/**
+ * Extract RFC number from various URL patterns
+ */
+function extractRfcNumber(url: URL): string | null {
+	const { hostname, pathname } = url;
+
+	// https://www.rfc-editor.org/rfc/rfc{number}
+	// https://www.rfc-editor.org/rfc/rfc{number}.html
+	// https://www.rfc-editor.org/rfc/rfc{number}.txt
+	if (hostname === "www.rfc-editor.org" || hostname === "rfc-editor.org") {
+		const match = pathname.match(/\/rfc\/rfc(\d+)(?:\.(?:html|txt|pdf))?$/i);
+		if (match) return match[1];
+	}
+
+	// https://datatracker.ietf.org/doc/rfc{number}/
+	// https://datatracker.ietf.org/doc/html/rfc{number}
+	if (hostname === "datatracker.ietf.org") {
+		const match = pathname.match(/\/doc\/(?:html\/)?rfc(\d+)\/?$/i);
+		if (match) return match[1];
+	}
+
+	// https://tools.ietf.org/html/rfc{number}
+	if (hostname === "tools.ietf.org") {
+		const match = pathname.match(/\/html\/rfc(\d+)$/i);
+		if (match) return match[1];
+	}
+
+	return null;
+}
+
+/**
+ * Clean up RFC plain text - remove page headers/footers and extra formatting
+ */
+function cleanRfcText(text: string): string {
+	const lines = text.split("\n");
+	const cleaned: string[] = [];
+	let skipNext = 0;
+
+	for (let i = 0; i < lines.length; i++) {
+		const line = lines[i];
+
+		// Skip lines we've marked to skip (form feeds and surrounding blank lines)
+		if (skipNext > 0) {
+			skipNext--;
+			continue;
+		}
+
+		// Skip form feed characters and page headers (RFC NNNN ... Month Year pattern)
+		if (line.includes("\f")) {
+			// Skip the form feed line and typically 2-3 following header lines
+			skipNext = 3;
+			continue;
+		}
+
+		// Skip page footer lines (typically just a page number or "[Page N]")
+		if (/^\s*\[Page \d+\]\s*$/.test(line)) {
+			continue;
+		}
+
+		cleaned.push(line);
+	}
+
+	return cleaned.join("\n").replace(/\n{4,}/g, "\n\n\n");
+}
+
+/**
+ * Handle RFC Editor URLs - fetches IETF RFCs
+ */
+export const handleRfc: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		const rfcNumber = extractRfcNumber(parsed);
+
+		if (!rfcNumber) return null;
+
+		const fetchedAt = new Date().toISOString();
+		const notes: string[] = [];
+
+		// Fetch metadata JSON and plain text in parallel
+		const metadataUrl = `https://www.rfc-editor.org/rfc/rfc${rfcNumber}.json`;
+		const textUrl = `https://www.rfc-editor.org/rfc/rfc${rfcNumber}.txt`;
+
+		const [metaResult, textResult] = await Promise.all([
+			loadPage(metadataUrl, { timeout: Math.min(timeout, 10), signal }),
+			loadPage(textUrl, { timeout, signal }),
+		]);
+
+		// We need at least the text content
+		if (!textResult.ok) return null;
+
+		let metadata: RfcMetadata | null = null;
+		if (metaResult.ok) {
+			try {
+				metadata = JSON.parse(metaResult.content);
+				notes.push("Metadata from RFC Editor JSON API");
+			} catch {
+				// JSON parse failed, continue without metadata
+			}
+		}
+
+		// Build markdown output
+		let md = "";
+
+		if (metadata) {
+			md += `# RFC ${rfcNumber}: ${metadata.title}\n\n`;
+
+			// Authors
+			if (metadata.authors?.length) {
+				const authorList = metadata.authors
+					.map(a => (a.affiliation ? `${a.name} (${a.affiliation})` : a.name))
+					.join(", ");
+				md += `**Authors:** ${authorList}\n`;
+			}
+
+			// Publication info
+			if (metadata.pub_date) md += `**Published:** ${metadata.pub_date}\n`;
+			if (metadata.current_status) md += `**Status:** ${metadata.current_status}\n`;
+			if (metadata.stream) md += `**Stream:** ${metadata.stream}\n`;
+			if (metadata.area) md += `**Area:** ${metadata.area}\n`;
+			if (metadata.wg_acronym) md += `**Working Group:** ${metadata.wg_acronym}\n`;
+			if (metadata.page_count) md += `**Pages:** ${metadata.page_count}\n`;
+
+			// Related RFCs
+			if (metadata.obsoletes?.length) {
+				md += `**Obsoletes:** ${metadata.obsoletes.join(", ")}\n`;
+			}
+			if (metadata.obsoleted_by?.length) {
+				md += `**Obsoleted by:** ${metadata.obsoleted_by.join(", ")}\n`;
+			}
+			if (metadata.updates?.length) {
+				md += `**Updates:** ${metadata.updates.join(", ")}\n`;
+			}
+			if (metadata.updated_by?.length) {
+				md += `**Updated by:** ${metadata.updated_by.join(", ")}\n`;
+			}
+
+			// Keywords
+			if (metadata.keywords?.length) {
+				md += `**Keywords:** ${metadata.keywords.join(", ")}\n`;
+			}
+
+			// Errata
+			if (metadata.errata_url) {
+				md += `**Errata:** ${metadata.errata_url}\n`;
+			}
+
+			md += "\n";
+
+			// Abstract from metadata
+			if (metadata.abstract) {
+				md += `## Abstract\n\n${metadata.abstract}\n\n`;
+			}
+
+			md += "---\n\n";
+		} else {
+			// No metadata, use simple header
+			md += `# RFC ${rfcNumber}\n\n`;
+			notes.push("Metadata not available, showing plain text only");
+		}
+
+		// Add full text content
+		md += "## Full Text\n\n";
+		md += "```\n";
+		md += cleanRfcText(textResult.content);
+		md += "\n```\n";
+
+		const output = finalizeOutput(md);
+		return {
+			url,
+			finalUrl: `https://www.rfc-editor.org/rfc/rfc${rfcNumber}`,
+			contentType: "text/markdown",
+			method: "rfc",
+			content: output.content,
+			fetchedAt,
+			truncated: output.truncated,
+			notes: notes.length ? notes : ["Fetched from RFC Editor"],
+		};
+	} catch {}
+
+	return null;
+};