@nghyane/arcane 0.1.0

package/src/web/scrapers/choosealicense.ts

@@ -0,0 +1,110 @@
+import { parseFrontmatter } from "../../utils/frontmatter";
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, loadPage } from "./types";
+
+const ALLOWED_HOSTS = new Set(["choosealicense.com", "www.choosealicense.com"]);
+const LICENSE_PATH = /^\/licenses\/([^/]+)\/?$/i;
+const APPENDIX_PATH = /^\/appendix\/?$/i;
+
+function asString(value: unknown): string | undefined {
+	if (typeof value !== "string") return undefined;
+	const trimmed = value.trim();
+	return trimmed.length > 0 ? trimmed : undefined;
+}
+
+function normalizeList(value: unknown): string[] {
+	if (Array.isArray(value)) {
+		return value
+			.filter((item): item is string => typeof item === "string")
+			.map(item => item.trim())
+			.filter(item => item.length > 0);
+	}
+	if (typeof value === "string") {
+		return value
+			.split(",")
+			.map(item => item.trim())
+			.filter(item => item.length > 0);
+	}
+	return [];
+}
+
+function formatLabel(value: string): string {
+	const cleaned = value.replace(/[-_]+/g, " ").replace(/\s+/g, " ").trim();
+	if (!cleaned) return value;
+	return cleaned.charAt(0).toUpperCase() + cleaned.slice(1);
+}
+
+function formatSection(title: string, items: string[]): string {
+	let md = `## ${title}\n\n`;
+	if (items.length === 0) {
+		md += "- None listed\n\n";
+		return md;
+	}
+	for (const item of items) {
+		md += `- ${formatLabel(item)}\n`;
+	}
+	md += "\n";
+	return md;
+}
+
+export const handleChooseALicense: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		if (!ALLOWED_HOSTS.has(parsed.hostname)) return null;
+
+		const licenseMatch = parsed.pathname.match(LICENSE_PATH);
+		const isAppendix = APPENDIX_PATH.test(parsed.pathname);
+		if (!licenseMatch && !isAppendix) return null;
+
+		const licenseSlug = licenseMatch ? decodeURIComponent(licenseMatch[1]).toLowerCase() : "appendix";
+		const rawUrl = licenseMatch
+			? `https://raw.githubusercontent.com/github/choosealicense.com/gh-pages/_licenses/${licenseSlug}.txt`
+			: "https://raw.githubusercontent.com/github/choosealicense.com/gh-pages/_pages/appendix.md";
+
+		const fetchedAt = new Date().toISOString();
+		const result = await loadPage(rawUrl, { timeout, headers: { Accept: "text/plain" }, signal });
+		if (!result.ok) return null;
+
+		const { frontmatter, body } = parseFrontmatter(result.content, { source: rawUrl });
+
+		const title = asString(frontmatter.title) ?? formatLabel(licenseSlug);
+		const spdxId = asString(frontmatter["spdx-id"]) ?? "Unknown";
+		const description = asString(frontmatter.description);
+		const permissions = normalizeList(frontmatter.permissions);
+		const conditions = normalizeList(frontmatter.conditions);
+		const limitations = normalizeList(frontmatter.limitations);
+
+		let md = `# ${title}\n\n`;
+		if (description) md += `${description}\n\n`;
+
+		md += `**SPDX ID:** ${spdxId}\n`;
+		md += `**Source:** https://choosealicense.com${isAppendix ? "/appendix" : `/licenses/${licenseSlug}/`}\n\n`;
+
+		md += formatSection("Permissions", permissions);
+		md += formatSection("Conditions", conditions);
+		md += formatSection("Limitations", limitations);
+
+		const licenseText = body.trim();
+		if (licenseText.length > 0) {
+			md += `---\n\n## License Text\n\n${licenseText}\n`;
+		}
+
+		const output = finalizeOutput(md);
+		return {
+			url,
+			finalUrl: url,
+			contentType: "text/markdown",
+			method: "choosealicense",
+			content: output.content,
+			fetchedAt,
+			truncated: output.truncated,
+			notes: ["Fetched via Choose a License"],
+		};
+	} catch {}
+
+	return null;
+};

package/src/web/scrapers/cisa-kev.ts

@@ -0,0 +1,100 @@
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, loadPage } from "./types";
+
+interface KevEntry {
+	cveID: string;
+	vendorProject?: string;
+	product?: string;
+	vulnerabilityName?: string;
+	shortDescription?: string;
+	requiredAction?: string;
+	dateAdded?: string;
+	dueDate?: string;
+}
+
+interface KevCatalog {
+	title?: string;
+	catalogVersion?: string;
+	dateReleased?: string;
+	count?: number;
+	vulnerabilities?: KevEntry[];
+}
+
+const CVE_PATTERN = /CVE-\d{4}-\d{4,7}/i;
+const KEV_FEED_URL = "https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json";
+
+/**
+ * Handle CISA Known Exploited Vulnerabilities (KEV) URLs
+ */
+export const handleCisaKev: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		const hostname = parsed.hostname.toLowerCase();
+		if (!hostname.endsWith("cisa.gov")) return null;
+
+		const path = parsed.pathname.toLowerCase();
+		if (!path.includes("known-exploited-vulnerabilities")) return null;
+
+		const cveMatch = parsed.pathname.match(CVE_PATTERN) ?? parsed.search.match(CVE_PATTERN);
+		if (!cveMatch) return null;
+
+		const cveId = cveMatch[0].toUpperCase();
+		const fetchedAt = new Date().toISOString();
+
+		const result = await loadPage(KEV_FEED_URL, {
+			timeout,
+			headers: { Accept: "application/json" },
+			signal,
+		});
+
+		if (!result.ok) return null;
+
+		let data: KevCatalog;
+		try {
+			data = JSON.parse(result.content) as KevCatalog;
+		} catch {
+			return null;
+		}
+
+		const entry = data.vulnerabilities?.find(item => item.cveID?.toUpperCase() === cveId);
+		if (!entry) return null;
+
+		let md = `# ${entry.cveID}\n\n`;
+		if (entry.vulnerabilityName) {
+			md += `${entry.vulnerabilityName}\n\n`;
+		}
+
+		md += "## Metadata\n\n";
+		if (entry.vendorProject) md += `**Vendor:** ${entry.vendorProject}\n`;
+		if (entry.product) md += `**Product:** ${entry.product}\n`;
+		if (entry.dateAdded) md += `**Date Added:** ${entry.dateAdded}\n`;
+		if (entry.dueDate) md += `**Due Date:** ${entry.dueDate}\n`;
+		md += "\n";
+
+		if (entry.shortDescription) {
+			md += `## Description\n\n${entry.shortDescription}\n\n`;
+		}
+
+		if (entry.requiredAction) {
+			md += `## Required Action\n\n${entry.requiredAction}\n\n`;
+		}
+
+		const output = finalizeOutput(md);
+		return {
+			url,
+			finalUrl: url,
+			contentType: "text/markdown",
+			method: "cisa-kev",
+			content: output.content,
+			fetchedAt,
+			truncated: output.truncated,
+			notes: ["Fetched via CISA KEV feed"],
+		};
+	} catch {}
+
+	return null;
+};

package/src/web/scrapers/clojars.ts

@@ -0,0 +1,180 @@
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, formatCount, loadPage } from "./types";
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+	return typeof value === "object" && value !== null;
+}
+
+function asString(value: unknown): string | null {
+	if (typeof value !== "string") return null;
+	const trimmed = value.trim();
+	return trimmed.length > 0 ? trimmed : null;
+}
+
+function asNumber(value: unknown): number | null {
+	return typeof value === "number" && Number.isFinite(value) ? value : null;
+}
+
+function formatLicenses(licenses: unknown): string[] {
+	if (!Array.isArray(licenses)) return [];
+	const output: string[] = [];
+	for (const license of licenses) {
+		if (typeof license === "string") {
+			const trimmed = license.trim();
+			if (trimmed) output.push(trimmed);
+			continue;
+		}
+		if (isRecord(license)) {
+			const name = asString(license.name);
+			const url = asString(license.url);
+			if (name && url) {
+				output.push(`${name} (${url})`);
+			} else if (name) {
+				output.push(name);
+			} else if (url) {
+				output.push(url);
+			}
+		}
+	}
+	return output;
+}
+
+function formatDependencies(deps: unknown): string[] {
+	const output: string[] = [];
+	if (Array.isArray(deps)) {
+		for (const dep of deps) {
+			if (typeof dep === "string") {
+				const trimmed = dep.trim();
+				if (trimmed) output.push(trimmed);
+				continue;
+			}
+			if (Array.isArray(dep)) {
+				const name = asString(dep[0]);
+				const version = asString(dep[1]);
+				if (name && version) {
+					output.push(`${name}: ${version}`);
+				} else if (name) {
+					output.push(name);
+				}
+				continue;
+			}
+			if (isRecord(dep)) {
+				const name = asString(dep.name) ?? asString(dep.artifact) ?? asString(dep.jar_name);
+				const version = asString(dep.version);
+				if (name && version) {
+					output.push(`${name}: ${version}`);
+				} else if (name) {
+					output.push(name);
+				}
+			}
+		}
+		return output;
+	}
+
+	if (isRecord(deps)) {
+		for (const [name, version] of Object.entries(deps)) {
+			const versionText = asString(version);
+			if (versionText) {
+				output.push(`${name}: ${versionText}`);
+			} else if (name.trim()) {
+				output.push(name);
+			}
+		}
+	}
+
+	return output;
+}
+
+/**
+ * Handle Clojars URLs via API
+ */
+export const handleClojars: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		if (parsed.hostname !== "clojars.org" && parsed.hostname !== "www.clojars.org") return null;
+
+		const path = parsed.pathname.replace(/^\/+|\/+$/g, "");
+		if (!path) return null;
+
+		const segments = path.split("/").filter(Boolean);
+		if (segments.length < 1 || segments.length > 2) return null;
+
+		const groupFromUrl = segments.length === 2 ? decodeURIComponent(segments[0]) : null;
+		const artifactFromUrl = decodeURIComponent(segments[segments.length - 1]);
+
+		const apiUrl =
+			segments.length === 2
+				? `https://clojars.org/api/artifacts/${encodeURIComponent(groupFromUrl ?? "")}/${encodeURIComponent(artifactFromUrl)}`
+				: `https://clojars.org/api/artifacts/${encodeURIComponent(artifactFromUrl)}`;
+
+		const fetchedAt = new Date().toISOString();
+
+		const result = await loadPage(apiUrl, {
+			timeout,
+			headers: { Accept: "application/json" },
+			signal,
+		});
+
+		if (!result.ok) return null;
+
+		let payload: unknown;
+		try {
+			payload = JSON.parse(result.content);
+		} catch {
+			return null;
+		}
+
+		const data = Array.isArray(payload) ? payload[0] : payload;
+		if (!isRecord(data)) return null;
+
+		const groupName = asString(data.group_name) ?? asString(data.group) ?? groupFromUrl;
+		const artifactName = asString(data.jar_name) ?? asString(data.artifact) ?? asString(data.name) ?? artifactFromUrl;
+		const version = asString(data.latest_version) ?? asString(data.version);
+		const description = asString(data.description) ?? asString(data.summary);
+		const downloads =
+			asNumber(data.downloads) ?? asNumber(data.downloads_total) ?? asNumber(data.total_downloads) ?? null;
+		const homepage = asString(data.homepage) ?? asString(data.url);
+		const licenses = formatLicenses(data.licenses);
+		const dependencies = formatDependencies(data.dependencies ?? data.deps);
+
+		const displayName =
+			groupName && artifactName && groupName !== artifactName
+				? `${groupName}/${artifactName}`
+				: (artifactName ?? groupName ?? "Clojars artifact");
+
+		let md = `# ${displayName}\n\n`;
+		if (description) md += `${description}\n\n`;
+
+		if (groupName) md += `**Group:** ${groupName}\n`;
+		if (artifactName) md += `**Artifact:** ${artifactName}\n`;
+		if (version) md += `**Latest:** ${version}\n`;
+		if (downloads !== null) md += `**Downloads:** ${formatCount(downloads)}\n`;
+		if (homepage) md += `**Homepage:** ${homepage}\n`;
+		if (licenses.length > 0) md += `**Licenses:** ${licenses.join(", ")}\n`;
+
+		if (dependencies.length > 0) {
+			md += "\n## Dependencies\n\n";
+			for (const dep of dependencies) {
+				md += `- ${dep}\n`;
+			}
+		}
+
+		const output = finalizeOutput(md);
+		return {
+			url,
+			finalUrl: url,
+			contentType: "text/markdown",
+			method: "clojars",
+			content: output.content,
+			fetchedAt,
+			truncated: output.truncated,
+			notes: ["Fetched via Clojars API"],
+		};
+	} catch {}
+
+	return null;
+};

package/src/web/scrapers/coingecko.ts

@@ -0,0 +1,184 @@
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, formatCount, loadPage } from "./types";
+
+interface CoinGeckoResponse {
+	id: string;
+	symbol: string;
+	name: string;
+	description?: { en?: string };
+	links?: {
+		homepage?: string[];
+		blockchain_site?: string[];
+		repos_url?: { github?: string[] };
+	};
+	market_data?: {
+		current_price?: { usd?: number };
+		market_cap?: { usd?: number };
+		total_volume?: { usd?: number };
+		price_change_percentage_24h?: number;
+		ath?: { usd?: number };
+		ath_date?: { usd?: string };
+		circulating_supply?: number;
+		total_supply?: number;
+		max_supply?: number;
+	};
+	categories?: string[];
+	genesis_date?: string;
+}
+
+/**
+ * Handle CoinGecko cryptocurrency URLs via API
+ */
+export const handleCoinGecko: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		if (!parsed.hostname.includes("coingecko.com")) return null;
+
+		// Extract coin ID from /coins/{id} or /en/coins/{id}
+		const match = parsed.pathname.match(/^(?:\/[a-z]{2})?\/coins\/([^/?#]+)/);
+		if (!match) return null;
+
+		const coinId = decodeURIComponent(match[1]);
+		const fetchedAt = new Date().toISOString();
+
+		// Fetch from CoinGecko API
+		const apiUrl = `https://api.coingecko.com/api/v3/coins/${coinId}?localization=false&tickers=false&community_data=false&developer_data=false`;
+		const result = await loadPage(apiUrl, {
+			timeout,
+			headers: { Accept: "application/json" },
+			signal,
+		});
+
+		if (!result.ok) return null;
+
+		let coin: CoinGeckoResponse;
+		try {
+			coin = JSON.parse(result.content);
+		} catch {
+			return null;
+		}
+
+		const market = coin.market_data;
+
+		let md = `# ${coin.name} (${coin.symbol.toUpperCase()})\n\n`;
+
+		// Price and market data
+		if (market?.current_price?.usd !== undefined) {
+			md += `**Price:** $${formatPrice(market.current_price.usd)}`;
+			if (market.price_change_percentage_24h !== undefined) {
+				const change = market.price_change_percentage_24h;
+				const sign = change >= 0 ? "+" : "";
+				md += ` (${sign}${change.toFixed(2)}% 24h)`;
+			}
+			md += "\n";
+		}
+
+		if (market?.market_cap?.usd) {
+			md += `**Market Cap:** $${formatLargeNumber(market.market_cap.usd)}\n`;
+		}
+
+		if (market?.total_volume?.usd) {
+			md += `**24h Volume:** $${formatLargeNumber(market.total_volume.usd)}\n`;
+		}
+
+		if (market?.ath?.usd !== undefined) {
+			md += `**All-Time High:** $${formatPrice(market.ath.usd)}`;
+			if (market.ath_date?.usd) {
+				const athDate = new Date(market.ath_date.usd).toLocaleDateString("en-US", {
+					year: "numeric",
+					month: "short",
+					day: "numeric",
+				});
+				md += ` (${athDate})`;
+			}
+			md += "\n";
+		}
+
+		md += "\n";
+
+		// Supply info
+		if (market?.circulating_supply) {
+			md += `**Circulating Supply:** ${formatCount(Math.round(market.circulating_supply))}`;
+			if (market.max_supply) {
+				const percent = ((market.circulating_supply / market.max_supply) * 100).toFixed(1);
+				md += ` / ${formatCount(Math.round(market.max_supply))} (${percent}%)`;
+			} else if (market.total_supply) {
+				md += ` / ${formatCount(Math.round(market.total_supply))} total`;
+			}
+			md += "\n";
+		}
+
+		if (coin.genesis_date) {
+			md += `**Launch Date:** ${coin.genesis_date}\n`;
+		}
+
+		if (coin.categories?.length) {
+			md += `**Categories:** ${coin.categories.join(", ")}\n`;
+		}
+
+		// Links
+		const links: string[] = [];
+		if (coin.links?.homepage?.[0]) {
+			links.push(`[Website](${coin.links.homepage[0]})`);
+		}
+		if (coin.links?.blockchain_site?.[0]) {
+			links.push(`[Explorer](${coin.links.blockchain_site[0]})`);
+		}
+		if (coin.links?.repos_url?.github?.[0]) {
+			links.push(`[GitHub](${coin.links.repos_url.github[0]})`);
+		}
+		if (links.length) {
+			md += `**Links:** ${links.join(" · ")}\n`;
+		}
+
+		// Description
+		if (coin.description?.en) {
+			const desc = coin.description.en
+				.replace(/<[^>]+>/g, "") // Strip HTML
+				.replace(/\r\n/g, "\n")
+				.trim();
+			if (desc) {
+				md += `\n## About\n\n${desc}\n`;
+			}
+		}
+
+		const output = finalizeOutput(md);
+		return {
+			url,
+			finalUrl: url,
+			contentType: "text/markdown",
+			method: "coingecko",
+			content: output.content,
+			fetchedAt,
+			truncated: output.truncated,
+			notes: ["Fetched via CoinGecko API"],
+		};
+	} catch {}
+
+	return null;
+};
+
+/**
+ * Format price with appropriate decimal places
+ */
+function formatPrice(price: number): string {
+	if (price >= 1000) return price.toLocaleString("en-US", { maximumFractionDigits: 2 });
+	if (price >= 1) return price.toFixed(2);
+	if (price >= 0.01) return price.toFixed(4);
+	if (price >= 0.0001) return price.toFixed(6);
+	return price.toFixed(8);
+}
+
+/**
+ * Format large numbers with B/M/K suffixes
+ */
+function formatLargeNumber(n: number): string {
+	if (n >= 1_000_000_000) return `${(n / 1_000_000_000).toFixed(2)}B`;
+	if (n >= 1_000_000) return `${(n / 1_000_000).toFixed(2)}M`;
+	if (n >= 1_000) return `${(n / 1_000).toFixed(2)}K`;
+	return n.toFixed(2);
+}

package/src/web/scrapers/crates-io.ts

@@ -0,0 +1,128 @@
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, loadPage } from "./types";
+
+/**
+ * Check if content looks like HTML
+ */
+function looksLikeHtml(content: string): boolean {
+	const trimmed = content.trim().toLowerCase();
+	return (
+		trimmed.startsWith("<!doctype") ||
+		trimmed.startsWith("<html") ||
+		trimmed.startsWith("<head") ||
+		trimmed.startsWith("<body")
+	);
+}
+
+/**
+ * Handle crates.io URLs via API
+ */
+export const handleCratesIo: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		if (parsed.hostname !== "crates.io" && parsed.hostname !== "www.crates.io") return null;
+
+		// Extract crate name from /crates/name or /crates/name/version
+		const match = parsed.pathname.match(/^\/crates\/([^/]+)/);
+		if (!match) return null;
+
+		const crateName = decodeURIComponent(match[1]);
+		const fetchedAt = new Date().toISOString();
+
+		// Fetch from crates.io API
+		const apiUrl = `https://crates.io/api/v1/crates/${crateName}`;
+		const result = await loadPage(apiUrl, {
+			timeout,
+			signal,
+			headers: { "User-Agent": "arc-web-fetch/1.0 (https://github.com/anthropics)" },
+		});
+
+		if (!result.ok) return null;
+
+		let data: {
+			crate: {
+				name: string;
+				description: string | null;
+				downloads: number;
+				recent_downloads: number;
+				max_version: string;
+				repository: string | null;
+				homepage: string | null;
+				documentation: string | null;
+				categories: string[];
+				keywords: string[];
+				created_at: string;
+				updated_at: string;
+			};
+			versions: Array<{
+				num: string;
+				downloads: number;
+				created_at: string;
+				license: string | null;
+				rust_version: string | null;
+			}>;
+		};
+
+		try {
+			data = JSON.parse(result.content);
+		} catch {
+			return null;
+		}
+
+		const crate = data.crate;
+		const latestVersion = data.versions?.[0];
+
+		// Format download counts
+		const formatDownloads = (n: number): string =>
+			n >= 1_000_000 ? `${(n / 1_000_000).toFixed(1)}M` : n >= 1_000 ? `${(n / 1_000).toFixed(1)}K` : String(n);
+
+		let md = `# ${crate.name}\n\n`;
+		if (crate.description) md += `${crate.description}\n\n`;
+
+		md += `**Latest:** ${crate.max_version}`;
+		if (latestVersion?.license) md += ` · **License:** ${latestVersion.license}`;
+		if (latestVersion?.rust_version) md += ` · **MSRV:** ${latestVersion.rust_version}`;
+		md += "\n";
+		md += `**Downloads:** ${formatDownloads(crate.downloads)} total · ${formatDownloads(crate.recent_downloads)} recent\n\n`;
+
+		if (crate.repository) md += `**Repository:** ${crate.repository}\n`;
+		if (crate.homepage && crate.homepage !== crate.repository) md += `**Homepage:** ${crate.homepage}\n`;
+		if (crate.documentation) md += `**Docs:** ${crate.documentation}\n`;
+		if (crate.keywords?.length) md += `**Keywords:** ${crate.keywords.join(", ")}\n`;
+		if (crate.categories?.length) md += `**Categories:** ${crate.categories.join(", ")}\n`;
+
+		// Show recent versions
+		if (data.versions?.length > 0) {
+			md += `\n## Recent Versions\n\n`;
+			for (const ver of data.versions.slice(0, 5)) {
+				const date = ver.created_at.split("T")[0];
+				md += `- **${ver.num}** (${date}) - ${formatDownloads(ver.downloads)} downloads\n`;
+			}
+		}
+
+		// Try to fetch README from docs.rs or repository
+		const docsRsUrl = `https://docs.rs/crate/${crateName}/${crate.max_version}/source/README.md`;
+		const readmeResult = await loadPage(docsRsUrl, { timeout: Math.min(timeout, 5), signal });
+		if (readmeResult.ok && readmeResult.content.length > 100 && !looksLikeHtml(readmeResult.content)) {
+			md += `\n---\n\n## README\n\n${readmeResult.content}\n`;
+		}
+
+		const output = finalizeOutput(md);
+		return {
+			url,
+			finalUrl: url,
+			contentType: "text/markdown",
+			method: "crates.io",
+			content: output.content,
+			fetchedAt,
+			truncated: output.truncated,
+			notes: ["Fetched via crates.io API"],
+		};
+	} catch {}
+
+	return null;
+};