@nghyane/arcane 0.1.0

package/src/web/scrapers/huggingface.ts

@@ -0,0 +1,385 @@
+import type { SpecialHandler } from "./types";
+import { finalizeOutput, formatCount, loadPage } from "./types";
+
+interface HfModelData {
+	modelId: string;
+	pipeline_tag?: string;
+	library_name?: string;
+	tags?: string[];
+	downloads?: number;
+	likes?: number;
+	private?: boolean;
+	gated?: boolean | string;
+	cardData?: {
+		license?: string;
+		language?: string | string[];
+		datasets?: string[];
+		metrics?: string[];
+	};
+}
+
+interface HfDatasetData {
+	id: string;
+	tags?: string[];
+	downloads?: number;
+	likes?: number;
+	private?: boolean;
+	gated?: boolean | string;
+	cardData?: {
+		license?: string;
+		language?: string | string[];
+		task_categories?: string[];
+		size_categories?: string[];
+	};
+	description?: string;
+}
+
+interface HfSpaceData {
+	id: string;
+	author?: string;
+	title?: string;
+	sdk?: string;
+	tags?: string[];
+	likes?: number;
+	private?: boolean;
+	cardData?: {
+		license?: string;
+		sdk?: string;
+		app_file?: string;
+	};
+}
+
+interface HfUserData {
+	avatarUrl?: string;
+	fullname?: string;
+	user?: string;
+	orgs?: Array<{ name: string }>;
+	numModels?: number;
+	numDatasets?: number;
+	numSpaces?: number;
+}
+
+/**
+ * Parse Hugging Face URL and determine type
+ */
+function parseHuggingFaceUrl(url: string): {
+	type: "model" | "dataset" | "space" | "model_or_user";
+	id: string; // Full ID (org/name or just name)
+} | null {
+	try {
+		const parsed = new URL(url);
+		if (parsed.hostname !== "huggingface.co") return null;
+
+		const parts = parsed.pathname.split("/").filter(Boolean);
+		if (parts.length === 0) return null;
+
+		// huggingface.co/datasets/{org}/{dataset} or huggingface.co/datasets/{dataset}
+		if (parts[0] === "datasets" && parts.length >= 2) {
+			const id = parts.slice(1).join("/");
+			return { type: "dataset", id };
+		}
+
+		// huggingface.co/spaces/{org}/{space}
+		if (parts[0] === "spaces" && parts.length >= 3) {
+			return { type: "space", id: `${parts[1]}/${parts[2]}` };
+		}
+
+		// Skip non-resource paths
+		const reservedPaths = ["docs", "blog", "pricing", "enterprise", "join", "login", "settings"];
+		if (reservedPaths.includes(parts[0])) {
+			return null;
+		}
+
+		// huggingface.co/{org}/{model} (two parts = definitely a model)
+		if (parts.length >= 2) {
+			return { type: "model", id: `${parts[0]}/${parts[1]}` };
+		}
+
+		// huggingface.co/{id} (single part = could be model or user, try model first)
+		if (parts.length === 1) {
+			return { type: "model_or_user", id: parts[0] };
+		}
+
+		return null;
+	} catch {
+		return null;
+	}
+}
+
+export const handleHuggingFace: SpecialHandler = async (url: string, timeout: number, signal?: AbortSignal) => {
+	const parsed = parseHuggingFaceUrl(url);
+	if (!parsed) return null;
+
+	const fetchedAt = new Date().toISOString();
+	const notes: string[] = [];
+
+	try {
+		switch (parsed.type) {
+			case "model": {
+				const apiUrl = `https://huggingface.co/api/models/${parsed.id}`;
+				const readmeUrl = `https://huggingface.co/${parsed.id}/raw/main/README.md`;
+
+				const [apiResult, readmeResult] = await Promise.all([
+					loadPage(apiUrl, { timeout, signal }),
+					loadPage(readmeUrl, { timeout: Math.min(timeout, 5), signal }),
+				]);
+
+				if (!apiResult.ok) return null;
+
+				let model: HfModelData;
+				try {
+					model = JSON.parse(apiResult.content);
+				} catch {
+					return null;
+				}
+
+				let md = `# ${model.modelId}\n\n`;
+
+				if (model.pipeline_tag) md += `**Task:** ${model.pipeline_tag}\n`;
+				if (model.library_name) md += `**Library:** ${model.library_name}\n`;
+				if (model.downloads !== undefined) md += `**Downloads:** ${formatCount(model.downloads)}\n`;
+				if (model.likes !== undefined) md += `**Likes:** ${formatCount(model.likes)}\n`;
+				if (model.private) md += `**Visibility:** Private\n`;
+				if (model.gated) md += `**Access:** Gated\n`;
+
+				if (model.cardData) {
+					if (model.cardData.license) md += `**License:** ${model.cardData.license}\n`;
+					if (model.cardData.language) {
+						const langs = Array.isArray(model.cardData.language)
+							? model.cardData.language.join(", ")
+							: model.cardData.language;
+						md += `**Language:** ${langs}\n`;
+					}
+					if (model.cardData.datasets?.length) {
+						md += `**Datasets:** ${model.cardData.datasets.join(", ")}\n`;
+					}
+					if (model.cardData.metrics?.length) {
+						md += `**Metrics:** ${model.cardData.metrics.join(", ")}\n`;
+					}
+				}
+
+				if (model.tags?.length) {
+					md += `**Tags:** ${model.tags.join(", ")}\n`;
+				}
+
+				md += "\n";
+
+				if (readmeResult.ok && readmeResult.content.trim()) {
+					md += `## Model Card\n\n${readmeResult.content}`;
+				}
+
+				const { content, truncated } = finalizeOutput(md);
+				return {
+					url,
+					finalUrl: apiResult.finalUrl,
+					contentType: "text/markdown",
+					method: "huggingface",
+					content,
+					fetchedAt,
+					truncated,
+					notes,
+				};
+			}
+
+			case "dataset": {
+				const apiUrl = `https://huggingface.co/api/datasets/${parsed.id}`;
+				const readmeUrl = `https://huggingface.co/datasets/${parsed.id}/raw/main/README.md`;
+
+				const [apiResult, readmeResult] = await Promise.all([
+					loadPage(apiUrl, { timeout, signal }),
+					loadPage(readmeUrl, { timeout: Math.min(timeout, 5), signal }),
+				]);
+
+				if (!apiResult.ok) return null;
+
+				let dataset: HfDatasetData;
+				try {
+					dataset = JSON.parse(apiResult.content);
+				} catch {
+					return null;
+				}
+
+				let md = `# ${dataset.id}\n\n`;
+				if (dataset.description) md += `${dataset.description}\n\n`;
+
+				if (dataset.downloads !== undefined) md += `**Downloads:** ${formatCount(dataset.downloads)}\n`;
+				if (dataset.likes !== undefined) md += `**Likes:** ${formatCount(dataset.likes)}\n`;
+				if (dataset.private) md += `**Visibility:** Private\n`;
+				if (dataset.gated) md += `**Access:** Gated\n`;
+
+				if (dataset.cardData) {
+					if (dataset.cardData.license) md += `**License:** ${dataset.cardData.license}\n`;
+					if (dataset.cardData.language) {
+						const langs = Array.isArray(dataset.cardData.language)
+							? dataset.cardData.language.join(", ")
+							: dataset.cardData.language;
+						md += `**Language:** ${langs}\n`;
+					}
+					if (dataset.cardData.task_categories?.length) {
+						md += `**Tasks:** ${dataset.cardData.task_categories.join(", ")}\n`;
+					}
+					if (dataset.cardData.size_categories?.length) {
+						md += `**Size:** ${dataset.cardData.size_categories.join(", ")}\n`;
+					}
+				}
+
+				if (dataset.tags?.length) {
+					md += `**Tags:** ${dataset.tags.join(", ")}\n`;
+				}
+
+				md += "\n";
+
+				if (readmeResult.ok && readmeResult.content.trim()) {
+					md += `## Dataset Card\n\n${readmeResult.content}`;
+				}
+
+				const { content, truncated } = finalizeOutput(md);
+				return {
+					url,
+					finalUrl: apiResult.finalUrl,
+					contentType: "text/markdown",
+					method: "huggingface",
+					content,
+					fetchedAt,
+					truncated,
+					notes,
+				};
+			}
+
+			case "space": {
+				const apiUrl = `https://huggingface.co/api/spaces/${parsed.id}`;
+				const readmeUrl = `https://huggingface.co/spaces/${parsed.id}/raw/main/README.md`;
+
+				const [apiResult, readmeResult] = await Promise.all([
+					loadPage(apiUrl, { timeout, signal }),
+					loadPage(readmeUrl, { timeout: Math.min(timeout, 5), signal }),
+				]);
+
+				if (!apiResult.ok) return null;
+
+				let space: HfSpaceData;
+				try {
+					space = JSON.parse(apiResult.content);
+				} catch {
+					return null;
+				}
+
+				let md = `# ${space.id}\n\n`;
+				if (space.title) md += `${space.title}\n\n`;
+
+				if (space.author) md += `**Author:** ${space.author}\n`;
+				if (space.sdk) md += `**SDK:** ${space.sdk}\n`;
+				if (space.likes !== undefined) md += `**Likes:** ${formatCount(space.likes)}\n`;
+				if (space.private) md += `**Visibility:** Private\n`;
+
+				if (space.cardData) {
+					if (space.cardData.license) md += `**License:** ${space.cardData.license}\n`;
+					if (space.cardData.app_file) md += `**App File:** ${space.cardData.app_file}\n`;
+				}
+
+				if (space.tags?.length) {
+					md += `**Tags:** ${space.tags.join(", ")}\n`;
+				}
+
+				md += "\n";
+
+				if (readmeResult.ok && readmeResult.content.trim()) {
+					md += `## Space Info\n\n${readmeResult.content}`;
+				}
+
+				const { content, truncated } = finalizeOutput(md);
+				return {
+					url,
+					finalUrl: apiResult.finalUrl,
+					contentType: "text/markdown",
+					method: "huggingface",
+					content,
+					fetchedAt,
+					truncated,
+					notes,
+				};
+			}
+
+			case "model_or_user": {
+				// Try model API first
+				const modelApiUrl = `https://huggingface.co/api/models/${parsed.id}`;
+				const modelResult = await loadPage(modelApiUrl, { timeout, signal });
+
+				if (modelResult.ok) {
+					let model: HfModelData | null = null;
+					try {
+						model = JSON.parse(modelResult.content);
+					} catch {
+						// Fall through to user check
+					}
+					if (model) {
+						const readmeUrl = `https://huggingface.co/${parsed.id}/raw/main/README.md`;
+						const readmeResult = await loadPage(readmeUrl, { timeout: Math.min(timeout, 5), signal });
+
+						let md = `# ${model.modelId}\n\n`;
+						if (model.pipeline_tag) md += `**Task:** ${model.pipeline_tag}\n`;
+						if (model.library_name) md += `**Library:** ${model.library_name}\n`;
+						if (model.downloads !== undefined) md += `**Downloads:** ${formatCount(model.downloads)}\n`;
+						if (model.likes !== undefined) md += `**Likes:** ${formatCount(model.likes)}\n`;
+						if (model.tags?.length) md += `**Tags:** ${model.tags.join(", ")}\n`;
+						md += "\n";
+						if (readmeResult.ok && readmeResult.content.trim()) {
+							md += `## Model Card\n\n${readmeResult.content}`;
+						}
+
+						const { content, truncated } = finalizeOutput(md);
+						return {
+							url,
+							finalUrl: modelResult.finalUrl,
+							contentType: "text/markdown",
+							method: "huggingface",
+							content,
+							fetchedAt,
+							truncated,
+							notes,
+						};
+					}
+				}
+
+				// Fall back to user API
+				const userApiUrl = `https://huggingface.co/api/users/${parsed.id}`;
+				const userResult = await loadPage(userApiUrl, { timeout, signal });
+				if (!userResult.ok) return null;
+
+				let user: HfUserData;
+				try {
+					user = JSON.parse(userResult.content);
+				} catch {
+					return null;
+				}
+
+				let md = `# ${user.user || parsed.id}\n\n`;
+				if (user.fullname) md += `**Name:** ${user.fullname}\n`;
+				if (user.numModels !== undefined) md += `**Models:** ${formatCount(user.numModels)}\n`;
+				if (user.numDatasets !== undefined) md += `**Datasets:** ${formatCount(user.numDatasets)}\n`;
+				if (user.numSpaces !== undefined) md += `**Spaces:** ${formatCount(user.numSpaces)}\n`;
+
+				if (user.orgs?.length) {
+					md += `**Organizations:** ${user.orgs.map(o => o.name).join(", ")}\n`;
+				}
+
+				const { content, truncated } = finalizeOutput(md);
+				return {
+					url,
+					finalUrl: userResult.finalUrl,
+					contentType: "text/markdown",
+					method: "huggingface",
+					content,
+					fetchedAt,
+					truncated,
+					notes,
+				};
+			}
+
+			default:
+				return null;
+		}
+	} catch {
+		return null;
+	}
+};

package/src/web/scrapers/iacr.ts

@@ -0,0 +1,86 @@
+import { parse as parseHtml } from "node-html-parser";
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, loadPage } from "./types";
+import { convertWithMarkitdown, fetchBinary } from "./utils";
+
+/**
+ * Handle IACR ePrint Archive URLs
+ */
+export const handleIacr: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		if (parsed.hostname !== "eprint.iacr.org") return null;
+
+		// Extract paper ID from /year/number or /year/number.pdf
+		const match = parsed.pathname.match(/\/(\d{4})\/(\d+)(?:\.pdf)?$/);
+		if (!match) return null;
+
+		const [, year, number] = match;
+		const paperId = `${year}/${number}`;
+		const fetchedAt = new Date().toISOString();
+		const notes: string[] = [];
+
+		// Fetch the HTML page for metadata
+		const pageUrl = `https://eprint.iacr.org/${paperId}`;
+		const result = await loadPage(pageUrl, { timeout, signal });
+
+		if (!result.ok) return null;
+
+		const doc = parseHtml(result.content);
+
+		// Extract metadata from the page
+		const title =
+			doc.querySelector("h3.mb-3")?.text?.trim() ||
+			doc.querySelector('meta[name="citation_title"]')?.getAttribute("content");
+		const authors = doc
+			.querySelectorAll('meta[name="citation_author"]')
+			.map(m => m.getAttribute("content"))
+			.filter(Boolean);
+		// Abstract is in <p> after <h5>Abstract</h5>
+		const abstractHeading = doc.querySelectorAll("h5").find(h => h.text?.includes("Abstract"));
+		const abstract =
+			abstractHeading?.parentNode?.querySelector("p")?.text?.trim() ||
+			doc.querySelector('meta[name="description"]')?.getAttribute("content");
+		const keywords = doc.querySelector(".keywords")?.text?.replace("Keywords:", "").trim();
+		const pubDate = doc.querySelector('meta[name="citation_publication_date"]')?.getAttribute("content");
+
+		let md = `# ${title || "IACR ePrint Paper"}\n\n`;
+		if (authors.length) md += `**Authors:** ${authors.join(", ")}\n`;
+		if (pubDate) md += `**Date:** ${pubDate}\n`;
+		md += `**ePrint:** ${paperId}\n`;
+		if (keywords) md += `**Keywords:** ${keywords}\n`;
+		md += `\n---\n\n## Abstract\n\n${abstract || "No abstract available."}\n\n`;
+
+		// If it was a PDF link, try to fetch and convert PDF
+		if (parsed.pathname.endsWith(".pdf")) {
+			const pdfUrl = `https://eprint.iacr.org/${paperId}.pdf`;
+			notes.push("Fetching PDF for full content...");
+			const pdfResult = await fetchBinary(pdfUrl, timeout, signal);
+			if (pdfResult.ok) {
+				const converted = await convertWithMarkitdown(pdfResult.buffer, ".pdf", timeout, signal);
+				if (converted.ok && converted.content.length > 500) {
+					md += `---\n\n## Full Paper\n\n${converted.content}\n`;
+					notes.push("PDF converted via markitdown");
+				}
+			}
+		}
+
+		const output = finalizeOutput(md);
+		return {
+			url,
+			finalUrl: url,
+			contentType: "text/markdown",
+			method: "iacr",
+			content: output.content,
+			fetchedAt,
+			truncated: output.truncated,
+			notes: notes.length ? notes : ["Fetched from IACR ePrint Archive"],
+		};
+	} catch {}
+
+	return null;
+};

package/src/web/scrapers/index.ts

@@ -0,0 +1,249 @@
+/**
+ * Web Fetch Special Handlers Index
+ *
+ * Exports all special handlers for site-specific content extraction.
+ */
+import { handleArtifactHub } from "./artifacthub";
+import { handleArxiv } from "./arxiv";
+import { handleAur } from "./aur";
+import { handleBiorxiv } from "./biorxiv";
+import { handleBluesky } from "./bluesky";
+import { handleBrew } from "./brew";
+import { handleCheatSh } from "./cheatsh";
+import { handleChocolatey } from "./chocolatey";
+import { handleChooseALicense } from "./choosealicense";
+import { handleCisaKev } from "./cisa-kev";
+import { handleClojars } from "./clojars";
+import { handleCoinGecko } from "./coingecko";
+import { handleCratesIo } from "./crates-io";
+import { handleCrossref } from "./crossref";
+import { handleDevTo } from "./devto";
+import { handleDiscogs } from "./discogs";
+import { handleDiscourse } from "./discourse";
+import { handleDockerHub } from "./dockerhub";
+import { handleFdroid } from "./fdroid";
+import { handleFirefoxAddons } from "./firefox-addons";
+import { handleFlathub } from "./flathub";
+import { fetchGitHubApi, handleGitHub } from "./github";
+import { handleGitHubGist } from "./github-gist";
+import { handleGitLab } from "./gitlab";
+import { handleGoPkg } from "./go-pkg";
+import { handleHackage } from "./hackage";
+import { handleHackerNews } from "./hackernews";
+import { handleHex } from "./hex";
+import { handleHuggingFace } from "./huggingface";
+import { handleIacr } from "./iacr";
+import { handleJetBrainsMarketplace } from "./jetbrains-marketplace";
+import { handleLemmy } from "./lemmy";
+import { handleLobsters } from "./lobsters";
+import { handleMastodon } from "./mastodon";
+import { handleMaven } from "./maven";
+import { handleMDN } from "./mdn";
+import { handleMetaCPAN } from "./metacpan";
+import { handleMusicBrainz } from "./musicbrainz";
+import { handleNpm } from "./npm";
+import { handleNuGet } from "./nuget";
+import { handleNvd } from "./nvd";
+import { handleOllama } from "./ollama";
+import { handleOpenVsx } from "./open-vsx";
+import { handleOpenCorporates } from "./opencorporates";
+import { handleOpenLibrary } from "./openlibrary";
+import { handleOrcid } from "./orcid";
+import { handleOsv } from "./osv";
+import { handlePackagist } from "./packagist";
+import { handlePubDev } from "./pub-dev";
+import { handlePubMed } from "./pubmed";
+import { handlePyPI } from "./pypi";
+import { handleRawg } from "./rawg";
+import { handleReadTheDocs } from "./readthedocs";
+import { handleReddit } from "./reddit";
+import { handleRepology } from "./repology";
+import { handleRfc } from "./rfc";
+import { handleRubyGems } from "./rubygems";
+import { handleSearchcode } from "./searchcode";
+import { handleSecEdgar } from "./sec-edgar";
+import { handleSemanticScholar } from "./semantic-scholar";
+import { handleSnapcraft } from "./snapcraft";
+import { handleSourcegraph } from "./sourcegraph";
+import { handleSpdx } from "./spdx";
+import { handleSpotify } from "./spotify";
+import { handleStackOverflow } from "./stackoverflow";
+import { handleTerraform } from "./terraform";
+import { handleTldr } from "./tldr";
+import { handleTwitter } from "./twitter";
+import type { SpecialHandler } from "./types";
+import { handleVimeo } from "./vimeo";
+import { handleVscodeMarketplace } from "./vscode-marketplace";
+import { handleW3c } from "./w3c";
+import { handleWikidata } from "./wikidata";
+import { handleWikipedia } from "./wikipedia";
+import { handleYouTube } from "./youtube";
+
+export type { RenderResult, SpecialHandler } from "./types";
+
+export {
+	fetchGitHubApi,
+	handleArtifactHub,
+	handleArxiv,
+	handleAur,
+	handleBiorxiv,
+	handleBluesky,
+	handleBrew,
+	handleCheatSh,
+	handleCisaKev,
+	handleChocolatey,
+	handleClojars,
+	handleChooseALicense,
+	handleCoinGecko,
+	handleCratesIo,
+	handleCrossref,
+	handleDevTo,
+	handleDiscogs,
+	handleDiscourse,
+	handleDockerHub,
+	handleFdroid,
+	handleFlathub,
+	handleFirefoxAddons,
+	handleGitHub,
+	handleGitHubGist,
+	handleGitLab,
+	handleGoPkg,
+	handleHackage,
+	handleHackerNews,
+	handleHex,
+	handleHuggingFace,
+	handleIacr,
+	handleJetBrainsMarketplace,
+	handleLemmy,
+	handleLobsters,
+	handleMastodon,
+	handleMaven,
+	handleMDN,
+	handleMetaCPAN,
+	handleMusicBrainz,
+	handleNpm,
+	handleNuGet,
+	handleNvd,
+	handleOllama,
+	handleOpenCorporates,
+	handleOpenLibrary,
+	handleOrcid,
+	handleOpenVsx,
+	handleOsv,
+	handlePackagist,
+	handlePubDev,
+	handlePubMed,
+	handlePyPI,
+	handleRawg,
+	handleReadTheDocs,
+	handleReddit,
+	handleRepology,
+	handleRfc,
+	handleRubyGems,
+	handleSecEdgar,
+	handleSearchcode,
+	handleSemanticScholar,
+	handleSnapcraft,
+	handleSourcegraph,
+	handleSpotify,
+	handleSpdx,
+	handleStackOverflow,
+	handleTerraform,
+	handleTldr,
+	handleTwitter,
+	handleVimeo,
+	handleVscodeMarketplace,
+	handleW3c,
+	handleWikidata,
+	handleWikipedia,
+	handleYouTube,
+};
+
+export const specialHandlers: SpecialHandler[] = [
+	// Git hosting
+	handleGitHubGist,
+	handleGitHub,
+	handleGitLab,
+	// Video/Media
+	handleYouTube,
+	handleVimeo,
+	handleSpotify,
+	handleDiscogs,
+	handleMusicBrainz,
+	// Games
+	handleRawg,
+	// Social/News
+	handleTwitter,
+	handleBluesky,
+	handleMastodon,
+	handleLemmy,
+	handleHackerNews,
+	handleLobsters,
+	handleReddit,
+	handleDiscourse,
+	// Developer content
+	handleStackOverflow,
+	handleDevTo,
+	handleMDN,
+	handleReadTheDocs,
+	handleSearchcode,
+	handleSourcegraph,
+	handleTldr,
+	handleCheatSh,
+	// Package registries
+	handleNpm,
+	handleFirefoxAddons,
+	handleVscodeMarketplace,
+	handleNuGet,
+	handleChocolatey,
+	handleClojars,
+	handleBrew,
+	handlePyPI,
+	handleCratesIo,
+	handleDockerHub,
+	handleFdroid,
+	handleFlathub,
+	handleGoPkg,
+	handleHex,
+	handlePackagist,
+	handlePubDev,
+	handleMaven,
+	handleJetBrainsMarketplace,
+	handleOpenVsx,
+	handleArtifactHub,
+	handleRubyGems,
+	handleTerraform,
+	handleAur,
+	handleHackage,
+	handleMetaCPAN,
+	handleRepology,
+	handleSnapcraft,
+	// ML/AI
+	handleHuggingFace,
+	handleOllama,
+	// Academic
+	handleArxiv,
+	handleBiorxiv,
+	handleCrossref,
+	handleIacr,
+	handleOrcid,
+	handleSemanticScholar,
+	handlePubMed,
+	handleRfc,
+	// Security
+	handleCisaKev,
+	handleNvd,
+	handleOsv,
+	// Crypto
+	handleCoinGecko,
+	// Business
+	handleOpenCorporates,
+	handleSecEdgar,
+	// Reference
+	handleOpenLibrary,
+	handleChooseALicense,
+	handleW3c,
+	handleSpdx,
+	handleWikidata,
+	handleWikipedia,
+];