@nghyane/arcane 0.1.0

package/src/web/scrapers/pub-dev.ts

@@ -0,0 +1,185 @@
+import { finalizeOutput, formatCount, loadPage, type SpecialHandler } from "./types";
+
+/**
+ * Handle pub.dev URLs via API
+ */
+export const handlePubDev: SpecialHandler = async (url: string, timeout: number, signal?: AbortSignal) => {
+	try {
+		const parsed = new URL(url);
+		if (parsed.hostname !== "pub.dev" && parsed.hostname !== "www.pub.dev") return null;
+
+		// Extract package name from /packages/{package}
+		const match = parsed.pathname.match(/^\/packages\/([^/]+)/);
+		if (!match) return null;
+
+		const packageName = decodeURIComponent(match[1]);
+		const fetchedAt = new Date().toISOString();
+
+		// Fetch from pub.dev API
+		const apiUrl = `https://pub.dev/api/packages/${encodeURIComponent(packageName)}`;
+		const result = await loadPage(apiUrl, { timeout, signal });
+
+		if (!result.ok) return null;
+
+		let data: {
+			name: string;
+			latest: {
+				version: string;
+				pubspec: {
+					description?: string;
+					homepage?: string;
+					repository?: string;
+					documentation?: string;
+					environment?: Record<string, string>;
+					dependencies?: Record<string, unknown>;
+					dev_dependencies?: Record<string, unknown>;
+				};
+			};
+			publisherId?: string;
+			metrics?: {
+				score?: {
+					likeCount?: number;
+					grantedPoints?: number;
+					maxPoints?: number;
+					popularityScore?: number;
+				};
+			};
+		};
+
+		try {
+			data = JSON.parse(result.content);
+		} catch {
+			return null;
+		}
+
+		const { name, latest, publisherId, metrics } = data;
+		const pubspec = latest.pubspec;
+
+		let md = `# ${name}\n\n`;
+		if (pubspec.description) md += `${pubspec.description}\n\n`;
+
+		md += `**Latest:** ${latest.version}`;
+		if (publisherId) md += ` · **Publisher:** ${publisherId}`;
+		md += "\n";
+
+		// Add metrics if available
+		const score = metrics?.score;
+		if (score) {
+			const likes = score.likeCount;
+			const points = score.grantedPoints;
+			const maxPoints = score.maxPoints;
+			const popularity = score.popularityScore;
+
+			if (likes !== undefined) md += `**Likes:** ${formatCount(likes)}`;
+			if (points !== undefined && maxPoints !== undefined) {
+				md += ` · **Pub Points:** ${points}/${maxPoints}`;
+			}
+			if (popularity !== undefined) {
+				md += ` · **Popularity:** ${Math.round(popularity * 100)}%`;
+			}
+			md += "\n";
+		}
+
+		md += "\n";
+
+		if (pubspec.homepage) md += `**Homepage:** ${pubspec.homepage}\n`;
+		if (pubspec.repository) md += `**Repository:** ${pubspec.repository}\n`;
+		if (pubspec.documentation) md += `**Documentation:** ${pubspec.documentation}\n`;
+
+		// SDK constraints
+		if (pubspec.environment) {
+			const constraints: string[] = [];
+			for (const [key, value] of Object.entries(pubspec.environment)) {
+				constraints.push(`${key}: ${value}`);
+			}
+			if (constraints.length > 0) {
+				md += `**SDK:** ${constraints.join(", ")}\n`;
+			}
+		}
+
+		md += "\n";
+
+		// Dependencies
+		if (pubspec.dependencies) {
+			const deps = Object.keys(pubspec.dependencies);
+			if (deps.length > 0) {
+				md += `## Dependencies (${deps.length})\n\n`;
+				for (const dep of deps.slice(0, 20)) {
+					const constraint = pubspec.dependencies[dep];
+					const constraintStr =
+						typeof constraint === "string" ? constraint : typeof constraint === "object" ? "complex" : "";
+					md += `- ${dep}`;
+					if (constraintStr) md += `: ${constraintStr}`;
+					md += "\n";
+				}
+				if (deps.length > 20) {
+					md += `\n*...and ${deps.length - 20} more*\n`;
+				}
+				md += "\n";
+			}
+		}
+
+		// Try to fetch README from pub.dev
+		const readmeUrl = `https://pub.dev/packages/${encodeURIComponent(packageName)}/versions/${encodeURIComponent(latest.version)}/readme`;
+		try {
+			const readmeResult = await loadPage(readmeUrl, { timeout: Math.min(timeout, 10), signal });
+			if (readmeResult.ok) {
+				// Extract README content from HTML
+				const readmeMatch = readmeResult.content.match(
+					/<div[^>]*class="[^"]*markdown-body[^"]*"[^>]*>([\s\S]*?)<\/div>/i,
+				);
+				if (readmeMatch) {
+					// Basic HTML to markdown conversion for README
+					const readme = readmeMatch[1]
+						.replace(/<h(\d)[^>]*>(.*?)<\/h\d>/gi, (_, level, text) => {
+							const stripped = text.replace(/<[^>]+>/g, "");
+							return `${"#".repeat(parseInt(level, 10))} ${stripped}\n\n`;
+						})
+						.replace(/<pre><code[^>]*>([\s\S]*?)<\/code><\/pre>/gi, (_, code) => {
+							const decoded = code
+								.replace(/&lt;/g, "<")
+								.replace(/&gt;/g, ">")
+								.replace(/&amp;/g, "&")
+								.replace(/&quot;/g, '"');
+							return `\n\`\`\`\n${decoded}\n\`\`\`\n\n`;
+						})
+						.replace(/<code[^>]*>(.*?)<\/code>/gi, "`$1`")
+						.replace(/<a[^>]*href="([^"]+)"[^>]*>(.*?)<\/a>/gi, "[$2]($1)")
+						.replace(/<strong[^>]*>(.*?)<\/strong>/gi, "**$1**")
+						.replace(/<em[^>]*>(.*?)<\/em>/gi, "*$1*")
+						.replace(/<li[^>]*>(.*?)<\/li>/gi, "- $1\n")
+						.replace(/<\/?(ul|ol|p|br)[^>]*>/gi, "\n")
+						.replace(/<[^>]+>/g, "")
+						.replace(/&lt;/g, "<")
+						.replace(/&gt;/g, ">")
+						.replace(/&amp;/g, "&")
+						.replace(/&quot;/g, '"')
+						.replace(/&#39;/g, "'")
+						.replace(/&nbsp;/g, " ")
+						.replace(/\n{3,}/g, "\n\n")
+						.trim();
+
+					if (readme.length > 100) {
+						md += `## README\n\n${readme}\n`;
+					}
+				}
+			}
+		} catch {
+			// README fetch failed, continue without it
+		}
+
+		const output = finalizeOutput(md);
+		return {
+			url,
+			finalUrl: url,
+			contentType: "text/markdown",
+			method: "pub.dev",
+			content: output.content,
+			fetchedAt,
+			truncated: output.truncated,
+			notes: ["Fetched via pub.dev API"],
+		};
+	} catch {}
+
+	return null;
+};

package/src/web/scrapers/pubmed.ts

@@ -0,0 +1,177 @@
+/**
+ * PubMed handler for web-fetch
+ */
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, loadPage } from "./types";
+
+/**
+ * Handle PubMed URLs - fetch article metadata, abstract, MeSH terms
+ */
+export const handlePubMed: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+
+		// Match pubmed.ncbi.nlm.nih.gov/{pmid} or ncbi.nlm.nih.gov/pubmed/{pmid}
+		if (
+			parsed.hostname !== "pubmed.ncbi.nlm.nih.gov" &&
+			!(parsed.hostname === "ncbi.nlm.nih.gov" && parsed.pathname.startsWith("/pubmed"))
+		) {
+			return null;
+		}
+
+		// Extract PMID from URL
+		let pmid: string | null = null;
+		if (parsed.hostname === "pubmed.ncbi.nlm.nih.gov") {
+			// Format: pubmed.ncbi.nlm.nih.gov/12345678/
+			const match = parsed.pathname.match(/\/(\d+)/);
+			if (match) pmid = match[1];
+		} else {
+			// Format: ncbi.nlm.nih.gov/pubmed/12345678
+			const match = parsed.pathname.match(/\/pubmed\/(\d+)/);
+			if (match) pmid = match[1];
+		}
+
+		if (!pmid) return null;
+
+		const fetchedAt = new Date().toISOString();
+		const notes: string[] = [];
+
+		// Fetch summary metadata
+		const summaryUrl = `https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esummary.fcgi?db=pubmed&id=${pmid}&retmode=json`;
+		const summaryResult = await loadPage(summaryUrl, { timeout, signal });
+
+		if (!summaryResult.ok) return null;
+
+		let summaryData: {
+			result?: {
+				[pmid: string]: {
+					title?: string;
+					authors?: Array<{ name: string }>;
+					fulljournalname?: string;
+					pubdate?: string;
+					volume?: string;
+					issue?: string;
+					pages?: string;
+					elocationid?: string; // DOI
+					articleids?: Array<{ idtype: string; value: string }>;
+				};
+			};
+		};
+
+		try {
+			summaryData = JSON.parse(summaryResult.content);
+		} catch {
+			return null;
+		}
+
+		const article = summaryData.result?.[pmid];
+		if (!article) return null;
+
+		// Fetch abstract
+		const abstractUrl = `https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?db=pubmed&id=${pmid}&rettype=abstract&retmode=text`;
+		const abstractResult = await loadPage(abstractUrl, { timeout, signal });
+
+		let abstractText = "";
+		if (abstractResult.ok) {
+			abstractText = abstractResult.content.trim();
+			notes.push("Fetched abstract via NCBI E-utilities");
+		}
+
+		// Extract DOI and PMCID
+		let doi = "";
+		let pmcid = "";
+		if (article.articleids) {
+			for (const id of article.articleids) {
+				if (id.idtype === "doi") doi = id.value;
+				if (id.idtype === "pmc") pmcid = id.value;
+			}
+		}
+		if (!doi && article.elocationid) {
+			doi = article.elocationid;
+		}
+
+		// Build markdown output
+		let md = `# ${article.title || "PubMed Article"}\n\n`;
+
+		// Authors
+		if (article.authors && article.authors.length > 0) {
+			const authorNames = article.authors.map(a => a.name).join(", ");
+			md += `**Authors:** ${authorNames}\n`;
+		}
+
+		// Journal info
+		if (article.fulljournalname) {
+			md += `**Journal:** ${article.fulljournalname}`;
+			if (article.pubdate) md += ` (${article.pubdate})`;
+			md += "\n";
+		}
+
+		// Volume/Issue/Pages
+		const citation: string[] = [];
+		if (article.volume) citation.push(`Vol ${article.volume}`);
+		if (article.issue) citation.push(`Issue ${article.issue}`);
+		if (article.pages) citation.push(`pp ${article.pages}`);
+		if (citation.length > 0) {
+			md += `**Citation:** ${citation.join(", ")}\n`;
+		}
+
+		// IDs
+		md += `**PMID:** ${pmid}\n`;
+		if (doi) md += `**DOI:** ${doi}\n`;
+		if (pmcid) md += `**PMCID:** ${pmcid}\n`;
+
+		md += "\n---\n\n";
+
+		// Abstract section
+		if (abstractText) {
+			md += `## Abstract\n\n${abstractText}\n`;
+		} else {
+			md += `## Abstract\n\nNo abstract available.\n`;
+		}
+
+		// Try to fetch MeSH terms
+		try {
+			const meshUrl = `https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?db=pubmed&id=${pmid}&rettype=medline&retmode=text`;
+			const meshResult = await loadPage(meshUrl, { timeout: Math.min(timeout, 5), signal });
+
+			if (meshResult.ok) {
+				const meshTerms: string[] = [];
+				const lines = meshResult.content.split("\n");
+				for (const line of lines) {
+					if (line.startsWith("MH  - ")) {
+						const term = line.slice(6).trim();
+						meshTerms.push(term);
+					}
+				}
+
+				if (meshTerms.length > 0) {
+					md += `\n## MeSH Terms\n\n`;
+					for (const term of meshTerms) {
+						md += `- ${term}\n`;
+					}
+					notes.push("Fetched MeSH terms via NCBI E-utilities");
+				}
+			}
+		} catch {
+			// MeSH terms are optional
+		}
+
+		const output = finalizeOutput(md);
+		return {
+			url,
+			finalUrl: url,
+			contentType: "text/markdown",
+			method: "pubmed",
+			content: output.content,
+			fetchedAt,
+			truncated: output.truncated,
+			notes: notes.length > 0 ? notes : ["Fetched via NCBI E-utilities"],
+		};
+	} catch {
+		return null;
+	}
+};

package/src/web/scrapers/pypi.ts

@@ -0,0 +1,129 @@
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, formatCount, loadPage } from "./types";
+
+/**
+ * Handle PyPI URLs via JSON API
+ */
+export const handlePyPI: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		if (parsed.hostname !== "pypi.org" && parsed.hostname !== "www.pypi.org") return null;
+
+		// Extract package name from /project/{package} or /project/{package}/{version}
+		const match = parsed.pathname.match(/^\/project\/([^/]+)/);
+		if (!match) return null;
+
+		const packageName = decodeURIComponent(match[1]);
+		const fetchedAt = new Date().toISOString();
+
+		// Fetch from PyPI JSON API
+		const apiUrl = `https://pypi.org/pypi/${packageName}/json`;
+		const downloadsUrl = `https://pypistats.org/api/packages/${packageName}/recent`;
+
+		// Fetch package info and download stats in parallel
+		const [result, downloadsResult] = await Promise.all([
+			loadPage(apiUrl, { timeout, signal }),
+			loadPage(downloadsUrl, { timeout: Math.min(timeout, 5), signal }),
+		]);
+
+		if (!result.ok) return null;
+
+		// Parse download stats
+		let weeklyDownloads: number | null = null;
+		if (downloadsResult.ok) {
+			try {
+				const dlData = JSON.parse(downloadsResult.content) as { data?: { last_week?: number } };
+				weeklyDownloads = dlData.data?.last_week ?? null;
+			} catch {}
+		}
+
+		let pkg: {
+			info: {
+				name: string;
+				version: string;
+				summary?: string;
+				description?: string;
+				author?: string;
+				author_email?: string;
+				license?: string;
+				home_page?: string;
+				project_urls?: Record<string, string>;
+				requires_python?: string;
+				keywords?: string;
+				classifiers?: string[];
+			};
+			urls?: Array<{ filename: string; size: number; upload_time: string }>;
+			releases?: Record<string, unknown>;
+			requires_dist?: string[];
+		};
+
+		try {
+			pkg = JSON.parse(result.content);
+		} catch {
+			return null; // JSON parse failed
+		}
+
+		const info = pkg.info;
+		let md = `# ${info.name}\n\n`;
+		if (info.summary) md += `${info.summary}\n\n`;
+
+		md += `**Latest:** ${info.version}`;
+		if (info.license) md += ` · **License:** ${info.license}`;
+		md += "\n";
+
+		if (weeklyDownloads !== null) {
+			md += `**Weekly Downloads:** ${formatCount(weeklyDownloads)}\n`;
+		}
+
+		md += "\n";
+
+		if (info.author) {
+			md += `**Author:** ${info.author}`;
+			if (info.author_email) md += ` <${info.author_email}>`;
+			md += "\n";
+		}
+
+		if (info.requires_python) md += `**Python:** ${info.requires_python}\n`;
+		if (info.home_page) md += `**Homepage:** ${info.home_page}\n`;
+
+		if (info.project_urls && Object.keys(info.project_urls).length > 0) {
+			md += "\n**Project URLs:**\n";
+			for (const [label, url] of Object.entries(info.project_urls)) {
+				md += `- ${label}: ${url}\n`;
+			}
+		}
+
+		if (info.keywords) md += `\n**Keywords:** ${info.keywords}\n`;
+
+		// Dependencies
+		if (pkg.requires_dist && pkg.requires_dist.length > 0) {
+			md += `\n## Dependencies\n\n`;
+			for (const dep of pkg.requires_dist) {
+				md += `- ${dep}\n`;
+			}
+		}
+
+		// README/Description
+		if (info.description) {
+			md += `\n---\n\n## Description\n\n${info.description}\n`;
+		}
+
+		const output = finalizeOutput(md);
+		return {
+			url,
+			finalUrl: url,
+			contentType: "text/markdown",
+			method: "pypi",
+			content: output.content,
+			fetchedAt,
+			truncated: output.truncated,
+			notes: ["Fetched via PyPI JSON API"],
+		};
+	} catch {}
+
+	return null;
+};

package/src/web/scrapers/rawg.ts

@@ -0,0 +1,124 @@
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, htmlToBasicMarkdown, loadPage } from "./types";
+
+interface RawgPlatformEntry {
+	platform?: {
+		name?: string;
+	};
+}
+
+interface RawgGenreEntry {
+	name?: string;
+}
+
+interface RawgGameResponse {
+	name?: string;
+	released?: string;
+	rating?: number;
+	platforms?: RawgPlatformEntry[];
+	genres?: RawgGenreEntry[];
+	description?: string;
+	description_raw?: string;
+	detail?: string;
+	error?: string;
+}
+
+export const handleRawg: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		if (!isRawgHostname(parsed.hostname)) return null;
+
+		const slug = extractGameSlug(parsed.pathname);
+		if (!slug) return null;
+
+		const fetchedAt = new Date().toISOString();
+		const apiUrl = `https://api.rawg.io/api/games/${encodeURIComponent(slug)}`;
+		const result = await loadPage(apiUrl, { timeout, signal, headers: { Accept: "application/json" } });
+
+		if (!result.ok) return null;
+
+		let game: RawgGameResponse;
+		try {
+			game = JSON.parse(result.content);
+		} catch {
+			return null;
+		}
+
+		if (requiresApiKey(game)) return null;
+
+		const title = game.name?.trim() || slug;
+		let md = `# ${title}\n\n`;
+
+		if (game.released) md += `**Released:** ${game.released}\n`;
+		if (typeof game.rating === "number" && !Number.isNaN(game.rating)) {
+			md += `**Rating:** ${game.rating.toFixed(2)} / 5\n`;
+		}
+
+		const platforms = collectNames(game.platforms?.map(entry => entry.platform?.name));
+		if (platforms.length) md += `**Platforms:** ${platforms.join(", ")}\n`;
+
+		const genres = collectNames(game.genres?.map(entry => entry.name));
+		if (genres.length) md += `**Genres:** ${genres.join(", ")}\n`;
+
+		md += `**RAWG:** https://rawg.io/games/${encodeURIComponent(slug)}\n`;
+		md += "\n";
+
+		const description = extractDescription(game);
+		if (description) {
+			md += `## Description\n\n${description}\n`;
+		}
+
+		const output = finalizeOutput(md);
+		return {
+			url,
+			finalUrl: url,
+			contentType: "text/markdown",
+			method: "rawg",
+			content: output.content,
+			fetchedAt,
+			truncated: output.truncated,
+			notes: ["Fetched via RAWG API"],
+		};
+	} catch {}
+
+	return null;
+};
+
+function isRawgHostname(hostname: string): boolean {
+	return hostname === "rawg.io" || hostname === "www.rawg.io";
+}
+
+function extractGameSlug(pathname: string): string | null {
+	const match = pathname.match(/^\/games\/([^/?#]+)/);
+	if (!match) return null;
+
+	const slug = decodeURIComponent(match[1]);
+	return slug ? slug.trim() : null;
+}
+
+function requiresApiKey(game: RawgGameResponse): boolean {
+	const detail = `${game.detail ?? ""} ${game.error ?? ""}`.toLowerCase();
+	return detail.includes("api key") || detail.includes("key is required") || detail.includes("apikey");
+}
+
+function extractDescription(game: RawgGameResponse): string | null {
+	if (game.description_raw) return game.description_raw.trim();
+	if (!game.description) return null;
+
+	const markdown = htmlToBasicMarkdown(game.description).trim();
+	return markdown || null;
+}
+
+function collectNames(values?: Array<string | undefined>): string[] {
+	if (!values?.length) return [];
+	const names = new Set<string>();
+	for (const value of values) {
+		const trimmed = value?.trim();
+		if (trimmed) names.add(trimmed);
+	}
+	return Array.from(names);
+}