@nghyane/arcane 0.1.0

package/src/web/scrapers/metacpan.ts

@@ -0,0 +1,253 @@
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, loadPage } from "./types";
+
+interface ModuleResponse {
+	name: string;
+	version: string;
+	abstract?: string;
+	author: string;
+	distribution: string;
+	release: string;
+	path: string;
+	pod?: string;
+}
+
+interface ReleaseResponse {
+	name: string;
+	version: string;
+	abstract?: string;
+	author: string;
+	distribution: string;
+	license?: string[];
+	stat?: { mtime: number };
+	download_url?: string;
+	dependency?: Array<{
+		module: string;
+		version: string;
+		phase: string;
+		relationship: string;
+	}>;
+	metadata?: {
+		resources?: {
+			repository?: { url?: string; web?: string };
+			homepage?: string;
+			bugtracker?: { web?: string };
+		};
+	};
+}
+
+/**
+ * Handle MetaCPAN URLs via fastapi.metacpan.org
+ */
+export const handleMetaCPAN: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		if (parsed.hostname !== "metacpan.org" && parsed.hostname !== "www.metacpan.org") return null;
+
+		const fetchedAt = new Date().toISOString();
+
+		// Match /pod/Module::Name pattern
+		const podMatch = parsed.pathname.match(/^\/pod\/(.+?)(?:\/|$)/);
+		if (podMatch) {
+			const moduleName = decodeURIComponent(podMatch[1]);
+			return await fetchModule(url, moduleName, timeout, fetchedAt, signal);
+		}
+
+		// Match /release/AUTHOR/Distribution pattern
+		const releaseMatch = parsed.pathname.match(/^\/release\/([^/]+)\/([^/]+)/);
+		if (releaseMatch) {
+			const distribution = decodeURIComponent(releaseMatch[2]);
+			return await fetchRelease(url, distribution, timeout, fetchedAt, signal);
+		}
+
+		// Match /release/Distribution pattern (without author)
+		const simpleReleaseMatch = parsed.pathname.match(/^\/release\/([^/]+)$/);
+		if (simpleReleaseMatch) {
+			const distribution = decodeURIComponent(simpleReleaseMatch[1]);
+			return await fetchRelease(url, distribution, timeout, fetchedAt, signal);
+		}
+
+		return null;
+	} catch {}
+
+	return null;
+};
+
+async function fetchModule(
+	url: string,
+	moduleName: string,
+	timeout: number,
+	fetchedAt: string,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> {
+	const apiUrl = `https://fastapi.metacpan.org/v1/module/${moduleName}`;
+	const result = await loadPage(apiUrl, { timeout, signal });
+
+	if (!result.ok) return null;
+
+	let module: ModuleResponse;
+	try {
+		module = JSON.parse(result.content);
+	} catch {
+		return null;
+	}
+
+	// Fetch additional release info for dependencies and metadata
+	const releaseUrl = `https://fastapi.metacpan.org/v1/release/${module.distribution}`;
+	const releaseResult = await loadPage(releaseUrl, { timeout: Math.min(timeout, 5), signal });
+
+	let release: ReleaseResponse | null = null;
+	if (releaseResult.ok) {
+		try {
+			release = JSON.parse(releaseResult.content);
+		} catch {}
+	}
+
+	const md = formatModuleMarkdown(module, release);
+	const output = finalizeOutput(md);
+
+	return {
+		url,
+		finalUrl: url,
+		contentType: "text/markdown",
+		method: "metacpan",
+		content: output.content,
+		fetchedAt,
+		truncated: output.truncated,
+		notes: ["Fetched via MetaCPAN API"],
+	};
+}
+
+async function fetchRelease(
+	url: string,
+	distribution: string,
+	timeout: number,
+	fetchedAt: string,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> {
+	const apiUrl = `https://fastapi.metacpan.org/v1/release/${distribution}`;
+	const result = await loadPage(apiUrl, { timeout, signal });
+
+	if (!result.ok) return null;
+
+	let release: ReleaseResponse;
+	try {
+		release = JSON.parse(result.content);
+	} catch {
+		return null;
+	}
+
+	const md = formatReleaseMarkdown(release);
+	const output = finalizeOutput(md);
+
+	return {
+		url,
+		finalUrl: url,
+		contentType: "text/markdown",
+		method: "metacpan",
+		content: output.content,
+		fetchedAt,
+		truncated: output.truncated,
+		notes: ["Fetched via MetaCPAN API"],
+	};
+}
+
+function formatModuleMarkdown(module: ModuleResponse, release: ReleaseResponse | null): string {
+	let md = `# ${module.name}\n\n`;
+	if (module.abstract) md += `${module.abstract}\n\n`;
+
+	md += `**Version:** ${module.version}`;
+	md += ` · **Distribution:** ${module.distribution}`;
+	md += ` · **Author:** [${module.author}](https://metacpan.org/author/${module.author})\n`;
+
+	if (release) {
+		if (release.license?.length) {
+			md += `**License:** ${release.license.join(", ")}\n`;
+		}
+
+		const resources = release.metadata?.resources;
+		if (resources?.repository?.web || resources?.repository?.url) {
+			const repoUrl = resources.repository.web || resources.repository.url;
+			md += `**Repository:** ${repoUrl}\n`;
+		}
+		if (resources?.homepage) {
+			md += `**Homepage:** ${resources.homepage}\n`;
+		}
+		if (resources?.bugtracker?.web) {
+			md += `**Issues:** ${resources.bugtracker.web}\n`;
+		}
+
+		// Show runtime dependencies
+		const runtimeDeps = release.dependency?.filter(
+			d => d.phase === "runtime" && d.relationship === "requires" && d.module !== "perl",
+		);
+		if (runtimeDeps?.length) {
+			md += `\n## Dependencies\n\n`;
+			for (const dep of runtimeDeps.slice(0, 20)) {
+				md += `- **${dep.module}**`;
+				if (dep.version && dep.version !== "0") md += ` >= ${dep.version}`;
+				md += "\n";
+			}
+			if (runtimeDeps.length > 20) {
+				md += `\n*...and ${runtimeDeps.length - 20} more*\n`;
+			}
+		}
+	}
+
+	md += `\n## Installation\n\n\`\`\`bash\ncpanm ${module.name}\n\`\`\`\n`;
+
+	return md;
+}
+
+function formatReleaseMarkdown(release: ReleaseResponse): string {
+	let md = `# ${release.distribution}\n\n`;
+	if (release.abstract) md += `${release.abstract}\n\n`;
+
+	md += `**Version:** ${release.version}`;
+	md += ` · **Author:** [${release.author}](https://metacpan.org/author/${release.author})\n`;
+
+	if (release.license?.length) {
+		md += `**License:** ${release.license.join(", ")}\n`;
+	}
+
+	if (release.stat?.mtime) {
+		const date = new Date(release.stat.mtime * 1000).toISOString().split("T")[0];
+		md += `**Released:** ${date}\n`;
+	}
+
+	const resources = release.metadata?.resources;
+	if (resources?.repository?.web || resources?.repository?.url) {
+		const repoUrl = resources.repository.web || resources.repository.url;
+		md += `**Repository:** ${repoUrl}\n`;
+	}
+	if (resources?.homepage) {
+		md += `**Homepage:** ${resources.homepage}\n`;
+	}
+	if (resources?.bugtracker?.web) {
+		md += `**Issues:** ${resources.bugtracker.web}\n`;
+	}
+
+	// Show runtime dependencies
+	const runtimeDeps = release.dependency?.filter(
+		d => d.phase === "runtime" && d.relationship === "requires" && d.module !== "perl",
+	);
+	if (runtimeDeps?.length) {
+		md += `\n## Dependencies\n\n`;
+		for (const dep of runtimeDeps.slice(0, 20)) {
+			md += `- **${dep.module}**`;
+			if (dep.version && dep.version !== "0") md += ` >= ${dep.version}`;
+			md += "\n";
+		}
+		if (runtimeDeps.length > 20) {
+			md += `\n*...and ${runtimeDeps.length - 20} more*\n`;
+		}
+	}
+
+	md += `\n## Installation\n\n\`\`\`bash\ncpanm ${release.distribution}\n\`\`\`\n`;
+
+	return md;
+}

package/src/web/scrapers/musicbrainz.ts

@@ -0,0 +1,272 @@
+/**
+ * MusicBrainz URL handler for artists, releases, and recordings
+ */
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, loadPage } from "./types";
+
+type MusicBrainzEntity = "artist" | "release" | "recording";
+
+interface MusicBrainzLifeSpan {
+	begin?: string;
+	end?: string;
+	ended?: boolean;
+}
+
+interface MusicBrainzArtist {
+	id: string;
+	name: string;
+	type?: string;
+	country?: string;
+	"life-span"?: MusicBrainzLifeSpan;
+}
+
+interface MusicBrainzArtistCredit {
+	name?: string;
+	artist?: {
+		id?: string;
+		name: string;
+	};
+}
+
+interface MusicBrainzRecording {
+	id: string;
+	title: string;
+	length?: number;
+	"artist-credit"?: MusicBrainzArtistCredit[];
+}
+
+interface MusicBrainzTrack {
+	id?: string;
+	title?: string;
+	number?: string;
+	position?: number;
+	length?: number;
+	recording?: {
+		title?: string;
+		length?: number;
+	};
+}
+
+interface MusicBrainzMedium {
+	position?: number;
+	format?: string;
+	"track-count"?: number;
+	tracks?: MusicBrainzTrack[];
+}
+
+interface MusicBrainzRelease {
+	id: string;
+	title: string;
+	"track-count"?: number;
+	media?: MusicBrainzMedium[];
+}
+
+const MUSICBRAINZ_HOSTS = new Set(["musicbrainz.org", "www.musicbrainz.org"]);
+const USER_AGENT = "arc-web-fetch/1.0 (https://github.com/anthropics)";
+const MAX_TRACKS = 50;
+
+function parseEntity(url: URL): { entity: MusicBrainzEntity; mbid: string } | null {
+	if (!MUSICBRAINZ_HOSTS.has(url.hostname)) return null;
+
+	const parts = url.pathname.split("/").filter(Boolean);
+	if (parts.length < 2) return null;
+
+	const entity = parts[0] as MusicBrainzEntity;
+	if (entity !== "artist" && entity !== "release" && entity !== "recording") return null;
+
+	const mbid = parts[1];
+	if (!/^[0-9a-fA-F-]{36}$/.test(mbid)) return null;
+
+	return { entity, mbid };
+}
+
+async function fetchJson<T>(apiUrl: string, timeout: number, signal?: AbortSignal): Promise<T | null> {
+	const result = await loadPage(apiUrl, {
+		timeout,
+		signal,
+		headers: {
+			"User-Agent": USER_AGENT,
+			Accept: "application/json",
+		},
+	});
+
+	if (!result.ok) return null;
+
+	try {
+		return JSON.parse(result.content) as T;
+	} catch {
+		return null;
+	}
+}
+
+function formatLifeSpan(life: MusicBrainzLifeSpan | undefined): string | null {
+	if (!life) return null;
+
+	const begin = life.begin?.trim();
+	const end = life.end?.trim();
+
+	if (begin && end) return `${begin} - ${end}`;
+	if (begin && !end) return `${begin} - ${life.ended ? "ended" : "present"}`;
+	if (!begin && end) return `? - ${end}`;
+	if (life.ended !== undefined) return life.ended ? "ended" : "present";
+
+	return null;
+}
+
+function formatDurationMs(lengthMs: number | undefined): string | null {
+	if (!lengthMs || lengthMs <= 0) return null;
+
+	const totalSeconds = Math.round(lengthMs / 1000);
+	const hours = Math.floor(totalSeconds / 3600);
+	const minutes = Math.floor((totalSeconds % 3600) / 60);
+	const seconds = totalSeconds % 60;
+
+	if (hours > 0) {
+		return `${hours}:${minutes.toString().padStart(2, "0")}:${seconds.toString().padStart(2, "0")}`;
+	}
+
+	return `${minutes}:${seconds.toString().padStart(2, "0")}`;
+}
+
+function formatArtistCredits(credits: MusicBrainzArtistCredit[] | undefined): string | null {
+	if (!credits?.length) return null;
+
+	const names = credits
+		.map(credit => credit.name || credit.artist?.name)
+		.filter((name): name is string => Boolean(name));
+
+	if (!names.length) return null;
+	return names.join(", ");
+}
+
+function formatTrack(track: MusicBrainzTrack): string {
+	const title = track.title || track.recording?.title || "Untitled";
+	const duration = formatDurationMs(track.length ?? track.recording?.length);
+	const number = track.number || (track.position ? String(track.position) : null);
+
+	const prefix = number ? `${number}. ` : "- ";
+	let line = `${prefix}${title}`;
+	if (duration) line += ` (${duration})`;
+	return line;
+}
+
+function buildMediumLabel(medium: MusicBrainzMedium, includePosition: boolean): string | null {
+	const parts: string[] = [];
+	if (includePosition && medium.position) parts.push(`Disc ${medium.position}`);
+	if (medium.format) parts.push(medium.format);
+	return parts.length ? parts.join(" - ") : null;
+}
+
+function buildArtistMarkdown(artist: MusicBrainzArtist): string {
+	let md = `# ${artist.name}\n\n`;
+	const meta: string[] = [];
+
+	if (artist.type) meta.push(`**Type**: ${artist.type}`);
+	if (artist.country) meta.push(`**Country**: ${artist.country}`);
+
+	const lifeSpan = formatLifeSpan(artist["life-span"]);
+	if (lifeSpan) meta.push(`**Life Span**: ${lifeSpan}`);
+
+	if (meta.length) md += `${meta.join("\n")}\n`;
+
+	return md;
+}
+
+function buildReleaseMarkdown(release: MusicBrainzRelease): string {
+	let md = `# ${release.title}\n\n`;
+
+	const media = release.media ?? [];
+	const totalTracks =
+		release["track-count"] ??
+		media.reduce((sum, medium) => sum + (medium["track-count"] ?? medium.tracks?.length ?? 0), 0);
+
+	if (totalTracks) {
+		md += `**Tracks**: ${totalTracks}\n\n`;
+	}
+
+	if (media.length) {
+		md += "## Tracks\n\n";
+		const includePosition = media.length > 1;
+
+		for (const medium of media) {
+			const label = buildMediumLabel(medium, includePosition);
+			if (label) md += `### ${label}\n\n`;
+
+			const tracks = medium.tracks ?? [];
+			if (tracks.length) {
+				const lines = tracks.slice(0, MAX_TRACKS).map(formatTrack).join("\n");
+				md += `${lines}\n\n`;
+
+				if (tracks.length > MAX_TRACKS) {
+					md += `_Showing first ${MAX_TRACKS} of ${tracks.length} tracks._\n\n`;
+				}
+			} else if (medium["track-count"]) {
+				md += `- ${medium["track-count"]} tracks (details unavailable)\n\n`;
+			}
+		}
+	}
+
+	return md;
+}
+
+function buildRecordingMarkdown(recording: MusicBrainzRecording): string {
+	let md = `# ${recording.title}\n\n`;
+	const meta: string[] = [];
+
+	const artists = formatArtistCredits(recording["artist-credit"]);
+	if (artists) meta.push(`**Artists**: ${artists}`);
+
+	const length = formatDurationMs(recording.length);
+	if (length) meta.push(`**Length**: ${length}`);
+
+	if (meta.length) md += `${meta.join("\n")}\n`;
+
+	return md;
+}
+
+export const handleMusicBrainz: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		const parsedEntity = parseEntity(parsed);
+		if (!parsedEntity) return null;
+
+		const { entity, mbid } = parsedEntity;
+		const fetchedAt = new Date().toISOString();
+		let md = "";
+
+		if (entity === "artist") {
+			const apiUrl = `https://musicbrainz.org/ws/2/artist/${mbid}?fmt=json&inc=url-rels`;
+			const artist = await fetchJson<MusicBrainzArtist>(apiUrl, timeout, signal);
+			if (!artist) return null;
+			md = buildArtistMarkdown(artist);
+		} else if (entity === "release") {
+			const apiUrl = `https://musicbrainz.org/ws/2/release/${mbid}?fmt=json&inc=recordings`;
+			const release = await fetchJson<MusicBrainzRelease>(apiUrl, timeout, signal);
+			if (!release) return null;
+			md = buildReleaseMarkdown(release);
+		} else {
+			const apiUrl = `https://musicbrainz.org/ws/2/recording/${mbid}?fmt=json`;
+			const recording = await fetchJson<MusicBrainzRecording>(apiUrl, timeout, signal);
+			if (!recording) return null;
+			md = buildRecordingMarkdown(recording);
+		}
+
+		const output = finalizeOutput(md);
+		return {
+			url,
+			finalUrl: url,
+			contentType: "text/markdown",
+			method: "musicbrainz-api",
+			content: output.content,
+			fetchedAt,
+			truncated: output.truncated,
+			notes: ["Fetched via MusicBrainz API"],
+		};
+	} catch {}
+
+	return null;
+};

package/src/web/scrapers/npm.ts

@@ -0,0 +1,114 @@
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, formatCount, loadPage } from "./types";
+
+/**
+ * Handle npm URLs via registry API
+ */
+export const handleNpm: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		if (parsed.hostname !== "www.npmjs.com" && parsed.hostname !== "npmjs.com") return null;
+
+		// Extract package name from /package/[scope/]name
+		const match = parsed.pathname.match(/^\/package\/(.+?)(?:\/|$)/);
+		if (!match) return null;
+
+		let packageName = decodeURIComponent(match[1]);
+		// Handle scoped packages: /package/@scope/name
+		if (packageName.startsWith("@")) {
+			const scopeMatch = parsed.pathname.match(/^\/package\/(@[^/]+\/[^/]+)/);
+			if (scopeMatch) packageName = decodeURIComponent(scopeMatch[1]);
+		}
+
+		const fetchedAt = new Date().toISOString();
+
+		// Fetch from npm registry - use /latest endpoint for smaller response
+		const latestUrl = `https://registry.npmjs.org/${packageName}/latest`;
+		const downloadsUrl = `https://api.npmjs.org/downloads/point/last-week/${encodeURIComponent(packageName)}`;
+
+		// Fetch package info and download stats in parallel
+		const [result, downloadsResult] = await Promise.all([
+			loadPage(latestUrl, { timeout, signal }),
+			loadPage(downloadsUrl, { timeout: Math.min(timeout, 5), signal }),
+		]);
+
+		if (!result.ok) return null;
+
+		// Parse download stats
+		let weeklyDownloads: number | null = null;
+		if (downloadsResult.ok) {
+			try {
+				const dlData = JSON.parse(downloadsResult.content) as { downloads?: number };
+				weeklyDownloads = dlData.downloads ?? null;
+			} catch {}
+		}
+
+		let pkg: {
+			name: string;
+			version: string;
+			description?: string;
+			license?: string | { type: string };
+			homepage?: string;
+			repository?: { url: string } | string;
+			keywords?: string[];
+			maintainers?: Array<{ name: string }>;
+			dependencies?: Record<string, string>;
+			readme?: string;
+		};
+
+		try {
+			pkg = JSON.parse(result.content);
+		} catch {
+			return null; // JSON parse failed (truncated response)
+		}
+
+		let md = `# ${pkg.name}\n\n`;
+		if (pkg.description) md += `${pkg.description}\n\n`;
+
+		md += `**Latest:** ${pkg.version || "unknown"}`;
+		if (pkg.license) {
+			const license = typeof pkg.license === "string" ? pkg.license : (pkg.license.type ?? String(pkg.license));
+			md += ` · **License:** ${license}`;
+		}
+		md += "\n";
+		if (weeklyDownloads !== null) {
+			md += `**Weekly Downloads:** ${formatCount(weeklyDownloads)}\n`;
+		}
+		md += "\n";
+
+		if (pkg.homepage) md += `**Homepage:** ${pkg.homepage}\n`;
+		const repoUrl = typeof pkg.repository === "string" ? pkg.repository : pkg.repository?.url;
+		if (repoUrl) md += `**Repository:** ${repoUrl.replace(/^git\+/, "").replace(/\.git$/, "")}\n`;
+		if (pkg.keywords?.length) md += `**Keywords:** ${pkg.keywords.join(", ")}\n`;
+		if (pkg.maintainers?.length) md += `**Maintainers:** ${pkg.maintainers.map(m => m.name).join(", ")}\n`;
+
+		if (pkg.dependencies && Object.keys(pkg.dependencies).length > 0) {
+			md += `\n## Dependencies\n\n`;
+			for (const [dep, version] of Object.entries(pkg.dependencies)) {
+				md += `- ${dep}: ${version}\n`;
+			}
+		}
+
+		if (pkg.readme) {
+			md += `\n---\n\n## README\n\n${pkg.readme}\n`;
+		}
+
+		const output = finalizeOutput(md);
+		return {
+			url,
+			finalUrl: url,
+			contentType: "text/markdown",
+			method: "npm",
+			content: output.content,
+			fetchedAt,
+			truncated: output.truncated,
+			notes: ["Fetched via npm registry"],
+		};
+	} catch {}
+
+	return null;
+};