@nghyane/arcane 0.1.0

package/src/web/scrapers/crossref.ts

@@ -0,0 +1,149 @@
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, htmlToBasicMarkdown, loadPage } from "./types";
+
+interface CrossrefAuthor {
+	given?: string;
+	family?: string;
+	name?: string;
+}
+
+interface CrossrefDate {
+	"date-parts"?: number[][];
+}
+
+interface CrossrefMessage {
+	title?: string[];
+	author?: CrossrefAuthor[];
+	"container-title"?: string[];
+	"short-container-title"?: string[];
+	publisher?: string;
+	published?: CrossrefDate;
+	"published-print"?: CrossrefDate;
+	"published-online"?: CrossrefDate;
+	issued?: CrossrefDate;
+	created?: CrossrefDate;
+	DOI?: string;
+	abstract?: string;
+	type?: string;
+}
+
+interface CrossrefResponse {
+	message?: CrossrefMessage;
+}
+
+const DOI_HOSTS = new Set(["doi.org", "dx.doi.org", "www.doi.org"]);
+
+function extractDoi(pathname: string): string | null {
+	const raw = pathname.replace(/^\/+/, "");
+	if (!raw) return null;
+	return decodeURIComponent(raw);
+}
+
+function formatAuthors(authors?: CrossrefAuthor[]): string | null {
+	if (!authors || authors.length === 0) return null;
+	const names = authors
+		.map(author => {
+			if (author.name) return author.name;
+			const parts = [author.given, author.family].filter(Boolean);
+			return parts.length > 0 ? parts.join(" ") : null;
+		})
+		.filter((name): name is string => Boolean(name));
+	if (names.length === 0) return null;
+	return names.join(", ");
+}
+
+function formatDate(date?: CrossrefDate): string | null {
+	const parts = date?.["date-parts"]?.[0];
+	if (!parts || parts.length === 0) return null;
+	const [year, month, day] = parts;
+	if (!year) return null;
+	const formatted = [
+		String(year),
+		month ? String(month).padStart(2, "0") : "",
+		day ? String(day).padStart(2, "0") : "",
+	].filter(Boolean);
+	return formatted.join("-");
+}
+
+function formatAbstract(abstract?: string): string | null {
+	if (!abstract) return null;
+	const normalized = abstract.replace(/<\/?jats:p[^>]*>/g, match => (match.startsWith("</") ? "</p>" : "<p>"));
+	const markdown = htmlToBasicMarkdown(normalized);
+	return markdown.trim().length > 0 ? markdown : null;
+}
+
+export const handleCrossref: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		if (!DOI_HOSTS.has(parsed.hostname.toLowerCase())) return null;
+
+		const doi = extractDoi(parsed.pathname);
+		if (!doi) return null;
+
+		const fetchedAt = new Date().toISOString();
+		const apiUrl = `https://api.crossref.org/works/${encodeURIComponent(doi)}`;
+		const result = await loadPage(apiUrl, {
+			timeout,
+			signal,
+			headers: {
+				Accept: "application/json",
+			},
+		});
+
+		if (!result.ok) return null;
+
+		let data: CrossrefResponse;
+		try {
+			data = JSON.parse(result.content);
+		} catch {
+			return null;
+		}
+
+		const message = data.message;
+		if (!message) return null;
+
+		const title = message.title?.[0]?.trim() || "CrossRef Record";
+		const authors = formatAuthors(message.author);
+		const journal = message["container-title"]?.[0] || message["short-container-title"]?.[0];
+		const publisher = message.publisher;
+		const published =
+			formatDate(message.published) ||
+			formatDate(message["published-print"]) ||
+			formatDate(message["published-online"]) ||
+			formatDate(message.issued) ||
+			formatDate(message.created);
+		const doiValue = message.DOI || doi;
+		const abstract = formatAbstract(message.abstract);
+		const type = message.type?.replace(/-/g, " ");
+
+		let md = `# ${title}\n\n`;
+		if (authors) md += `**Authors:** ${authors}\n`;
+		if (journal) md += `**Journal:** ${journal}\n`;
+		if (publisher) md += `**Publisher:** ${publisher}\n`;
+		if (published) md += `**Published:** ${published}\n`;
+		md += `**DOI:** ${doiValue}\n`;
+		if (type) md += `**Type:** ${type}\n`;
+		md += "\n---\n\n";
+		md += "## Abstract\n\n";
+		md += abstract || "No abstract available.";
+		md += "\n";
+
+		const output = finalizeOutput(md);
+		return {
+			url,
+			finalUrl: url,
+			contentType: "text/markdown",
+			method: "crossref",
+			content: output.content,
+			fetchedAt,
+			truncated: output.truncated,
+			notes: ["Fetched via CrossRef API"],
+		};
+	} catch {}
+
+	return null;
+};

package/src/web/scrapers/devto.ts

@@ -0,0 +1,177 @@
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, formatCount, htmlToBasicMarkdown, loadPage } from "./types";
+
+interface DevToArticle {
+	title: string;
+	description?: string;
+	published_at: string;
+	published_timestamp?: string;
+	tags: string[];
+	tag_list?: string[];
+	reading_time_minutes?: number;
+	public_reactions_count?: number;
+	positive_reactions_count?: number;
+	comments_count?: number;
+	user?: {
+		name: string;
+		username: string;
+	};
+	body_markdown?: string;
+	body_html?: string;
+}
+
+/**
+ * Handle dev.to URLs via API
+ */
+export const handleDevTo: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		if (parsed.hostname !== "dev.to") return null;
+
+		const fetchedAt = new Date().toISOString();
+		const notes: string[] = [];
+
+		// Parse URL patterns
+		const pathParts = parsed.pathname.split("/").filter(Boolean);
+
+		// Tag page: /t/{tag}
+		if (pathParts[0] === "t" && pathParts.length >= 2) {
+			const tag = pathParts[1];
+			const apiUrl = `https://dev.to/api/articles?tag=${encodeURIComponent(tag)}&per_page=20`;
+
+			const result = await loadPage(apiUrl, { timeout, signal });
+			if (!result.ok) return null;
+
+			const articles = JSON.parse(result.content) as DevToArticle[];
+			if (!articles?.length) return null;
+
+			let md = `# dev.to/t/${tag}\n\n`;
+			md += `## Recent Articles (${articles.length})\n\n`;
+
+			for (const article of articles) {
+				const tags = article.tag_list || article.tags || [];
+				const reactions = article.positive_reactions_count ?? article.public_reactions_count ?? 0;
+				const readTime = article.reading_time_minutes ? ` · ${article.reading_time_minutes} min read` : "";
+				const reactStr = reactions > 0 ? ` · ${formatCount(reactions)} reactions` : "";
+
+				md += `### ${article.title}\n\n`;
+				md += `by **${article.user?.name || "Unknown"}** (@${article.user?.username || "unknown"})`;
+				md += `${readTime}${reactStr}\n`;
+				md += `*${new Date(article.published_at || article.published_timestamp || "").toISOString().split("T")[0]}*\n`;
+				if (tags.length > 0) md += `Tags: ${tags.map(t => `#${t}`).join(", ")}\n`;
+				if (article.description) md += `\n${article.description}\n`;
+				md += `\n---\n\n`;
+			}
+
+			notes.push("Fetched via dev.to API");
+			const output = finalizeOutput(md);
+			return {
+				url,
+				finalUrl: url,
+				contentType: "text/markdown",
+				method: "devto",
+				content: output.content,
+				fetchedAt,
+				truncated: output.truncated,
+				notes,
+			};
+		}
+
+		// User profile: /{username} (only if single path segment)
+		if (pathParts.length === 1) {
+			const username = pathParts[0];
+			const apiUrl = `https://dev.to/api/articles?username=${encodeURIComponent(username)}&per_page=20`;
+
+			const result = await loadPage(apiUrl, { timeout, signal });
+			if (!result.ok) return null;
+
+			const articles = JSON.parse(result.content) as DevToArticle[];
+			if (!articles?.length) return null;
+
+			let md = `# dev.to/${username}\n\n`;
+			md += `## Recent Articles (${articles.length})\n\n`;
+
+			for (const article of articles) {
+				const tags = article.tag_list || article.tags || [];
+				const reactions = article.positive_reactions_count ?? article.public_reactions_count ?? 0;
+				const readTime = article.reading_time_minutes ? ` · ${article.reading_time_minutes} min read` : "";
+				const reactStr = reactions > 0 ? ` · ${formatCount(reactions)} reactions` : "";
+
+				md += `### ${article.title}\n\n`;
+				md += `${readTime.substring(3)}${reactStr}\n`;
+				md += `*${new Date(article.published_at || article.published_timestamp || "").toISOString().split("T")[0]}*\n`;
+				if (tags.length > 0) md += `Tags: ${tags.map(t => `#${t}`).join(", ")}\n`;
+				if (article.description) md += `\n${article.description}\n`;
+				md += `\n---\n\n`;
+			}
+
+			notes.push("Fetched via dev.to API");
+			const output = finalizeOutput(md);
+			return {
+				url,
+				finalUrl: url,
+				contentType: "text/markdown",
+				method: "devto",
+				content: output.content,
+				fetchedAt,
+				truncated: output.truncated,
+				notes,
+			};
+		}
+
+		// Article: /{username}/{slug}
+		if (pathParts.length >= 2) {
+			const username = pathParts[0];
+			const slug = pathParts[1];
+			const apiUrl = `https://dev.to/api/articles/${encodeURIComponent(username)}/${encodeURIComponent(slug)}`;
+
+			const result = await loadPage(apiUrl, { timeout, signal });
+			if (!result.ok) return null;
+
+			const article = JSON.parse(result.content) as DevToArticle;
+			if (!article?.title) return null;
+
+			const tags = article.tag_list || article.tags || [];
+			const reactions = article.positive_reactions_count ?? article.public_reactions_count ?? 0;
+			const comments = article.comments_count ?? 0;
+			const readTime = article.reading_time_minutes ?? 0;
+
+			let md = `# ${article.title}\n\n`;
+			md += `**Author:** ${article.user?.name || "Unknown"} (@${article.user?.username || username})\n`;
+			md += `**Published:** ${new Date(article.published_at || article.published_timestamp || "").toISOString().split("T")[0]}\n`;
+			if (readTime > 0) md += `**Reading time:** ${readTime} min\n`;
+			if (reactions > 0) md += `**Reactions:** ${formatCount(reactions)}\n`;
+			if (comments > 0) md += `**Comments:** ${formatCount(comments)}\n`;
+			if (tags.length > 0) md += `**Tags:** ${tags.map(t => `#${t}`).join(", ")}\n`;
+			md += `\n---\n\n`;
+
+			// Prefer body_markdown over body_html
+			if (article.body_markdown) {
+				md += article.body_markdown;
+			} else if (article.body_html) {
+				md += htmlToBasicMarkdown(article.body_html);
+			}
+
+			notes.push("Fetched via dev.to API");
+			const output = finalizeOutput(md);
+			return {
+				url,
+				finalUrl: url,
+				contentType: "text/markdown",
+				method: "devto",
+				content: output.content,
+				fetchedAt,
+				truncated: output.truncated,
+				notes,
+			};
+		}
+
+		return null;
+	} catch {
+		return null;
+	}
+};

package/src/web/scrapers/discogs.ts

@@ -0,0 +1,307 @@
+/**
+ * Discogs URL handler for music releases and masters
+ *
+ * Uses the Discogs API to extract structured metadata about releases.
+ * API docs: https://www.discogs.com/developers
+ */
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, loadPage } from "./types";
+
+interface DiscogsArtist {
+	name: string;
+	anv?: string; // artist name variation
+	role?: string;
+	join?: string;
+}
+
+interface DiscogsTrack {
+	position: string;
+	title: string;
+	duration?: string;
+	artists?: DiscogsArtist[];
+	extraartists?: DiscogsArtist[];
+}
+
+interface DiscogsLabel {
+	name: string;
+	catno?: string; // catalog number
+}
+
+interface DiscogsFormat {
+	name: string;
+	qty?: string;
+	descriptions?: string[];
+}
+
+interface DiscogsRelease {
+	id: number;
+	title: string;
+	artists?: DiscogsArtist[];
+	year?: number;
+	released?: string;
+	country?: string;
+	genres?: string[];
+	styles?: string[];
+	labels?: DiscogsLabel[];
+	formats?: DiscogsFormat[];
+	tracklist?: DiscogsTrack[];
+	extraartists?: DiscogsArtist[];
+	notes?: string;
+	uri?: string;
+	master_id?: number;
+	master_url?: string;
+}
+
+interface DiscogsMaster {
+	id: number;
+	title: string;
+	artists?: DiscogsArtist[];
+	year?: number;
+	genres?: string[];
+	styles?: string[];
+	tracklist?: DiscogsTrack[];
+	notes?: string;
+	uri?: string;
+	main_release?: number;
+	main_release_url?: string;
+	versions_url?: string;
+	num_for_sale?: number;
+	lowest_price?: number;
+}
+
+/**
+ * Format artist names, handling name variations
+ */
+function formatArtists(artists: DiscogsArtist[] | undefined): string {
+	if (!artists?.length) return "Unknown Artist";
+	return artists
+		.map(a => {
+			const name = a.anv || a.name;
+			const join = a.join || ", ";
+			return name + (a.join ? ` ${join} ` : "");
+		})
+		.join("")
+		.replace(/[,&]\s*$/, "")
+		.trim();
+}
+
+/**
+ * Format a single track
+ */
+function formatTrack(track: DiscogsTrack): string {
+	let line = track.position ? `${track.position}. ` : "- ";
+	line += track.title;
+	if (track.duration) line += ` (${track.duration})`;
+	if (track.artists?.length) {
+		line += ` - ${formatArtists(track.artists)}`;
+	}
+	return line;
+}
+
+/**
+ * Format credits/extraartists grouped by role
+ */
+function formatCredits(extraartists: DiscogsArtist[] | undefined): string {
+	if (!extraartists?.length) return "";
+
+	const byRole: Record<string, string[]> = {};
+	for (const artist of extraartists) {
+		const role = artist.role || "Other";
+		if (!byRole[role]) byRole[role] = [];
+		byRole[role].push(artist.anv || artist.name);
+	}
+
+	const lines: string[] = [];
+	for (const [role, names] of Object.entries(byRole)) {
+		lines.push(`- **${role}**: ${names.join(", ")}`);
+	}
+	return lines.join("\n");
+}
+
+/**
+ * Format formats (e.g., "2×LP, Album, Reissue")
+ */
+function formatFormats(formats: DiscogsFormat[] | undefined): string {
+	if (!formats?.length) return "";
+
+	return formats
+		.map(f => {
+			const parts: string[] = [];
+			if (f.qty && parseInt(f.qty, 10) > 1) parts.push(`${f.qty}×`);
+			parts.push(f.name);
+			if (f.descriptions?.length) parts.push(f.descriptions.join(", "));
+			return parts.join(" ");
+		})
+		.join(" + ");
+}
+
+/**
+ * Format labels with catalog numbers
+ */
+function formatLabels(labels: DiscogsLabel[] | undefined): string {
+	if (!labels?.length) return "";
+	return labels
+		.map(l => {
+			if (l.catno && l.catno !== "none") return `${l.name} (${l.catno})`;
+			return l.name;
+		})
+		.join(", ");
+}
+
+/**
+ * Build markdown for a release
+ */
+function buildReleaseMarkdown(release: DiscogsRelease): string {
+	const sections: string[] = [];
+
+	// Title with artist
+	const artist = formatArtists(release.artists);
+	sections.push(`# ${artist} - ${release.title}\n`);
+
+	// Metadata
+	const meta: string[] = [];
+	if (release.year) meta.push(`**Year**: ${release.year}`);
+	if (release.country) meta.push(`**Country**: ${release.country}`);
+
+	const format = formatFormats(release.formats);
+	if (format) meta.push(`**Format**: ${format}`);
+
+	const labels = formatLabels(release.labels);
+	if (labels) meta.push(`**Label**: ${labels}`);
+
+	if (release.genres?.length) meta.push(`**Genre**: ${release.genres.join(", ")}`);
+	if (release.styles?.length) meta.push(`**Style**: ${release.styles.join(", ")}`);
+
+	if (release.master_id) {
+		meta.push(`**Master Release**: [${release.master_id}](https://www.discogs.com/master/${release.master_id})`);
+	}
+
+	if (meta.length) sections.push(`${meta.join("\n")}\n`);
+
+	// Tracklist
+	if (release.tracklist?.length) {
+		sections.push("## Tracklist\n");
+		const tracks = release.tracklist.map(formatTrack);
+		sections.push(`${tracks.join("\n")}\n`);
+	}
+
+	// Credits
+	const credits = formatCredits(release.extraartists);
+	if (credits) {
+		sections.push("## Credits\n");
+		sections.push(`${credits}\n`);
+	}
+
+	// Notes
+	if (release.notes) {
+		sections.push("## Notes\n");
+		sections.push(`${release.notes}\n`);
+	}
+
+	return sections.join("\n");
+}
+
+/**
+ * Build markdown for a master release
+ */
+function buildMasterMarkdown(master: DiscogsMaster): string {
+	const sections: string[] = [];
+
+	// Title with artist
+	const artist = formatArtists(master.artists);
+	sections.push(`# ${artist} - ${master.title}\n`);
+	sections.push("*Master Release*\n");
+
+	// Metadata
+	const meta: string[] = [];
+	if (master.year) meta.push(`**Year**: ${master.year}`);
+	if (master.genres?.length) meta.push(`**Genre**: ${master.genres.join(", ")}`);
+	if (master.styles?.length) meta.push(`**Style**: ${master.styles.join(", ")}`);
+
+	if (master.main_release) {
+		meta.push(`**Main Release**: [${master.main_release}](https://www.discogs.com/release/${master.main_release})`);
+	}
+
+	if (master.num_for_sale !== undefined && master.num_for_sale > 0) {
+		meta.push(`**For Sale**: ${master.num_for_sale} copies`);
+		if (master.lowest_price !== undefined) {
+			meta.push(`**Lowest Price**: $${master.lowest_price.toFixed(2)}`);
+		}
+	}
+
+	if (meta.length) sections.push(`${meta.join("\n")}\n`);
+
+	// Tracklist
+	if (master.tracklist?.length) {
+		sections.push("## Tracklist\n");
+		const tracks = master.tracklist.map(formatTrack);
+		sections.push(`${tracks.join("\n")}\n`);
+	}
+
+	// Notes
+	if (master.notes) {
+		sections.push("## Notes\n");
+		sections.push(`${master.notes}\n`);
+	}
+
+	return sections.join("\n");
+}
+
+export const handleDiscogs: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		if (!parsed.hostname.includes("discogs.com")) return null;
+
+		// Match release or master URLs
+		// Patterns: /release/{id}, /master/{id}
+		// Also handles: /release/{id}-Artist-Title, /master/{id}-Artist-Title
+		const releaseMatch = parsed.pathname.match(/\/release\/(\d+)/);
+		const masterMatch = parsed.pathname.match(/\/master\/(\d+)/);
+
+		if (!releaseMatch && !masterMatch) return null;
+
+		const fetchedAt = new Date().toISOString();
+		const isRelease = !!releaseMatch;
+		const id = isRelease ? releaseMatch[1] : masterMatch![1];
+
+		const apiUrl = isRelease ? `https://api.discogs.com/releases/${id}` : `https://api.discogs.com/masters/${id}`;
+
+		const result = await loadPage(apiUrl, {
+			timeout,
+			signal,
+			headers: {
+				Accept: "application/json",
+				"User-Agent": "CodingAgent/1.0 +https://github.com/nghyane/arcane",
+			},
+		});
+
+		if (!result.ok) return null;
+
+		let md: string;
+		if (isRelease) {
+			const release = JSON.parse(result.content) as DiscogsRelease;
+			md = buildReleaseMarkdown(release);
+		} else {
+			const master = JSON.parse(result.content) as DiscogsMaster;
+			md = buildMasterMarkdown(master);
+		}
+
+		const output = finalizeOutput(md);
+		return {
+			url,
+			finalUrl: url,
+			contentType: "text/markdown",
+			method: "discogs",
+			content: output.content,
+			fetchedAt,
+			truncated: output.truncated,
+			notes: [`Fetched via Discogs API (${isRelease ? "release" : "master"})`],
+		};
+	} catch {}
+
+	return null;
+};